diff --git a/.githooks/_/install.sh b/.githooks/_/install.sh deleted file mode 100755 index a90c9cb65..000000000 --- a/.githooks/_/install.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -DIR="$(dirname "$0")/.." - -FLAG_FILE="$DIR/_/.setup" - -if [ ! -f "$FLAG_FILE" ]; then - echo "Linking Git Hooks 🐶..." - git config core.hooksPath "$DIR" - touch "$FLAG_FILE" -fi diff --git a/.githooks/commit-msg b/.githooks/commit-msg deleted file mode 100755 index 9745496f9..000000000 --- a/.githooks/commit-msg +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Regex for Conventional Commits (Plus Git Vernacular for Merges and Reverts) -CONVENTIONAL_COMMITS_REGEX="^((Merge[ a-z-]* branch.*)|(Revert*)|((build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test)(\(.*\))?!?: .*))" - -# Get the commit message -COMMIT_MSG=$(cat "$1") - -# Check if the commit message matches the Conventional Commits format -if [[ ! $COMMIT_MSG =~ $CONVENTIONAL_COMMITS_REGEX ]]; then - echo "❌ Error: Commit message does not follow the Conventional Commits format." - echo "" - echo "Expected format: (): " - echo "" - echo "✅ Examples:" - echo " feat(parser): add ability to parse arrays" - echo " fix(login): handle edge case with empty passwords" - echo " docs: update README with installation instructions" - echo "" - echo "Allowed types: build, chore, ci, docs, feat, fix, perf, refactor, revert, style, test" - exit 1 -fi - -# If the commit message is valid, allow the commit -exit 0 diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml new file mode 100644 index 000000000..8024b54dc --- /dev/null +++ b/.github/workflows/build-deploy-docs.yml @@ -0,0 +1,204 @@ +name: 🥘 Build & Deploy Docs HB + +on: + pull_request: + branches: + - main + paths: + # Trigger on changes to docs, mkdocs config, or the workflow itself + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/build-deploy-docs.yml" + push: + branches: + - main + paths: + # Trigger on changes to docs, mkdocs config, or the workflow itself + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/build-deploy-docs.yml" + + # Perform a release using a workflow dispatch + workflow_dispatch: + +defaults: + run: + shell: bash + +jobs: + # Run the build as part of PRs to confirm the site properly builds + check_build: + if: ${{ startsWith(github.ref, 'refs/pull/') }} + runs-on: ubuntu-22.04 + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v3 + + # Setup Python environment + - name: 🐍 Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' # Use a recent Python 3 version + + # Install Erlang OTP 27 using kerl + - name: Install Erlang OTP 27 + run: | + sudo apt-get update + sudo apt-get install -y build-essential autoconf libncurses5-dev libssl-dev + git clone https://github.com/kerl/kerl.git + ./kerl/kerl build 27.0 otp-27.0 + ./kerl/kerl install otp-27.0 ~/otp-27.0 + echo '. ~/otp-27.0/activate' >> ~/.bashrc + . ~/otp-27.0/activate + echo "Erlang version:" + erl -eval 'io:format("~s~n", [erlang:system_info(otp_release)]), halt().' + # Install system dependencies needed for HyperBEAM + - name: Install system dependencies + run: | + sudo apt-get update && sudo apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + pkg-config \ + ncurses-dev \ + libssl-dev \ + ca-certificates + # Debug step - display the region with syntax error + - name: Debug syntax error region + run: | + echo "Showing the region with syntax error in hb_message.erl:" + sed -n '1440,1460p' src/hb_message.erl || echo "File not found or cannot be read" + echo "Checking for syntax error fix files:" + find . -name "*.erl.fix" -o -name "hb_message.erl.*" | grep -v ".beam" || echo "No fix files found" + echo "Erlang version:" + . ~/otp-27.0/activate && erl -eval 'io:format("~s~n", [erlang:system_info(otp_release)]), halt().' + # Install rebar3 + - name: Install rebar3 + run: | + . ~/otp-27.0/activate + mkdir -p ~/.config/rebar3 + curl -O https://s3.amazonaws.com/rebar3/rebar3 && chmod +x rebar3 + sudo mv rebar3 /usr/local/bin/rebar3 + . ~/otp-27.0/activate && rebar3 --version + # Install Rust toolchain (needed for WASM components) + - name: Install Rust and Cargo + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + source "$HOME/.cargo/env" + # Setup Node.js + - name: ⎔ Setup Node + uses: actions/setup-node@v3 + with: + node-version: 22 # Or your preferred version + + # Install pip dependencies and cache them + - name: 📦 Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install mkdocs mkdocs-material mkdocs-git-revision-date-localized-plugin + - name: 🛠 Build Docs + run: | + . ~/otp-27.0/activate + SKIP_COMPILE=1 SKIP_EDOC=1 ./docs/build-all.sh -v + # Build and deploy the artifacts to Arweave via ArDrive + deploy: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-22.04 + # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. + # However, do NOT cancel in-progress runs as we want to allow these deployments to complete. + concurrency: + group: deploy + cancel-in-progress: false + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v3 + + # Setup Python environment + - name: 🐍 Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + # Install Erlang OTP 27 using kerl + - name: Install Erlang OTP 27 + run: | + sudo apt-get update + sudo apt-get install -y build-essential autoconf libncurses5-dev libssl-dev + git clone https://github.com/kerl/kerl.git + ./kerl/kerl build 27.0 otp-27.0 + ./kerl/kerl install otp-27.0 ~/otp-27.0 + echo '. ~/otp-27.0/activate' >> ~/.bashrc + . ~/otp-27.0/activate + echo "Erlang version:" + erl -eval 'io:format("~s~n", [erlang:system_info(otp_release)]), halt().' + # Install system dependencies needed for HyperBEAM + - name: Install system dependencies + run: | + sudo apt-get update && sudo apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + pkg-config \ + ncurses-dev \ + libssl-dev \ + ca-certificates + # Debug step - display the region with syntax error + - name: Debug syntax error region + run: | + echo "Showing the region with syntax error in hb_message.erl:" + sed -n '1440,1460p' src/hb_message.erl || echo "File not found or cannot be read" + echo "Checking for syntax error fix files:" + find . -name "*.erl.fix" -o -name "hb_message.erl.*" | grep -v ".beam" || echo "No fix files found" + echo "Erlang version:" + . ~/otp-27.0/activate && erl -eval 'io:format("~s~n", [erlang:system_info(otp_release)]), halt().' + # Install rebar3 + - name: Install rebar3 + run: | + . ~/otp-27.0/activate + mkdir -p ~/.config/rebar3 + curl -O https://s3.amazonaws.com/rebar3/rebar3 && chmod +x rebar3 + sudo mv rebar3 /usr/local/bin/rebar3 + . ~/otp-27.0/activate && rebar3 --version + # Install Rust toolchain (needed for WASM components) + - name: Install Rust and Cargo + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + source "$HOME/.cargo/env" + # Install pip dependencies and cache them + - name: 📦 Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install mkdocs mkdocs-material mkdocs-git-revision-date-localized-plugin + # Setup Node.js (needed for npx deploy command) + - name: ⎔ Setup Node + uses: actions/setup-node@v3 + with: + node-version: 22 # Or your preferred version + + - name: 👀 Env + run: | + echo "Event name: ${{ github.event_name }}" + echo "Git ref: ${{ github.ref }}" + echo "GH actor: ${{ github.actor }}" + echo "SHA: ${{ github.sha }}" + VER=`node --version`; echo "Node ver: $VER" + VER=`npm --version`; echo "npm ver: $VER" + . ~/otp-27.0/activate && erl -eval 'io:format("Erlang OTP version: ~s~n", [erlang:system_info(otp_release)]), halt().' + - name: 🛠 Build Docs + id: build_artifacts + run: | + . ~/otp-27.0/activate + SKIP_COMPILE=1 SKIP_EDOC=1 ./docs/build-all.sh -v + touch mkdocs-site/.nojekyll + echo "artifacts_output_dir=mkdocs-site" >> $GITHUB_OUTPUT + - name: 💾 Publish to Arweave + id: publish_artifacts + run: | + npx permaweb-deploy \ + --arns-name=dps-testing-facility \ + --ant-process=${{ secrets.ANT_PROCESS }} \ + --deploy-folder=${ARTIFACTS_OUTPUT_DIR} + env: + DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }} + ARTIFACTS_OUTPUT_DIR: ${{ steps.build_artifacts.outputs.artifacts_output_dir }} + ANT_PROCESS: ${{ secrets.ANT_PROCESS }} diff --git a/.gitignore b/.gitignore index ff540b3bf..5823721c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +config.* .rebar3 _build _checkouts @@ -19,20 +20,29 @@ logs *.iml rebar3.crashdump *~ +/venv *.json -node_modules !.vscode/* *.bin -!.vscode/* +.vscode/c_cpp_properties.json + +native/hb_beamr/*.o +native/hb_beamr/*.d -c_src/*.o -c_src/*.d priv/* .DS_STORE -TEST-data* -test-cache/* -TEST-cache-** +cache-* + +*.dot +*.svg + +cu/ +mkdocs-site/ +mkdocs-site-id.txt +mkdocs-site-manifest.csv -.githooks/_/.setup +!test/admissible-report-wallet.json +!test/admissible-report.json +!test/config.json \ No newline at end of file diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile deleted file mode 100644 index 064ce1706..000000000 --- a/.gitpod.Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM gitpod/workspace-full - -RUN git clone https://github.com/erlang/otp.git && cd otp && git checkout maint-27 && ./configure && make && sudo make install -RUN git clone https://github.com/erlang/rebar3.git && cd rebar3 && ./bootstrap && sudo mv rebar3 /usr/local/bin/ diff --git a/.gitpod.yml b/.gitpod.yml deleted file mode 100644 index 8b4ff02c7..000000000 --- a/.gitpod.yml +++ /dev/null @@ -1,16 +0,0 @@ -# This configuration file was automatically generated by Gitpod. -# Please adjust to your needs (see https://www.gitpod.io/docs/introduction/learn-gitpod/gitpod-yaml) -# and commit this file to your remote git repository to share the goodness with others. - -# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: - file: .gitpod.Dockerfile - -tasks: - - init: make - -vscode: - extensions: - - "editorconfig.editorconfig" - - "pgourlain.erlang" - diff --git a/.vscode/launch.json b/.vscode/launch.json index fb4b63075..4d956ca88 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,14 +2,28 @@ "version": "0.2.0", "configurations": [ { - "name": "Launch Erlang with Dependencies", + "name": "Launch debugger on a function.", "type": "erlang", "request": "launch", "cwd": "${workspaceRoot}", + "projectnode": "hb", + "cookie": "hb-debug", "preLaunchTask": "Rebar3 Compile", - "arguments": "-pa _build/default/lib/*/ebin -eval \"ssl:start(), application:ensure_all_started(hb).\"", - "stopOnEntry": false, - "internalConsoleOptions": "openOnSessionStart" + "postDebugTask": "Stop HyperBEAM", + "stopOnEntry": true, + "internalConsoleOptions": "openOnSessionStart", + "module": "hb_debugger", + "function": "start_and_break", + "args": "[${input:moduleName}, ${input:functionName}, [${input:funcArgs}], <<\"${input:debuggerScope}\">>]" + }, + { + "name": "Attach to a 'rebar3 debugger' node.", + "type": "erlang", + "request": "attach", + "projectnode": "hb", + "cookie": "hb-debug", + "timeout": 10, + "cwd": "${workspaceRoot}" }, { "name": "Attach C Debugger to beam.smp", @@ -35,5 +49,27 @@ }, "internalConsoleOptions": "neverOpen" } + ], + "inputs": [ + { + "id": "moduleName", + "type": "promptString", + "description": "Enter module to break in:" + }, + { + "id": "functionName", + "type": "promptString", + "description": "Enter function to invoke:" + }, + { + "id": "funcArgs", + "type": "promptString", + "description": "(Optional) Pass arguments to the function:" + }, + { + "id": "debuggerScope", + "type": "promptString", + "description": "(Optional) Additional modules/prefixes for debugger scope:" + } ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 7b9c135d2..f8cbdec3e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { "editor.detectIndentation": false, - "editor.insertSpaces": false, + "editor.insertSpaces": true, "editor.tabSize": 4 -} \ No newline at end of file +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 1750d5cdf..4433eda60 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -11,5 +11,40 @@ }, "problemMatcher": "$erlang" }, + { + "label": "Stop HyperBEAM", + "type": "shell", + "command": "lsof -i tcp:8734 | tail -n 1 | awk '{print $2}' | xargs kill -9" + }, + { + "label": "Generate a flame graph for a function.", + "type": "shell", + "command": "rebar3 as eflame shell --eval \"hb_debugger:profile_and_stop(fun() -> ${input:moduleName}:${input:functionName}(${input:funcArgs}) end).\"", + "group": "test", + "problemMatcher": "$erlang", + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "new" + } + } + ], + "inputs": [ + { + "id": "moduleName", + "type": "promptString", + "description": "Enter module:" + }, + { + "id": "functionName", + "type": "promptString", + "description": "Enter an exported function name:" + }, + { + "id": "funcArgs", + "type": "promptString", + "description": "(Optional) Pass arguments to the function:" + } ] } \ No newline at end of file diff --git a/DEPLOY-notes.md b/DEPLOY-notes.md deleted file mode 100644 index 597719c3d..000000000 --- a/DEPLOY-notes.md +++ /dev/null @@ -1,39 +0,0 @@ -# Deploy notes - -This document describes the notes for deploying HyperBEAM using packer. - -Packer is a tool that enables a single build to create images for multiple -distribution networks. - -The current version is setup for gcp, so you will need to install your gcp. - - -* https://cloud.google.com/sdk/docs/install - -* https://cloud.google.com/sdk/docs/initializing - -* https://cloud.google.com/docs/authentication/provide-credentials-adc#how-to - -Install Packer - -* https://developer.hashicorp.com/packer/tutorials/docker-get-started/get-started-install-cli - -```sh -rebar3 clean -rebar3 get-deps -rebar3 compile -rebar3 release -``` - -```sh -packer init . -packer validate . -packer build . -``` - -Install go-tpm-tools -Install erlinit - -possibly build ao project using nerves - -replace /sbin/init with erlinit \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..a78f0137b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM --platform=linux/amd64 ubuntu:22.04 + +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + git \ + pkg-config \ + ncurses-dev \ + libssl-dev \ + sudo + +RUN git clone https://github.com/erlang/otp.git && \ + cd otp && \ + git checkout maint-27 && \ + ./configure && \ + make -j16 && \ + sudo make install + +RUN git clone https://github.com/erlang/rebar3.git && \ + cd rebar3 && \ + ./bootstrap && \ + sudo mv rebar3 /usr/local/bin/ + +RUN git clone https://github.com/rust-lang/rust.git && \ + cd rust && \ + ./configure && \ + make && \ + sudo make install + +COPY . /app + +RUN cd /app && \ + rebar3 compile + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/GCP-notes.md b/GCP-notes.md deleted file mode 100644 index 695312854..000000000 --- a/GCP-notes.md +++ /dev/null @@ -1,125 +0,0 @@ -# GCP & GoTPM Tools - -## Table of Contents - -- [GCP \& GoTPM Tools](#gcp--gotpm-tools) - - [Table of Contents](#table-of-contents) - - [Variables](#variables) - - [Create an AMD SEV-SNP Instance](#create-an-amd-sev-snp-instance) - - [Create an Intel TDX Instance](#create-an-intel-tdx-instance) - - [Connecting to the Instance](#connecting-to-the-instance) - - [Useful Commands](#useful-commands) - - [List Available Machine Types](#list-available-machine-types) - - [Generate Random Nonces](#generate-random-nonces) - - [Run `gotpm` Attestation](#run-gotpm-attestation) - - [Notes](#notes) - -## Variables - -```sh -export GCI_NAME=PETES-SEV-SNP-TEST-0 -export GCI_PROJECT=arweave-437622 -export GCI_IMAGE=packer-1730219529 -export GCI_ZONE=us-central1-a -``` - -## Create an AMD SEV-SNP Instance - -```sh -gcloud compute instances create $GCI_NAME \ - --zone=$GCI_ZONE \ - --machine-type=n2d-standard-2 \ - --min-cpu-platform="AMD Milan" \ - --confidential-compute-type=SEV_SNP \ - --maintenance-policy=TERMINATE \ - --image-family=ubuntu-2404-lts-amd64 \ - --image-project=ubuntu-os-cloud \ - --project=$GCI_PROJECT \ - --network-interface=network-tier=PREMIUM,nic-type=GVNIC,stack-type=IPV4_ONLY,subnet=default \ - --tags=http-server,https-server \ - --shielded-secure-boot \ - --shielded-vtpm \ - --shielded-integrity-monitoring \ - --create-disk=auto-delete=yes,boot=yes,device-name=instance-20241030-131350,image=projects/$GCI_PROJECT/global/images/$GCI_IMAGE,mode=rw,size=20,type=pd-balanced -``` - -## Create an Intel TDX Instance - -```sh -gcloud compute instances create $GCI_NAME \ - --zone=$GCI_ZONE \ - --machine-type=c3-standard-4 \ - --confidential-compute-type=TDX \ - --maintenance-policy=TERMINATE \ - --image-family=ubuntu-2204-lts \ - --image-project=ubuntu-os-cloud \ - --project=$GCI_PROJECT \ - --network-interface=network-tier=PREMIUM,nic-type=GVNIC,stack-type=IPV4_ONLY,subnet=default \ - --tags=http-server,https-server \ - --shielded-secure-boot \ - --shielded-vtpm \ - --shielded-integrity-monitoring \ - --create-disk=auto-delete=yes,boot=yes,device-name=instance-20241030-131350,image=projects/$GCI_PROJECT/global/images/$GCI_IMAGE,mode=rw,size=20,type=pd-balanced -``` - -## Connecting to the Instance - -```sh -gcloud compute ssh --zone "$GCI_ZONE" "$GCI_NAME" --project "$GCI_PROJECT" -``` - -## Useful Commands - -### List Available Machine Types - -```sh -gcloud compute machine-types list --zones=$GCI_ZONE -``` - -### Generate Random Nonces - -- **32-byte Nonce** (for `--nonce`): - - ```sh - head -c 32 /dev/urandom | xxd -p -c 64 - ``` - -- **64-byte TEE Nonce** (for `--tee-nonce`): - - ```sh - head -c 64 /dev/urandom | xxd -p -c 128 - ``` - -### Run `gotpm` Attestation - -```sh -sudo gotpm attest --key AK --nonce <32 bytes (64 hex characters)> --tee-nonce <64 bytes (128 hex characters)> --tee-technology -``` - -## Notes - -> [!NOTE] -> The requirement to include both `--nonce` and `--tee-nonce` for the `gotpm attest` command, even when `--tee-technology` (e.g., `sev-snp` or `tdx`) is specified, indicates that **both TPM and TEE layers** of attestation are being validated in this command. Here’s why this is the case: -> -> ### Dual-Layer Attestation in Confidential VMs -> -> **TPM Attestation Layer**: -> -> - The **`--nonce`** parameter is required for the **TPM (Trusted Platform Module) attestation**. It acts as a freshness mechanism for the TPM-based portion of the attestation, preventing replay attacks by ensuring the response is unique to each request. -> - Even when the TEE technology is specified, `gotpm` still performs TPM-based attestation, which includes the TPM nonce (`--nonce`) in the attestation report. -> -> **TEE-Specific Attestation Layer**: -> -> - The **`--tee-nonce`** parameter is required for the **TEE (Trusted Execution Environment) attestation**. This layer provides hardware-backed isolation (e.g., Intel TDX or AMD SEV-SNP), and the larger 64-byte nonce uniquely identifies the TEE attestation report. -> - The `--tee-technology` option (e.g., `sev-snp` or `tdx`) specifies which TEE environment to use, and `--tee-nonce` is essential for proving the freshness of the TEE report in that specific environment. -> -> ### Why Both Nonces Are Required Together -> -> Since the command produces an attestation report containing **both TPM and TEE layers**, each layer requires its own nonce: -> -> - **`--nonce` (TPM layer)**: Required for the general TPM quote in the attestation. -> - **`--tee-nonce` (TEE layer)**: Required specifically for the TEE report tied to `sev-snp` or `tdx`. -> -> This design ensures that the attestation report is comprehensive and includes **freshness proofs** for both the TPM and TEE components, preventing replay attacks across both security layers. This dual nonce requirement is specific to environments where both TPM and TEE attestation are requested together, as in a Confidential VM using TEE. - ---- diff --git a/Makefile b/Makefile index 20cfdea6b..6d074e578 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,12 @@ compile: rebar3 compile -WAMR_VERSION = 2.1.2 +WAMR_VERSION = 2.2.0 WAMR_DIR = _build/wamr -# commented out to remove NFR blocking commits -# GITHOOKS_DIR = .githooks +GENESIS_WASM_BRANCH = feat/http-checkpoint +GENESIS_WASM_REPO = https://github.com/permaweb/ao.git +GENESIS_WASM_SERVER_DIR = _build/genesis_wasm/genesis-wasm-server ifdef HB_DEBUG WAMR_FLAGS = -DWAMR_ENABLE_LOG=1 -DWAMR_BUILD_DUMP_CALL_STACK=1 -DCMAKE_BUILD_TYPE=Debug @@ -30,8 +31,6 @@ else WAMR_BUILD_TARGET = X86_64 endif -# githooks: $(GITHOOKS_DIR)/_/setup - wamr: $(WAMR_DIR)/lib/libvmlib.a debug: debug-clean $(WAMR_DIR) @@ -51,29 +50,57 @@ $(WAMR_DIR): --single-branch $(WAMR_DIR)/lib/libvmlib.a: $(WAMR_DIR) + sed -i '742a tbl_inst->is_table64 = 1;' ./_build/wamr/core/iwasm/aot/aot_runtime.c; \ cmake \ $(WAMR_FLAGS) \ -S $(WAMR_DIR) \ -B $(WAMR_DIR)/lib \ -DWAMR_BUILD_TARGET=$(WAMR_BUILD_TARGET) \ -DWAMR_BUILD_PLATFORM=$(WAMR_BUILD_PLATFORM) \ - -DWAMR_BUILD_LIBC_WASI=0 \ -DWAMR_BUILD_MEMORY64=1 \ -DWAMR_DISABLE_HW_BOUND_CHECK=1 \ -DWAMR_BUILD_EXCE_HANDLING=1 \ -DWAMR_BUILD_SHARED_MEMORY=0 \ - -DWAMR_BUILD_AOT=0 \ + -DWAMR_BUILD_AOT=1 \ + -DWAMR_BUILD_LIBC_WASI=0 \ -DWAMR_BUILD_FAST_INTERP=0 \ -DWAMR_BUILD_INTERP=1 \ - -DWAMR_BUILD_JIT=0 - make -C $(WAMR_DIR)/lib - -# $(GITHOOKS_DIR)/_/setup: -# @sh ./$(GITHOOKS_DIR)/_/install.sh + -DWAMR_BUILD_JIT=0 \ + -DWAMR_BUILD_FAST_JIT=0 \ + -DWAMR_BUILD_DEBUG_AOT=1 \ + -DWAMR_BUILD_TAIL_CALL=1 \ + -DWAMR_BUILD_AOT_STACK_FRAME=1 \ + -DWAMR_BUILD_MEMORY_PROFILING=1 \ + -DWAMR_BUILD_DUMP_CALL_STACK=1 + make -C $(WAMR_DIR)/lib -j8 clean: rebar3 clean # Add a new target to print the library path print-lib-path: - @echo $(CURDIR)/lib/libvmlib.a \ No newline at end of file + @echo $(CURDIR)/lib/libvmlib.a + +$(GENESIS_WASM_SERVER_DIR): + mkdir -p $(GENESIS_WASM_SERVER_DIR) + @echo "Cloning genesis-wasm repository..." && \ + tmp_dir=$$(mktemp -d) && \ + git clone --depth=1 -b $(GENESIS_WASM_BRANCH) $(GENESIS_WASM_REPO) $$tmp_dir && \ + mkdir -p $(GENESIS_WASM_SERVER_DIR) && \ + cp -r $$tmp_dir/servers/cu/* $(GENESIS_WASM_SERVER_DIR) && \ + rm -rf $$tmp_dir && \ + echo "Extracted servers/genesis-wasm to $(GENESIS_WASM_SERVER_DIR)" + +# Set up genesis-wasm@1.0 environment +setup-genesis-wasm: $(GENESIS_WASM_SERVER_DIR) + @cp native/genesis-wasm/launch-monitored.sh $(GENESIS_WASM_SERVER_DIR) && \ + if ! command -v node > /dev/null; then \ + echo "Error: Node.js is not installed. Please install Node.js before continuing."; \ + echo "For Ubuntu/Debian, you can install it with:"; \ + echo " curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - && \\"; \ + echo " apt-get install -y nodejs=22.16.0-1nodesource1 --allow-downgrades && \\"; \ + echo " node -v && npm -v"; \ + exit 1; \ + fi + @cd $(GENESIS_WASM_SERVER_DIR) && npm install > /dev/null 2>&1 && \ + echo "Installed genesis-wasm@1.0 server." diff --git a/README.md b/README.md index 8cfa547e1..dc6bf80e1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,300 @@ - +![hyperbeam_logo-thin-3](https://github.com/user-attachments/assets/fcca891c-137e-4022-beff-360eb2a0d05e) -The original HyperBEAM (and `SuperSU`) implementation, preserved for historical purposes. +This repository contains a reference implementation of AO-Core, along with an +Erlang-based (BEAM) client implementing a number of devices for the protocol. -Unless you are curious about how the design of AO-Core evolved from a simple implementation of AO testnet's original SU specification, you want to be [here](https://github.com/permaweb/HyperBEAM). +AO-Core is a protocol built to enable decentralized computations, offering a +series of universal primitives to achieve this end. Instead of enforcing a single, +monolithic architecture, AO-Core provides a framework into which any number of +different computational models, encapsulated as primitive `devices`, can be attached. + +AO-Core's protocol offers a framework for decentralized computations, built upon the +following fundamental primitives: + +1. Hashpaths: A mechanism for referencing locations in a program's state-space +prior to execution. These state-space `links` are represented as Merklized lists of +programs inputs and initial states. +2. A unified data structure for representing program states as HTTP documents, +as described in the [HTTP Semantics RFC](https://www.rfc-editor.org/rfc/rfc9110.html). +3. A unified protocol for expressing `commitments` of the `states` found at +particular `hashpaths`. These commitments allow nodes to participate in varied +economic and cryptographic mechanisms to prove and challenge each-other's +representations regarding the programs that operate inside the AO-Core protocol. +4. A meta-VM that allows any number of different virtual machines and computational +models (`devices`) to be executed inside the AO-Core protocol, while enabling their +states and inputs to be calculated and committed to in a unified format. + +## What is HyperBeam? + +HyperBeam is a client implementation of the AO-Core protocol, written in Erlang. +It can be seen as the 'node' software for the decentralized operating system that +AO enables; abstracting hardware provisioning and details from the execution of +individual programs. + +HyperBEAM node operators can offer the services of their machine to others inside +the network by electing to execute any number of different `devices`, charging +users for their computation as necessary. + +Each HyperBEAM node is configured using the `~meta@1.0` device, which provides +an interface for specifying the node's hardware, supported devices, metering and +payments information, amongst other configuration options. + +## Getting Started + +To begin using HyperBeam, you will need to install: + +- The Erlang runtime (OTP 27) +- Rebar3 +- Git +- Docker (optional, for containerized deployment) + +You will also need: +- A wallet and it's keyfile *(generate a new wallet and keyfile with https://www.wander.app)* + +Then you can clone the HyperBEAM source and build it: + +```bash +git clone https://github.com/permaweb/HyperBEAM.git +cd HyperBEAM +rebar3 compile +``` + +If you would prefer to execute HyperBEAM in a containerized environment, you +can use the provided Dockerfile to build a container image. + +```bash +docker build -t hyperbeam . +``` + +If you intend to offer TEE-based computation of AO-Core devices, please see the +[`HyperBEAM OS`](https://github.com/permaweb/hb-os) repo for details on configuration and deployment. + +## Running HyperBEAM + +Once the code is compiled, you can start HyperBEAM with: + +```bash +# Start with default configuration +rebar3 shell +``` + +The default configuration uses settings from `hb_opts.erl`, which preloads +all devices and sets up default stores on port 10000. + +### Optional Build Profiles + +HyperBEAM supports several optional build profiles that enable additional features: + +- `genesis_wasm`: Enables Genesis WebAssembly support +- `rocksdb`: Enables RocksDB storage backend (adds RocksDB v1.8.0 dependency) +- `http3`: Enables HTTP/3 support via QUIC protocol + + +Using these profiles allows you to optimize HyperBEAM for your specific use case without adding unnecessary dependencies to the base installation. + +To start a shell with profiles: + +```bash +# Single profile +rebar3 as rocksdb shell + +# Multiple profiles +rebar3 as rocksdb, genesis_wasm shell +``` + +To create a release with profiles: + +```bash +# Create release with profiles +rebar3 as rocksdb,genesis_wasm release +``` + +Note: Profiles modify compile-time options that get baked into the release. Choose the profiles you need before starting HyperBEAM. + +### Verify Installation + +To verify that your HyperBEAM node is running correctly, check: + +```bash +curl http://localhost:10000/~meta@1.0/info +``` + +If you receive a response with node information, your HyperBEAM +installation is working properly. + +## Configuration + +HyperBEAM can be configured using a `~meta@1.0` device, which is initialized + using either command line arguments or a configuration file. + +### Configuration with `config.flat` + +The simplest way to configure HyperBEAM is using the `config.flat` file: + +1. A file named `config.flat` is already included in the project directory +2. Update to include your configuration values: + +``` +port: 10000 +priv_key_location: /path/to/wallet.json +``` + +3. Start HyperBEAM with `rebar3 shell` + +HyperBEAM will automatically load your configuration and display the active +settings in the startup log. + +### Creating a Release + +For production environments, you can create a standalone release: + +```bash +rebar3 release +``` + +This creates a release in `_build/default/rel/hb` that can be deployed independently. + +### Runtime Configuration Changes + +Additionally, if you would like to modify a running node's configuration, you can + do so by sending a HTTP Signed Message using any RFC-9421 compatible client + in the following form: + +``` +POST /~meta@1.0/info +Your-Config-Tag: Your-Config-Tag +``` + +The individual headers provided in the message will each be interpreted as additional +configuration options for the node. + +## Messages + +HyperBEAM describes every piece of data as a `message`, which can be interpreted as +a binary term or as collection of named functions aka. a `Map` of functions. + +Every message _may_ specify a `device` which is interpreted by the AO-Core compatible +system in order to operate upon the message's contents, which to say read it, or +execute it. Executing a named function within a message, providing a map of arguments, +results in another `message`. + +In this way, `messages` in AO-Core always _beget_ further `messages`, giving rise +to a vast computational space, leveraging function application and composition at its core. +For those familiar with the concept, this programming model is similar to that +described by traditional `combinator` systems. + +> Notably, this computation does not require the computor of a message +> to know the values of all the keys contained therin. In other words, keys +> may be _lazily_ evaluated, and only by computors that are interested +> in their outputs, or even _sharded_ across arbitrary sets of nodes, as necessary + +If a `message` does not explicitly specify a `device`, its implied `device` is a + `message@1.0`, which simply returns the binary or `message` at a given named function. + +## Devices + +HyperBeam supports a number of different devices, each of which enable different +services to be offered by the node. There are presently 25 different devices +included in the `preloaded_devices` of a HyperBEAM node, although it is possible +to add and remove devices as necessary. + +### Preloaded Devices + +The following devices are included in the `preloaded_devices` of a HyperBEAM node: + +- `~meta@1.0`: The `~meta@1.0` device is used to configure the node's hardware, +supported devices, metering and payments information, amongst other configuration options. +Additionally, this device allows external clients to find and validate the configuration +of nodes in the network. + +- `~relay@1.0`: The `~relay@1.0` device is used to relay messages between nodes +and the wider HTTP network. It offers an interface for sending and receiving messages +to and from nodes in the network, using a variety of execution strategies. + +- `~wasm64@1.0`: The `~wasm64@1.0` device is used to execute WebAssembly code, using +the [Web Assembly Micro-Runtime (WAMR)](https://github.com/bytecodealliance/wasm-micro-runtime) +under-the-hood. WASM modules can be called from any other device, and can also be +used to execute `devices` written in languages such as Rust, C, and C++. + +- `~json-iface@1.0`: The `~json-iface@1.0` device offers a translation layer between +the JSON-encoded message format used by AOS 2.0 and prior versions, to HyperBEAM's +native HTTP message format. + +- `~compute-lite@1.0`: The `~compute-lite@1.0` device is a lightweight device wrapping +a local WASM executor, used for executing legacynet AO processes inside HyperBEAM. +See the [HyperBEAM OS](https://github.com/permaweb/hb-os) repository for an +example setup with co-executing HyperBEAM and legacy-CU nodes. + +- `~snp@1.0`: The `~snp@1.0` device is used to generate and validate proofs that +the local node, or another node in the network, is executing inside a [Trusted Execution +Environment (TEE)](https://en.wikipedia.org/wiki/Trusted_execution_environment). +Nodes executing inside these environments use an ephemeral key pair, provably +only existing inside the TEE, and can be signed commitments of AO-Core executions +in a trust-minimized way. + +- `p4@1.0`: The `p4@1.0` device runs as a `pre-processor` and `post-processor` in +the framework provided by `~meta@1.0`, enabling a framework for node operators to +sell usage of their machine's hardware to execute AO-Core devices. The `p4@1.0` +framework offers two additional hooks, allowing node operators flexibility in how +their hardware is offered: A `pricing` device, and a `ledger` device. + +- `~simple-pay@1.0`: Implements a simple, flexible pricing device that can be used +in conjunction with `p4@1.0` to offer flat-fees for the execution of AO-Core messages. + +- `~faff@1.0`: A simple pricing (and ledger) device for `p4@1.0`, allowing nodes +to offer access to their services only to a specific set of users. This device is +useful if you intend to operate your node onmly for personal use, or for a specific +subset of users (servicing an individual app, for example). + +- `scheduler@1.0`: The `scheduler@1.0` device is used to assign a linear hashpath +to an execution, such that all users may access it with a deterministic ordering. +When used in conjunction with other AO-Core devices, this allows for the creation +of executions that mirror the behaviour of traditional smart contracting networks. + +- `stack@1.0`: The `stack@1.0` device is used to execute an ordered set of devices, +over the same inputs. This device allows its users to create complex combinations of +other devices and apply them as a single unit, with a single hashpath. + +- `~process@1.0`: Processes enable users to create persistent, shared executions +that can be accessed by any number of users, each of whom may add additional inputs +to its hashpath. The `~process@1.0` allows users to customize the `execution` and +`scheduler` devices that they choose for their process, such that a variety of different +execution patterns can be created. In addition, the `~process@1.0` device offers a +`push` key, which moves messages from a process's execution `outbox` into the +schedule of another execution. + +Details on other devices found in the pre-loaded set can be located in their +respective documentation. + +## Documentation + +HyperBEAM uses [MkDocs](https://www.mkdocs.org/) with the [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) theme to build its documentation site. All documentation source files are located in the `docs/` directory. + +To build and view the documentation locally: + +```bash +# Create and activate a virtual environment (optional but recommended) +python3 -m venv venv +source venv/bin/activate # (macOS/Linux) On Windows use `venv\Scripts\activate` + +# Install required packages +pip3 install mkdocs mkdocs-material mkdocs-git-revision-date-localized-plugin + +# Build the docs +./docs/build-all.sh + +# Serve the docs +cd mkdocs-site +python3 -m http.server 8000 +# Then open http://127.0.0.1:8000/ in your browser +``` + +For more details on the documentation structure, how to contribute, and other information, please see the [full documentation README](./docs/README.md). + +## Contributing + +HyperBEAM is developed as an open source implementation of the AO-Core protocol +by [Forward Research](https://fwd.arweave.net). Pull Requests are always welcome! + +To get started building on HyperBEAM, check out the [hacking on HyperBEAM](./docs/misc/hacking-on-hyperbeam.md) +guide. diff --git a/c_src/hb_beamr.c b/c_src/hb_beamr.c deleted file mode 100644 index addebfb62..000000000 --- a/c_src/hb_beamr.c +++ /dev/null @@ -1,957 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -typedef struct { - ErlDrvMutex* response_ready; - ErlDrvCond* cond; - int ready; - char* error_message; - ei_term* result_terms; - int result_length; -} ImportResponse; - -typedef struct { - wasm_engine_t* engine; - wasm_instance_t* instance; - wasm_module_t* module; - wasm_store_t* store; - ErlDrvPort port; - ErlDrvTermData port_term; - ErlDrvMutex* is_running; - char* current_function; - ei_term* current_args; - int current_args_length; - ImportResponse* current_import; - ErlDrvTermData pid; - int is_initialized; - time_t start_time; -} Proc; - -typedef struct { - char* module_name; - char* field_name; - char* signature; - Proc* proc; - wasm_func_t* stub_func; -} ImportHook; - -typedef struct { - void* binary; - long size; - Proc* proc; -} LoadWasmReq; - -static ErlDrvTermData atom_ok; -static ErlDrvTermData atom_error; -static ErlDrvTermData atom_import; -static ErlDrvTermData atom_execution_result; - -#ifndef HB_DEBUG -#define HB_DEBUG 0 -#endif - -#define DRV_DEBUG(format, ...) beamr_print(HB_DEBUG, __FILE__, __LINE__, format, ##__VA_ARGS__) -#define DRV_PRINT(format, ...) beamr_print(1, __FILE__, __LINE__, format, ##__VA_ARGS__) - -void beamr_print(int print, const char* file, int line, const char* format, ...) { - va_list args; - va_start(args, format); - if(print) { - pthread_t thread_id = pthread_self(); - printf("[DBG#%p @ %s:%d] ", thread_id, file, line); - vprintf(format, args); - printf("\r\n"); - } - va_end(args); -} - -const char* wasm_externtype_to_kind_string(const wasm_externtype_t* type) { - switch (wasm_externtype_kind(type)) { - case WASM_EXTERN_FUNC: return "func"; - case WASM_EXTERN_GLOBAL: return "global"; - case WASM_EXTERN_TABLE: return "table"; - case WASM_EXTERN_MEMORY: return "memory"; - default: return "unknown"; - } -} - -// Helper function to convert wasm_valtype_t to char -char wasm_valtype_kind_to_char(const wasm_valtype_t* valtype) { - switch (wasm_valtype_kind(valtype)) { - case WASM_I32: return 'i'; - case WASM_I64: return 'I'; - case WASM_F32: return 'f'; - case WASM_F64: return 'F'; - case WASM_EXTERNREF: return 'e'; - case WASM_V128: return 'v'; - case WASM_FUNCREF: return 'f'; - default: return 'u'; - } -} - -int wasm_val_to_erl_term(ErlDrvTermData* term, const wasm_val_t* val) { - DRV_DEBUG("Adding wasm val to erl term"); - DRV_DEBUG("Val of: %d", val->of.i32); - switch (val->kind) { - case WASM_I32: - term[0] = ERL_DRV_INT; - term[1] = val->of.i32; - return 2; - case WASM_I64: - term[0] = ERL_DRV_INT64; - term[1] = (ErlDrvTermData) &val->of.i64; - return 2; - case WASM_F32: - term[0] = ERL_DRV_FLOAT; - term[1] = (ErlDrvTermData) &val->of.f32; - return 2; - case WASM_F64: - term[0] = ERL_DRV_FLOAT; - term[1] = (ErlDrvTermData) &val->of.f64; - return 2; - default: - DRV_DEBUG("Unsupported result type: %d", val->kind); - return 0; - } -} - -int erl_term_to_wasm_val(wasm_val_t* val, ei_term* term) { - DRV_DEBUG("Converting erl term to wasm val. Term: %d. Size: %d", term->value.i_val, term->size); - switch (val->kind) { - case WASM_I32: - val->of.i32 = (int) term->value.i_val; - break; - case WASM_I64: - val->of.i64 = (long) term->value.i_val; - break; - case WASM_F32: - val->of.f32 = (float) term->value.d_val; - break; - case WASM_F64: - val->of.f64 = term->value.d_val; - break; - default: - DRV_DEBUG("Unsupported parameter type: %d", val->kind); - return -1; - } - return 0; -} - -int erl_terms_to_wasm_vals(wasm_val_vec_t* vals, ei_term* terms) { - DRV_DEBUG("Converting erl terms to wasm vals"); - DRV_DEBUG("Vals: %d", vals->size); - for(int i = 0; i < vals->size; i++) { - DRV_DEBUG("Converting term %d: %p", i, &vals->data[i]); - int res = erl_term_to_wasm_val(&vals->data[i], &terms[i]); - if(res == -1) { - DRV_DEBUG("Failed to convert term to wasm val"); - return -1; - } - } - return 0; -} - -ei_term* decode_list(char* buff, int* index) { - int arity, type; - - if(ei_get_type(buff, index, &type, &arity) == -1) { - DRV_DEBUG("Failed to get type"); - return NULL; - } - DRV_DEBUG("Decoded header. Arity: %d", arity); - - ei_term* res = driver_alloc(sizeof(ei_term) * arity); - - if(type == ERL_LIST_EXT) { - //DRV_DEBUG("Decoding list"); - ei_decode_list_header(buff, index, &arity); - //DRV_DEBUG("Decoded list header. Arity: %d", arity); - for(int i = 0; i < arity; i++) { - ei_decode_ei_term(buff, index, &res[i]); - DRV_DEBUG("Decoded term (assuming int) %d: %d", i, res[i].value.i_val); - } - } - else if(type == ERL_STRING_EXT) { - //DRV_DEBUG("Decoding list encoded as string"); - unsigned char* str = driver_alloc(arity * sizeof(char) + 1); - ei_decode_string(buff, index, str); - for(int i = 0; i < arity; i++) { - res[i].ei_type = ERL_INTEGER_EXT; - res[i].value.i_val = (long) str[i]; - DRV_DEBUG("Decoded term %d: %d", i, res[i].value.i_val); - } - driver_free(str); - } - else { - DRV_DEBUG("Unknown type: %d", type); - return NULL; - } - - return res; -} - -int get_function_sig(const wasm_externtype_t* type, char* type_str) { - if (wasm_externtype_kind(type) == WASM_EXTERN_FUNC) { - const wasm_functype_t* functype = wasm_externtype_as_functype_const(type); - const wasm_valtype_vec_t* params = wasm_functype_params(functype); - const wasm_valtype_vec_t* results = wasm_functype_results(functype); - - if(!params || !results) { - DRV_DEBUG("Export function params/results are NULL"); - return 0; - } - - type_str[0] = '('; - size_t offset = 1; - - for (size_t i = 0; i < params->size; i++) { - type_str[offset++] = wasm_valtype_kind_to_char(params->data[i]); - } - type_str[offset++] = ')'; - - for (size_t i = 0; i < results->size; i++) { - type_str[offset++] = wasm_valtype_kind_to_char(results->data[i]); - } - type_str[offset] = '\0'; - - return 1; - } - return 0; -} - -void drv_lock(ErlDrvMutex* mutex) { - DRV_DEBUG("Locking: %s", erl_drv_mutex_name(mutex)); - erl_drv_mutex_lock(mutex); - DRV_DEBUG("Locked: %s", erl_drv_mutex_name(mutex)); -} - -void drv_unlock(ErlDrvMutex* mutex) { - DRV_DEBUG("Unlocking: %s", erl_drv_mutex_name(mutex)); - erl_drv_mutex_unlock(mutex); - DRV_DEBUG("Unlocked: %s", erl_drv_mutex_name(mutex)); -} - -void drv_signal(ErlDrvMutex* mut, ErlDrvCond* cond, int* ready) { - DRV_DEBUG("Signaling: %s. Pre-signal ready state: %d", erl_drv_cond_name(cond), *ready); - drv_lock(mut); - *ready = 1; - erl_drv_cond_signal(cond); - drv_unlock(mut); - DRV_DEBUG("Signaled: %s. Post-signal ready state: %d", erl_drv_cond_name(cond), *ready); -} - -void drv_wait(ErlDrvMutex* mut, ErlDrvCond* cond, int* ready) { - DRV_DEBUG("Started to wait: %s. Ready: %d", erl_drv_cond_name(cond), *ready); - DRV_DEBUG("Mutex: %s", erl_drv_mutex_name(mut)); - drv_lock(mut); - while (!*ready) { - DRV_DEBUG("Waiting: %s", erl_drv_cond_name(cond)); - erl_drv_cond_wait(cond, mut); - DRV_DEBUG("Woke up: Ready: %d", *ready); - } - drv_unlock(mut); - DRV_DEBUG("Finish waiting: %s", erl_drv_cond_name(cond)); -} - -wasm_func_t* get_exported_function(Proc* proc, const char* target_name) { - wasm_extern_vec_t exports; - wasm_instance_exports(proc->instance, &exports); - wasm_exporttype_vec_t export_types; - wasm_module_exports(proc->module, &export_types); - wasm_func_t* func = NULL; - - for (size_t i = 0; i < exports.size; ++i) { - wasm_extern_t* ext = exports.data[i]; - if (wasm_extern_kind(ext) == WASM_EXTERN_FUNC) { - const wasm_name_t* exp_name = wasm_exporttype_name(export_types.data[i]); - if (exp_name && exp_name->size == strlen(target_name) + 1 && - strncmp(exp_name->data, target_name, exp_name->size - 1) == 0) { - func = wasm_extern_as_func(ext); - break; - } - } - } - - return func; -} - -wasm_memory_t* get_memory(Proc* proc) { - wasm_extern_vec_t exports; - wasm_instance_exports(proc->instance, &exports); - for (size_t i = 0; i < exports.size; i++) { - if (wasm_extern_kind(exports.data[i]) == WASM_EXTERN_MEMORY) { - return wasm_extern_as_memory(exports.data[i]); - } - } - return NULL; -} - -long get_memory_size(Proc* proc) { - wasm_memory_t* memory = get_memory(proc); - return wasm_memory_size(memory) * 65536; -} - -void send_error(Proc* proc, const char* message_fmt, ...) { - va_list args; - va_start(args, message_fmt); - char* message = driver_alloc(256); - vsnprintf(message, 256, message_fmt, args); - DRV_DEBUG("Sending error message: %s", message); - ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * 7); - int msg_index = 0; - msg[msg_index++] = ERL_DRV_ATOM; - msg[msg_index++] = atom_error; - msg[msg_index++] = ERL_DRV_STRING; - msg[msg_index++] = (ErlDrvTermData)message; - msg[msg_index++] = strlen(message); - msg[msg_index++] = ERL_DRV_TUPLE; - msg[msg_index++] = 2; - - int msg_res = erl_drv_output_term(proc->port_term, msg, msg_index); - DRV_DEBUG("Sent error message. Res: %d", msg_res); - driver_free(message); - driver_free(msg); - va_end(args); -} - -wasm_trap_t* generic_import_handler(void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results) { - DRV_DEBUG("generic_import_handler called"); - ImportHook* import_hook = (ImportHook*)env; - Proc* proc = import_hook->proc; - DRV_DEBUG("Proc: %p. Args size: %d", proc, args->size); - DRV_DEBUG("Import name: %s.%s [%s]", import_hook->module_name, import_hook->field_name, import_hook->signature); - - // Initialize the message object - ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * ((2+(2*3)) + ((args->size + 1) * 2) + ((results->size + 1) * 2) + 2)); - int msg_index = 0; - msg[msg_index++] = ERL_DRV_ATOM; - msg[msg_index++] = atom_import; - msg[msg_index++] = ERL_DRV_STRING; - msg[msg_index++] = (ErlDrvTermData) import_hook->module_name; - msg[msg_index++] = strlen(import_hook->module_name); - msg[msg_index++] = ERL_DRV_STRING; - msg[msg_index++] = (ErlDrvTermData) import_hook->field_name; - msg[msg_index++] = strlen(import_hook->field_name); - - // Encode args - for (size_t i = 0; i < args->size; i++) { - msg_index += wasm_val_to_erl_term(&msg[msg_index], &args->data[i]); - } - msg[msg_index++] = ERL_DRV_NIL; - msg[msg_index++] = ERL_DRV_LIST; - msg[msg_index++] = args->size + 1; - - // Encode function signature - msg[msg_index++] = ERL_DRV_STRING; - msg[msg_index++] = (ErlDrvTermData) import_hook->signature; - msg[msg_index++] = strlen(import_hook->signature) - 1; - - // Prepare the message to send to the Erlang side - msg[msg_index++] = ERL_DRV_TUPLE; - msg[msg_index++] = 5; - - // Initialize the result vector and set the required result types - proc->current_import = driver_alloc(sizeof(ImportResponse)); - - // Create and initialize a is_running and condition variable for the response - proc->current_import->response_ready = erl_drv_mutex_create("response_mutex"); - proc->current_import->cond = erl_drv_cond_create("response_cond"); - proc->current_import->ready = 0; - - DRV_DEBUG("Sending %d terms...", msg_index); - // Send the message to the caller process - int msg_res = erl_drv_output_term(proc->port_term, msg, msg_index); - driver_free(msg); - // Wait for the response (we set this directly after the message was sent - // so we have the lock, before Erlang sends us data back) - drv_wait(proc->current_import->response_ready, proc->current_import->cond, &proc->current_import->ready); - - DRV_DEBUG("Response ready"); - - // Handle error in the response - if (proc->current_import->error_message) { - DRV_DEBUG("Import execution failed. Error message: %s", proc->current_import->error_message); - wasm_name_t message; - wasm_name_new_from_string_nt(&message, proc->current_import->error_message); - wasm_trap_t* trap = wasm_trap_new(proc->store, &message); - // TODO: check where the error_message is allocated - driver_free(proc->current_import->error_message); - driver_free(proc->current_import); - proc->current_import = NULL; - return trap; - } - - // Convert the response back to WASM values - const wasm_valtype_vec_t* result_types = wasm_functype_results(wasm_func_type(import_hook->stub_func)); - for(int i = 0; i < proc->current_import->result_length; i++) { - results->data[i].kind = wasm_valtype_kind(result_types->data[i]); - } - int res = erl_terms_to_wasm_vals(results, proc->current_import->result_terms); - if(res == -1) { - DRV_DEBUG("Failed to convert terms to wasm vals"); - return NULL; - } - - results->num_elems = result_types->num_elems; - - // Clean up - DRV_DEBUG("Cleaning up import response"); - erl_drv_cond_destroy(proc->current_import->cond); - erl_drv_mutex_destroy(proc->current_import->response_ready); - if (proc->current_import->result_terms) { - driver_free(proc->current_import->result_terms); - } - driver_free(proc->current_import); - - proc->current_import = NULL; - return NULL; -} - -// Async initialization function -static void async_init(void* raw) { - DRV_DEBUG("Initializing WASM module"); - LoadWasmReq* mod_bin = (LoadWasmReq*)raw; - Proc* proc = mod_bin->proc; - drv_lock(proc->is_running); - // Initialize WASM engine, store, etc. - -#if HB_DEBUG==1 - wasm_runtime_set_log_level(WASM_LOG_LEVEL_VERBOSE); -#else - wasm_runtime_set_log_level(WASM_LOG_LEVEL_ERROR); -#endif - - wasm_runtime_set_default_running_mode(Mode_Interp); - - proc->engine = wasm_engine_new(); - DRV_DEBUG("Created engine"); - proc->store = wasm_store_new(proc->engine); - DRV_DEBUG("Created store"); - - // Load WASM module - wasm_byte_vec_t binary; - wasm_byte_vec_new(&binary, mod_bin->size, (const wasm_byte_t*)mod_bin->binary); - - proc->module = wasm_module_new(proc->store, &binary); - DRV_DEBUG("Module created: %p", proc->module); - if (!proc->module) { - DRV_DEBUG("Failed to create module"); - send_error(proc, "Failed to create module."); - wasm_byte_vec_delete(&binary); - wasm_store_delete(proc->store); - wasm_engine_delete(proc->engine); - drv_unlock(proc->is_running); - return; - } - //wasm_byte_vec_delete(&binary); - DRV_DEBUG("Created module"); - - // Get imports - wasm_importtype_vec_t imports; - wasm_module_imports(proc->module, &imports); - DRV_DEBUG("Imports size: %d", imports.size); - wasm_extern_t *stubs[imports.size]; - - // Get exports - wasm_exporttype_vec_t exports; - wasm_module_exports(proc->module, &exports); - - // Create Erlang lists for imports - //DRV_DEBUG("Exports size: %d", exports.size); - ErlDrvTermData* init_msg = driver_alloc(sizeof(ErlDrvTermData) * (2 + (13 * imports.size) + (11 * exports.size))); - //DRV_DEBUG("Allocated init message"); - int msg_i = 0; - init_msg[msg_i++] = ERL_DRV_ATOM; - init_msg[msg_i++] = atom_execution_result; - - // Process imports - for (int i = 0; i < imports.size; ++i) { - //DRV_DEBUG("Processing import %d", i); - const wasm_importtype_t* import = imports.data[i]; - const wasm_name_t* module_name = wasm_importtype_module(import); - const wasm_name_t* name = wasm_importtype_name(import); - const wasm_externtype_t* type = wasm_importtype_type(import); - - //DRV_DEBUG("Import: %s.%s", module_name->data, name->data); - - char* type_str = driver_alloc(256); - // TODO: What happpens here? - if(!get_function_sig(type, type_str)) { - // TODO: Handle other types of imports? - continue; - } - - init_msg[msg_i++] = ERL_DRV_ATOM; - init_msg[msg_i++] = driver_mk_atom((char*)wasm_externtype_to_kind_string(type)); - init_msg[msg_i++] = ERL_DRV_STRING; - init_msg[msg_i++] = (ErlDrvTermData)module_name->data; - init_msg[msg_i++] = module_name->size - 1; - init_msg[msg_i++] = ERL_DRV_STRING; - init_msg[msg_i++] = (ErlDrvTermData)name->data; - init_msg[msg_i++] = name->size - 1; - init_msg[msg_i++] = ERL_DRV_STRING; - init_msg[msg_i++] = (ErlDrvTermData)type_str; - init_msg[msg_i++] = strlen(type_str); - init_msg[msg_i++] = ERL_DRV_TUPLE; - init_msg[msg_i++] = 4; - - DRV_DEBUG("Creating callback for %s.%s [%s]", module_name->data, name->data, type_str); - ImportHook* hook = driver_alloc(sizeof(ImportHook)); - hook->module_name = module_name->data; - hook->field_name = name->data; - hook->proc = proc; - hook->signature = type_str; - - hook->stub_func = - wasm_func_new_with_env( - proc->store, - wasm_externtype_as_functype_const(type), - generic_import_handler, - hook, - NULL - ); - stubs[i] = wasm_func_as_extern(hook->stub_func); - } - - init_msg[msg_i++] = ERL_DRV_NIL; - init_msg[msg_i++] = ERL_DRV_LIST; - init_msg[msg_i++] = imports.size + 1; - - // Create proc! - wasm_extern_vec_t externs; - wasm_extern_vec_new(&externs, imports.size, stubs); - wasm_trap_t* trap = NULL; - proc->instance = wasm_instance_new_with_args(proc->store, proc->module, &externs, &trap, 0x10000, 0x10000); - if (!proc->instance) { - DRV_DEBUG("Failed to create WASM instance"); - send_error(proc, "Failed to create WASM instance (although module was created)."); - drv_unlock(proc->is_running); - return; - } - - // Refresh the exports now that we have an instance - wasm_module_exports(proc->module, &exports); - for (size_t i = 0; i < exports.size; i++) { - //DRV_DEBUG("Processing export %d", i); - const wasm_exporttype_t* export = exports.data[i]; - const wasm_name_t* name = wasm_exporttype_name(export); - const wasm_externtype_t* type = wasm_exporttype_type(export); - char* kind_str = (char*) wasm_externtype_to_kind_string(type); - - char* type_str = driver_alloc(256); - get_function_sig(type, type_str); - DRV_DEBUG("Export: %s [%s] -> %s", name->data, kind_str, type_str); - - init_msg[msg_i++] = ERL_DRV_ATOM; - init_msg[msg_i++] = driver_mk_atom(kind_str); - init_msg[msg_i++] = ERL_DRV_STRING; - init_msg[msg_i++] = (ErlDrvTermData)name->data; - init_msg[msg_i++] = name->size - 1; - init_msg[msg_i++] = ERL_DRV_STRING; - init_msg[msg_i++] = (ErlDrvTermData)type_str; - init_msg[msg_i++] = strlen(type_str); - init_msg[msg_i++] = ERL_DRV_TUPLE; - init_msg[msg_i++] = 3; - } - - init_msg[msg_i++] = ERL_DRV_NIL; - init_msg[msg_i++] = ERL_DRV_LIST; - init_msg[msg_i++] = (exports.size) + 1; - init_msg[msg_i++] = ERL_DRV_TUPLE; - init_msg[msg_i++] = 3; - - DRV_DEBUG("Sending init message to Erlang. Elements: %d", msg_i); - - int send_res = erl_drv_output_term(proc->port_term, init_msg, msg_i); - DRV_DEBUG("Send result: %d", send_res); - // TODO: init_msg is not freed up, but probably should live as long as the driver - // What happens during stop? - - proc->current_import = NULL; - proc->is_initialized = 1; - drv_unlock(proc->is_running); -} - -static void async_call(void* raw) { - Proc* proc = (Proc*)raw; - DRV_DEBUG("Calling function: %s", proc->current_function); - drv_lock(proc->is_running); - char* function_name = proc->current_function; - - // Find the function in the exports - wasm_func_t* func = get_exported_function(proc, function_name); - if (!func) { - send_error(proc, "Function not found: %s", function_name); - drv_unlock(proc->is_running); - return; - } - DRV_DEBUG("Func: %p", func); - - const wasm_functype_t* func_type = wasm_func_type(func); - const wasm_valtype_vec_t* param_types = wasm_functype_params(func_type); - const wasm_valtype_vec_t* result_types = wasm_functype_results(func_type); - - wasm_val_vec_t args, results; - wasm_val_vec_new_uninitialized(&args, param_types->size); - args.num_elems = param_types->num_elems; - // CONV: ei_term* -> wasm_val_vec_t - for(int i = 0; i < param_types->size; i++) { - args.data[i].kind = wasm_valtype_kind(param_types->data[i]); - } - int res = erl_terms_to_wasm_vals(&args, proc->current_args); - - for(int i = 0; i < args.size; i++) { - DRV_DEBUG("Arg %d: %d", i, args.data[i].of.i64); - DRV_DEBUG("Source term: %d", proc->current_args[i].value.i_val); - } - - if(res == -1) { - send_error(proc, "Failed to convert terms to wasm vals"); - drv_unlock(proc->is_running); - return; - } - - wasm_val_vec_new_uninitialized(&results, result_types->size); - results.num_elems = result_types->num_elems; - for (size_t i = 0; i < result_types->size; i++) { - results.data[i].kind = wasm_valtype_kind(result_types->data[i]); - } - - // Call the function - DRV_DEBUG("Calling function: %s", function_name); - wasm_trap_t* trap = wasm_func_call(func, &args, &results); - - if (trap) { - wasm_message_t trap_msg; - wasm_trap_message(trap, &trap_msg); - // wasm_frame_t* origin = wasm_trap_origin(trap); - // int32_t func_index = wasm_frame_func_index(origin); - // int32_t func_offset = wasm_frame_func_offset(origin); - // char* func_name; - - // DRV_DEBUG("WASM Exception: [func_index: %d, func_offset: %d] %.*s", func_index, func_offset, trap_msg.size, trap_msg.data); - send_error(proc, "%.*s", trap_msg.size, trap_msg.data); - drv_unlock(proc->is_running); - return; - } - - // Send the results back to Erlang - DRV_DEBUG("Results size: %d", results.size); - ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * (7 + (results.size * 2))); - DRV_DEBUG("Allocated msg"); - int msg_index = 0; - msg[msg_index++] = ERL_DRV_ATOM; - msg[msg_index++] = atom_execution_result; - for (size_t i = 0; i < results.size; i++) { - DRV_DEBUG("Processing result %d", i); - DRV_DEBUG("Result type: %d", results.data[i].kind); - switch(results.data[i].kind) { - case WASM_I32: - DRV_DEBUG("Value: %d", results.data[i].of.i32); - break; - case WASM_I64: - DRV_DEBUG("Value: %ld", results.data[i].of.i64); - break; - case WASM_F32: - DRV_DEBUG("Value: %f", results.data[i].of.f32); - break; - case WASM_F64: - DRV_DEBUG("Value: %f", results.data[i].of.f64); - break; - default: - DRV_DEBUG("Unknown result type.", results.data[i].kind); - break; - } - - int res_size = wasm_val_to_erl_term(&msg[msg_index], &results.data[i]); - msg_index += res_size; - } - msg[msg_index++] = ERL_DRV_NIL; - msg[msg_index++] = ERL_DRV_LIST; - msg[msg_index++] = results.size + 1; - msg[msg_index++] = ERL_DRV_TUPLE; - msg[msg_index++] = 2; - DRV_DEBUG("Sending %d terms", msg_index); - int response_msg_res = erl_drv_output_term(proc->port_term, msg, msg_index); - driver_free(msg); - - DRV_DEBUG("Msg: %d", response_msg_res); - - driver_free(proc->current_args); - driver_free(proc->current_function); - - wasm_val_vec_delete(&results); - proc->current_import = NULL; - - drv_unlock(proc->is_running); -} - -static ErlDrvData wasm_driver_start(ErlDrvPort port, char *buff) { - ErlDrvSysInfo info; - driver_system_info(&info, sizeof(info)); - DRV_DEBUG("Starting WASM driver"); - DRV_DEBUG("Port: %p", port); - DRV_DEBUG("Buff: %s", buff); - DRV_DEBUG("Caller PID: %d", driver_caller(port)); - DRV_DEBUG("ERL_DRV_EXTENDED_MAJOR_VERSION: %d", ERL_DRV_EXTENDED_MAJOR_VERSION); - DRV_DEBUG("ERL_DRV_EXTENDED_MINOR_VERSION: %d", ERL_DRV_EXTENDED_MINOR_VERSION); - DRV_DEBUG("ERL_DRV_FLAG_USE_PORT_LOCKING: %d", ERL_DRV_FLAG_USE_PORT_LOCKING); - DRV_DEBUG("info.major_version: %d", info.driver_major_version); - DRV_DEBUG("info.minor_version: %d", info.driver_major_version); - DRV_DEBUG("info.thread_support: %d", info.thread_support); - DRV_DEBUG("info.smp_support: %d", info.smp_support); - DRV_DEBUG("info.async_threads: %d", info.async_threads); - DRV_DEBUG("info.scheduler_threads: %d", info.scheduler_threads); - DRV_DEBUG("info.nif_major_version: %d", info.nif_major_version); - DRV_DEBUG("info.nif_minor_version: %d", info.nif_minor_version); - DRV_DEBUG("info.dirty_scheduler_support: %d", info.dirty_scheduler_support); - DRV_DEBUG("info.erts_version: %s", info.erts_version); - DRV_DEBUG("info.otp_release: %s", info.otp_release); - Proc* proc = driver_alloc(sizeof(Proc)); - proc->port = port; - DRV_DEBUG("Port: %p", proc->port); - proc->port_term = driver_mk_port(proc->port); - DRV_DEBUG("Port term: %p", proc->port_term); - proc->is_running = erl_drv_mutex_create("wasm_instance_mutex"); - proc->is_initialized = 0; - proc->start_time = time(NULL); - return (ErlDrvData)proc; -} - -static void wasm_driver_stop(ErlDrvData raw) { - Proc* proc = (Proc*)raw; - DRV_DEBUG("Stopping WASM driver"); - - // TODO: We should probably lock a mutex here and in the import_response function. - if(proc->current_import) { - DRV_DEBUG("Shutting down during import response..."); - proc->current_import->error_message = "WASM driver unloaded during import response"; - proc->current_import->ready = 1; - DRV_DEBUG("Signalling import_response with error"); - drv_signal(proc->current_import->response_ready, proc->current_import->cond, &proc->current_import->ready); - DRV_DEBUG("Signalled worker to fail. Locking is_running mutex to shutdown"); - } - - // We need to first grab the lock, then unlock it and destroy it. Must be a better way... - DRV_DEBUG("Grabbing is_running mutex to shutdown..."); - drv_lock(proc->is_running); - drv_unlock(proc->is_running); - DRV_DEBUG("Destroying is_running mutex"); - erl_drv_mutex_destroy(proc->is_running); - // Cleanup WASM resources - DRV_DEBUG("Cleaning up WASM resources"); - if (proc->is_initialized) { - DRV_DEBUG("Deleting WASM instance"); - wasm_instance_delete(proc->instance); - DRV_DEBUG("Deleted WASM instance"); - wasm_module_delete(proc->module); - DRV_DEBUG("Deleted WASM module"); - wasm_store_delete(proc->store); - DRV_DEBUG("Deleted WASM store"); - } - DRV_DEBUG("Freeing proc"); - driver_free(proc); - DRV_DEBUG("Freed proc"); -} - -static void wasm_driver_output(ErlDrvData raw, char *buff, ErlDrvSizeT bufflen) { - DRV_DEBUG("WASM driver output received"); - Proc* proc = (Proc*)raw; - //DRV_DEBUG("Port: %p", proc->port); - //DRV_DEBUG("Port term: %p", proc->port_term); - - int index = 0; - int version; - if(ei_decode_version(buff, &index, &version) != 0) { - send_error(proc, "Failed to decode message header (version)."); - return; - } - //DRV_DEBUG("Received term has version: %d", version); - //DRV_DEBUG("Index: %d. buff_len: %d. buff: %p", index, bufflen, buff); - int arity; - ei_decode_tuple_header(buff, &index, &arity); - //DRV_DEBUG("Term arity: %d", arity); - - char command[MAXATOMLEN]; - ei_decode_atom(buff, &index, command); - DRV_DEBUG("Port %p received command: %s, arity: %d", proc->port, command, arity); - - if (strcmp(command, "init") == 0) { - // Start async initialization - proc->pid = driver_caller(proc->port); - //DRV_DEBUG("Caller PID: %d", proc->pid); - int size, type; - ei_get_type(buff, &index, &type, &size); - //DRV_DEBUG("WASM binary size: %d bytes. Type: %c", size, type); - void* wasm_binary = driver_alloc(size); - long size_l = (long)size; - ei_decode_binary(buff, &index, wasm_binary, &size_l); - LoadWasmReq* mod_bin = driver_alloc(sizeof(LoadWasmReq)); - mod_bin->proc = proc; - mod_bin->binary = wasm_binary; - mod_bin->size = size; - //DRV_DEBUG("Calling for async thread to init"); - driver_async(proc->port, NULL, async_init, mod_bin, NULL); - } else if (strcmp(command, "call") == 0) { - if (!proc->is_initialized) { - send_error(proc, "Cannot run WASM function as module not initialized."); - return; - } - // Extract the function name and the args from the Erlang term and generate the wasm_val_vec_t - char* function_name = driver_alloc(MAXATOMLEN); - ei_decode_string(buff, &index, function_name); - //DRV_DEBUG("Function name: %s", function_name); - proc->current_function = function_name; - - //DRV_DEBUG("Decoding args. Buff: %p. Index: %d", buff, index); - proc->current_args = decode_list(buff, &index); - - driver_async(proc->port, NULL, async_call, proc, NULL); - } else if (strcmp(command, "import_response") == 0) { - // Handle import response - // TODO: We should probably start a mutex on the current_import object here. - // At the moment current_import->response_ready must not be locked so that signalling can happen. - DRV_DEBUG("Import response received. Providing..."); - if (proc->current_import) { - DRV_DEBUG("Decoding import response from Erlang..."); - proc->current_import->result_terms = decode_list(buff, &index); - proc->current_import->error_message = NULL; - - // Signal that the response is ready - drv_signal( - proc->current_import->response_ready, - proc->current_import->cond, - &proc->current_import->ready); - } else { - DRV_DEBUG("[error] No pending import response waiting"); - send_error(proc, "No pending import response waiting"); - } - } else if (strcmp(command, "write") == 0) { - DRV_DEBUG("Write received"); - long ptr, size; - int type; - ei_decode_tuple_header(buff, &index, &arity); - ei_decode_long(buff, &index, &ptr); - ei_get_type(buff, &index, &type, &size); - long size_l = (long)size; - char* wasm_binary; - int res = ei_decode_bitstring(buff, &index, &wasm_binary, NULL, &size_l); - DRV_DEBUG("Decoded binary. Res: %d. Size (bits): %ld", res, size_l); - long size_bytes = size_l / 8; - DRV_DEBUG("Write received. Ptr: %ld. Bytes: %ld", ptr, size_bytes); - long memory_size = get_memory_size(proc); - if(ptr + size_bytes > memory_size) { - DRV_DEBUG("Write request out of bounds."); - send_error(proc, "Write request out of bounds"); - return; - } - byte_t* memory_data = wasm_memory_data(get_memory(proc)); - DRV_DEBUG("Memory location to write to: %p", ptr+memory_data); - - memcpy(memory_data + ptr, wasm_binary, size_bytes); - DRV_DEBUG("Write complete"); - - ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * 2); - msg[0] = ERL_DRV_ATOM; - msg[1] = atom_ok; - erl_drv_output_term(proc->port_term, msg, 2); - driver_free(msg); - } - else if (strcmp(command, "read") == 0) { - DRV_DEBUG("Read received"); - long ptr, size; - ei_decode_tuple_header(buff, &index, &arity); - ei_decode_long(buff, &index, &ptr); - ei_decode_long(buff, &index, &size); - long size_l = (long)size; - long memory_size = get_memory_size(proc); - DRV_DEBUG("Read received. Ptr: %ld. Size: %ld. Memory size: %ld", ptr, size_l, memory_size); - if(ptr + size_l > memory_size) { - DRV_DEBUG("Read request out of bounds."); - send_error(proc, "Read request out of bounds"); - return; - } - DRV_DEBUG("Read received. Ptr: %ld. Size: %ld", ptr, size_l); - byte_t* memory_data = wasm_memory_data(get_memory(proc)); - DRV_DEBUG("Memory location to read from: %p", memory_data + ptr); - - char* out_binary = driver_alloc(size_l); - memcpy(out_binary, memory_data + ptr, size_l); - - DRV_DEBUG("Read complete. Binary: %p", out_binary); - - ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * 7); - int msg_index = 0; - msg[msg_index++] = ERL_DRV_ATOM; - msg[msg_index++] = atom_execution_result; - msg[msg_index++] = ERL_DRV_BUF2BINARY; - msg[msg_index++] = (ErlDrvTermData)out_binary; - msg[msg_index++] = size_l; - msg[msg_index++] = ERL_DRV_TUPLE; - msg[msg_index++] = 2; - - int msg_res = erl_drv_output_term(proc->port_term, msg, msg_index); - DRV_DEBUG("Read response sent: %d", msg_res); - driver_free(out_binary); - driver_free(msg); - } - else if (strcmp(command, "size") == 0) { - DRV_DEBUG("Size received"); - long size = get_memory_size(proc); - DRV_DEBUG("Size: %ld", size); - - ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * 6); - int msg_index = 0; - msg[msg_index++] = ERL_DRV_ATOM; - msg[msg_index++] = atom_execution_result; - msg[msg_index++] = ERL_DRV_INT; - msg[msg_index++] = size; - msg[msg_index++] = ERL_DRV_TUPLE; - msg[msg_index++] = 2; - erl_drv_output_term(proc->port_term, msg, msg_index); - } - else { - DRV_DEBUG("Unknown command: %s", command); - send_error(proc, "Unknown command"); - } -} - -static ErlDrvEntry wasm_driver_entry = { - NULL, - wasm_driver_start, - wasm_driver_stop, - wasm_driver_output, - NULL, - NULL, - "hb_beamr", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - ERL_DRV_FLAG_USE_PORT_LOCKING, - NULL, - NULL, - NULL -}; - -DRIVER_INIT(wasm_driver) { - atom_ok = driver_mk_atom("ok"); - atom_error = driver_mk_atom("error"); - atom_import = driver_mk_atom("import"); - atom_execution_result = driver_mk_atom("execution_result"); - return &wasm_driver_entry; -} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..4168a5bed --- /dev/null +++ b/docs/README.md @@ -0,0 +1,115 @@ + +## Documentation + +HyperBEAM uses [MkDocs](https://www.mkdocs.org/) with the [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) theme to build its documentation site. + +Building the documentation requires Python 3 and pip. It's recommended to use a virtual environment: + +```bash +# Create and activate a virtual environment (optional but recommended) +python3 -m venv venv +source venv/bin/activate # (macOS/Linux) On Windows use `venv\Scripts\activate` + +# Install required packages +pip3 install mkdocs mkdocs-material mkdocs-git-revision-date-localized-plugin + +# Deactivate the virtual environment when done +# deactivate +``` + +- **Source Files:** All documentation source files (Markdown `.md`, images, CSS) are located in the `docs/` directory. +- **Source Code Docs:** Erlang source code documentation is generated using `rebar3 edoc` (with the `edown_doclet` plugin) into the `docs/source-code-docs/` directory as Markdown files. These are then incorporated into the main MkDocs site. +- **Build Script:** The entire process (compiling, generating edoc, processing source docs, building the site) is handled by the `./docs/build-all.sh` script. + +To build the documentation locally: + +1. Ensure you are in the project root directory. +2. If using a virtual environment, make sure it's activated. +3. Run the build script: + ```bash + ./docs/build-all.sh + ``` + +This script performs the following steps: +- Compiles the Erlang project (`rebar3 compile`). +- Generates Markdown documentation from source code comments (`rebar3 edoc`) into `docs/source-code-docs/`. +- Processes the generated source code Markdown files (updates index, cleans up TOCs). +- Builds the MkDocs site into the `mkdocs-site` directory (`mkdocs build`). + +To view the built documentation locally: + +1. Navigate to the site directory: + ```bash + cd mkdocs-site + ``` +2. Start a simple Python HTTP server: + ```bash + python3 -m http.server 8000 + ``` +3. Open your web browser and go to `http://127.0.0.1:8000/`. + +Press `Ctrl+C` in the terminal where the server is running to stop it. + +The final static site is generated in the `mkdocs-site` directory, as configured in `mkdocs.yml` (`site_dir: mkdocs-site`). + +### Contributing to the Documentation + +To contribute documentation to HyperBEAM, follow these steps: + +1. **Fork the Repository** + - Fork the [HyperBEAM repository](https://github.com/permaweb/HyperBEAM) to your GitHub account + +2. **Choose the Right Location** + - Review the existing documentation structure in `./docs/` to determine the appropriate location for your content + - Documentation is organized into several main sections: + - `overview/`: High-level concepts and architecture + - `installation-core/`: Setup and configuration guides + - `components/`: Detailed component documentation + - `usage/`: Tutorials and usage guides + - `resources/`: Reference materials and source code documentation + - `community/`: Contribution guidelines and community resources + +3. **Create Your Documentation** + - Create a new Markdown file (`.md`) in the appropriate directory + - Follow the existing documentation style and format + - Use proper Markdown syntax and include: + - Clear headings and subheadings + - Code blocks with appropriate language specification + - Links to related documentation + - Images (if needed) in the `docs/assets/` directory + +4. **Update the Navigation** + - Edit `mkdocs.yml` to add your documentation to the navigation + - Place your entry in the appropriate section under the `nav:` configuration + - Follow the existing indentation and format + +5. **Test Your Changes** + - Set up a local development environment: + ```bash + python3 -m venv venv + source venv/bin/activate + pip3 install mkdocs mkdocs-material mkdocs-git-revision-date-localized-plugin + ``` + - Run the build script to verify your changes: + ```bash + ./docs/build-all.sh + ``` + - View the documentation locally at `http://127.0.0.1:8000/` + +6. **Submit a Pull Request** + - Create a new branch for your documentation changes + - Commit your changes with a descriptive message + - Submit a PR with: + - A clear title describing the documentation addition + - A detailed description explaining: + - The purpose of the new documentation + - Why it should be added to the official docs + - Any related issues or discussions + - Screenshots of the rendered documentation (if applicable) + +7. **Review Process** + - The HyperBEAM team will review your PR + - Be prepared to make adjustments based on feedback + - Once approved, your documentation will be merged into the main repository + +For more detailed contribution guidelines, see the [Community Guidelines](./docs/misc/community/guidelines.md) and [Development Setup](./docs/misc/community/setup.md) documentation. diff --git a/docs/assets/images/Power-web2-web3-fig.mp4 b/docs/assets/images/Power-web2-web3-fig.mp4 new file mode 100644 index 000000000..ff11d1511 Binary files /dev/null and b/docs/assets/images/Power-web2-web3-fig.mp4 differ diff --git a/docs/assets/images/create-new-devices-fig.png b/docs/assets/images/create-new-devices-fig.png new file mode 100644 index 000000000..c65714ea8 Binary files /dev/null and b/docs/assets/images/create-new-devices-fig.png differ diff --git a/docs/assets/images/favicon.ico b/docs/assets/images/favicon.ico new file mode 100644 index 000000000..67ababaff Binary files /dev/null and b/docs/assets/images/favicon.ico differ diff --git a/docs/assets/images/favicon.png b/docs/assets/images/favicon.png new file mode 100644 index 000000000..dedcaa2ef Binary files /dev/null and b/docs/assets/images/favicon.png differ diff --git a/docs/assets/images/monetize-fig.mp4 b/docs/assets/images/monetize-fig.mp4 new file mode 100644 index 000000000..2b69faa52 Binary files /dev/null and b/docs/assets/images/monetize-fig.mp4 differ diff --git a/docs/assets/images/monetize-your-hardware-fig.png b/docs/assets/images/monetize-your-hardware-fig.png new file mode 100644 index 000000000..04fb303a2 Binary files /dev/null and b/docs/assets/images/monetize-your-hardware-fig.png differ diff --git a/docs/assets/images/rock-solid-fig.png b/docs/assets/images/rock-solid-fig.png new file mode 100644 index 000000000..5de564c07 Binary files /dev/null and b/docs/assets/images/rock-solid-fig.png differ diff --git a/docs/assets/images/what-is-hyperbeam-fig.mp4 b/docs/assets/images/what-is-hyperbeam-fig.mp4 new file mode 100644 index 000000000..364d598f3 Binary files /dev/null and b/docs/assets/images/what-is-hyperbeam-fig.mp4 differ diff --git a/docs/assets/style.css b/docs/assets/style.css new file mode 100644 index 000000000..faf217687 --- /dev/null +++ b/docs/assets/style.css @@ -0,0 +1,880 @@ +/* General Text Styles */ +h1 { + font-size: clamp(1.6rem, 1.5vw, 1.7rem) !important; + color: rgba(60, 60, 67) !important; + font-weight: 600 !important; +} + +h2 { + font-size: clamp(1.2rem, 1.5vw, 1.3rem) !important; + color: rgba(60, 60, 67); + ; + /* Semi-transparent black */ +} + +p { + font-size: clamp(0.6rem, 1.5vw, 0.7rem); + line-height: 1.75; +} + +li { + font-size: clamp(0.6rem, 1.5vw, 0.75rem); + line-height: 1.75; +} + +img { + user-select: none; +} + +input { + font-size: clamp(0.47rem, 1.5vw, 0.5rem) !important; +} + +body { + --docs-max-width: 60rem; + --homepage-max-width: 90rem; + --sections-max-width: 80rem; + --parallax-perspective: 2rem; + --md-accent-fg-color: #555555 !important; + --md-default-fg-color--light: #bebebe !important; +} + +.md-nav__item--section>.md-nav__link { + color: black !important; + margin-bottom: 8px; +} + +/* +h1, h2, h3, h4, h5, h6 { + border-bottom: 1px solid #ccc; + padding-bottom: 10px; +} +*/ + +.md-content h2, +h3, +h4, +h5, +h6 { + font-weight: 400 !important; +} + +.md-content h2 { + border-top: 1px solid #ccc; + padding-top: 1.5rem; +} + +.md-content h3, +h4, +h5, +h6 { + border-top: 1px solid #e5e5e58a; + padding-top: 1rem; +} + +/* Body and Header Customization */ +.md-header { + box-shadow: none !important; + z-index: 100; + transition: transform 0.3s ease-in-out; + position: fixed; + width: 100%; + transform: translateY(0); +} + +.header-hidden { + transform: translateY(-100%); + transition: transform 0.15s ease-in-out; +} + +.md-main { + transition: padding-top 0.3s ease; +} + +.header-hidden + .md-main { + padding-top: 0; +} + +[dir=ltr] .md-sidebar--primary { + left: -15rem !important; +} + +[data-md-toggle=drawer]:checked~.md-container .md-sidebar--primary { + transform: translateX(15rem) !important; +} + +.md-grid { + /* transition: all 1s; this occurs on initial load, causing strange UI artifact */ + max-width: var(--docs-max-width); +} + +.md-main__inner { + gap: 2.25rem; +} + +.custom-homepage-header { + position: fixed; + filter: invert(1); + top: 0; + z-index: 20 !important; + background: linear-gradient(0deg, rgba(255, 255, 255, 0) 0%, #ffffff 100%); + border-bottom: 0px solid; +} + + + +.custom-homepage-header .md-grid { + max-width: var(--homepage-max-width); +} + +.custom-homepage-header .md-tabs { + background: #ffffff00 !important; + border-bottom: 0; +} + +/* Logo Customization */ +.md-logo img { + height: 0.9rem !important; +} + +/* Header Topic Visibility */ +.md-header__topic { + display: none; +} + +/* Navigation Styles */ +.md-nav__title, +.md-nav__item { + font-size: clamp(0.6rem, 1.5vw, 0.65rem) !important; + box-shadow: none !important; +} + +.md-nav__title { + color: #000000 !important; +} + +.md-nav__link { + padding: 4px 16px; + border-radius: 6px; + margin: 0; + margin-top: 4px; +} + +.md-tabs__item--active .md-tabs__link { + position: relative; + font-weight: 700; +} + +.md-tabs__item--active .md-tabs__link::after { + content: ""; + position: absolute; + bottom: -9px; + left: 5%; + width: 90%; + height: 2px; + background-color: black; + border-radius: 1px; + transition: all 0.3s ease; +} + + +.md-nav__link:hover { + background: rgb(243, 243, 243); +} + +.md-nav__link--active { + color: #000000 !important; + /* Active link color */ + font-weight: 700; + background: rgb(237, 237, 237); +} + +.md-sidebar { + width: 15rem; +} + +[dir=ltr] .md-sidebar__inner { + padding-right: calc(100% - 15rem); +} + +.md-nav--secondary { + border-left: 0.05rem solid lightgray !important; +} + +/* Tab Navigation */ +.md-tabs__link { + font-size: clamp(0.65rem, 1.5vw, 0.65rem) !important; +} + +.md-tabs__list { + justify-content: space-between; +} + +.md-tabs__item { + + height: 1.5rem; + +} + +.md-tabs__item a { + margin-top: 0px; +} + +/* Search Form Customization */ +.md-search__form { + height: 32px !important; + /* Adjust search form height */ + padding: 0 8px !important; +} + +.md-search { + margin-left: 1.8rem; +} + +.md-search__inner { + width: 8rem; +} + +.md-search__input { + padding-left: 1rem !important; + padding-right: 1rem !important; +} + +/* Search Results */ +.md-search-result__item h1, +.md-search-result__item h2 { + font-size: clamp(0.7rem, 1.5vw, 0.8rem) !important; +} + +.md-search-result__item h2 { + font-weight: 500 !important; +} + +.md-search-result__item summary div { + font-size: clamp(0.5rem, 1.5vw, 0.55rem) !important; +} + +.md-search-result__icon { + width: 0.8rem; + height: 0.8rem; +} + +.md-search__icon svg { + width: 16px; + height: 16px; +} + +.md-search__icon { + top: 0.4rem !important; + left: 0.3rem !important; +} + +/* Source Citation Styles */ + +.md-header__source { + width: 10rem; + padding: 0.6rem; +} + +.md-source { + font-size: clamp(0.45rem, 1.5vw, 0.5rem); + display: flex; + justify-content: end; + align-items: center; +} + +.md-source__fact { + font-size: clamp(0.35rem, 1.5vw, 0.4rem); +} + +/* Tagline Styles */ +.tagline { + font-size: 0.9rem; + letter-spacing: 1px; + margin: 0; + color: var(--md-default-fg-color--light); +} + +/* Color Scheme Customization for Slate */ +[data-md-color-scheme="slate"] .logo-text h1 { + color: #FFF; +} + +[data-md-color-scheme="slate"] .tagline { + color: #AAA; +} + +/* Hero Customization */ +.custom-homepage-main { + perspective: var(--parallax-perspective); + overflow: hidden auto; + scroll-behavior: smooth; + height: 100vh; + width: 100vw; +} + +.hero-container { + position: relative; + height: 160vh; + transform-style: preserve-3d; +} + + +.hero-content-wrapper { + + height: inherit; +} + +.hero-floating-wrapper { + display: flex; + flex-direction: column; + justify-content: end; + position: sticky; + top: 0; + z-index: 11; + height: 100vh; + margin-bottom: -100vh; + +} + +.hero-detail { + display: flex; + flex-direction: column; +} + +.hero-detail h1 { + font-size: clamp(1.3rem, 1.5vw, 1.45rem) !important; + margin: 0 !important; + font-weight: 500 !important; + color: white !important; +} + +.hero-detail span { + color: white !important; + margin: 0; +} + +.hero-inner-content-middle { + display: flex; + + margin-left: auto; + margin-right: auto; + justify-content: space-between; + align-items: center; + /* padding: 2rem 0.8rem; */ + transition: all 1s; + max-width: var(--homepage-max-width); + width: 100%; + padding: 2rem 0.8rem; + +} + +.hero-inner-content-bottom { + display: flex; + max-width: var(--homepage-max-width); + margin-left: auto; + margin-right: auto; + transition: all 1s; + width: 100%; + padding: 1rem 0.8rem; +} + +.hero-text-container { + display: flex; + flex-direction: column; + flex: 1 0 0; + transition: all 1s; + align-items: start; + justify-content: space-between; +} + +.hero-button-cards-container { + display: flex; + flex: 1.5 0 0; + gap: 0.5rem; + height: 40%; + min-height: 7rem; + backdrop-filter: blur(50px); + +} + +.hero-button-card { + position: relative; + display: flex; + flex-direction: column; + flex: 1 0 0; + border-radius: 4px; + border: 1px solid #2b2b2b; + background: rgba(0, 0, 0, 0.75); + justify-content: space-between; + align-items: start; + padding: 8px; + cursor: pointer; + transition: all 300ms; + height: 100%; +} + +.hero-button-card:hover { + background: rgba(0, 0, 0, 1); + border: 1px solid rgb(149, 149, 149); +} + +.hero-main-heading h1 { + font-size: clamp(1.8rem, 1.5vw, 2.0rem) !important; + margin: 0 !important; + font-weight: 500 !important; + color: white !important; +} + +.hero-button-card h2 { + font-size: clamp(0.9rem, 1.5vw, 1rem) !important; + margin: 0 !important; + font-weight: 500 !important; + color: white !important; +} + +.hero-button-card p { + font-size: clamp(0.65rem, 1.5vw, 0.7rem) !important; + color: #c0c0c0 !important; + font-weight: 500; +} + +.hero-main-heading h2, +.hero-button-card p { + font-size: clamp(0.55rem, 1.5vw, 0.6rem) !important; + line-height: normal; + margin: 0; + font-weight: 500; + text-align: left; +} + +.hero-main-heading h2 { + color: white !important; +} + +/* Hero Rocks */ + +.rocks { + position: absolute; + top: 0; + left: 0; + height: 100%; + width: 100%; + object-fit: cover; + background-position: center; + background-size: cover; + transform: + translateZ(calc(var(--parallax-perspective) * (var(--depth, 0) * -1))) scale(calc(1 + var(--depth, 1))); + transform-origin: 50vw 50vh; + will-change: transform; + pointer-events: none; + z-index: calc(10 - var(--depth, 0)); +} + +.rocks img { + width: 100%; + height: 100%; + object-fit: cover; +} + +.flicker-img { + animation: flickerLight 10s infinite; + will-change: filter; +} + +.foreground-rocks { + z-index: 3; + padding-bottom: 200px; + +} + +.foreground-rocks-2 { + z-index: 2; + +} + +.mid-rocks { + z-index: 2; + +} + +.background-rocks { + position: absolute; + z-index: 1; + height: 100%; + object-fit: cover; +} + +.background-chroma { + position: absolute; + z-index: 0; + height: 100%; + width: 100%; + object-fit: cover; +} + +.background-chroma img { + object-fit: cover; +} + +.background-rocks img { + object-fit: cover; + transform: translateZ(0); + image-rendering: smooth; + /* filter: brightness(0.5); */ +} + +.scroll-button span { + margin-left: 5px; +} + + + +.dark-bottom { + position: absolute; + bottom: 0; + background: linear-gradient(0deg, #000000 0%, rgba(255, 179, 0, 0) 100%); + height: 100vh; + width: 100%; + z-index: 10; +} + +/* Sections Customization */ +.section-container { + position: relative; + display: flex; + width: 100%; +} + +.section-container-background-primary { + background: white; +} + +.section-container-background-secondary { + background: #F9F9F9; +} + +.section-inner-content h1 { + font-size: clamp(1.1rem, 1.5vw, 1.2rem) !important; + margin: 0 !important; + font-weight: 500 !important; +} + +.section-inner-content h2 { + font-size: clamp(0.65rem, 1.5vw, 0.7rem) !important; + color: #6E6E6E; + font-weight: 500; +} + +.section-inner-content span { + margin: 0; +} + +.section-header { + display: flex; + width: 100%; + justify-content: space-between; +} + +.section-header h1 { + margin-block-start: 0.7em !important; +} + +.divider { + height: 1px; + width: 100%; + background: #D4D4D4; +} + +.section-inner-content { + display: flex; + flex-direction: column; + width: 100%; + max-width: var(--sections-max-width); + z-index: 2; + margin-left: auto; + margin-right: auto; + padding: 2rem 0.8rem; + transition: all 1s; + min-height: 70vh; +} + +.section-monetize { + justify-content: space-between; + position: relative; + overflow: hidden; +} + +.section-monetize>*:not(:last-child) { + z-index: 3; +} + +.double-column-content { + display: flex; + width: 100%; + padding: 48px 0px; + height: 100%; +} + +.double-single-grid-content { + display: grid; + grid-template-columns: repeat(2, 1fr); + grid-template-rows: auto auto; + gap: 2rem; + width: 100%; + padding: 48px 0px; +} + +.column-container { + width: 100%; + height: 100%; + min-height: 70vh; +} + +.grid-container { + position: relative; + display: flex; + min-height: 450px; + width: 100%; + background: white; + overflow: hidden; + border-radius: 4px; +} + +.full-span { + grid-column: span 2; + min-height: 600px; + overflow: hidden; +} + +.column-text { + display: flex; + flex-direction: column; + height: 100%; + width: 100%; + justify-content: space-between; + padding: 16px; +} + +.feature-cards { + display: grid; + width: 100%; + grid-template-columns: repeat(3, 1fr); + gap: 0.5rem; + margin-top: 4rem; +} + +.card { + display: flex; + flex-direction: column; + justify-content: space-between; + background: #F9F9F9; + border: 1px solid #E6E6E6; + border-radius: 4px; + min-height: 130px; + padding: 8px; + overflow: hidden; +} + +.card p { + font-size: clamp(0.65rem, 1.5vw, 0.7rem) !important; + color: #6E6E6E !important; + font-weight: 500; + line-height: normal; +} + +.transparent-card { + min-height: 125px; + display: flex; + width: 100%; + background: rgba(255, 255, 255, 0.559); +} + +.transparent-card p { + line-height: normal; +} + +.card-row { + display: flex; + width: 100%; + overflow: hidden; +} + +.grid-card-header { + display: flex; + flex-direction: column; + gap: 4px; +} + +.card span { + /* background: white; */ + width: fit-content; +} + +.card p { + margin: 0; +} + +.cta-wrapper { + display: flex; + gap: 8px; + width: 100%; + + align-items: center; +} + +.cta-right { + justify-content: end; +} + +.cta-left { + justify-content: start; +} + +.main-button { + display: flex; + align-items: center; + justify-content: center; + background: #E4EABB; + padding: 12.5px 60px; + border-radius: 4px; + font-size: clamp(0.6rem, 1.5vw, 0.7rem); + cursor: pointer; +} + +.fig-container { + position: relative; + height: 100%; + display: flex; + width: 100%; + justify-content: center; + overflow: hidden; +} + + +.fig { + position: absolute; + width: 100%; + max-width: 400px; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); +} + +.what-is-hyperbeam-fig { + position: absolute; + width: 100%; + top: 50%; + left: 50%; + max-width: 600px; + transform: translate(-50%, -50%); +} + +.power-web2-web3-fig { + position: absolute; + width: 100%; + top: 0%; + right: 0%; + max-width: 700px; + transform: translate(-10%, -10%); + +} + + + +.monetize-fig { + position: absolute; + top: 0%; + left: 0%; + + transform: translate(0%, 0%); + z-index: 1; + opacity: 20%; + +} + +/* Footer Customization */ +.md-footer-meta { + display: none !important; +} + +.md-footer__link { + margin: 0; +} + +.md-footer { + border-top: .05rem solid #00000012; + background-color: white; + color: black; +} + +.md-footer__direction { + font-size: clamp(0.35rem, 1.5vw, 0.45rem); +} + +.md-footer__title { + font-size: clamp(0.75rem, 1.5vw, 0.85rem); +} + +/* Last Updated Date */ +.md-source-file { + border-top: 1px solid #ddd; + /* Adds a divider above the text */ + padding-top: 10px; + /* Adds some space above the text */ + font-size: 0.9em; + /* Makes the font size slightly smaller */ + font-style: italic; + /* Makes the text italic */ + margin-top: 2rem !important; + margin-bottom: 2rem !important; +} + +.md-source-file .md-source-file__fact { + display: flex; + align-items: center; + gap: 0; +} + +.md-source-file .md-icon { + width: 1em; + /* Adjusts the icon size proportionally */ + height: 1em; + /* Adjusts the icon size proportionally */ + margin-right: 0.5em; + /* Adds space between the icon and the text */ +} + +.md-source-file .git-revision-date-localized-plugin::before { + content: "Last updated: "; + /* Adds the prefix before the date */ +} + + +@keyframes flickerLight { + + 0%, + 100% { + filter: brightness(1.2); + } + + 10% { + filter: brightness(1); + } + + 30% { + filter: brightness(1.1); + } + + 50% { + filter: brightness(0.9); + } + + 70% { + filter: brightness(1.1); + } + + 90% { + filter: brightness(1.3); + } +} \ No newline at end of file diff --git a/docs/build-all.sh b/docs/build-all.sh new file mode 100755 index 000000000..80f075d5c --- /dev/null +++ b/docs/build-all.sh @@ -0,0 +1,365 @@ +#!/bin/bash + +# Script to build HyperBEAM documentation in one seamless command +# +# Usage: ./docs/build-all.sh [-v | --verbose] +# -v, --verbose: Show detailed output from rebar3 and mkdocs commands. + +# --- Color Definitions --- +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' +BOLD='\033[1m' +NC='\033[0m' # No Color + +# HyperBEAM Logo Colors +NEON_GREEN='\033[38;5;46m' +CYAN='\033[38;5;51m' +BRIGHT_YELLOW='\033[38;5;226m' +MAGENTA='\033[38;5;201m' +BRIGHT_RED='\033[38;5;196m' +BLACK='\033[38;5;0m' +GRAY='\033[38;5;245m' + +# --- Helper Functions --- +log_success() { + echo -e "${GREEN}✓ $1${NC}" +} + +log_info() { + echo -e "${BLUE}→ $1${NC}" +} + +log_step() { + echo -e "\n${YELLOW}${BOLD}$1${NC}" +} + +log_error() { + echo -e "${RED}✗ $1${NC}" +} + +# --- Variable Defaults --- +VERBOSE=false + +# --- Parse Command Line Arguments --- +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + -v|--verbose) + VERBOSE=true + log_info "Verbose mode enabled" + shift # past argument + ;; + *) + # unknown option + log_error "Unknown option: $1" + # Optionally, show usage here and exit + exit 1 + ;; + esac +done + +# --- Display HyperBEAM ASCII Logo --- +display_logo() { + echo -e " +${NEON_GREEN} ++ ${BLACK}${BOLD} ${NC} +${NEON_GREEN} +++ ${BLACK}${BOLD} _ ${NC} +${NEON_GREEN} ++++* ${BLACK}${BOLD}| |__ _ _ _ __ ___ _ __ ${NC} +${NEON_GREEN} :+++*${BRIGHT_YELLOW}## ${BLACK}${BOLD} | '_ \\| | | | '_ \\ / _ \\ '__| ${NC} +${NEON_GREEN} ++**${BRIGHT_YELLOW}#### ${BLACK}${BOLD} | | | | |_| | |_) | __/ | ${NC} +${NEON_GREEN} +++${BRIGHT_YELLOW}####${NEON_GREEN}*** ${BLACK}${BOLD} |_| |_|\\__, | .__/ \\___|_| ${NC} +${NEON_GREEN} +*${BRIGHT_YELLOW}##${NEON_GREEN}****${MAGENTA}+-- ${BLACK}${BOLD} |___/|_| ${NC} +${MAGENTA} -**${BRIGHT_YELLOW}##${NEON_GREEN}**${MAGENTA}+------ ${BLACK}${BOLD} BEAM.${NC} +${MAGENTA} -##${NEON_GREEN}*+${BRIGHT_RED}---::::::: +${GRAY} =${GRAY}%%${NEON_GREEN}*+${BRIGHT_RED}=-:::::::::${GRAY} DECENTRALIZED OPERATING SYSTEM${NC} +" +} + +# --- Script Start --- +display_logo +log_step "DOCUMENTATION BUILD" + +# Ensure we're in the root directory of the project +ROOT_DIR="$(dirname "$(realpath "$0")")/.." +cd "$ROOT_DIR" || { log_error "Failed to change to root directory"; exit 1; } + +# --- Step 1: Compile the project with rebar3 --- +log_step "Compiling project" +if [ "$VERBOSE" = true ]; then + rebar3 compile || { log_error "rebar3 compile failed"; exit 1; } +else + rebar3 compile > /dev/null 2>&1 || { log_error "rebar3 compile failed"; exit 1; } +fi +log_success "Compilation completed" + +# --- Step 2: Generate edoc documentation --- +log_step "Generating edoc documentation" +if [ "$VERBOSE" = true ]; then + rebar3 edoc || { log_error "rebar3 edoc failed"; exit 1; } +else + rebar3 edoc > /dev/null 2>&1 || { log_error "rebar3 edoc failed"; exit 1; } +fi +log_success "Edoc generation completed" + +# --- Step 3: Process source code documentation --- +log_step "Processing source code documentation" +DOCS_DIR="$ROOT_DIR/docs/resources/source-code" +INDEX_FILE="$DOCS_DIR/index.md" + +# Check if the directory and index file exist +if [ ! -d "$DOCS_DIR" ]; then + log_error "Source code docs directory not found at $DOCS_DIR" + exit 1 +fi + +if [ ! -f "$INDEX_FILE" ]; then + log_error "Source code index file not found at $INDEX_FILE" + exit 1 +fi + +# --- Step 3.1: Remove ToC entries for Function Index and Function Details --- +log_info "Cleaning module files" + +find "$DOCS_DIR" -maxdepth 1 -type f -name "*.md" -not -name "index.md" -not -name "README.md" | while read -r file; do + TEMP_MODULE_FILE=$(mktemp) + + awk ' + /^\* \[Description\]\(#description\)$/ { next; } + /^\* \[Function Index\]\(#index\)$/ { next; } + /^\* \[Function Details\]\(#functions\)$/ { next; } + /^\* \[Data Types\]\(#types\)$/ { next; } + { print; } + ' "$file" > "$TEMP_MODULE_FILE" + + mv "$TEMP_MODULE_FILE" "$file" +done + +# --- Step 3.2: Add GitHub links to source code files --- +log_info "Adding GitHub repository links to source code files" + +# Base GitHub repository URL +GITHUB_BASE_URL="https://github.com/permaweb/HyperBEAM/blob/main/src" + +# Process only files in the resources/source-code directory +find "$DOCS_DIR" -maxdepth 1 -type f -name "*.md" -not -name "index.md" -not -name "README.md" | while read -r file; do + TEMP_MODULE_FILE_CLEANED=$(mktemp) + TEMP_MODULE_FILE_FINAL=$(mktemp) + + # Get the module name from the filename + module_name=$(basename "$file" .md) + + # Define the exact header pattern to remove + # Note: Assumes module names are simple enough not to need complex regex escaping. + header_pattern="^# Module ${module_name} #$" + + # Remove the old header line using sed + # Use -i option for in-place editing on a temporary copy first to avoid issues with read/write on same file descriptor + cp "$file" "$TEMP_MODULE_FILE_CLEANED" + sed -i'' -e "/${header_pattern}/d" "$TEMP_MODULE_FILE_CLEANED" + + # Add the new GitHub link header at the top of the final temp file + # Using the user's updated format with "Module" text + echo "# [Module $module_name.erl]($GITHUB_BASE_URL/$module_name.erl)" > "$TEMP_MODULE_FILE_FINAL" + echo "" >> "$TEMP_MODULE_FILE_FINAL" + + # Append the cleaned content (without the old header) + cat "$TEMP_MODULE_FILE_CLEANED" >> "$TEMP_MODULE_FILE_FINAL" + + # Replace the original file + mv "$TEMP_MODULE_FILE_FINAL" "$file" + + # Clean up the intermediate temp file + rm "$TEMP_MODULE_FILE_CLEANED" +done + +log_success "GitHub links added and old headers removed" + +# --- Step 3.3: Update mkdocs.yml navigation with current module list --- +# log_info "Updating mkdocs.yml navigation" + +# # Create temporary file for the new mkdocs.yml +# MKDOCS_TEMP=$(mktemp) +# MKDOCS_FILE="$ROOT_DIR/mkdocs.yml" + +# # Process mkdocs.yml file to remove old modules +# awk ' +# BEGIN { in_modules = 0; skip_modules = 0; } +# /^ *- Modules:/ { +# print $0; +# in_modules = 1; +# skip_modules = 1; +# next; +# } +# { +# if (skip_modules == 0) { +# print $0; +# } +# if (in_modules == 1 && $0 ~ /^ *-/) { +# if ($0 !~ /^ *- Modules:/) { +# in_modules = 0; +# skip_modules = 0; +# print $0; +# } +# } +# } +# ' "$MKDOCS_FILE" > "$MKDOCS_TEMP" + +# # Find the position to insert module entries +# INSERT_LINE=$(grep -n "^ *- Modules:" "$MKDOCS_TEMP" | cut -d: -f1) + +# if [ -z "$INSERT_LINE" ]; then +# log_error "Could not find '- Modules:' section in mkdocs.yml" +# # Clean up temp file before exiting +# rm -f "$MKDOCS_TEMP" +# exit 1 +# fi + +# # Prepare head and tail parts +# head -n "$INSERT_LINE" "$MKDOCS_TEMP" > "${MKDOCS_TEMP}.head" +# tail -n +$((INSERT_LINE + 1)) "$MKDOCS_TEMP" > "${MKDOCS_TEMP}.tail" + +# # Use an associative array to track added modules +# declare -A added_modules +# MODULE_LINES="" # Accumulate module lines here + +# # Use process substitution to read modules without a subshell per iteration +# while IFS= read -r module_file; do +# # Check if module_file is empty or not a file (safety check) +# if [[ -z "$module_file" || ! -f "$module_file" ]]; then +# continue +# fi + +# module_name=$(basename "$module_file" .md) + +# # Only add the module if its basename hasn't been added yet +# if [[ -z "${added_modules[$module_name]}" ]]; then +# # Append the line to a variable instead of echoing directly +# MODULE_LINES+=" - $module_name: 'resources/source-code/$module_name.md'\n" +# added_modules[$module_name]=1 +# fi +# # Feed the loop using process substitution <(...) +# done < <(find "$DOCS_DIR" -maxdepth 1 -type f -name "*.md" -not -name "index.md" -not -name "README.md" | sort -u) + +# # Assemble the final mkdocs.yml +# { +# cat "${MKDOCS_TEMP}.head" +# # Echo the accumulated module lines (use printf for robustness) +# printf "%b" "$MODULE_LINES" +# cat "${MKDOCS_TEMP}.tail" +# } > "$MKDOCS_FILE" + +# # Clean up temporary files +# rm -f "$MKDOCS_TEMP" "${MKDOCS_TEMP}.head" "${MKDOCS_TEMP}.tail" + +# log_success "mkdocs.yml navigation updated" + +# --- Step 4: Build and serve mkdocs --- +log_step "Building mkdocs documentation" +if [ "$VERBOSE" = true ]; then + mkdocs build || { log_error "mkdocs build failed"; exit 1; } +else + mkdocs build > /dev/null 2>&1 || { log_error "mkdocs build failed"; exit 1; } +fi + +# Find the latest CSS files with their hashes +MAIN_CSS=$(find ./mkdocs-site/assets/stylesheets -name "main.*.min.css" | sort | tail -n 1) +PALETTE_CSS=$(find ./mkdocs-site/assets/stylesheets -name "palette.*.min.css" | sort | tail -n 1) + +# Extract just the filenames from the paths +MAIN_CSS_FILE=$(basename "$MAIN_CSS") +PALETTE_CSS_FILE=$(basename "$PALETTE_CSS") + +# Find all HTML files and replace the CSS references in each one +log_info "Updating CSS references in HTML files" +find ./mkdocs-site -type f -name "*.html" | while read -r html_file; do + sed -i'' -e "s|MAIN\.CSS|assets/stylesheets/$MAIN_CSS_FILE|g" "$html_file" + sed -i'' -e "s|MAIN_PALETTE\.CSS|assets/stylesheets/$PALETTE_CSS_FILE|g" "$html_file" +done + +# Remove .html-e files +find ./mkdocs-site -type f -name "*.html-e" -delete + +log_success "MkDocs build completed" + +# --- Step 5: Generate LLM context files --- +log_step "Generating LLM context files" + +LLM_SUMMARY_FILE="$ROOT_DIR/docs/llms.txt" +LLM_FULL_FILE="$ROOT_DIR/docs/llms-full.txt" +DOC_DIRS=( + "$ROOT_DIR/docs/introduction" + "$ROOT_DIR/docs/run" + "$ROOT_DIR/docs/build" + "$ROOT_DIR/docs/devices" + "$ROOT_DIR/docs/resources" +) + +# Get current timestamp +GENERATION_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +# Generate llms.txt (routes and summary) +log_info "Creating summary and routes file" +cat > "$LLM_SUMMARY_FILE" <> "$LLM_SUMMARY_FILE" + echo "### $SECTION_HEADING" >> "$LLM_SUMMARY_FILE" + echo "" >> "$LLM_SUMMARY_FILE" + + find "$DOC_DIR" -type f -name "*.md" -print | + sed "s|^$ROOT_DIR/||" | + sed 's/^docs\///' | + sed 's/\.md$//' | + sort | + while IFS= read -r base_path; do + html_path="${base_path}.html" + md_path_relative="docs/${base_path}.md" + md_file_path="$ROOT_DIR/$md_path_relative" + + if [ -f "$md_file_path" ]; then + title=$(grep -m 1 '^# ' "$md_file_path" 2>/dev/null | sed 's/^# //') + else + title="" + fi + + if [ -z "$title" ]; then + title=$(basename "$base_path" | sed -e 's/-/ /g' -e 's/\b\(.\)/\u\1/g') + fi + + echo "* [$title](./$html_path)" >> "$LLM_SUMMARY_FILE" + done +done + +# Generate llms-full.txt (concatenated content) +log_info "Creating full documentation file" +echo "Generated: $GENERATION_TIMESTAMP" > "$LLM_FULL_FILE" +echo "" >> "$LLM_FULL_FILE" + +find "${DOC_DIRS[@]}" -type f -name "*.md" | sort | while read -r doc_file; do + relative_path="${doc_file#$ROOT_DIR/}" + echo "--- START OF FILE: $relative_path ---" >> "$LLM_FULL_FILE" + cat "$doc_file" >> "$LLM_FULL_FILE" + echo "" >> "$LLM_FULL_FILE" + echo "--- END OF FILE: $relative_path ---" >> "$LLM_FULL_FILE" + echo "" >> "$LLM_FULL_FILE" +done + +log_success "LLM context files generated" + +# --- Final success message --- +echo -e "\n${GREEN}${BOLD}✓ Documentation build completed successfully${NC}\n" diff --git a/docs/build/exposing-process-state.md b/docs/build/exposing-process-state.md new file mode 100644 index 000000000..a15e8985e --- /dev/null +++ b/docs/build/exposing-process-state.md @@ -0,0 +1,109 @@ +# Exposing Process State with the Patch Device + +The [`~patch@1.0`](../resources/source-code/dev_patch.md) device provides a mechanism for AO processes to expose parts of their internal state, making it readable via direct HTTP GET requests along the process's HyperPATH. + +## Why Use the Patch Device? + +Standard AO process execution typically involves sending a message to a process, letting it compute, and then potentially reading results from its outbox or state after the computation is scheduled and finished. This is asynchronous. + +The `patch` device allows for a more direct, synchronous-like read pattern. A process can use it to "patch" specific data elements from its internal state into a location that becomes directly accessible via a HyperPATH GET request *before* the full asynchronous scheduling might complete. + +This is particularly useful for: + +* **Web Interfaces:** Building frontends that need to quickly read specific data points from an AO process without waiting for a full message round-trip. +* **Data Feeds:** Exposing specific metrics or state variables for monitoring or integration with other systems. +* **Caching:** Allowing frequently accessed data to be retrieved efficiently via simple HTTP GETs. + +## How it Works + +1. **Process Logic:** Inside your AO process code (e.g., in Lua or WASM), when you want to expose data, you construct an **Outbound Message** targeted at the [`~patch@1.0`](../resources/source-code/dev_patch.md) device. +2. **Patch Message Format:** This outbound message typically includes tags that specify: + * `device = 'patch@1.0'` + * A `cache` tag containing a table. The **keys** within this table become the final segments in the HyperPATH used to access the data, and the **values** are the data itself. + * Example Lua using `aos`: `Send({ Target = ao.id, device = 'patch@1.0', cache = { mydatakey = MyValue } })` +3. **HyperBEAM Execution:** When HyperBEAM executes the process schedule and encounters this outbound message: + * It invokes the `dev_patch` module. + * `dev_patch` inspects the message. + * It takes the keys from the `cache` table (`mydatakey` in the example) and their associated values (`MyValue`) and makes these values available under the `/cache/` path segment. +4. **HTTP Access:** You (or any HTTP client) can now access this data directly using a GET request: + ``` + GET /~process@1.0/compute/cache/ + # Or potentially using /now/ + GET /~process@1.0/now/cache/ + ``` + The HyperBEAM node serving the request will resolve the path up to `/compute/cache` (or `/now/cache`), then use the logic associated with the patched data (`mydatakey`) to return the `MyValue` directly. + +## Initial State Sync (Optional) + +It can be beneficial to expose the initial state of your process via the `patch` device as soon as the process is loaded or spawned. This makes key data points immediately accessible via HTTP GET requests without requiring an initial interaction message to trigger a `Send` to the patch device. + +This pattern typically involves checking a flag within your process state to ensure the initial sync only happens once. Here's an example from the Token Blueprint, demonstrating how to sync `Balances` and `TotalSupply` right after the process starts: + +```lua +-- Place this logic at the top level of your process script, +-- outside of specific handlers, so it runs on load. + +-- Initialize the sync flag if it doesn't exist +InitialSync = InitialSync or 'INCOMPLETE' + +-- Sync state on spawn/load if not already done +if InitialSync == 'INCOMPLETE' then + -- Send the relevant state variables to the patch device + Send({ device = 'patch@1.0', cache = { balances = Balances, totalsupply = TotalSupply } }) + -- Update the flag to prevent re-syncing on subsequent executions + InitialSync = 'COMPLETE' + print("Initial state sync complete. Balances and TotalSupply patched.") +end +``` + +**Explanation:** + +1. `InitialSync = InitialSync or 'INCOMPLETE'`: This line ensures the `InitialSync` variable exists in the process state, initializing it to `'INCOMPLETE'` if it's the first time the code runs. +2. `if InitialSync == 'INCOMPLETE' then`: The code proceeds only if the initial sync hasn't been marked as complete. +3. `Send(...)`: The relevant state (`Balances`, `TotalSupply`) is sent to the `patch` device, making it available under `/cache/balances` and `/cache/totalsupply`. +4. `InitialSync = 'COMPLETE'`: The flag is updated, so this block won't execute again in future message handlers within the same process lifecycle. + +This ensures that clients or frontends can immediately query essential data like token balances as soon as the process ID is known, improving the responsiveness of applications built on AO. + +## Example (Lua in `aos`) + +```lua +-- In your process code (e.g., loaded via .load) +Handlers.add( + "PublishData", + Handlers.utils.hasMatchingTag("Action", "PublishData"), + function (msg) + local dataToPublish = "Some important state: " .. math.random() + -- Expose 'currentstatus' key under the 'cache' path + Send({ device = 'patch@1.0', cache = { currentstatus = dataToPublish } }) + print("Published data to /cache/currentstatus") + end +) + +-- Spawning and interacting +[aos]> MyProcess = spawn(MyModule) + +[aos]> Send({ Target = MyProcess, Action = "PublishData" }) +-- Wait a moment for scheduling + +``` + +## Avoiding Key Conflicts + +When defining keys within the `cache` table (e.g., `cache = { mydatakey = MyValue }`), these keys become path segments under `/cache/` (e.g., `/compute/cache/mydatakey` or `/now/cache/mydatakey`). It's important to choose keys that do not conflict with existing, reserved path segments used by HyperBEAM or the `~process` device itself for state access. + +Using reserved keywords as your cache keys can lead to routing conflicts or prevent you from accessing your patched data as expected. While the exact list can depend on device implementations, it's wise to avoid keys commonly associated with state access, such as: `now`, `compute`, `state`, `info`, `test`. + +It's recommended to use descriptive and specific keys for your cached data to prevent clashes with the underlying HyperPATH routing mechanisms. For example, instead of `cache = { state = ... }`, prefer `cache = { myappstate = ... }` or `cache = { usercount = ... }`. + +!!! warning + Be aware that HTTP path resolution is case-insensitive and automatically normalizes paths to lowercase. While the `patch` device itself stores keys with case sensitivity (e.g., distinguishing `MyKey` from `mykey`), accessing them via an HTTP GET request will treat `/cache/MyKey` and `/cache/mykey` as the same path. This means that using keys that only differ in case (like `MyKey` and `mykey` in your `cache` table) will result in unpredictable behavior or data overwrites when accessed via HyperPATH. To prevent these issues, it is **strongly recommended** to use **consistently lowercase keys** within the `cache` table (e.g., `mykey`, `usercount`, `appstate`). + +## Key Points + +* **Path Structure:** The data is exposed under the `/cache/` path segment. The tag name you use *inside* the `cache` table in the `Send` call (e.g., `currentstatus`) becomes the final segment in the accessible HyperPATH (e.g., `/compute/cache/currentstatus`). +* **Data Types:** The `patch` device typically handles basic data types (strings, numbers) within the `cache` table effectively. Complex nested tables might require specific encoding or handling. +* **`compute` vs `now`:** Accessing patched data via `/compute/cache/...` typically serves the last known patched value quickly. Accessing via `/now/cache/...` might involve more computation to ensure the absolute latest state before checking for the patched key under `/cache/`. +* **Not a Replacement for State:** Patching is primarily for *exposing* reads. It doesn't replace the core state management within your process handler logic. + +By using the `patch` device, you can make parts of your AO process state easily and efficiently readable over standard HTTP, bridging the gap between decentralized computation and web-based applications. \ No newline at end of file diff --git a/docs/build/extending-hyperbeam.md b/docs/build/extending-hyperbeam.md new file mode 100644 index 000000000..c033a8d93 --- /dev/null +++ b/docs/build/extending-hyperbeam.md @@ -0,0 +1,83 @@ +# Extending HyperBEAM + +HyperBEAM's modular design, built on AO-Core principles and Erlang/OTP, makes it highly extensible. You can add new functionalities or modify existing behaviors primarily by creating new **Devices** or implementing **Pre/Post-Processors**. + +!!! warning "Advanced Topic" + Extending HyperBEAM requires a good understanding of Erlang/OTP, the AO-Core protocol, and HyperBEAM's internal architecture. This guide provides a high-level overview; detailed implementation requires deeper exploration of the source code. + +## Approach 1: Creating New Devices + +This is the most common way to add significant new capabilities. +A Device is essentially an Erlang module (typically named `dev_*.erl`) that processes AO-Core messages. + +**Steps:** + +1. **Define Purpose:** Clearly define what your device will do. What kind of messages will it process? What state will it manage (if any)? What functions (keys) will it expose? +2. **Create Module:** Create a new Erlang module (e.g., `src/dev_my_new_device.erl`). +3. **Implement `info/0..2` (Optional but Recommended):** Define an `info` function to signal capabilities and requirements to HyperBEAM (e.g., exported keys, variant/version ID). + ```erlang + info() -> + #{ + variant => <<"MyNewDevice/1.0">>, + exports => [<<"do_something">>, <<"get_status">>] + }. + ``` +4. **Implement Key Functions:** Create Erlang functions corresponding to the keys your device exposes. These functions typically take `StateMessage`, `InputMessage`, and `Environment` as arguments and return `{ok, NewMessage}` or `{error, Reason}`. + ```erlang + do_something(StateMsg, InputMsg, Env) -> + % ... perform action based on InputMsg ... + NewState = ..., % Calculate new state + {ok, NewState}. + + get_status(StateMsg, _InputMsg, _Env) -> + % ... read status from StateMsg ... + StatusData = ..., + {ok, StatusData}. + ``` +5. **Handle State (If Applicable):** Devices can be stateless or stateful. Stateful devices manage their state within the `StateMessage` passed between function calls. +6. **Register Device:** Ensure HyperBEAM knows about your device. This might involve adding it to build configurations or potentially a dynamic registration mechanism if available. +7. **Testing:** Write EUnit tests for your device's functions. + +**Example Idea:** A device that bridges to another blockchain network, allowing AO processes to read data or trigger transactions on that chain. + +## Approach 2: Building Pre/Post-Processors + +Pre/post-processors allow you to intercept incoming requests *before* they reach the target device/process (`preprocess`) or modify the response *after* execution (`postprocess`). These are often implemented using the `dev_stack` device or specific hooks within the request handling pipeline. + +**Use Cases:** + +* **Authentication/Authorization:** Checking signatures or permissions before allowing execution. +* **Request Modification:** Rewriting requests, adding metadata, or routing based on specific criteria. +* **Response Formatting:** Changing the structure or content type of the response. +* **Metering/Logging:** Recording request details or charging for usage before or after execution. + +**Implementation:** + +Processors often involve checking specific conditions (like request path or headers) and then either: + +a. Passing the request through unchanged. +b. Modifying the request/response message structure. +c. Returning an error or redirect. + + +**Example Idea:** A preprocessor that automatically adds a timestamp tag to all incoming messages for a specific process. + + +## Approach 3: Custom Routing Strategies + +While `dev_router` provides basic strategies (round-robin, etc.), you could potentially implement a custom load balancing or routing strategy module that `dev_router` could be configured to use. This would involve understanding the interfaces expected by `dev_router`. + +**Example Idea:** A routing strategy that queries worker nodes for their specific capabilities before forwarding a request. + +## Getting Started + +1. **Familiarize Yourself:** Deeply understand Erlang/OTP and the HyperBEAM codebase (`src/` directory), especially [`hb_ao.erl`](../resources/source-code/hb_ao.md), [`hb_message.erl`](../resources/source-code/hb_message.md), and existing `dev_*.erl` modules relevant to your idea. +2. **Study Examples:** Look at simple devices like `dev_patch.erl` or more complex ones like `dev_process.erl` to understand patterns. +3. **Start Small:** Implement a minimal version of your idea first. +4. **Test Rigorously:** Use `rebar3 eunit` extensively. +5. **Engage Community:** Ask questions in developer channels if you get stuck. + +Extending HyperBEAM allows you to tailor the AO network's capabilities to specific needs, contributing to its rich and evolving ecosystem. diff --git a/docs/build/get-started-building-on-ao-core.md b/docs/build/get-started-building-on-ao-core.md new file mode 100644 index 000000000..117b923da --- /dev/null +++ b/docs/build/get-started-building-on-ao-core.md @@ -0,0 +1,132 @@ +# Getting Started Building on AO-Core + +Welcome to building on AO, the decentralized supercomputer! + +AO combines the permanent storage of Arweave with the flexible, scalable computation enabled by the AO-Core protocol and its HyperBEAM implementation. This allows you to create truly autonomous applications, agents, and services that run trustlessly and permissionlessly. + +## Core Idea: Processes & Messages + +At its heart, building on AO involves: + +1. **Creating Processes:** Think of these as independent programs or stateful contracts. Each process has a unique ID and maintains its own state. +2. **Sending Messages:** You interact with processes by sending them messages. These messages trigger computations, update state, or cause the process to interact with other processes or the outside world. + +Messages are processed by [Devices](../begin/ao-devices.md), which define *how* the computation happens (e.g., running WASM code, executing Lua scripts, managing state transitions). + +## Starting `aos`: Your Development Environment + +The primary tool for interacting with AO and developing processes is `aos`, a command-line interface and development environment. + +=== "npm" + ```bash + npm i -g https://get_ao.arweave.net + ``` + +=== "bun" + ```bash + bun install -g https://get_ao.arweave.net + ``` + +=== "pnpm" + ```bash + pnpm add -g https://get_ao.arweave.net + ``` + +**Starting `aos`:** + +Simply run the command in your terminal: + +```bash +aos +``` + +This connects you to an interactive Lua environment running within a **process** on the AO network. This process acts as your command-line interface (CLI) to the AO network, allowing you to interact with other processes, manage your wallet, and develop new AO processes. By default, it connects to a process running on the mainnet Compute Unit (CU). + +**What `aos` is doing:** + +* **Connecting:** Establishes a connection from your terminal to a remote process running the `aos` environment. +* **Loading Wallet:** Looks for a default Arweave key file (usually `~/.aos.json` or specified via arguments) to load into the remote process context for signing outgoing messages. +* **Providing Interface:** Gives you a Lua prompt (`[aos]>`) within the remote process where you can: + * Load code for new persistent processes on the network. + * Send messages to existing network processes. + * Inspect process state. + * Manage your local environment. + +## Your First Interaction: Assigning a Variable + +From the `aos` prompt, you can assign a variable. Let's assign a basic Lua process that just holds some data: + +```lua +[aos]> myVariable = "Hello from aos!" +-- This assigns the string "Hello from aos!" to the variable 'myVariable' +-- within the current process's Lua environment. + +[aos]> myVariable +-- Displays the content of 'myVariable' +Hello from aos! +``` + + +## Your First Handler + +Follow these steps to create and interact with your first message handler in AO: + +1. **Create a Lua File to Handle Messages:** + Create a new file named `main.lua` in your local directory and add the following Lua code: + + ```lua + Handlers.add( + "HelloWorld", + function(msg) + -- This function gets called when a message with Action = "HelloWorld" arrives. + print("Handler triggered by message from: " .. msg.From) + -- It replies to the sender with a new message containing the specified data. + msg.reply({ Data = "Hello back from your process!" }) + end + ) + + print("HelloWorld handler loaded.") -- Confirmation message + ``` + + * `Handlers.add`: Registers a function to handle incoming messages. + * `"HelloWorld"`: The name of this handler. It will be triggered by messages with `Action = "HelloWorld"`. + * `function(msg)`: The function that executes when the handler is triggered. `msg` contains details about the incoming message (like `msg.From`, the sender's process ID). + * `msg.reply({...})`: Sends a response message back to the original sender. The response must be a Lua table, typically containing a `Data` field. + +2. **Load the Handler into `aos`:** + From your `aos` prompt, load the handler code into your running process: + + ```lua + [aos]> .load main.lua + ``` + +3. **Send a Message to Trigger the Handler:** + Now, send a message to your own process (`ao.id` refers to the current process ID) with the action that matches your handler's name: + + ```lua + [aos]> Send({ Target = ao.id, Action = "HelloWorld" }) + ``` + +4. **Observe the Output:** + You should see two things happen in your `aos` terminal: + * The `print` statement from your handler: `Handler triggered by message from: ` + * A notification about the reply message: `New Message From : Data = Hello back from your process!` + +5. **Inspect the Reply Message:** + The reply message sent by your handler is now in your process's inbox. You can inspect its data like this: + + ```lua + [aos]> Inbox[#Inbox].Data + ``` + This should output: `"Hello back from your process!"` + +You've successfully created a handler, loaded it into your AO process, triggered it with a message, and received a reply! + +## Next Steps + +This is just the beginning. To dive deeper: + +* **AO Cookbook:** Explore practical examples and recipes for common tasks: [AO Cookbook](https://cookbook_ao.arweave.net/) +* **Expose Process State:** Learn how to make your process data accessible via HTTP using the `patch` device: [Exposing Process State](./exposing-process-state.md) +* **Serverless Compute:** Discover how to run WASM or Lua computations within your processes: [Serverless Decentralized Compute](./serverless-decentralized-compute.md) +* **aos Documentation:** Refer to the official `aos` documentation for detailed commands and usage. diff --git a/docs/build/serverless-decentralized-compute.md b/docs/build/serverless-decentralized-compute.md new file mode 100644 index 000000000..cb1c589c6 --- /dev/null +++ b/docs/build/serverless-decentralized-compute.md @@ -0,0 +1,82 @@ +# Serverless Decentralized Compute on AO + +AO enables powerful "serverless" computation patterns by allowing you to run code (WASM, Lua) directly within decentralized processes, triggered by messages. Furthermore, if computations are performed on nodes running in Trusted Execution Environments (TEEs), you can obtain cryptographic attestations verifying the execution integrity. + +## Core Concept: Compute Inside Processes + +Instead of deploying code to centralized servers, you deploy code *to* the Arweave permaweb and instantiate it as an AO process. Interactions happen by sending messages to this process ID. + +* **Code Deployment:** Your WASM binary or Lua script is uploaded to Arweave, getting a permanent transaction ID. +* **Process Spawning:** You create an AO process, associating it with your code's transaction ID and specifying the appropriate compute device ([`~wasm64@1.0`](../devices/wasm64-at-1-0.md) or [`~lua@5.3a`](../devices/lua-at-5-3a.md)). +* **Execution via Messages:** Sending a message to the process ID triggers the HyperBEAM node (that picks up the message) to: + 1. Load the process state. + 2. Fetch the associated WASM/Lua code from Arweave. + 3. Execute the code using the relevant device ([`dev_wasm`](../resources/source-code/dev_wasm.md) or [`dev_lua`](../resources/source-code/dev_lua.md)), passing the message data and current state. + 4. Update the process state based on the execution results. + + +## TEE Attestations (via [`~snp@1.0`](../resources/source-code/dev_snp.md)) + +If a HyperBEAM node performing these computations runs within a supported Trusted Execution Environment (like AMD SEV-SNP), it can provide cryptographic proof of execution. + +* **How it works:** The [`~snp@1.0`](../resources/source-code/dev_snp.md) device interacts with the TEE hardware. +* **Signed Responses:** When a TEE-enabled node processes your message (e.g., executes your WASM function), the HTTP response containing the result can be cryptographically signed by a key that *provably* only exists inside the TEE. +* **Verification:** Clients receiving this response can verify the signature against the TEE platform's attestation mechanism (e.g., AMD's KDS) to gain high confidence that the computation was performed correctly and confidentially within the secure environment, untampered by the node operator. + +**Obtaining Attested Responses:** + +This usually involves interacting with nodes specifically advertised as TEE-enabled. The exact mechanism for requesting and verifying attestations depends on the specific TEE technology and node configuration. + +* The HTTP response headers might contain specific signature or attestation data (e.g., using HTTP Message Signatures RFC-9421 via [`dev_codec_httpsig`](../resources/source-code/dev_codec_httpsig.md)). +* You might query the [`~snp@1.0`](../resources/source-code/dev_snp.md) device directly on the node to get its attestation report. + +Refer to documentation on [TEE Nodes](./run/tee-nodes.md) and the [`~snp@1.0`](../resources/source-code/dev_snp.md) device for details. + +By leveraging WASM, Lua, and optional TEE attestations, AO provides a powerful platform for building complex, verifiable, and truly decentralized serverless applications. diff --git a/docs/devices/json-at-1-0.md b/docs/devices/json-at-1-0.md new file mode 100644 index 000000000..eec5e7985 --- /dev/null +++ b/docs/devices/json-at-1-0.md @@ -0,0 +1,42 @@ +# Device: ~json@1.0 + +## Overview + +The [`~json@1.0`](../resources/source-code/dev_json_iface.md) device provides a mechanism to interact with JSON (JavaScript Object Notation) data structures using HyperPATHs. It allows treating a JSON document or string as a stateful entity against which HyperPATH queries can be executed. + +This device is useful for: + +* Serializing and deserializing JSON data. +* Querying and modifying JSON objects. +* Integrating with other devices and operations via HyperPATH chaining. + +## Core Functions (Keys) + +### Serialization + +* **`GET /~json@1.0/serialize` (Direct Serialize Action)** + * **Action:** Serializes the input message or data into a JSON string. + * **Example:** `GET /~json@1.0/serialize` - serializes the current message as JSON. + * **HyperPATH:** The path segment `/serialize` directly follows the device identifier. + +* **`GET //~json@1.0/serialize` (Chained Serialize Action)** + * **Action:** Takes arbitrary data output from `` (another device or operation) and returns its serialized JSON string representation. + * **Example:** `GET /~meta@1.0/info/~json@1.0/serialize` - fetches node info from the meta device and then pipes it to the JSON device to serialize the result as JSON. + * **HyperPATH:** This segment (`/~json@1.0/serialize`) is appended to a previous HyperPATH segment. + +## HyperPATH Chaining Example + +The JSON device is particularly useful in HyperPATH chains to convert output from other devices into JSON format: + +``` +GET /~meta@1.0/info/~json@1.0/serialize +``` + +This retrieves the node configuration from the meta device and serializes it to JSON. + +## See Also + +- [Message Device](../resources/source-code/dev_message.md) - Works well with JSON serialization +- [Meta Device](../resources/source-code/dev_meta.md) - Can provide configuration data to serialize + +[json module](../resources/source-code/dev_codec_json.md) \ No newline at end of file diff --git a/docs/devices/lua-at-5-3a.md b/docs/devices/lua-at-5-3a.md new file mode 100644 index 000000000..4c961bca2 --- /dev/null +++ b/docs/devices/lua-at-5-3a.md @@ -0,0 +1,70 @@ +# Device: ~lua@5.3a + +## Overview + +The [`~lua@5.3a`](../resources/source-code/dev_lua.md) device enables the execution of Lua scripts within the HyperBEAM environment. It provides an isolated sandbox where Lua code can process incoming messages, interact with other devices, and manage state. + +## Core Concept: Lua Script Execution + +This device allows processes to perform computations defined in Lua scripts. Similar to the [`~wasm64@1.0`](../resources/source-code/dev_wasm.md) device, it manages the lifecycle of a Lua execution state associated with the process. + +## Key Functions (Keys) + +These keys are typically used within an execution stack (managed by [`dev_stack`](../resources/source-code/dev_stack.md)) for an AO process. + +* **`init`** + * **Action:** Initializes the Lua environment for the process. It finds and loads the Lua script(s) associated with the process, creates a `luerl` state, applies sandboxing rules if specified, installs the [`dev_lua_lib`](../resources/source-code/dev_lua_lib.md) (providing AO-specific functions like `ao.send`), and stores the initialized state in the process's private area (`priv/state`). + * **Inputs (Expected in Process Definition or `init` Message):** + * `script`: Can be: + * An Arweave Transaction ID of the Lua script file. + * A list of script IDs or script message maps. + * A message map containing the Lua script in its `body` tag (Content-Type `application/lua` or `text/x-lua`). + * A map where keys are module names and values are script IDs/messages. + * `sandbox`: (Optional) Controls Lua sandboxing. Can be `true` (uses default sandbox list), `false` (no sandbox), or a map/list specifying functions to disable and their return values. + * **Outputs (Stored in `priv/`):** + * `state`: The initialized `luerl` state handle. +* **`` (Default Handler - `compute`)** + * **Action:** Executes a specific function within the loaded Lua script(s). This is the default handler; if a key matching a Lua function name is called on the device, this logic runs. + * **Inputs (Expected in Process State or Incoming Message):** + * `priv/state`: The Lua state obtained during `init`. + * The **key** being accessed (used as the default function name). + * `function` or `body/function`: (Optional) Overrides the function name derived from the key. + * `parameters` or `body/parameters`: (Optional) Arguments to pass to the Lua function. Defaults to a list containing the process message, the request message, and an empty options map. + * **Response:** The results returned by the Lua function call, typically encoded. The device also updates the `priv/state` with the Lua state after execution. +* **`snapshot`** + * **Action:** Captures the current state of the running Lua environment. `luerl` state is serializable. + * **Inputs:** `priv/state`. + * **Outputs:** A message containing the serialized Lua state, typically tagged with `[Prefix]/State`. +* **`normalize` (Internal Helper)** + * **Action:** Ensures a consistent state representation by loading a Lua state from a snapshot (`[Prefix]/State`) if a live state (`priv/state`) isn't already present. +* **`functions`** + * **Action:** Returns a list of all globally defined functions within the current Lua state. + * **Inputs:** `priv/state`. + * **Response:** A list of function names. + +## Sandboxing + +The `sandbox` option in the process definition restricts potentially harmful Lua functions (like file I/O, OS commands, loading arbitrary code). By default (`sandbox = true`), common dangerous functions are disabled. You can customize the sandbox rules. + +## AO Library (`dev_lua_lib`) + +The `init` function automatically installs a helper library ([`dev_lua_lib`](../resources/source-code/dev_lua_lib.md)) into the Lua state. This library typically provides functions for interacting with the AO environment from within the Lua script, such as: + +* `ao.send({ Target = ..., ... })`: To send messages from the process. +* Access to message tags and data. + +## Usage within `dev_stack` + +Like [`~wasm64@1.0`](../resources/source-code/dev_wasm.md), the `~lua@5.3a` device is typically used within an execution stack. + +```text +# Example Process Definition Snippet +Execution-Device: stack@1.0 +Execution-Stack: scheduler@1.0, lua@5.3a +Script: +Sandbox: true +``` + +This device offers a lightweight, integrated scripting capability for AO processes, suitable for a wide range of tasks from simple logic to more complex state management and interactions. + +[lua module](../resources/source-code/dev_lua.md) diff --git a/docs/devices/message-at-1-0.md b/docs/devices/message-at-1-0.md new file mode 100644 index 000000000..000bdb860 --- /dev/null +++ b/docs/devices/message-at-1-0.md @@ -0,0 +1,74 @@ +# Device: ~message@1.0 + +## Overview + +The [`~message@1.0`](../resources/source-code/dev_message.md) device is a fundamental built-in device in HyperBEAM. It serves as the identity device for standard AO-Core messages, which are represented as Erlang maps internally. Its primary function is to allow manipulation and inspection of these message maps directly via HyperPATH requests, without needing a persistent process state. + +This device is particularly useful for: + +* Creating and modifying transient messages on the fly using query parameters. +* Retrieving specific values from a message map. +* Inspecting the keys of a message. +* Handling message commitments and verification (though often delegated to specialized commitment devices like [`httpsig@1.0`](../resources/source-code/dev_codec_httpsig.md)). + +## Core Functionality + +The `message@1.0` device treats the message itself as the state it operates on. Key operations are accessed via path segments in the HyperPATH. + +### Key Access (`/key`) + +To retrieve the value associated with a specific key in the message map, simply append the key name to the path. Key lookup is case-insensitive. + +**Example:** + +``` +GET /~message@1.0&hello=world&Key=Value/key +``` + +**Response:** + +``` +"Value" +``` + +### Reserved Keys + +The `message@1.0` device reserves several keys for specific operations: + +* **`get`**: (Default operation if path segment matches a key in the map) Retrieves the value of a specified key. Behaves identically to accessing `/key` directly. +* **`set`**: Modifies the message by adding or updating key-value pairs. Requires additional parameters (usually in the request body or subsequent path segments/query params, depending on implementation specifics). + * Supports deep merging of maps. + * Setting a key to `unset` removes it. + * Overwriting keys that are part of existing commitments will typically remove those commitments unless the new value matches the old one. +* **`set_path`**: A special case for setting the `path` key itself, which cannot be done via the standard `set` operation. +* **`remove`**: Removes one or more specified keys from the message. Requires an `item` or `items` parameter. +* **`keys`**: Returns a list of all public (non-private) keys present in the message map. +* **`id`**: Calculates and returns the ID (hash) of the message. Considers active commitments based on specified `committers`. May delegate ID calculation to a device specified by the message\'s `id-device` key or the default ([`httpsig@1.0`](../resources/source-code/dev_codec_httpsig.md)). +* **`commit`**: Creates a commitment (e.g., a signature) for the message. Requires parameters like `commitment-device` and potentially committer information. Delegates the actual commitment generation to the specified device (default [`httpsig@1.0`](../resources/source-code/dev_codec_httpsig.md)). +* **`committers`**: Returns a list of committers associated with the commitments in the message. Can be filtered by request parameters. +* **`commitments`**: Used internally and in requests to filter or specify which commitments to operate on (e.g., for `id` or `verify`). +* **`verify`**: Verifies the commitments attached to the message. Can be filtered by `committers` or specific `commitment` IDs in the request. Delegates verification to the device specified in each commitment (`commitment-device`). + +### Private Keys + +Keys prefixed with `priv` (e.g., `priv_key`, `private.data`) are considered private and cannot be accessed or listed via standard `get` or `keys` operations. + +## HyperPATH Example + +This example demonstrates creating a transient message and retrieving a value: + +``` +GET /~message@1.0&hello=world&k=v/k +``` + +**Breakdown:** + +1. `~message@1.0`: Sets the root device. +2. `&hello=world&k=v`: Query parameters create the initial message: `#{ <<"hello">> => <<"world">>, <<"k">> => <<"v">> }`. +3. `/k`: The path segment requests the value for the key `k`. + +**Response:** + +``` +"v" +``` \ No newline at end of file diff --git a/docs/devices/meta-at-1-0.md b/docs/devices/meta-at-1-0.md new file mode 100644 index 000000000..b448a135b --- /dev/null +++ b/docs/devices/meta-at-1-0.md @@ -0,0 +1,55 @@ +# Device: ~meta@1.0 + +## Overview + +The [`~meta@1.0`](../resources/source-code/dev_meta.md) device provides access to metadata and configuration information about the local HyperBEAM node and the broader AO network. + +This device is essential for: + +## Core Functions (Keys) + +### `info` + +Retrieves or modifies the node's configuration message (often referred to as `NodeMsg` internally). + +* **`GET /~meta@1.0/info`** + * **Action:** Returns the current node configuration message. + * **Response:** A message map containing the node's settings. Sensitive keys (like private wallets) are filtered out. Dynamically generated keys like the node's public `address` are added if a wallet is configured. +* **`POST /~meta@1.0/info`** + * **Action:** Updates the node's configuration message. Requires the request to be signed by the node's configured `operator` key/address. + * **Request Body:** A message map containing the configuration keys and values to update. + * **Response:** Confirmation message indicating success or failure. + * **Note:** Once a node's configuration is marked as `initialized = permanent`, it cannot be changed via this method. + +## Key Configuration Parameters Managed by `~meta` + +While the `info` key is the primary interaction point, the `NodeMsg` managed by `~meta` holds crucial configuration parameters affecting the entire node's behavior, including (but not limited to): + +* `port`: HTTP server port. +* `priv_wallet` / `key_location`: Path to the node's Arweave key file. +* `operator`: The address designated as the node operator (defaults to the address derived from `priv_wallet`). +* `initialized`: Status indicating if the node setup is temporary or permanent. +* `preprocessor` / `postprocessor`: Optional messages defining pre/post-processing logic for requests. +* `routes`: Routing table used by [`dev_router`](../resources/source-code/dev_router.md). +* `store`: Configuration for data storage. +* `trace`: Debug tracing options. +* `p4_*`: Payment configuration. +* `faff_*`: Access control lists. + +*(Refer to `hb_opts.erl` for a comprehensive list of options.)* + +## Utility Functions (Internal/Module Level) + +The [`dev_meta.erl`](../resources/source-code/dev_meta.md) module also contains helper functions used internally or callable from other Erlang modules: + +* `is_operator(, ) -> boolean()`: Checks if the signer of `RequestMsg` matches the configured `operator` in `NodeMsg`. + +## Pre/Post-Processing Hooks + +The `~meta` device applies the node's configured `preprocessor` message before resolving the main request and the `postprocessor` message after obtaining the result, allowing for global interception and modification of requests/responses. + +## Initialization + +Before a node can process general requests, it usually needs to be initialized. Attempts to access devices other than `~meta@1.0/info` before initialization typically result in an error. Initialization often involves setting essential parameters like the operator key via a `POST` to `info`. + +[meta module](../resources/source-code/dev_meta.md) \ No newline at end of file diff --git a/docs/devices/overview.md b/docs/devices/overview.md new file mode 100644 index 000000000..5a9ce0d1b --- /dev/null +++ b/docs/devices/overview.md @@ -0,0 +1,26 @@ +# Devices + +Devices are the core functional units within HyperBEAM and AO-Core. They define how messages are processed and what actions can be performed. + +Each device listed here represents a specific capability available to AO processes and nodes. Understanding these devices is key to building complex applications and configuring your HyperBEAM node effectively. + +## Available Devices + +Below is a list of documented built-in devices. Each page details the device's purpose, available functions (keys), and usage examples where applicable. + +* **[`~message@1.0`](./message-at-1-0.md):** Base message handling and manipulation. +* **[`~meta@1.0`](./meta-at-1-0.md):** Node configuration and metadata. +* **[`~process@1.0`](./process-at-1-0.md):** Persistent, shared process execution environment. +* **[`~scheduler@1.0`](./scheduler-at-1-0.md):** Message scheduling and execution ordering for processes. +* **[`~wasm64@1.0`](./wasm64-at-1-0.md):** WebAssembly (WASM) execution engine. +* **[`~lua@5.3a`](./lua-at-5-3a.md):** Lua script execution engine. +* **[`~relay@1.0`](./relay-at-1-0.md):** Relaying messages to other nodes or HTTP endpoints. +* **[`~json@1.0`](./json-at-1-0.md):** Provides access to JSON data structures using HyperPATHs. + +*(More devices will be documented here as specifications are finalized and reviewed.)* + +## Device Naming and Versioning + +Devices are typically referenced using a name and version, like `~@` (e.g., `~process@1.0`). The tilde (`~`) often indicates a primary, user-facing device, while internal or utility devices might use a `dev_` prefix in the source code (e.g., `dev_router`). + +Versioning indicates the specific interface and behavior of the device. Changes to a device that break backward compatibility usually result in a version increment. diff --git a/docs/devices/process-at-1-0.md b/docs/devices/process-at-1-0.md new file mode 100644 index 000000000..090d5d6d0 --- /dev/null +++ b/docs/devices/process-at-1-0.md @@ -0,0 +1,72 @@ +# Device: ~process@1.0 + +## Overview + +The [`~process@1.0`](../resources/source-code/dev_process.md) device represents a persistent, shared execution environment within HyperBEAM, analogous to a process or actor in other systems. It allows for stateful computation and interaction over time. + +## Core Concept: Orchestration + +A message tagged with `Device: process@1.0` (the "Process Definition Message") doesn't typically perform computation itself. Instead, it defines *which other devices* should be used for key aspects of its lifecycle: + +* **Scheduler Device:** Determines the order of incoming messages (assignments) to be processed. (Defaults to [`~scheduler@1.0`](../resources/source-code/dev_scheduler.md)). +* **Execution Device:** Executes the actual computation based on the current state and the scheduled message. Often configured as [`dev_stack`](../resources/source-code/dev_stack.md) to allow multiple computational steps (e.g., running WASM, applying cron jobs, handling proofs). +* **Push Device:** Handles the injection of new messages into the process\'s schedule. (Defaults to [`~push@1.0`](../resources/source-code/dev_push.md)). + +The `~process@1.0` device acts as a router, intercepting requests and delegating them to the appropriate configured device (scheduler, executor, etc.) by temporarily swapping the device tag on the message before resolving. + +## Key Functions (Keys) + +These keys are accessed via HyperPATHs relative to the Process Definition Message ID (``). + +* **`GET /~process@1.0/schedule`** + * **Action:** Delegates to the configured Scheduler Device (via the process's `schedule/3` function) to retrieve the current schedule or state. + * **Response:** Depends on the Scheduler Device implementation (e.g., list of message IDs). +* **`POST /~process@1.0/schedule`** + * **Action:** Delegates to the configured Push Device (via the process's `push/3` function) to add a new message to the process's schedule. + * **Request Body:** The message to be added. + * **Response:** Confirmation or result from the Push Device. +* **`GET /~process@1.0/compute/`** + * **Action:** Computes the process state up to a specific point identified by `` (either a slot number or a message ID within the schedule). It retrieves assignments from the Scheduler Device and applies them sequentially using the configured Execution Device. + * **Response:** The process state message after executing up to the target slot/message. + * **Caching:** Results are cached aggressively (see [`dev_process_cache`](../resources/source-code/dev_process_cache.md)) to avoid recomputation. +* **`GET /~process@1.0/now`** + * **Action:** Computes and returns the `Results` key from the *latest* known state of the process. This typically involves computing all pending assignments. + * **Response:** The value of the `Results` key from the final state. +* **`GET /~process@1.0/slot`** + * **Action:** Delegates to the configured Scheduler Device to query information about a specific slot or the current slot number. + * **Response:** Depends on the Scheduler Device implementation. +* **`GET /~process@1.0/snapshot`** + * **Action:** Delegates to the configured Execution Device to generate a snapshot of the current process state. This often involves running the execution stack in a specific "map" mode to gather state from different components. + * **Response:** A message representing the process snapshot, often marked for caching. + +## Process Definition Example + +A typical process definition message might look like this (represented conceptually): + +```text +Device: process@1.0 +Scheduler-Device: [`scheduler@1.0`](../resources/source-code/dev_scheduler.md) +Execution-Device: [`stack@1.0`](../resources/source-code/dev_stack.md) +Execution-Stack: "[`scheduler@1.0`](../resources/source-code/dev_scheduler.md)", "[`cron@1.0`](../resources/source-code/dev_cron.md)", "[`wasm64@1.0`](../resources/source-code/dev_wasm.md)", "[`PoDA@1.0`](../resources/source-code/dev_poda.md)" +Cron-Frequency: 10-Minutes +WASM-Image: +PoDA: + Device: [`PoDA/1.0`](../resources/source-code/dev_poda.md) + Authority: + Authority: + Quorum: 2 +``` + +This defines a process that uses: +* The standard scheduler. +* A stack executor that runs scheduling logic, cron jobs, a WASM module, and a Proof-of-Data-Availability check. + +## State Management & Caching + +`~process@1.0` relies heavily on caching ([`dev_process_cache`](../resources/source-code/dev_process_cache.md)) to optimize performance. Full state snapshots and intermediate results are cached periodically (configurable via `Cache-Frequency` and `Cache-Keys` options) to avoid recomputing the entire history for every request. + +## Initialization (`init`) + +Processes often require an initialization step before they can process messages. This is typically triggered by calling the `init` key on the configured Execution Device via the process path (`/~process@1.0/init`). This allows components within the execution stack (like WASM modules) to set up their initial state. + +[process module](../resources/source-code/dev_process.md) diff --git a/docs/devices/relay-at-1-0.md b/docs/devices/relay-at-1-0.md new file mode 100644 index 000000000..9d432568c --- /dev/null +++ b/docs/devices/relay-at-1-0.md @@ -0,0 +1,46 @@ +# Device: ~relay@1.0 + +## Overview + +The [`~relay@1.0`](../resources/source-code/dev_relay.md) device enables HyperBEAM nodes to send messages to external HTTP endpoints or other AO nodes. + +## Core Concept: Message Forwarding + +This device acts as an HTTP client within the AO ecosystem. It allows a node or process to make outbound HTTP requests. + +## Key Functions (Keys) + +* **`call`** + * **Action:** Sends an HTTP request to a specified target and waits synchronously for the response. + * **Inputs (from Request Message or Base Message M1):** + * `target`: (Optional) A message map defining the request to be sent. Defaults to the original incoming request (`Msg2` or `M1`). + * `relay-path` or `path`: The URL/path to send the request to. + * `relay-method` or `method`: The HTTP method (GET, POST, etc.). + * `relay-body` or `body`: The request body. + * `requires-sign`: (Optional, boolean) If true, the request message (`target`) will be signed using the node's key before sending. Defaults to `false`. + * `http-client`: (Optional) Specify a custom HTTP client module to use (defaults to node's configured `relay_http_client`). + * **Response:** `{ok, }` where `` is the full message received from the remote peer, or `{error, Reason}`. + * **Example HyperPATH:** + ``` + GET /~relay@1.0/call?method=GET&path=https://example.com + ``` +* **`cast`** + * **Action:** Sends an HTTP request asynchronously. The device returns immediately after spawning a process to send the request; it does not wait for or return the response from the remote peer. + * **Inputs:** Same as `call`. + * **Response:** `{ok, <<"OK">>}`. +* **`preprocess`** + * **Action:** This function is designed to be used as a node's global `preprocessor` (configured via [`~meta@1.0`](../resources/source-code/dev_meta.md)). When configured, it intercepts *all* incoming requests to the node and automatically rewrites them to be relayed via the `call` key. This effectively turns the node into a pure forwarding proxy, using its routing table ([`dev_router`](../resources/source-code/dev_router.md)) to determine the destination. + * **Response:** A message structure that invokes `/~relay@1.0/call` with the original request as the target body. + +## Use Cases + +* **Inter-Node Communication:** Sending messages between HyperBEAM nodes. +* **External API Calls:** Allowing AO processes to interact with traditional web APIs. +* **Routing Nodes:** Nodes configured with the `preprocess` key act as dedicated routers/proxies. +* **Client-Side Relaying:** A local HyperBEAM instance can use `~relay@1.0` to forward requests to public compute nodes. + +## Interaction with Routing + +When `call` or `cast` is invoked, the actual HTTP request dispatch is handled by `hb_http:request/2`. This function often utilizes the node's routing configuration ([`dev_router`](../resources/source-code/dev_router.md)) to determine the specific peer/URL to send the request to, especially if the target path is an AO process ID or another internal identifier rather than a full external URL. + +[relay module](../resources/source-code/dev_relay.md) diff --git a/docs/devices/scheduler-at-1-0.md b/docs/devices/scheduler-at-1-0.md new file mode 100644 index 000000000..2922d699c --- /dev/null +++ b/docs/devices/scheduler-at-1-0.md @@ -0,0 +1,65 @@ +# Device: ~scheduler@1.0 + +## Overview + +The [`~scheduler@1.0`](../resources/source-code/dev_scheduler.md) device manages the queueing and ordering of messages targeted at a specific process ([`~process@1.0`](../resources/source-code/dev_process.md)). It ensures that messages are processed according to defined scheduling rules. + +## Core Concept: Message Ordering + +When messages are sent to an AO process (typically via the [`~push@1.0`](../resources/source-code/dev_push.md) device or a `POST` to the process's `/schedule` endpoint), they are added to a queue managed by the Scheduler Device associated with that process. The scheduler ensures that messages are processed one after another in a deterministic order, typically based on arrival time and potentially other factors like message nonces or timestamps (depending on the specific scheduler implementation details). + +The [`~process@1.0`](../resources/source-code/dev_process.md) device interacts with its configured Scheduler Device (which defaults to `~scheduler@1.0`) primarily through the `next` key to retrieve the next message to be executed. + +## Slot System + +Slots are a fundamental concept in the `~scheduler@1.0` device, providing a structured mechanism for organizing and sequencing computation. + +* **Sequential Ordering:** Slots act as numbered containers (starting at 0) that hold specific messages or tasks to be processed in a deterministic order. +* **State Tracking:** The `at-slot` key in a process's state (or a similar internal field like `current-slot` within the scheduler itself) tracks execution progress, indicating which messages have been processed and which are pending. The `slot` function can be used to query this. +* **Assignment Storage:** Each slot contains an "assignment" - the cryptographically verified message waiting to be executed. These assignments are retrieved using the `schedule` function or internally via `next`. +* **Schedule Organization:** The collection of all slots for a process forms its "schedule". +* **Application Scenarios:** + * **Scheduling Messages:** When a message is posted to a process (e.g., via `register`), it's assigned to the next available slot. + * **Status Monitoring:** Clients can query a process's current slot (via the `slot` function) to check progress. + * **Task Retrieval:** Processes find their next task by requesting the next assignment via the `next` function, which implicitly uses the next slot number based on the current state. + * **Distributed Consistency:** Slots ensure deterministic execution order across nodes, crucial for maintaining consistency in AO. + +This slotting mechanism is central to AO processes built on HyperBEAM, allowing for deterministic, verifiable computation. + +## Key Functions (Keys) + +These keys are typically accessed via the [`~process@1.0`](../resources/source-code/dev_process.md) device, which delegates the calls to its configured scheduler. + +* **`schedule` (Handler for `GET /~process@1.0/schedule`)** + * **Action:** Retrieves the list of pending assignments (messages) for the process. May support cursor-based traversal for long schedules. + * **Response:** A message map containing the assignments, often keyed by slot number or message ID. +* **`register` (Handler for `POST /~process@1.0/schedule`)** + * **Action:** Adds/registers a new message to the process's schedule. If this is the first message for a process, it might initialize the scheduler state. + * **Request Body:** The message to schedule. + * **Response:** Confirmation, potentially including the assigned slot or message ID. +* **`slot` (Handler for `GET /~process@1.0/slot`)** + * **Action:** Queries the current or a specific slot number within the process's schedule. + * **Response:** Information about the requested slot, such as the current highest slot number. +* **`status` (Handler for `GET /~process@1.0/status`)** + * **Action:** Retrieves status information about the scheduler for the process. + * **Response:** A status message. +* **`next` (Internal Key used by [`~process@1.0`](../resources/source-code/dev_process.md))** + * **Action:** Retrieves the next assignment message from the schedule based on the process's current `at-slot` state. + * **State Management:** Requires the current process state (`Msg1`) containing the `at-slot` key. + * **Response:** `{ok, #{ "body" => , "state" => }}` or `{error, Reason}` if no next assignment is found. + * **Caching & Lookahead:** The implementation uses internal caching (`dev_scheduler_cache`, `priv/assignments`) and potentially background lookahead workers to optimize fetching subsequent assignments. +* **`init` (Internal Key)** + * **Action:** Initializes the scheduler state for a process, often called when the process itself is initialized. +* **`checkpoint` (Internal Key)** + * **Action:** Triggers the scheduler to potentially persist its current state or perform other checkpointing operations. + +## Interaction with Other Components + +* **[`~process@1.0`](../resources/source-code/dev_process.md):** The primary user of the scheduler, calling `next` to drive process execution. +* **[`~push@1.0`](../resources/source-code/dev_push.md):** Often used to add messages to the schedule via `POST /schedule`. +* **`dev_scheduler_cache`:** Internal module used for caching assignments locally on the node to reduce latency. +* **Scheduling Unit (SU):** Schedulers may interact with external entities (like Arweave gateways or dedicated SU nodes) to fetch or commit schedules, although `~scheduler@1.0` aims for a simpler, often node-local or SU-client model. + +`~scheduler@1.0` provides the fundamental mechanism for ordered, sequential execution within the potentially asynchronous and parallel environment of AO. + +[scheduler module](../resources/source-code/dev_scheduler.md) diff --git a/docs/devices/wasm64-at-1-0.md b/docs/devices/wasm64-at-1-0.md new file mode 100644 index 000000000..492d86e40 --- /dev/null +++ b/docs/devices/wasm64-at-1-0.md @@ -0,0 +1,63 @@ +# Device: ~wasm64@1.0 + +## Overview + +The [`~wasm64@1.0`](../resources/source-code/dev_wasm.md) device enables the execution of 64-bit WebAssembly (WASM) code within the HyperBEAM environment. It provides a sandboxed environment for running compiled code from various languages (like Rust, C++, Go) that target WASM. + +## Core Concept: WASM Execution + +This device allows AO processes to perform complex computations defined in WASM modules, which can be written in languages like Rust, C++, C, Go, etc., and compiled to WASM. + +The device manages the lifecycle of a WASM instance associated with the process state. + +## Key Functions (Keys) + +These keys are typically used within an execution stack (managed by [`dev_stack`](../resources/source-code/dev_stack.md)) for an AO process. + +* **`init`** + * **Action:** Initializes the WASM environment for the process. It locates the WASM image (binary), starts a WAMR instance, and stores the instance handle and helper functions (for reading/writing WASM memory) in the process's private state (`priv/...`). + * **Inputs (Expected in Process Definition or `init` Message):** + * `[Prefix]/image`: The Arweave Transaction ID of the WASM binary, or the WASM binary itself, or a message containing the WASM binary in its body. + * `[Prefix]/Mode`: (Optional) Specifies execution mode (`WASM` (default) or `AOT` if allowed by node config). + * **Outputs (Stored in `priv/`):** + * `[Prefix]/instance`: The handle to the running WAMR instance. + * `[Prefix]/write`: A function to write data into the WASM instance's memory. + * `[Prefix]/read`: A function to read data from the WASM instance's memory. + * `[Prefix]/import-resolver`: A function used to handle calls *from* the WASM module back *to* the AO environment (imports). +* **`compute`** + * **Action:** Executes a function within the initialized WASM instance. It retrieves the target function name and parameters from the incoming message or process definition and calls the WASM instance via `hb_beamr`. + * **Inputs (Expected in Process State or Incoming Message):** + * `priv/[Prefix]/instance`: The handle obtained during `init`. + * `function` or `body/function`: The name of the WASM function to call. + * `parameters` or `body/parameters`: A list of parameters to pass to the WASM function. + * **Outputs (Stored in `results/`):** + * `results/[Prefix]/type`: The result type returned by the WASM function. + * `results/[Prefix]/output`: The actual result value returned by the WASM function. +* **`import`** + * **Action:** Handles calls originating *from* the WASM module (imports). The default implementation (`default_import_resolver`) resolves these calls by treating them as sub-calls within the AO environment, allowing WASM code to invoke other AO device functions or access process state via the `hb_ao:resolve` mechanism. + * **Inputs (Provided by `hb_beamr`):** Module name, function name, arguments, signature. + * **Response:** Returns the result of the resolved AO call back to the WASM instance. +* **`snapshot`** + * **Action:** Captures the current memory state of the running WASM instance. This is used for checkpointing and restoring process state. + * **Inputs:** `priv/[Prefix]/instance`. + * **Outputs:** A message containing the raw binary snapshot of the WASM memory state, typically tagged with `[Prefix]/State`. +* **`normalize` (Internal Helper)** + * **Action:** Ensures a consistent state representation for computation, primarily by loading a WASM instance from a snapshot (`[Prefix]/State`) if a live instance (`priv/[Prefix]/instance`) isn't already present. This allows resuming execution from a cached state. +* **`terminate`** + * **Action:** Stops and cleans up the running WASM instance associated with the process. + * **Inputs:** `priv/[Prefix]/instance`. + +## Usage within `dev_stack` + +The `~wasm64@1.0` device is almost always used as part of an execution stack configured in the Process Definition Message and managed by [`dev_stack`](../resources/source-code/dev_stack.md). [`dev_stack`](../resources/source-code/dev_stack.md) ensures that `init` is called on the first pass, `compute` on subsequent passes, and potentially `snapshot` or `terminate` as needed. + +```text +# Example Process Definition Snippet +Execution-Device: [`stack@1.0`](../resources/source-code/dev_stack.md) +Execution-Stack: "[`scheduler@1.0`](../resources/source-code/dev_scheduler.md)", "wasm64@1.0" +WASM-Image: +``` + +This setup allows AO processes to leverage the computational power and language flexibility offered by WebAssembly in a decentralized, verifiable manner. + +[wasm module](../resources/source-code/dev_wasm.md) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ + diff --git a/docs/introduction/ao-devices.md b/docs/introduction/ao-devices.md new file mode 100644 index 000000000..818e0d42d --- /dev/null +++ b/docs/introduction/ao-devices.md @@ -0,0 +1,60 @@ +# AO Devices + +In AO-Core and its implementation HyperBEAM, **Devices** are modular components responsible for processing and interpreting [Messages](./what-is-ao-core.md#core-concepts). They define the specific logic for how computations are performed, data is handled, or interactions occur within the AO ecosystem. + +Think of Devices as specialized engines or services that can be plugged into the AO framework. This modularity is key to AO's flexibility and extensibility. + +## Purpose of Devices + +* **Define Computation:** Devices dictate *how* a message's instructions are executed. One device might run WASM code, another might manage process state, and yet another might simply relay data. +* **Enable Specialization:** Nodes running HyperBEAM can choose which Devices to support, allowing them to specialize in certain tasks (e.g., high-compute tasks, storage-focused tasks, secure TEE operations). +* **Promote Modularity:** New functionalities can be added to AO by creating new Devices, without altering the core protocol. +* **Distribute Workload:** Different Devices can handle different parts of a complex task, enabling parallel processing and efficient resource utilization across the network. + +## Familiar Examples + +HyperBEAM includes many preloaded devices that provide core functionality. Some key examples include: + +* **[`~meta@1.0`](../devices/meta-at-1-0.md):** Configures the node itself (hardware specs, supported devices, payment info). +* **[`~process@1.0`](../devices/process-at-1-0.md):** Manages persistent, shared computational states (like traditional smart contracts, but more flexible). +* **[`~scheduler@1.0`](../devices/scheduler-at-1-0.md):** Handles the ordering and execution of messages within a process. +* **[`~wasm64@1.0`](../devices/wasm64-at-1-0.md):** Executes WebAssembly (WASM) code, allowing for complex computations written in languages like Rust, C++, etc. +* **[`~lua@5.3a`](../devices/lua-at-5-3a.md):** Executes Lua scripts. +* **[`~relay@1.0`](../devices/relay-at-1-0.md):** Forwards messages between AO nodes or to external HTTP endpoints. +* **[`~json@1.0`](../devices/json-at-1-0.md):** Provides access to JSON data structures using HyperPATHs. +* **[`~message@1.0`](../devices/message-at-1-0.md):** Manages message state and processing. +* **[`~patch@1.0`](../guides/exposing-process-state.md):** Applies state updates directly to a process, often used for migrating or managing process data. + +## Beyond the Basics + +Devices aren't limited to just computation or state management. They can represent more abstract concepts: + +* **Security Devices ([`~snp@1.0`](../resources/source-code/dev_snp.md), [`dev_codec_httpsig`](../resources/source-code/dev_codec_httpsig.md)):** Handle tasks related to Trusted Execution Environments (TEEs) or message signing, adding layers of security and verification. +* **Payment/Access Control Devices ([`~p4@1.0`](../resources/source-code/dev_p4.md), [`~faff@1.0`](../resources/source-code/dev_faff.md)):** Manage metering, billing, or access control for node services. +* **Workflow/Utility Devices ([`dev_cron`](../resources/source-code/dev_cron.md), [`dev_stack`](../resources/source-code/dev_stack.md), [`dev_monitor`](../resources/source-code/dev_monitor.md)):** Coordinate complex execution flows, schedule tasks, or monitor process activity. + +## Using Devices + +Devices are typically invoked via [HyperPATHs](./pathing-in-ao-core.md). The path specifies which Device should interpret the subsequent parts of the path or the request body. + +``` +# Example: Execute the 'now' key on the process device for a specific process +/~process@1.0/now + +# Example: Relay a GET request via the relay device +/~relay@1.0/call?method=GET&path=https://example.com +``` + +The specific functions or 'keys' available for each Device are documented individually. See the [Devices section](../devices/index.md) for details on specific built-in devices. + +## The Potential of Devices + +The modular nature of AO Devices opens up vast possibilities for future expansion and innovation. The current set of preloaded and community devices is just the beginning. As the AO ecosystem evolves, we can anticipate the development of new devices catering to increasingly specialized needs: + +* **Specialized Hardware Integration:** Devices could be created to interface directly with specialized hardware accelerators like GPUs (for AI/ML tasks such as running large language models), TPUs, or FPGAs, allowing AO processes to leverage high-performance computing resources securely and verifiably. +* **Advanced Cryptography:** New devices could implement cutting-edge cryptographic techniques, such as zero-knowledge proofs (ZKPs) or fully homomorphic encryption (FHE), enabling enhanced privacy and complex computations on encrypted data. +* **Cross-Chain & Off-Chain Bridges:** Devices could act as secure bridges to other blockchain networks or traditional Web2 APIs, facilitating seamless interoperability and data exchange between AO and the wider digital world. +* **AI/ML Specific Devices:** Beyond raw GPU access, specialized devices could offer higher-level AI/ML functionalities, like optimized model inference engines or distributed training frameworks. +* **Domain-Specific Logic:** Communities or organizations could develop devices tailored to specific industries or use cases, such as decentralized finance (DeFi) primitives, scientific computing libraries, or decentralized identity management systems. + +The Device framework ensures that AO can adapt and grow, incorporating new technologies and computational paradigms without requiring fundamental changes to the core protocol. This extensibility is key to AO's long-term vision of becoming a truly global, decentralized computer. diff --git a/docs/introduction/pathing-in-ao-core.md b/docs/introduction/pathing-in-ao-core.md new file mode 100644 index 000000000..2c1a82be4 --- /dev/null +++ b/docs/introduction/pathing-in-ao-core.md @@ -0,0 +1,132 @@ +# Pathing in AO-Core + +## Overview + +Understanding how to construct and interpret paths in AO-Core is fundamental to working with HyperBEAM. This guide explains the structure and components of AO-Core paths, enabling you to effectively interact with processes and access their data. + +## HyperPATH Structure + +Let's examine a typical HyperBEAM endpoint piece-by-piece: + +``` +https://router-1.forward.computer/~process@1.0/now +``` + +### Node URL (`router-1.forward.computer`) + +The HTTP response from this node includes a signature from the host's key. By accessing the [`~snp@1.0`](../resources/source-code/dev_snp.md) device, you can verify that the node is running in a genuine Trusted Execution Environment (TEE), ensuring computation integrity. You can replace `router-1.forward.computer` with any HyperBEAM TEE node operated by any party while maintaining trustless guarantees. + +### Process Path (`/~process@1.0`) + +Every path in AO-Core represents a program. Think of the URL bar as a Unix-style command-line interface, providing access to AO's trustless and verifiable compute. Each path component (between `/` characters) represents a step in the computation. In this example, we instruct the AO-Core node to: + +1. Load a specific message from its caches (local, another node, or Arweave) +2. Interpret it with the [`~process@1.0`](../devices/process-at-1-0.md) device +3. The process device implements a shared computing environment with consistent state between users + +### State Access (`/now` or `/compute`) + +Devices in AO-Core expose keys accessible via path components. Each key executes a function on the device: + +- `now`: Calculates real-time process state +- `compute`: Serves the latest known state (faster than checking for new messages) + +Under the surface, these keys represent AO-Core messages. As we progress through the path, AO-Core applies each message to the existing state. You can access the full process state by visiting: +``` +/~process@1.0/now +``` + +### State Navigation + +You can browse through sub-messages and data fields by accessing them as keys. For example, if a process stores its interaction count in a field named `cache`, you can access it like this: +``` +/~process@1.0/compute/cache +``` +This shows the 'cache' of your process. Each response is: + +- A message with a signature attesting to its correctness +- A hashpath describing its generation +- Transferable to other AO-Core nodes for uninterrupted execution + +### Query Parameters and Type Casting + +Beyond path segments, HyperBEAM URLs can include query parameters that utilize a special type casting syntax. This allows specifying the desired data type for a parameter directly within the URL using the format `key+type=value`. + +- **Syntax**: A `+` symbol separates the parameter key from its intended type (e.g., `count+integer=42`, `items+list="apple",7`). +- **Mechanism**: The HyperBEAM node identifies the `+type` suffix (e.g., `+integer`, `+list`, `+map`, `+float`, `+atom`, `+resolve`). It then uses internal functions ([`hb_singleton:maybe_typed`](../resources/source-code/hb_singleton.md) and [`dev_codec_structured:decode_value`](../resources/source-code/dev_codec_structured.md)) to decode and cast the provided value string into the corresponding Erlang data type before incorporating it into the message. +- **Supported Types**: Common types include `integer`, `float`, `list`, `map`, `atom`, `binary` (often implicit), and `resolve` (for path resolution). List values often follow the [HTTP Structured Fields format (RFC 8941)](https://www.rfc-editor.org/rfc/rfc8941.html). + +This powerful feature enables the expression of complex data structures directly in URLs. + +## Examples + +The following examples illustrate using HyperPATH with various AO-Core processes and devices. While these cover a few specific use cases, HyperBEAM's extensible nature allows interaction with any device or process via HyperPATH. For a deeper understanding, we encourage exploring the [source code](https://github.com/permaweb/hyperbeam) and experimenting with different paths. + +### Example 1: Accessing Full Process State + +To get the complete, real-time state of a process identified by ``, use the `/now` path component with the [`~process@1.0`](../devices/process-at-1-0.md) device: + +``` +GET /~process@1.0/now +``` + +This instructs the AO-Core node to load the process and execute the `now` function on the [`~process@1.0`](../devices/process-at-1-0.md) device. + +### Example 2: Navigating to Specific Process Data + +If a process maintains its state in a map and you want to access a specific field, like `at-slot`, using the faster `/compute` endpoint: + +``` +GET /~process@1.0/compute/cache +``` + +This accesses the `compute` key on the [`~process@1.0`](../devices/process-at-1-0.md) device and then navigates to the `cache` key within the resulting state map. Using this path, you will see the latest 'cache' of your process (the number of interactions it has received). Every piece of relevant information about your process can be accessed similarly, effectively providing a native API. + +(Note: This represents direct navigation within the process state structure. For accessing data specifically published via the `~patch@1.0` device, see the documentation on [Exposing Process State](../build/exposing-process-state.md), which typically uses the `/cache/` path.) + +### Example 3: Basic `~message@1.0` Usage + +Here's a simple example of using [`~message@1.0`](../devices/message-at-1-0.md) to create a message and retrieve a value: + +``` +GET /~message@1.0&greeting="Hello"&count+integer=42/count +``` + +1. **Base:** `/` - The base URL of the HyperBEAM node. +2. **Root Device:** [`~message@1.0`](../devices/message-at-1-0.md) +3. **Query Params:** `greeting="Hello"` (binary) and `count+integer=42` (integer), forming the message `#{ <<"greeting">> => <<"Hello">>, <<"count">> => 42 }`. +4. **Path:** `/count` tells `~message@1.0` to retrieve the value associated with the key `count`. + +**Response:** The integer `42`. + +### Example 4: Using the `~message@1.0` Device with Type Casting + +The [`~message@1.0`](../devices/message-at-1-0.md) device can be used to construct and query transient messages, utilizing type casting in query parameters. + +Consider the following URL: + +``` +GET /~message@1.0&name="Alice"&age+integer=30&items+list="apple",1,"banana"&config+map=key1="val1";key2=true/[PATH] +``` + +HyperBEAM processes this as follows: + +1. **Base:** `/` - The base URL of the HyperBEAM node. +2. **Root Device:** [`~message@1.0`](../devices/message-at-1-0.md) +3. **Query Parameters (with type casting):** + * `name="Alice"` -> `#{ <<"name">> => <<"Alice">> }` (binary) + * `age+integer=30` -> `#{ <<"age">> => 30 }` (integer) + * `items+list="apple",1,"banana"` -> `#{ <<"items">> => [<<"apple">>, 1, <<"banana">>] }` (list) + * `config+map=key1="val1";key2=true` -> `#{ <<"config">> => #{<<"key1">> => <<"val1">>, <<"key2">> => true} }` (map) +4. **Initial Message Map:** A combination of the above key-value pairs. +5. **Path Evaluation:** + * If `[PATH]` is `/items/1`, the response is the integer `1`. + * If `[PATH]` is `/config/key1`, the response is the binary `<<"val1">>`. + +## Best Practices + +1. Always verify cryptographic signatures on responses +2. Use appropriate caching strategies for frequently accessed data +3. Implement proper error handling for network requests +4. Consider rate limits and performance implications +5. Keep sensitive data secure and use appropriate authentication methods \ No newline at end of file diff --git a/docs/introduction/what-is-ao-core.md b/docs/introduction/what-is-ao-core.md new file mode 100644 index 000000000..11e59a6bc --- /dev/null +++ b/docs/introduction/what-is-ao-core.md @@ -0,0 +1,22 @@ +# What is AO-Core? + +AO-Core is the foundational protocol underpinning the [AO Computer](https://ao.arweave.net). It defines a minimal, generalized model for decentralized computation built around standard web technologies like HTTP. Think of it as a way to interpret the Arweave permaweb not just as static storage, but as a dynamic, programmable, and infinitely scalable computing environment. + +## Core Concepts + +AO-Core revolves around three fundamental components: + +1. **Messages:** The smallest units of data and computation. Messages can be simple data blobs or maps of named functions. They are the primary means of communication and triggering execution within the system. Messages are cryptographically linked, forming a verifiable computation graph. +2. **Devices:** Modules responsible for interpreting and processing messages. Each device defines specific logic for how messages are handled (e.g., executing WASM, storing data, relaying information). This modular design allows nodes to specialize and the system to be highly extensible. +3. **Paths:** Structures that link messages over time, creating a verifiable history of computations. Paths allow users to navigate the computation graph and access specific states or results. They leverage `HashPaths`, cryptographic fingerprints representing the sequence of operations leading to a specific message state, ensuring traceability and integrity. + +## Key Principles + +* **Minimalism:** AO-Core provides the simplest possible representation of data and computation, avoiding prescriptive consensus mechanisms or specific VM requirements. +* **HTTP Native:** Designed for compatibility with HTTP protocols, making it accessible via standard web tools and infrastructure. +* **Scalability:** By allowing parallel message processing and modular device execution, AO-Core enables hyper-parallel computing, overcoming the limitations of traditional sequential blockchains. +* **Permissionlessness & Trustlessness:** While AO-Core itself is minimal, it provides the framework upon which higher-level protocols like AO can build systems that allow anyone to participate (`permissionlessness`) without needing to trust intermediaries (`trustlessness`). Users can choose their desired security and performance trade-offs. + +AO-Core transforms the permanent data storage of Arweave into a global, shared computation space, enabling the creation of complex, autonomous, and scalable decentralized applications. + + \ No newline at end of file diff --git a/docs/introduction/what-is-hyperbeam.md b/docs/introduction/what-is-hyperbeam.md new file mode 100644 index 000000000..3d9f669c4 --- /dev/null +++ b/docs/introduction/what-is-hyperbeam.md @@ -0,0 +1,40 @@ +# What is HyperBEAM? + +HyperBEAM is the primary, production-ready implementation of the [AO-Core protocol](./what-is-ao-core.md), built on the robust Erlang/OTP framework. It serves as a decentralized operating system, powering the AO Computer—a scalable, trust-minimized, distributed supercomputer built on permanent storage. HyperBEAM provides the runtime environment and essential services to execute AO-Core computations across a network of distributed nodes. + +## Why HyperBEAM Matters + +HyperBEAM transforms the abstract concepts of AO-Core—such as [Messages](./what-is-ao-core.md#core-concepts), [Devices](./what-is-ao-core.md#core-concepts), and [Paths](./what-is-ao-core.md#core-concepts)—into a concrete, operational system. Here's why it's pivotal to the AO ecosystem: + +- **Modularity via Devices:** HyperBEAM introduces a uniquely modular architecture centered around [Devices](./ao-devices.md). These pluggable components define specific computational logic (like running WASM, managing state, or relaying data), allowing for unprecedented flexibility, specialization, and extensibility in a decentralized system. +- **Decentralized OS:** It equips nodes with the infrastructure to join the AO network, manage resources, execute computations, and communicate seamlessly. +- **Erlang/OTP Powerhouse:** Leveraging the BEAM virtual machine, HyperBEAM inherits Erlang's concurrency, fault tolerance, and scalability—perfect for distributed systems with lightweight processes and message passing. +- **Hardware Independence:** It abstracts underlying hardware, allowing diverse nodes to contribute resources without compatibility issues. +- **Node Coordination:** It governs how nodes join the network, offer services through specific Devices, and interact with one another. +- **Verifiable Computation:** Through hashpaths and the Converge Protocol, HyperBEAM ensures computation results are cryptographically verified and trustworthy. + +In essence, HyperBEAM is the engine that drives the AO Computer, enabling a vision of decentralized, verifiable computing at scale. + +## Core Components & Features + +- **Pluggable Devices:** The heart of HyperBEAM's extensibility. It includes essential built-in devices like [`~meta`](../devices/meta-at-1-0.md), [`~relay`](../devices/relay-at-1-0.md), [`~process`](../devices/process-at-1-0.md), [`~scheduler`](../devices/scheduler-at-1-0.md), and [`~wasm64`](../devices/wasm64-at-1-0.md) for core functionality, but the system is designed for easy addition of new custom devices. +- **Message System:** Everything in HyperBEAM is a "Message"—a map of named functions or binary data that can be processed, transformed, and cryptographically verified. +- **HTTP Interface:** Nodes expose an HTTP server for interaction via standard web requests and HyperPATHs, structured URLs that represent computation paths. +- **Modularity:** Its design supports easy extension, allowing new devices and functionalities to be added effortlessly. + +## Architecture + +* **Initialization Flow:** When a HyperBEAM node starts, it initializes the name service, scheduler registry, timestamp server, and HTTP server, establishing core services for process management, timing, communication, and storage. +* **Compute Model:** Computation follows the pattern \`Message1(Message2) => Message3\`, where messages are resolved through their devices and [paths](./pathing-in-ao-core.md). The integrity and history of these computations are ensured by **hashpaths**, which serves as a cryptographic audit trail. +* **Scheduler System:** The scheduler component manages execution order using ["slots"](../devices/scheduler-at-1-0.md#slot-system) — sequential positions that guarantee deterministic computation. +* **Process Slots:** Each process has numbered slots starting from 0 that track message execution order, ensuring consistent computation even across distributed nodes. + +## HTTP API and Pathing + +HyperBEAM exposes a powerful HTTP API that allows for interacting with processes and accessing data through structured URL patterns. We call URLs that represent computation paths "HyperPATHs". The URL bar effectively functions as a command-line interface for AO's trustless and verifiable compute. + +For a comprehensive guide on constructing and interpreting paths in AO-Core, including detailed examples and best practices, see [Pathing in AO-Core](./pathing-in-ao-core.md). + +In essence, HyperBEAM is the engine that powers the AO Computer, enabling the vision of a scalable, trust-minimized, decentralized supercomputer built on permanent storage. + +*See also: [HyperBEAM GitHub Repository](https://github.com/permaweb/HyperBEAM)* diff --git a/docs/js/custom-header.js b/docs/js/custom-header.js new file mode 100644 index 000000000..0f84e44b1 --- /dev/null +++ b/docs/js/custom-header.js @@ -0,0 +1,75 @@ +(function () { + let currentPath = window.location.pathname; + + function updateHeaderAndMainClass() { + const header = document.querySelector(".md-header"); + const main = document.querySelector("main"); + const tabs = document.querySelector(".md-tabs"); + + const segments = window.location.pathname.split("/").filter(Boolean); + const arweavePath = segments.length === 1 && segments[0].length === 43; + const isHomepage = segments.length === 0 || arweavePath; + + if (!header || !main) return; + + if (isHomepage) { + header.classList.add("custom-homepage-header"); + main.classList.add("custom-homepage-main"); + main.classList.remove("md-main"); + if (tabs) tabs.style.display = "none"; + } else { + header.classList.remove("custom-homepage-header"); + main.classList.remove("custom-homepage-main"); + main.classList.add("md-main"); + if (tabs) tabs.style.display = ""; + } + } + + // Initial run + updateHeaderAndMainClass(); + + // Watch for URL changes + const observer = new MutationObserver(() => { + if (window.location.pathname !== currentPath) { + currentPath = window.location.pathname; + updateHeaderAndMainClass(); + } + }); + + observer.observe(document.body, { childList: true, subtree: true }); + + window.addEventListener("popstate", updateHeaderAndMainClass); +})(); + +document.addEventListener("DOMContentLoaded", function () { + function updateMainClass() { + const mainElement = document.querySelector("main"); + const isHomepage = window.location.pathname === "/"; + + // Apply the homepage class if on the homepage, else remove it + if (isHomepage) { + mainElement.classList.add("custom-homepage-main"); + mainElement.classList.remove("md-main"); + } else { + mainElement.classList.add("md-main"); + mainElement.classList.remove("custom-homepage-main"); + } + } + + // Initial update on page load + updateMainClass(); + + // Listen for link clicks and update the class after navigation + const links = document.querySelectorAll("a"); + links.forEach((link) => { + link.addEventListener("click", function (event) { + // Small delay to ensure the page has started loading + setTimeout(updateMainClass, 0); + }); + }); + + // Listen for popstate events (back/forward navigation) + window.addEventListener("popstate", function () { + setTimeout(updateMainClass, 500); + }); +}); diff --git a/docs/js/header-scroll.js b/docs/js/header-scroll.js new file mode 100644 index 000000000..1a27e8e35 --- /dev/null +++ b/docs/js/header-scroll.js @@ -0,0 +1,91 @@ +document.addEventListener('DOMContentLoaded', function() { + const header = document.querySelector('.md-header'); + const paddingTargetElement = document.querySelector('.md-content'); // Element for padding adjustments + const contentVisibilityTargetElement = document.querySelector('.md-main__inner.md-grid'); // Element to hide/show with transition + + if (!header || !paddingTargetElement || !contentVisibilityTargetElement) { + console.error('Header scroll: Required elements (.md-header, .md-content, or .md-main__inner.md-grid) not found.'); + return; + } + + const HIDING_CLASS = 'content--initializing'; + + // Function to inject CSS for transition and initial hiding + function injectTransitionStyles() { + const styleId = 'md-content-transition-style'; + if (document.getElementById(styleId)) { + return; // Style already added + } + const styleElement = document.createElement('style'); + styleElement.id = styleId; + styleElement.textContent = ` + .md-main__inner.md-grid { /* Style for the element to be shown with transition */ + opacity: 0; + transition: opacity 200ms ease-in-out; /* Tiny transition */ + } + .${HIDING_CLASS} { /* Class to initially hide the content */ + display: none !important; + opacity: 0 !important; /* Ensure opacity is 0 when hidden */ + } + `; + document.head.appendChild(styleElement); + } + + // Initially hide the content and set up for transition + injectTransitionStyles(); + contentVisibilityTargetElement.classList.add(HIDING_CLASS); + + let headerHeight = 0; + + // Function to update paddings based on header state + function updatePaddings() { + const currentHeaderHeight = header.offsetHeight; + if (currentHeaderHeight > 0) { + headerHeight = currentHeaderHeight; + } + + if (header.classList.contains('header-hidden')) { + if (paddingTargetElement) paddingTargetElement.style.paddingTop = '75px'; + document.documentElement.style.scrollPaddingTop = '0'; + } else { + if (paddingTargetElement) paddingTargetElement.style.paddingTop = headerHeight + 'px'; + document.documentElement.style.scrollPaddingTop = headerHeight + 'px'; + } + } + + // Function to initialize header state and reveal content + function initializeHeaderState() { + headerHeight = header.offsetHeight; + updatePaddings(); // Apply padding to paddingTargetElement + + // Make content displayable (it's still opacity 0 due to injected styles) + contentVisibilityTargetElement.classList.remove(HIDING_CLASS); + + // Trigger the opacity transition to fade in the content + requestAnimationFrame(() => { + contentVisibilityTargetElement.style.opacity = 1; + }); + } + + window.addEventListener('load', initializeHeaderState); + + window.addEventListener('scroll', function() { + const scrollTop = window.scrollY || document.documentElement.scrollTop; + + if (scrollTop > 0) { // When scrolling down / header should be hidden + if (!header.classList.contains('header-hidden')) { + header.classList.add('header-hidden'); + updatePaddings(); + } + } else { // When at the top / header should be visible + if (header.classList.contains('header-hidden')) { + header.classList.remove('header-hidden'); + updatePaddings(); + } + } + }); + + window.addEventListener('resize', function() { + updatePaddings(); + }); +}); \ No newline at end of file diff --git a/docs/js/parallax.js b/docs/js/parallax.js new file mode 100644 index 000000000..a09767c5f --- /dev/null +++ b/docs/js/parallax.js @@ -0,0 +1,39 @@ +document.addEventListener("DOMContentLoaded", () => { + const header = document.querySelector(".custom-homepage-header"); + const scrollContainer = document.querySelector(".custom-homepage-main"); + + if (!header || !scrollContainer) + return console.log("Missing header or scroll container"); + + let needsUpdate = false; + + function updateHeaderFade() { + needsUpdate = false; + const scrollTop = scrollContainer.scrollTop; + + const fadeStart = window.innerHeight * 1.35; // fade starts 80% into hero + const fadeEnd = window.innerHeight * 1.45; // fade finishes at 120% + + let opacity; + if (scrollTop <= fadeStart) { + opacity = 0; + } else if (scrollTop >= fadeEnd) { + opacity = 1; + } else { + opacity = (scrollTop - fadeStart) / (fadeEnd - fadeStart); + } + + header.style.backgroundColor = `rgba(255, 255, 255, ${opacity})`; + header.style.filter = `invert(${1 - opacity})`; + } + + scrollContainer.addEventListener("scroll", () => { + if (!needsUpdate) { + needsUpdate = true; + requestAnimationFrame(updateHeaderFade); + } + }); + + window.addEventListener("resize", updateHeaderFade); + requestAnimationFrame(updateHeaderFade); // run on load +}); diff --git a/docs/js/toc-highlight.js b/docs/js/toc-highlight.js new file mode 100644 index 000000000..1d4e1416d --- /dev/null +++ b/docs/js/toc-highlight.js @@ -0,0 +1,124 @@ +document.addEventListener("DOMContentLoaded", function () { + /** + * Fixes navigation highlighting in MkDocs Material Theme: + * 1. If a list item has both an active label and an active link, remove active from label + * 2. If a parent item has active children, remove active from the parent's links + * 3. When scroll position is at the top, reactivate the parent navigation item + */ + function fixNavigationHighlighting() { + // First fix case where both label and anchor in same item are active + document.querySelectorAll(".md-nav__item").forEach(function (item) { + const label = item.querySelector("label.md-nav__link--active"); + const link = item.querySelector("a.md-nav__link--active"); + + // If both exist in the same item, keep only the link active + if (label && link) { + label.classList.remove("md-nav__link--active"); + } + }); + + // Check if scroll position is at the top + const atTop = window.scrollY === 0; + + // Now fix nested navigation (parent sections shouldn't be active when children are, unless at top) + document + .querySelectorAll(".md-nav__item--active") + .forEach(function (activeItem) { + // Check if this active item contains other active items + const hasActiveChildren = activeItem.querySelector( + ".md-nav__link--active", + ); + + // console.log("Has Active Children:", hasActiveChildren); + + if (hasActiveChildren && !atTop) { + // Remove active class from parent's links + const parentLinks = activeItem.querySelectorAll( + ":scope > a.md-nav__link--active, :scope > label.md-nav__link--active", + ); + parentLinks.forEach(function (link) { + link.classList.remove("md-nav__link--active"); + }); + } else if (!hasActiveChildren && atTop) { + // Reactivate parent link if at top and no active children + const parentLinks = activeItem.querySelectorAll( + ":scope > a.md-nav__link, :scope > label.md-nav__link", + ); + parentLinks.forEach(function (link) { + link.classList.add("md-nav__link--active"); + }); + } + }); + } + + // Initial run + fixNavigationHighlighting(); + + // Set up a mutation observer to detect changes + const observer = new MutationObserver(function (mutations) { + let shouldUpdate = false; + + for (const mutation of mutations) { + if ( + mutation.type === "attributes" && + mutation.attributeName === "class" && + (mutation.target.classList.contains("md-nav__link--active") || + mutation.target.classList.contains("md-nav__item--active")) + ) { + shouldUpdate = true; + break; + } + } + + if (shouldUpdate) { + fixNavigationHighlighting(); + } + }); + + // Observe all navigation elements + document + .querySelectorAll(".md-nav__item, .md-nav__link") + .forEach(function (el) { + observer.observe(el, { attributes: true }); + }); + + // Update on navigation events + window.addEventListener("popstate", function () { + setTimeout(fixNavigationHighlighting, 100); + }); + + window.addEventListener("load", fixNavigationHighlighting); + + // Update on scroll with throttling + let scrollTimeout; + window.addEventListener("scroll", function () { + if (!scrollTimeout) { + scrollTimeout = setTimeout(function () { + fixNavigationHighlighting(); + scrollTimeout = null; + }, 50); + } + }); + + document.addEventListener("click", function (e) { + if (e.target.closest(".md-nav__link")) { + setTimeout(fixNavigationHighlighting, 50); + } + }); + + // Add click event handling for navigation tabs + const tabLinks = document.querySelectorAll('nav.md-tabs .md-tabs__list .md-tabs__item a'); + tabLinks.forEach(link => { + link.addEventListener('click', function(event) { + // Basic check if it's an internal link + if (link.hostname === window.location.hostname || !link.hostname.length) { + console.log('Tab clicked, forcing full reload for:', link.href); + console.log('Updating navigation highlighting before navigation'); + fixNavigationHighlighting(); + event.preventDefault(); + event.stopPropagation(); + window.location.href = link.href; + } + }, true); // Use capture phase to catch the event before the theme's handler + }); +}); diff --git a/docs/js/utc-time.js b/docs/js/utc-time.js new file mode 100644 index 000000000..13ab1fb74 --- /dev/null +++ b/docs/js/utc-time.js @@ -0,0 +1,21 @@ +window.addEventListener("DOMContentLoaded", () => { + const timeDiv = document.createElement("div"); + timeDiv.id = "utc-time"; + timeDiv.style.cssText = ` + font-size: clamp(0.4rem, 1.5vw, 0.5rem); + user-select: none; + `; + + const updateTime = () => { + const now = new Date(); + timeDiv.textContent = "UTC " + now.toISOString().substring(11, 19); // HH:MM:SS + }; + + updateTime(); + setInterval(updateTime, 1000); // update every second + + const headerOptions = document.querySelector(".md-header__option"); + if (headerOptions) { + headerOptions.replaceWith(timeDiv); // Replace existing element + } +}); diff --git a/docs/llms-full.txt b/docs/llms-full.txt new file mode 100644 index 000000000..29c722720 --- /dev/null +++ b/docs/llms-full.txt @@ -0,0 +1,21424 @@ +Generated: 2025-05-15T13:32:25Z + +--- START OF FILE: docs/build/exposing-process-state.md --- +# Exposing Process State with the Patch Device + +The [`~patch@1.0`](../resources/source-code/dev_patch.md) device provides a mechanism for AO processes to expose parts of their internal state, making it readable via direct HTTP GET requests along the process's HyperPATH. + +## Why Use the Patch Device? + +Standard AO process execution typically involves sending a message to a process, letting it compute, and then potentially reading results from its outbox or state after the computation is scheduled and finished. This is asynchronous. + +The `patch` device allows for a more direct, synchronous-like read pattern. A process can use it to "patch" specific data elements from its internal state into a location that becomes directly accessible via a HyperPATH GET request *before* the full asynchronous scheduling might complete. + +This is particularly useful for: + +* **Web Interfaces:** Building frontends that need to quickly read specific data points from an AO process without waiting for a full message round-trip. +* **Data Feeds:** Exposing specific metrics or state variables for monitoring or integration with other systems. +* **Caching:** Allowing frequently accessed data to be retrieved efficiently via simple HTTP GETs. + +## How it Works + +1. **Process Logic:** Inside your AO process code (e.g., in Lua or WASM), when you want to expose data, you construct an **Outbound Message** targeted at the [`~patch@1.0`](../resources/source-code/dev_patch.md) device. +2. **Patch Message Format:** This outbound message typically includes tags that specify: + * `device = 'patch@1.0'` + * A `cache` tag containing a table. The **keys** within this table become the final segments in the HyperPATH used to access the data, and the **values** are the data itself. + * Example Lua using `aos`: `Send({ Target = ao.id, device = 'patch@1.0', cache = { mydatakey = MyValue } })` +3. **HyperBEAM Execution:** When HyperBEAM executes the process schedule and encounters this outbound message: + * It invokes the `dev_patch` module. + * `dev_patch` inspects the message. + * It takes the keys from the `cache` table (`mydatakey` in the example) and their associated values (`MyValue`) and makes these values available under the `/cache/` path segment. +4. **HTTP Access:** You (or any HTTP client) can now access this data directly using a GET request: + ``` + GET /~process@1.0/compute/cache/ + # Or potentially using /now/ + GET /~process@1.0/now/cache/ + ``` + The HyperBEAM node serving the request will resolve the path up to `/compute/cache` (or `/now/cache`), then use the logic associated with the patched data (`mydatakey`) to return the `MyValue` directly. + +## Initial State Sync (Optional) + +It can be beneficial to expose the initial state of your process via the `patch` device as soon as the process is loaded or spawned. This makes key data points immediately accessible via HTTP GET requests without requiring an initial interaction message to trigger a `Send` to the patch device. + +This pattern typically involves checking a flag within your process state to ensure the initial sync only happens once. Here's an example from the Token Blueprint, demonstrating how to sync `Balances` and `TotalSupply` right after the process starts: + +```lua +-- Place this logic at the top level of your process script, +-- outside of specific handlers, so it runs on load. + +-- Initialize the sync flag if it doesn't exist +InitialSync = InitialSync or 'INCOMPLETE' + +-- Sync state on spawn/load if not already done +if InitialSync == 'INCOMPLETE' then + -- Send the relevant state variables to the patch device + Send({ device = 'patch@1.0', cache = { balances = Balances, totalsupply = TotalSupply } }) + -- Update the flag to prevent re-syncing on subsequent executions + InitialSync = 'COMPLETE' + print("Initial state sync complete. Balances and TotalSupply patched.") +end +``` + +**Explanation:** + +1. `InitialSync = InitialSync or 'INCOMPLETE'`: This line ensures the `InitialSync` variable exists in the process state, initializing it to `'INCOMPLETE'` if it's the first time the code runs. +2. `if InitialSync == 'INCOMPLETE' then`: The code proceeds only if the initial sync hasn't been marked as complete. +3. `Send(...)`: The relevant state (`Balances`, `TotalSupply`) is sent to the `patch` device, making it available under `/cache/balances` and `/cache/totalsupply`. +4. `InitialSync = 'COMPLETE'`: The flag is updated, so this block won't execute again in future message handlers within the same process lifecycle. + +This ensures that clients or frontends can immediately query essential data like token balances as soon as the process ID is known, improving the responsiveness of applications built on AO. + +## Example (Lua in `aos`) + +```lua +-- In your process code (e.g., loaded via .load) +Handlers.add( + "PublishData", + Handlers.utils.hasMatchingTag("Action", "PublishData"), + function (msg) + local dataToPublish = "Some important state: " .. math.random() + -- Expose 'currentstatus' key under the 'cache' path + Send({ device = 'patch@1.0', cache = { currentstatus = dataToPublish } }) + print("Published data to /cache/currentstatus") + end +) + +-- Spawning and interacting +[aos]> MyProcess = spawn(MyModule) + +[aos]> Send({ Target = MyProcess, Action = "PublishData" }) +-- Wait a moment for scheduling + +``` + +## Avoiding Key Conflicts + +When defining keys within the `cache` table (e.g., `cache = { mydatakey = MyValue }`), these keys become path segments under `/cache/` (e.g., `/compute/cache/mydatakey` or `/now/cache/mydatakey`). It's important to choose keys that do not conflict with existing, reserved path segments used by HyperBEAM or the `~process` device itself for state access. + +Using reserved keywords as your cache keys can lead to routing conflicts or prevent you from accessing your patched data as expected. While the exact list can depend on device implementations, it's wise to avoid keys commonly associated with state access, such as: `now`, `compute`, `state`, `info`, `test`. + +It's recommended to use descriptive and specific keys for your cached data to prevent clashes with the underlying HyperPATH routing mechanisms. For example, instead of `cache = { state = ... }`, prefer `cache = { myappstate = ... }` or `cache = { usercount = ... }`. + +!!! warning + Be aware that HTTP path resolution is case-insensitive and automatically normalizes paths to lowercase. While the `patch` device itself stores keys with case sensitivity (e.g., distinguishing `MyKey` from `mykey`), accessing them via an HTTP GET request will treat `/cache/MyKey` and `/cache/mykey` as the same path. This means that using keys that only differ in case (like `MyKey` and `mykey` in your `cache` table) will result in unpredictable behavior or data overwrites when accessed via HyperPATH. To prevent these issues, it is **strongly recommended** to use **consistently lowercase keys** within the `cache` table (e.g., `mykey`, `usercount`, `appstate`). + +## Key Points + +* **Path Structure:** The data is exposed under the `/cache/` path segment. The tag name you use *inside* the `cache` table in the `Send` call (e.g., `currentstatus`) becomes the final segment in the accessible HyperPATH (e.g., `/compute/cache/currentstatus`). +* **Data Types:** The `patch` device typically handles basic data types (strings, numbers) within the `cache` table effectively. Complex nested tables might require specific encoding or handling. +* **`compute` vs `now`:** Accessing patched data via `/compute/cache/...` typically serves the last known patched value quickly. Accessing via `/now/cache/...` might involve more computation to ensure the absolute latest state before checking for the patched key under `/cache/`. +* **Not a Replacement for State:** Patching is primarily for *exposing* reads. It doesn't replace the core state management within your process handler logic. + +By using the `patch` device, you can make parts of your AO process state easily and efficiently readable over standard HTTP, bridging the gap between decentralized computation and web-based applications. +--- END OF FILE: docs/build/exposing-process-state.md --- + +--- START OF FILE: docs/build/extending-hyperbeam.md --- +# Extending HyperBEAM + +HyperBEAM's modular design, built on AO-Core principles and Erlang/OTP, makes it highly extensible. You can add new functionalities or modify existing behaviors primarily by creating new **Devices** or implementing **Pre/Post-Processors**. + +!!! warning "Advanced Topic" + Extending HyperBEAM requires a good understanding of Erlang/OTP, the AO-Core protocol, and HyperBEAM's internal architecture. This guide provides a high-level overview; detailed implementation requires deeper exploration of the source code. + +## Approach 1: Creating New Devices + +This is the most common way to add significant new capabilities. +A Device is essentially an Erlang module (typically named `dev_*.erl`) that processes AO-Core messages. + +**Steps:** + +1. **Define Purpose:** Clearly define what your device will do. What kind of messages will it process? What state will it manage (if any)? What functions (keys) will it expose? +2. **Create Module:** Create a new Erlang module (e.g., `src/dev_my_new_device.erl`). +3. **Implement `info/0..2` (Optional but Recommended):** Define an `info` function to signal capabilities and requirements to HyperBEAM (e.g., exported keys, variant/version ID). + ```erlang + info() -> + #{ + variant => <<"MyNewDevice/1.0">>, + exports => [<<"do_something">>, <<"get_status">>] + }. + ``` +4. **Implement Key Functions:** Create Erlang functions corresponding to the keys your device exposes. These functions typically take `StateMessage`, `InputMessage`, and `Environment` as arguments and return `{ok, NewMessage}` or `{error, Reason}`. + ```erlang + do_something(StateMsg, InputMsg, Env) -> + % ... perform action based on InputMsg ... + NewState = ..., % Calculate new state + {ok, NewState}. + + get_status(StateMsg, _InputMsg, _Env) -> + % ... read status from StateMsg ... + StatusData = ..., + {ok, StatusData}. + ``` +5. **Handle State (If Applicable):** Devices can be stateless or stateful. Stateful devices manage their state within the `StateMessage` passed between function calls. +6. **Register Device:** Ensure HyperBEAM knows about your device. This might involve adding it to build configurations or potentially a dynamic registration mechanism if available. +7. **Testing:** Write EUnit tests for your device's functions. + +**Example Idea:** A device that bridges to another blockchain network, allowing AO processes to read data or trigger transactions on that chain. + +## Approach 2: Building Pre/Post-Processors + +Pre/post-processors allow you to intercept incoming requests *before* they reach the target device/process (`preprocess`) or modify the response *after* execution (`postprocess`). These are often implemented using the `dev_stack` device or specific hooks within the request handling pipeline. + +**Use Cases:** + +* **Authentication/Authorization:** Checking signatures or permissions before allowing execution. +* **Request Modification:** Rewriting requests, adding metadata, or routing based on specific criteria. +* **Response Formatting:** Changing the structure or content type of the response. +* **Metering/Logging:** Recording request details or charging for usage before or after execution. + +**Implementation:** + +Processors often involve checking specific conditions (like request path or headers) and then either: + +a. Passing the request through unchanged. +b. Modifying the request/response message structure. +c. Returning an error or redirect. + + +**Example Idea:** A preprocessor that automatically adds a timestamp tag to all incoming messages for a specific process. + + +## Approach 3: Custom Routing Strategies + +While `dev_router` provides basic strategies (round-robin, etc.), you could potentially implement a custom load balancing or routing strategy module that `dev_router` could be configured to use. This would involve understanding the interfaces expected by `dev_router`. + +**Example Idea:** A routing strategy that queries worker nodes for their specific capabilities before forwarding a request. + +## Getting Started + +1. **Familiarize Yourself:** Deeply understand Erlang/OTP and the HyperBEAM codebase (`src/` directory), especially [`hb_ao.erl`](../resources/source-code/hb_ao.md), [`hb_message.erl`](../resources/source-code/hb_message.md), and existing `dev_*.erl` modules relevant to your idea. +2. **Study Examples:** Look at simple devices like `dev_patch.erl` or more complex ones like `dev_process.erl` to understand patterns. +3. **Start Small:** Implement a minimal version of your idea first. +4. **Test Rigorously:** Use `rebar3 eunit` extensively. +5. **Engage Community:** Ask questions in developer channels if you get stuck. + +Extending HyperBEAM allows you to tailor the AO network's capabilities to specific needs, contributing to its rich and evolving ecosystem. + +--- END OF FILE: docs/build/extending-hyperbeam.md --- + +--- START OF FILE: docs/build/get-started-building-on-ao-core.md --- +# Getting Started Building on AO-Core + +Welcome to building on AO, the decentralized supercomputer! + +AO combines the permanent storage of Arweave with the flexible, scalable computation enabled by the AO-Core protocol and its HyperBEAM implementation. This allows you to create truly autonomous applications, agents, and services that run trustlessly and permissionlessly. + +## Core Idea: Processes & Messages + +At its heart, building on AO involves: + +1. **Creating Processes:** Think of these as independent programs or stateful contracts. Each process has a unique ID and maintains its own state. +2. **Sending Messages:** You interact with processes by sending them messages. These messages trigger computations, update state, or cause the process to interact with other processes or the outside world. + +Messages are processed by [Devices](../begin/ao-devices.md), which define *how* the computation happens (e.g., running WASM code, executing Lua scripts, managing state transitions). + +## Starting `aos`: Your Development Environment + +The primary tool for interacting with AO and developing processes is `aos`, a command-line interface and development environment. + +=== "npm" + ```bash + npm i -g https://get_ao.arweave.net + ``` + +=== "bun" + ```bash + bun install -g https://get_ao.arweave.net + ``` + +=== "pnpm" + ```bash + pnpm add -g https://get_ao.arweave.net + ``` + +**Starting `aos`:** + +Simply run the command in your terminal: + +```bash +aos +``` + +This connects you to an interactive Lua environment running within a **process** on the AO network. This process acts as your command-line interface (CLI) to the AO network, allowing you to interact with other processes, manage your wallet, and develop new AO processes. By default, it connects to a process running on the mainnet Compute Unit (CU). + +**What `aos` is doing:** + +* **Connecting:** Establishes a connection from your terminal to a remote process running the `aos` environment. +* **Loading Wallet:** Looks for a default Arweave key file (usually `~/.aos.json` or specified via arguments) to load into the remote process context for signing outgoing messages. +* **Providing Interface:** Gives you a Lua prompt (`[aos]>`) within the remote process where you can: + * Load code for new persistent processes on the network. + * Send messages to existing network processes. + * Inspect process state. + * Manage your local environment. + +## Your First Interaction: Assigning a Variable + +From the `aos` prompt, you can assign a variable. Let's assign a basic Lua process that just holds some data: + +```lua +[aos]> myVariable = "Hello from aos!" +-- This assigns the string "Hello from aos!" to the variable 'myVariable' +-- within the current process's Lua environment. + +[aos]> myVariable +-- Displays the content of 'myVariable' +Hello from aos! +``` + + +## Your First Handler + +Follow these steps to create and interact with your first message handler in AO: + +1. **Create a Lua File to Handle Messages:** + Create a new file named `main.lua` in your local directory and add the following Lua code: + + ```lua + Handlers.add( + "HelloWorld", + function(msg) + -- This function gets called when a message with Action = "HelloWorld" arrives. + print("Handler triggered by message from: " .. msg.From) + -- It replies to the sender with a new message containing the specified data. + msg.reply({ Data = "Hello back from your process!" }) + end + ) + + print("HelloWorld handler loaded.") -- Confirmation message + ``` + + * `Handlers.add`: Registers a function to handle incoming messages. + * `"HelloWorld"`: The name of this handler. It will be triggered by messages with `Action = "HelloWorld"`. + * `function(msg)`: The function that executes when the handler is triggered. `msg` contains details about the incoming message (like `msg.From`, the sender's process ID). + * `msg.reply({...})`: Sends a response message back to the original sender. The response must be a Lua table, typically containing a `Data` field. + +2. **Load the Handler into `aos`:** + From your `aos` prompt, load the handler code into your running process: + + ```lua + [aos]> .load main.lua + ``` + +3. **Send a Message to Trigger the Handler:** + Now, send a message to your own process (`ao.id` refers to the current process ID) with the action that matches your handler's name: + + ```lua + [aos]> Send({ Target = ao.id, Action = "HelloWorld" }) + ``` + +4. **Observe the Output:** + You should see two things happen in your `aos` terminal: + * The `print` statement from your handler: `Handler triggered by message from: ` + * A notification about the reply message: `New Message From : Data = Hello back from your process!` + +5. **Inspect the Reply Message:** + The reply message sent by your handler is now in your process's inbox. You can inspect its data like this: + + ```lua + [aos]> Inbox[#Inbox].Data + ``` + This should output: `"Hello back from your process!"` + +You've successfully created a handler, loaded it into your AO process, triggered it with a message, and received a reply! + +## Next Steps + +This is just the beginning. To dive deeper: + +* **AO Cookbook:** Explore practical examples and recipes for common tasks: [AO Cookbook](https://cookbook_ao.arweave.net/) +* **Expose Process State:** Learn how to make your process data accessible via HTTP using the `patch` device: [Exposing Process State](./exposing-process-state.md) +* **Serverless Compute:** Discover how to run WASM or Lua computations within your processes: [Serverless Decentralized Compute](./serverless-decentralized-compute.md) +* **aos Documentation:** Refer to the official `aos` documentation for detailed commands and usage. + +--- END OF FILE: docs/build/get-started-building-on-ao-core.md --- + +--- START OF FILE: docs/build/serverless-decentralized-compute.md --- +# Serverless Decentralized Compute on AO + +AO enables powerful "serverless" computation patterns by allowing you to run code (WASM, Lua) directly within decentralized processes, triggered by messages. Furthermore, if computations are performed on nodes running in Trusted Execution Environments (TEEs), you can obtain cryptographic attestations verifying the execution integrity. + +## Core Concept: Compute Inside Processes + +Instead of deploying code to centralized servers, you deploy code *to* the Arweave permaweb and instantiate it as an AO process. Interactions happen by sending messages to this process ID. + +* **Code Deployment:** Your WASM binary or Lua script is uploaded to Arweave, getting a permanent transaction ID. +* **Process Spawning:** You create an AO process, associating it with your code's transaction ID and specifying the appropriate compute device ([`~wasm64@1.0`](../devices/wasm64-at-1-0.md) or [`~lua@5.3a`](../devices/lua-at-5-3a.md)). +* **Execution via Messages:** Sending a message to the process ID triggers the HyperBEAM node (that picks up the message) to: + 1. Load the process state. + 2. Fetch the associated WASM/Lua code from Arweave. + 3. Execute the code using the relevant device ([`dev_wasm`](../resources/source-code/dev_wasm.md) or [`dev_lua`](../resources/source-code/dev_lua.md)), passing the message data and current state. + 4. Update the process state based on the execution results. + + +## TEE Attestations (via [`~snp@1.0`](../resources/source-code/dev_snp.md)) + +If a HyperBEAM node performing these computations runs within a supported Trusted Execution Environment (like AMD SEV-SNP), it can provide cryptographic proof of execution. + +* **How it works:** The [`~snp@1.0`](../resources/source-code/dev_snp.md) device interacts with the TEE hardware. +* **Signed Responses:** When a TEE-enabled node processes your message (e.g., executes your WASM function), the HTTP response containing the result can be cryptographically signed by a key that *provably* only exists inside the TEE. +* **Verification:** Clients receiving this response can verify the signature against the TEE platform's attestation mechanism (e.g., AMD's KDS) to gain high confidence that the computation was performed correctly and confidentially within the secure environment, untampered by the node operator. + +**Obtaining Attested Responses:** + +This usually involves interacting with nodes specifically advertised as TEE-enabled. The exact mechanism for requesting and verifying attestations depends on the specific TEE technology and node configuration. + +* The HTTP response headers might contain specific signature or attestation data (e.g., using HTTP Message Signatures RFC-9421 via [`dev_codec_httpsig`](../resources/source-code/dev_codec_httpsig.md)). +* You might query the [`~snp@1.0`](../resources/source-code/dev_snp.md) device directly on the node to get its attestation report. + +Refer to documentation on [TEE Nodes](./run/tee-nodes.md) and the [`~snp@1.0`](../resources/source-code/dev_snp.md) device for details. + +By leveraging WASM, Lua, and optional TEE attestations, AO provides a powerful platform for building complex, verifiable, and truly decentralized serverless applications. + +--- END OF FILE: docs/build/serverless-decentralized-compute.md --- + +--- START OF FILE: docs/devices/json-at-1-0.md --- +# Device: ~json@1.0 + +## Overview + +The [`~json@1.0`](../resources/source-code/dev_json_iface.md) device provides a mechanism to interact with JSON (JavaScript Object Notation) data structures using HyperPATHs. It allows treating a JSON document or string as a stateful entity against which HyperPATH queries can be executed. + +This device is useful for: + +* Serializing and deserializing JSON data. +* Querying and modifying JSON objects. +* Integrating with other devices and operations via HyperPATH chaining. + +## Core Functions (Keys) + +### Serialization + +* **`GET /~json@1.0/serialize` (Direct Serialize Action)** + * **Action:** Serializes the input message or data into a JSON string. + * **Example:** `GET /~json@1.0/serialize` - serializes the current message as JSON. + * **HyperPATH:** The path segment `/serialize` directly follows the device identifier. + +* **`GET //~json@1.0/serialize` (Chained Serialize Action)** + * **Action:** Takes arbitrary data output from `` (another device or operation) and returns its serialized JSON string representation. + * **Example:** `GET /~meta@1.0/info/~json@1.0/serialize` - fetches node info from the meta device and then pipes it to the JSON device to serialize the result as JSON. + * **HyperPATH:** This segment (`/~json@1.0/serialize`) is appended to a previous HyperPATH segment. + +## HyperPATH Chaining Example + +The JSON device is particularly useful in HyperPATH chains to convert output from other devices into JSON format: + +``` +GET /~meta@1.0/info/~json@1.0/serialize +``` + +This retrieves the node configuration from the meta device and serializes it to JSON. + +## See Also + +- [Message Device](../resources/source-code/dev_message.md) - Works well with JSON serialization +- [Meta Device](../resources/source-code/dev_meta.md) - Can provide configuration data to serialize + +[json module](../resources/source-code/dev_codec_json.md) +--- END OF FILE: docs/devices/json-at-1-0.md --- + +--- START OF FILE: docs/devices/lua-at-5-3a.md --- +# Device: ~lua@5.3a + +## Overview + +The [`~lua@5.3a`](../resources/source-code/dev_lua.md) device enables the execution of Lua scripts within the HyperBEAM environment. It provides an isolated sandbox where Lua code can process incoming messages, interact with other devices, and manage state. + +## Core Concept: Lua Script Execution + +This device allows processes to perform computations defined in Lua scripts. Similar to the [`~wasm64@1.0`](../resources/source-code/dev_wasm.md) device, it manages the lifecycle of a Lua execution state associated with the process. + +## Key Functions (Keys) + +These keys are typically used within an execution stack (managed by [`dev_stack`](../resources/source-code/dev_stack.md)) for an AO process. + +* **`init`** + * **Action:** Initializes the Lua environment for the process. It finds and loads the Lua script(s) associated with the process, creates a `luerl` state, applies sandboxing rules if specified, installs the [`dev_lua_lib`](../resources/source-code/dev_lua_lib.md) (providing AO-specific functions like `ao.send`), and stores the initialized state in the process's private area (`priv/state`). + * **Inputs (Expected in Process Definition or `init` Message):** + * `script`: Can be: + * An Arweave Transaction ID of the Lua script file. + * A list of script IDs or script message maps. + * A message map containing the Lua script in its `body` tag (Content-Type `application/lua` or `text/x-lua`). + * A map where keys are module names and values are script IDs/messages. + * `sandbox`: (Optional) Controls Lua sandboxing. Can be `true` (uses default sandbox list), `false` (no sandbox), or a map/list specifying functions to disable and their return values. + * **Outputs (Stored in `priv/`):** + * `state`: The initialized `luerl` state handle. +* **`` (Default Handler - `compute`)** + * **Action:** Executes a specific function within the loaded Lua script(s). This is the default handler; if a key matching a Lua function name is called on the device, this logic runs. + * **Inputs (Expected in Process State or Incoming Message):** + * `priv/state`: The Lua state obtained during `init`. + * The **key** being accessed (used as the default function name). + * `function` or `body/function`: (Optional) Overrides the function name derived from the key. + * `parameters` or `body/parameters`: (Optional) Arguments to pass to the Lua function. Defaults to a list containing the process message, the request message, and an empty options map. + * **Response:** The results returned by the Lua function call, typically encoded. The device also updates the `priv/state` with the Lua state after execution. +* **`snapshot`** + * **Action:** Captures the current state of the running Lua environment. `luerl` state is serializable. + * **Inputs:** `priv/state`. + * **Outputs:** A message containing the serialized Lua state, typically tagged with `[Prefix]/State`. +* **`normalize` (Internal Helper)** + * **Action:** Ensures a consistent state representation by loading a Lua state from a snapshot (`[Prefix]/State`) if a live state (`priv/state`) isn't already present. +* **`functions`** + * **Action:** Returns a list of all globally defined functions within the current Lua state. + * **Inputs:** `priv/state`. + * **Response:** A list of function names. + +## Sandboxing + +The `sandbox` option in the process definition restricts potentially harmful Lua functions (like file I/O, OS commands, loading arbitrary code). By default (`sandbox = true`), common dangerous functions are disabled. You can customize the sandbox rules. + +## AO Library (`dev_lua_lib`) + +The `init` function automatically installs a helper library ([`dev_lua_lib`](../resources/source-code/dev_lua_lib.md)) into the Lua state. This library typically provides functions for interacting with the AO environment from within the Lua script, such as: + +* `ao.send({ Target = ..., ... })`: To send messages from the process. +* Access to message tags and data. + +## Usage within `dev_stack` + +Like [`~wasm64@1.0`](../resources/source-code/dev_wasm.md), the `~lua@5.3a` device is typically used within an execution stack. + +```text +# Example Process Definition Snippet +Execution-Device: stack@1.0 +Execution-Stack: scheduler@1.0, lua@5.3a +Script: +Sandbox: true +``` + +This device offers a lightweight, integrated scripting capability for AO processes, suitable for a wide range of tasks from simple logic to more complex state management and interactions. + +[lua module](../resources/source-code/dev_lua.md) + +--- END OF FILE: docs/devices/lua-at-5-3a.md --- + +--- START OF FILE: docs/devices/message-at-1-0.md --- +# Device: ~message@1.0 + +## Overview + +The [`~message@1.0`](../resources/source-code/dev_message.md) device is a fundamental built-in device in HyperBEAM. It serves as the identity device for standard AO-Core messages, which are represented as Erlang maps internally. Its primary function is to allow manipulation and inspection of these message maps directly via HyperPATH requests, without needing a persistent process state. + +This device is particularly useful for: + +* Creating and modifying transient messages on the fly using query parameters. +* Retrieving specific values from a message map. +* Inspecting the keys of a message. +* Handling message commitments and verification (though often delegated to specialized commitment devices like [`httpsig@1.0`](../resources/source-code/dev_codec_httpsig.md)). + +## Core Functionality + +The `message@1.0` device treats the message itself as the state it operates on. Key operations are accessed via path segments in the HyperPATH. + +### Key Access (`/key`) + +To retrieve the value associated with a specific key in the message map, simply append the key name to the path. Key lookup is case-insensitive. + +**Example:** + +``` +GET /~message@1.0&hello=world&Key=Value/key +``` + +**Response:** + +``` +"Value" +``` + +### Reserved Keys + +The `message@1.0` device reserves several keys for specific operations: + +* **`get`**: (Default operation if path segment matches a key in the map) Retrieves the value of a specified key. Behaves identically to accessing `/key` directly. +* **`set`**: Modifies the message by adding or updating key-value pairs. Requires additional parameters (usually in the request body or subsequent path segments/query params, depending on implementation specifics). + * Supports deep merging of maps. + * Setting a key to `unset` removes it. + * Overwriting keys that are part of existing commitments will typically remove those commitments unless the new value matches the old one. +* **`set_path`**: A special case for setting the `path` key itself, which cannot be done via the standard `set` operation. +* **`remove`**: Removes one or more specified keys from the message. Requires an `item` or `items` parameter. +* **`keys`**: Returns a list of all public (non-private) keys present in the message map. +* **`id`**: Calculates and returns the ID (hash) of the message. Considers active commitments based on specified `committers`. May delegate ID calculation to a device specified by the message\'s `id-device` key or the default ([`httpsig@1.0`](../resources/source-code/dev_codec_httpsig.md)). +* **`commit`**: Creates a commitment (e.g., a signature) for the message. Requires parameters like `commitment-device` and potentially committer information. Delegates the actual commitment generation to the specified device (default [`httpsig@1.0`](../resources/source-code/dev_codec_httpsig.md)). +* **`committers`**: Returns a list of committers associated with the commitments in the message. Can be filtered by request parameters. +* **`commitments`**: Used internally and in requests to filter or specify which commitments to operate on (e.g., for `id` or `verify`). +* **`verify`**: Verifies the commitments attached to the message. Can be filtered by `committers` or specific `commitment` IDs in the request. Delegates verification to the device specified in each commitment (`commitment-device`). + +### Private Keys + +Keys prefixed with `priv` (e.g., `priv_key`, `private.data`) are considered private and cannot be accessed or listed via standard `get` or `keys` operations. + +## HyperPATH Example + +This example demonstrates creating a transient message and retrieving a value: + +``` +GET /~message@1.0&hello=world&k=v/k +``` + +**Breakdown:** + +1. `~message@1.0`: Sets the root device. +2. `&hello=world&k=v`: Query parameters create the initial message: `#{ <<"hello">> => <<"world">>, <<"k">> => <<"v">> }`. +3. `/k`: The path segment requests the value for the key `k`. + +**Response:** + +``` +"v" +``` +--- END OF FILE: docs/devices/message-at-1-0.md --- + +--- START OF FILE: docs/devices/meta-at-1-0.md --- +# Device: ~meta@1.0 + +## Overview + +The [`~meta@1.0`](../resources/source-code/dev_meta.md) device provides access to metadata and configuration information about the local HyperBEAM node and the broader AO network. + +This device is essential for: + +## Core Functions (Keys) + +### `info` + +Retrieves or modifies the node's configuration message (often referred to as `NodeMsg` internally). + +* **`GET /~meta@1.0/info`** + * **Action:** Returns the current node configuration message. + * **Response:** A message map containing the node's settings. Sensitive keys (like private wallets) are filtered out. Dynamically generated keys like the node's public `address` are added if a wallet is configured. +* **`POST /~meta@1.0/info`** + * **Action:** Updates the node's configuration message. Requires the request to be signed by the node's configured `operator` key/address. + * **Request Body:** A message map containing the configuration keys and values to update. + * **Response:** Confirmation message indicating success or failure. + * **Note:** Once a node's configuration is marked as `initialized = permanent`, it cannot be changed via this method. + +## Key Configuration Parameters Managed by `~meta` + +While the `info` key is the primary interaction point, the `NodeMsg` managed by `~meta` holds crucial configuration parameters affecting the entire node's behavior, including (but not limited to): + +* `port`: HTTP server port. +* `priv_wallet` / `key_location`: Path to the node's Arweave key file. +* `operator`: The address designated as the node operator (defaults to the address derived from `priv_wallet`). +* `initialized`: Status indicating if the node setup is temporary or permanent. +* `preprocessor` / `postprocessor`: Optional messages defining pre/post-processing logic for requests. +* `routes`: Routing table used by [`dev_router`](../resources/source-code/dev_router.md). +* `store`: Configuration for data storage. +* `trace`: Debug tracing options. +* `p4_*`: Payment configuration. +* `faff_*`: Access control lists. + +*(Refer to `hb_opts.erl` for a comprehensive list of options.)* + +## Utility Functions (Internal/Module Level) + +The [`dev_meta.erl`](../resources/source-code/dev_meta.md) module also contains helper functions used internally or callable from other Erlang modules: + +* `is_operator(, ) -> boolean()`: Checks if the signer of `RequestMsg` matches the configured `operator` in `NodeMsg`. + +## Pre/Post-Processing Hooks + +The `~meta` device applies the node's configured `preprocessor` message before resolving the main request and the `postprocessor` message after obtaining the result, allowing for global interception and modification of requests/responses. + +## Initialization + +Before a node can process general requests, it usually needs to be initialized. Attempts to access devices other than `~meta@1.0/info` before initialization typically result in an error. Initialization often involves setting essential parameters like the operator key via a `POST` to `info`. + +[meta module](../resources/source-code/dev_meta.md) +--- END OF FILE: docs/devices/meta-at-1-0.md --- + +--- START OF FILE: docs/devices/overview.md --- +# Devices + +Devices are the core functional units within HyperBEAM and AO-Core. They define how messages are processed and what actions can be performed. + +Each device listed here represents a specific capability available to AO processes and nodes. Understanding these devices is key to building complex applications and configuring your HyperBEAM node effectively. + +## Available Devices + +Below is a list of documented built-in devices. Each page details the device's purpose, available functions (keys), and usage examples where applicable. + +* **[`~message@1.0`](./message-at-1-0.md):** Base message handling and manipulation. +* **[`~meta@1.0`](./meta-at-1-0.md):** Node configuration and metadata. +* **[`~process@1.0`](./process-at-1-0.md):** Persistent, shared process execution environment. +* **[`~scheduler@1.0`](./scheduler-at-1-0.md):** Message scheduling and execution ordering for processes. +* **[`~wasm64@1.0`](./wasm64-at-1-0.md):** WebAssembly (WASM) execution engine. +* **[`~lua@5.3a`](./lua-at-5-3a.md):** Lua script execution engine. +* **[`~relay@1.0`](./relay-at-1-0.md):** Relaying messages to other nodes or HTTP endpoints. +* **[`~json@1.0`](./json-at-1-0.md):** Provides access to JSON data structures using HyperPATHs. + +*(More devices will be documented here as specifications are finalized and reviewed.)* + +## Device Naming and Versioning + +Devices are typically referenced using a name and version, like `~@` (e.g., `~process@1.0`). The tilde (`~`) often indicates a primary, user-facing device, while internal or utility devices might use a `dev_` prefix in the source code (e.g., `dev_router`). + +Versioning indicates the specific interface and behavior of the device. Changes to a device that break backward compatibility usually result in a version increment. + +--- END OF FILE: docs/devices/overview.md --- + +--- START OF FILE: docs/devices/process-at-1-0.md --- +# Device: ~process@1.0 + +## Overview + +The [`~process@1.0`](../resources/source-code/dev_process.md) device represents a persistent, shared execution environment within HyperBEAM, analogous to a process or actor in other systems. It allows for stateful computation and interaction over time. + +## Core Concept: Orchestration + +A message tagged with `Device: process@1.0` (the "Process Definition Message") doesn't typically perform computation itself. Instead, it defines *which other devices* should be used for key aspects of its lifecycle: + +* **Scheduler Device:** Determines the order of incoming messages (assignments) to be processed. (Defaults to [`~scheduler@1.0`](../resources/source-code/dev_scheduler.md)). +* **Execution Device:** Executes the actual computation based on the current state and the scheduled message. Often configured as [`dev_stack`](../resources/source-code/dev_stack.md) to allow multiple computational steps (e.g., running WASM, applying cron jobs, handling proofs). +* **Push Device:** Handles the injection of new messages into the process\'s schedule. (Defaults to [`~push@1.0`](../resources/source-code/dev_push.md)). + +The `~process@1.0` device acts as a router, intercepting requests and delegating them to the appropriate configured device (scheduler, executor, etc.) by temporarily swapping the device tag on the message before resolving. + +## Key Functions (Keys) + +These keys are accessed via HyperPATHs relative to the Process Definition Message ID (``). + +* **`GET /~process@1.0/schedule`** + * **Action:** Delegates to the configured Scheduler Device (via the process's `schedule/3` function) to retrieve the current schedule or state. + * **Response:** Depends on the Scheduler Device implementation (e.g., list of message IDs). +* **`POST /~process@1.0/schedule`** + * **Action:** Delegates to the configured Push Device (via the process's `push/3` function) to add a new message to the process's schedule. + * **Request Body:** The message to be added. + * **Response:** Confirmation or result from the Push Device. +* **`GET /~process@1.0/compute/`** + * **Action:** Computes the process state up to a specific point identified by `` (either a slot number or a message ID within the schedule). It retrieves assignments from the Scheduler Device and applies them sequentially using the configured Execution Device. + * **Response:** The process state message after executing up to the target slot/message. + * **Caching:** Results are cached aggressively (see [`dev_process_cache`](../resources/source-code/dev_process_cache.md)) to avoid recomputation. +* **`GET /~process@1.0/now`** + * **Action:** Computes and returns the `Results` key from the *latest* known state of the process. This typically involves computing all pending assignments. + * **Response:** The value of the `Results` key from the final state. +* **`GET /~process@1.0/slot`** + * **Action:** Delegates to the configured Scheduler Device to query information about a specific slot or the current slot number. + * **Response:** Depends on the Scheduler Device implementation. +* **`GET /~process@1.0/snapshot`** + * **Action:** Delegates to the configured Execution Device to generate a snapshot of the current process state. This often involves running the execution stack in a specific "map" mode to gather state from different components. + * **Response:** A message representing the process snapshot, often marked for caching. + +## Process Definition Example + +A typical process definition message might look like this (represented conceptually): + +```text +Device: process@1.0 +Scheduler-Device: [`scheduler@1.0`](../resources/source-code/dev_scheduler.md) +Execution-Device: [`stack@1.0`](../resources/source-code/dev_stack.md) +Execution-Stack: "[`scheduler@1.0`](../resources/source-code/dev_scheduler.md)", "[`cron@1.0`](../resources/source-code/dev_cron.md)", "[`wasm64@1.0`](../resources/source-code/dev_wasm.md)", "[`PoDA@1.0`](../resources/source-code/dev_poda.md)" +Cron-Frequency: 10-Minutes +WASM-Image: +PoDA: + Device: [`PoDA/1.0`](../resources/source-code/dev_poda.md) + Authority: + Authority: + Quorum: 2 +``` + +This defines a process that uses: +* The standard scheduler. +* A stack executor that runs scheduling logic, cron jobs, a WASM module, and a Proof-of-Data-Availability check. + +## State Management & Caching + +`~process@1.0` relies heavily on caching ([`dev_process_cache`](../resources/source-code/dev_process_cache.md)) to optimize performance. Full state snapshots and intermediate results are cached periodically (configurable via `Cache-Frequency` and `Cache-Keys` options) to avoid recomputing the entire history for every request. + +## Initialization (`init`) + +Processes often require an initialization step before they can process messages. This is typically triggered by calling the `init` key on the configured Execution Device via the process path (`/~process@1.0/init`). This allows components within the execution stack (like WASM modules) to set up their initial state. + +[process module](../resources/source-code/dev_process.md) + +--- END OF FILE: docs/devices/process-at-1-0.md --- + +--- START OF FILE: docs/devices/relay-at-1-0.md --- +# Device: ~relay@1.0 + +## Overview + +The [`~relay@1.0`](../resources/source-code/dev_relay.md) device enables HyperBEAM nodes to send messages to external HTTP endpoints or other AO nodes. + +## Core Concept: Message Forwarding + +This device acts as an HTTP client within the AO ecosystem. It allows a node or process to make outbound HTTP requests. + +## Key Functions (Keys) + +* **`call`** + * **Action:** Sends an HTTP request to a specified target and waits synchronously for the response. + * **Inputs (from Request Message or Base Message M1):** + * `target`: (Optional) A message map defining the request to be sent. Defaults to the original incoming request (`Msg2` or `M1`). + * `relay-path` or `path`: The URL/path to send the request to. + * `relay-method` or `method`: The HTTP method (GET, POST, etc.). + * `relay-body` or `body`: The request body. + * `requires-sign`: (Optional, boolean) If true, the request message (`target`) will be signed using the node's key before sending. Defaults to `false`. + * `http-client`: (Optional) Specify a custom HTTP client module to use (defaults to node's configured `relay_http_client`). + * **Response:** `{ok, }` where `` is the full message received from the remote peer, or `{error, Reason}`. + * **Example HyperPATH:** + ``` + GET /~relay@1.0/call?method=GET&path=https://example.com + ``` +* **`cast`** + * **Action:** Sends an HTTP request asynchronously. The device returns immediately after spawning a process to send the request; it does not wait for or return the response from the remote peer. + * **Inputs:** Same as `call`. + * **Response:** `{ok, <<"OK">>}`. +* **`preprocess`** + * **Action:** This function is designed to be used as a node's global `preprocessor` (configured via [`~meta@1.0`](../resources/source-code/dev_meta.md)). When configured, it intercepts *all* incoming requests to the node and automatically rewrites them to be relayed via the `call` key. This effectively turns the node into a pure forwarding proxy, using its routing table ([`dev_router`](../resources/source-code/dev_router.md)) to determine the destination. + * **Response:** A message structure that invokes `/~relay@1.0/call` with the original request as the target body. + +## Use Cases + +* **Inter-Node Communication:** Sending messages between HyperBEAM nodes. +* **External API Calls:** Allowing AO processes to interact with traditional web APIs. +* **Routing Nodes:** Nodes configured with the `preprocess` key act as dedicated routers/proxies. +* **Client-Side Relaying:** A local HyperBEAM instance can use `~relay@1.0` to forward requests to public compute nodes. + +## Interaction with Routing + +When `call` or `cast` is invoked, the actual HTTP request dispatch is handled by `hb_http:request/2`. This function often utilizes the node's routing configuration ([`dev_router`](../resources/source-code/dev_router.md)) to determine the specific peer/URL to send the request to, especially if the target path is an AO process ID or another internal identifier rather than a full external URL. + +[relay module](../resources/source-code/dev_relay.md) + +--- END OF FILE: docs/devices/relay-at-1-0.md --- + +--- START OF FILE: docs/devices/scheduler-at-1-0.md --- +# Device: ~scheduler@1.0 + +## Overview + +The [`~scheduler@1.0`](../resources/source-code/dev_scheduler.md) device manages the queueing and ordering of messages targeted at a specific process ([`~process@1.0`](../resources/source-code/dev_process.md)). It ensures that messages are processed according to defined scheduling rules. + +## Core Concept: Message Ordering + +When messages are sent to an AO process (typically via the [`~push@1.0`](../resources/source-code/dev_push.md) device or a `POST` to the process's `/schedule` endpoint), they are added to a queue managed by the Scheduler Device associated with that process. The scheduler ensures that messages are processed one after another in a deterministic order, typically based on arrival time and potentially other factors like message nonces or timestamps (depending on the specific scheduler implementation details). + +The [`~process@1.0`](../resources/source-code/dev_process.md) device interacts with its configured Scheduler Device (which defaults to `~scheduler@1.0`) primarily through the `next` key to retrieve the next message to be executed. + +## Slot System + +Slots are a fundamental concept in the `~scheduler@1.0` device, providing a structured mechanism for organizing and sequencing computation. + +* **Sequential Ordering:** Slots act as numbered containers (starting at 0) that hold specific messages or tasks to be processed in a deterministic order. +* **State Tracking:** The `at-slot` key in a process's state (or a similar internal field like `current-slot` within the scheduler itself) tracks execution progress, indicating which messages have been processed and which are pending. The `slot` function can be used to query this. +* **Assignment Storage:** Each slot contains an "assignment" - the cryptographically verified message waiting to be executed. These assignments are retrieved using the `schedule` function or internally via `next`. +* **Schedule Organization:** The collection of all slots for a process forms its "schedule". +* **Application Scenarios:** + * **Scheduling Messages:** When a message is posted to a process (e.g., via `register`), it's assigned to the next available slot. + * **Status Monitoring:** Clients can query a process's current slot (via the `slot` function) to check progress. + * **Task Retrieval:** Processes find their next task by requesting the next assignment via the `next` function, which implicitly uses the next slot number based on the current state. + * **Distributed Consistency:** Slots ensure deterministic execution order across nodes, crucial for maintaining consistency in AO. + +This slotting mechanism is central to AO processes built on HyperBEAM, allowing for deterministic, verifiable computation. + +## Key Functions (Keys) + +These keys are typically accessed via the [`~process@1.0`](../resources/source-code/dev_process.md) device, which delegates the calls to its configured scheduler. + +* **`schedule` (Handler for `GET /~process@1.0/schedule`)** + * **Action:** Retrieves the list of pending assignments (messages) for the process. May support cursor-based traversal for long schedules. + * **Response:** A message map containing the assignments, often keyed by slot number or message ID. +* **`register` (Handler for `POST /~process@1.0/schedule`)** + * **Action:** Adds/registers a new message to the process's schedule. If this is the first message for a process, it might initialize the scheduler state. + * **Request Body:** The message to schedule. + * **Response:** Confirmation, potentially including the assigned slot or message ID. +* **`slot` (Handler for `GET /~process@1.0/slot`)** + * **Action:** Queries the current or a specific slot number within the process's schedule. + * **Response:** Information about the requested slot, such as the current highest slot number. +* **`status` (Handler for `GET /~process@1.0/status`)** + * **Action:** Retrieves status information about the scheduler for the process. + * **Response:** A status message. +* **`next` (Internal Key used by [`~process@1.0`](../resources/source-code/dev_process.md))** + * **Action:** Retrieves the next assignment message from the schedule based on the process's current `at-slot` state. + * **State Management:** Requires the current process state (`Msg1`) containing the `at-slot` key. + * **Response:** `{ok, #{ "body" => , "state" => }}` or `{error, Reason}` if no next assignment is found. + * **Caching & Lookahead:** The implementation uses internal caching (`dev_scheduler_cache`, `priv/assignments`) and potentially background lookahead workers to optimize fetching subsequent assignments. +* **`init` (Internal Key)** + * **Action:** Initializes the scheduler state for a process, often called when the process itself is initialized. +* **`checkpoint` (Internal Key)** + * **Action:** Triggers the scheduler to potentially persist its current state or perform other checkpointing operations. + +## Interaction with Other Components + +* **[`~process@1.0`](../resources/source-code/dev_process.md):** The primary user of the scheduler, calling `next` to drive process execution. +* **[`~push@1.0`](../resources/source-code/dev_push.md):** Often used to add messages to the schedule via `POST /schedule`. +* **`dev_scheduler_cache`:** Internal module used for caching assignments locally on the node to reduce latency. +* **Scheduling Unit (SU):** Schedulers may interact with external entities (like Arweave gateways or dedicated SU nodes) to fetch or commit schedules, although `~scheduler@1.0` aims for a simpler, often node-local or SU-client model. + +`~scheduler@1.0` provides the fundamental mechanism for ordered, sequential execution within the potentially asynchronous and parallel environment of AO. + +[scheduler module](../resources/source-code/dev_scheduler.md) + +--- END OF FILE: docs/devices/scheduler-at-1-0.md --- + +--- START OF FILE: docs/devices/wasm64-at-1-0.md --- +# Device: ~wasm64@1.0 + +## Overview + +The [`~wasm64@1.0`](../resources/source-code/dev_wasm.md) device enables the execution of 64-bit WebAssembly (WASM) code within the HyperBEAM environment. It provides a sandboxed environment for running compiled code from various languages (like Rust, C++, Go) that target WASM. + +## Core Concept: WASM Execution + +This device allows AO processes to perform complex computations defined in WASM modules, which can be written in languages like Rust, C++, C, Go, etc., and compiled to WASM. + +The device manages the lifecycle of a WASM instance associated with the process state. + +## Key Functions (Keys) + +These keys are typically used within an execution stack (managed by [`dev_stack`](../resources/source-code/dev_stack.md)) for an AO process. + +* **`init`** + * **Action:** Initializes the WASM environment for the process. It locates the WASM image (binary), starts a WAMR instance, and stores the instance handle and helper functions (for reading/writing WASM memory) in the process's private state (`priv/...`). + * **Inputs (Expected in Process Definition or `init` Message):** + * `[Prefix]/image`: The Arweave Transaction ID of the WASM binary, or the WASM binary itself, or a message containing the WASM binary in its body. + * `[Prefix]/Mode`: (Optional) Specifies execution mode (`WASM` (default) or `AOT` if allowed by node config). + * **Outputs (Stored in `priv/`):** + * `[Prefix]/instance`: The handle to the running WAMR instance. + * `[Prefix]/write`: A function to write data into the WASM instance's memory. + * `[Prefix]/read`: A function to read data from the WASM instance's memory. + * `[Prefix]/import-resolver`: A function used to handle calls *from* the WASM module back *to* the AO environment (imports). +* **`compute`** + * **Action:** Executes a function within the initialized WASM instance. It retrieves the target function name and parameters from the incoming message or process definition and calls the WASM instance via `hb_beamr`. + * **Inputs (Expected in Process State or Incoming Message):** + * `priv/[Prefix]/instance`: The handle obtained during `init`. + * `function` or `body/function`: The name of the WASM function to call. + * `parameters` or `body/parameters`: A list of parameters to pass to the WASM function. + * **Outputs (Stored in `results/`):** + * `results/[Prefix]/type`: The result type returned by the WASM function. + * `results/[Prefix]/output`: The actual result value returned by the WASM function. +* **`import`** + * **Action:** Handles calls originating *from* the WASM module (imports). The default implementation (`default_import_resolver`) resolves these calls by treating them as sub-calls within the AO environment, allowing WASM code to invoke other AO device functions or access process state via the `hb_ao:resolve` mechanism. + * **Inputs (Provided by `hb_beamr`):** Module name, function name, arguments, signature. + * **Response:** Returns the result of the resolved AO call back to the WASM instance. +* **`snapshot`** + * **Action:** Captures the current memory state of the running WASM instance. This is used for checkpointing and restoring process state. + * **Inputs:** `priv/[Prefix]/instance`. + * **Outputs:** A message containing the raw binary snapshot of the WASM memory state, typically tagged with `[Prefix]/State`. +* **`normalize` (Internal Helper)** + * **Action:** Ensures a consistent state representation for computation, primarily by loading a WASM instance from a snapshot (`[Prefix]/State`) if a live instance (`priv/[Prefix]/instance`) isn't already present. This allows resuming execution from a cached state. +* **`terminate`** + * **Action:** Stops and cleans up the running WASM instance associated with the process. + * **Inputs:** `priv/[Prefix]/instance`. + +## Usage within `dev_stack` + +The `~wasm64@1.0` device is almost always used as part of an execution stack configured in the Process Definition Message and managed by [`dev_stack`](../resources/source-code/dev_stack.md). [`dev_stack`](../resources/source-code/dev_stack.md) ensures that `init` is called on the first pass, `compute` on subsequent passes, and potentially `snapshot` or `terminate` as needed. + +```text +# Example Process Definition Snippet +Execution-Device: [`stack@1.0`](../resources/source-code/dev_stack.md) +Execution-Stack: "[`scheduler@1.0`](../resources/source-code/dev_scheduler.md)", "wasm64@1.0" +WASM-Image: +``` + +This setup allows AO processes to leverage the computational power and language flexibility offered by WebAssembly in a decentralized, verifiable manner. + +[wasm module](../resources/source-code/dev_wasm.md) + +--- END OF FILE: docs/devices/wasm64-at-1-0.md --- + +--- START OF FILE: docs/introduction/ao-devices.md --- +# AO Devices + +In AO-Core and its implementation HyperBEAM, **Devices** are modular components responsible for processing and interpreting [Messages](./what-is-ao-core.md#core-concepts). They define the specific logic for how computations are performed, data is handled, or interactions occur within the AO ecosystem. + +Think of Devices as specialized engines or services that can be plugged into the AO framework. This modularity is key to AO's flexibility and extensibility. + +## Purpose of Devices + +* **Define Computation:** Devices dictate *how* a message's instructions are executed. One device might run WASM code, another might manage process state, and yet another might simply relay data. +* **Enable Specialization:** Nodes running HyperBEAM can choose which Devices to support, allowing them to specialize in certain tasks (e.g., high-compute tasks, storage-focused tasks, secure TEE operations). +* **Promote Modularity:** New functionalities can be added to AO by creating new Devices, without altering the core protocol. +* **Distribute Workload:** Different Devices can handle different parts of a complex task, enabling parallel processing and efficient resource utilization across the network. + +## Familiar Examples + +HyperBEAM includes many preloaded devices that provide core functionality. Some key examples include: + +* **[`~meta@1.0`](../devices/meta-at-1-0.md):** Configures the node itself (hardware specs, supported devices, payment info). +* **[`~process@1.0`](../devices/process-at-1-0.md):** Manages persistent, shared computational states (like traditional smart contracts, but more flexible). +* **[`~scheduler@1.0`](../devices/scheduler-at-1-0.md):** Handles the ordering and execution of messages within a process. +* **[`~wasm64@1.0`](../devices/wasm64-at-1-0.md):** Executes WebAssembly (WASM) code, allowing for complex computations written in languages like Rust, C++, etc. +* **[`~lua@5.3a`](../devices/lua-at-5-3a.md):** Executes Lua scripts. +* **[`~relay@1.0`](../devices/relay-at-1-0.md):** Forwards messages between AO nodes or to external HTTP endpoints. +* **[`~json@1.0`](../devices/json-at-1-0.md):** Provides access to JSON data structures using HyperPATHs. +* **[`~message@1.0`](../devices/message-at-1-0.md):** Manages message state and processing. +* **[`~patch@1.0`](../guides/exposing-process-state.md):** Applies state updates directly to a process, often used for migrating or managing process data. + +## Beyond the Basics + +Devices aren't limited to just computation or state management. They can represent more abstract concepts: + +* **Security Devices ([`~snp@1.0`](../resources/source-code/dev_snp.md), [`dev_codec_httpsig`](../resources/source-code/dev_codec_httpsig.md)):** Handle tasks related to Trusted Execution Environments (TEEs) or message signing, adding layers of security and verification. +* **Payment/Access Control Devices ([`~p4@1.0`](../resources/source-code/dev_p4.md), [`~faff@1.0`](../resources/source-code/dev_faff.md)):** Manage metering, billing, or access control for node services. +* **Workflow/Utility Devices ([`dev_cron`](../resources/source-code/dev_cron.md), [`dev_stack`](../resources/source-code/dev_stack.md), [`dev_monitor`](../resources/source-code/dev_monitor.md)):** Coordinate complex execution flows, schedule tasks, or monitor process activity. + +## Using Devices + +Devices are typically invoked via [HyperPATHs](./pathing-in-ao-core.md). The path specifies which Device should interpret the subsequent parts of the path or the request body. + +``` +# Example: Execute the 'now' key on the process device for a specific process +/~process@1.0/now + +# Example: Relay a GET request via the relay device +/~relay@1.0/call?method=GET&path=https://example.com +``` + +The specific functions or 'keys' available for each Device are documented individually. See the [Devices section](../devices/index.md) for details on specific built-in devices. + +## The Potential of Devices + +The modular nature of AO Devices opens up vast possibilities for future expansion and innovation. The current set of preloaded and community devices is just the beginning. As the AO ecosystem evolves, we can anticipate the development of new devices catering to increasingly specialized needs: + +* **Specialized Hardware Integration:** Devices could be created to interface directly with specialized hardware accelerators like GPUs (for AI/ML tasks such as running large language models), TPUs, or FPGAs, allowing AO processes to leverage high-performance computing resources securely and verifiably. +* **Advanced Cryptography:** New devices could implement cutting-edge cryptographic techniques, such as zero-knowledge proofs (ZKPs) or fully homomorphic encryption (FHE), enabling enhanced privacy and complex computations on encrypted data. +* **Cross-Chain & Off-Chain Bridges:** Devices could act as secure bridges to other blockchain networks or traditional Web2 APIs, facilitating seamless interoperability and data exchange between AO and the wider digital world. +* **AI/ML Specific Devices:** Beyond raw GPU access, specialized devices could offer higher-level AI/ML functionalities, like optimized model inference engines or distributed training frameworks. +* **Domain-Specific Logic:** Communities or organizations could develop devices tailored to specific industries or use cases, such as decentralized finance (DeFi) primitives, scientific computing libraries, or decentralized identity management systems. + +The Device framework ensures that AO can adapt and grow, incorporating new technologies and computational paradigms without requiring fundamental changes to the core protocol. This extensibility is key to AO's long-term vision of becoming a truly global, decentralized computer. + +--- END OF FILE: docs/introduction/ao-devices.md --- + +--- START OF FILE: docs/introduction/pathing-in-ao-core.md --- +# Pathing in AO-Core + +## Overview + +Understanding how to construct and interpret paths in AO-Core is fundamental to working with HyperBEAM. This guide explains the structure and components of AO-Core paths, enabling you to effectively interact with processes and access their data. + +## HyperPATH Structure + +Let's examine a typical HyperBEAM endpoint piece-by-piece: + +``` +https://router-1.forward.computer/~process@1.0/now +``` + +### Node URL (`router-1.forward.computer`) + +The HTTP response from this node includes a signature from the host's key. By accessing the [`~snp@1.0`](../resources/source-code/dev_snp.md) device, you can verify that the node is running in a genuine Trusted Execution Environment (TEE), ensuring computation integrity. You can replace `router-1.forward.computer` with any HyperBEAM TEE node operated by any party while maintaining trustless guarantees. + +### Process Path (`/~process@1.0`) + +Every path in AO-Core represents a program. Think of the URL bar as a Unix-style command-line interface, providing access to AO's trustless and verifiable compute. Each path component (between `/` characters) represents a step in the computation. In this example, we instruct the AO-Core node to: + +1. Load a specific message from its caches (local, another node, or Arweave) +2. Interpret it with the [`~process@1.0`](../devices/process-at-1-0.md) device +3. The process device implements a shared computing environment with consistent state between users + +### State Access (`/now` or `/compute`) + +Devices in AO-Core expose keys accessible via path components. Each key executes a function on the device: + +- `now`: Calculates real-time process state +- `compute`: Serves the latest known state (faster than checking for new messages) + +Under the surface, these keys represent AO-Core messages. As we progress through the path, AO-Core applies each message to the existing state. You can access the full process state by visiting: +``` +/~process@1.0/now +``` + +### State Navigation + +You can browse through sub-messages and data fields by accessing them as keys. For example, if a process stores its interaction count in a field named `cache`, you can access it like this: +``` +/~process@1.0/compute/cache +``` +This shows the 'cache' of your process. Each response is: + +- A message with a signature attesting to its correctness +- A hashpath describing its generation +- Transferable to other AO-Core nodes for uninterrupted execution + +### Query Parameters and Type Casting + +Beyond path segments, HyperBEAM URLs can include query parameters that utilize a special type casting syntax. This allows specifying the desired data type for a parameter directly within the URL using the format `key+type=value`. + +- **Syntax**: A `+` symbol separates the parameter key from its intended type (e.g., `count+integer=42`, `items+list="apple",7`). +- **Mechanism**: The HyperBEAM node identifies the `+type` suffix (e.g., `+integer`, `+list`, `+map`, `+float`, `+atom`, `+resolve`). It then uses internal functions ([`hb_singleton:maybe_typed`](../resources/source-code/hb_singleton.md) and [`dev_codec_structured:decode_value`](../resources/source-code/dev_codec_structured.md)) to decode and cast the provided value string into the corresponding Erlang data type before incorporating it into the message. +- **Supported Types**: Common types include `integer`, `float`, `list`, `map`, `atom`, `binary` (often implicit), and `resolve` (for path resolution). List values often follow the [HTTP Structured Fields format (RFC 8941)](https://www.rfc-editor.org/rfc/rfc8941.html). + +This powerful feature enables the expression of complex data structures directly in URLs. + +## Examples + +The following examples illustrate using HyperPATH with various AO-Core processes and devices. While these cover a few specific use cases, HyperBEAM's extensible nature allows interaction with any device or process via HyperPATH. For a deeper understanding, we encourage exploring the [source code](https://github.com/permaweb/hyperbeam) and experimenting with different paths. + +### Example 1: Accessing Full Process State + +To get the complete, real-time state of a process identified by ``, use the `/now` path component with the [`~process@1.0`](../devices/process-at-1-0.md) device: + +``` +GET /~process@1.0/now +``` + +This instructs the AO-Core node to load the process and execute the `now` function on the [`~process@1.0`](../devices/process-at-1-0.md) device. + +### Example 2: Navigating to Specific Process Data + +If a process maintains its state in a map and you want to access a specific field, like `at-slot`, using the faster `/compute` endpoint: + +``` +GET /~process@1.0/compute/cache +``` + +This accesses the `compute` key on the [`~process@1.0`](../devices/process-at-1-0.md) device and then navigates to the `cache` key within the resulting state map. Using this path, you will see the latest 'cache' of your process (the number of interactions it has received). Every piece of relevant information about your process can be accessed similarly, effectively providing a native API. + +(Note: This represents direct navigation within the process state structure. For accessing data specifically published via the `~patch@1.0` device, see the documentation on [Exposing Process State](../build/exposing-process-state.md), which typically uses the `/cache/` path.) + +### Example 3: Basic `~message@1.0` Usage + +Here's a simple example of using [`~message@1.0`](../devices/message-at-1-0.md) to create a message and retrieve a value: + +``` +GET /~message@1.0&greeting="Hello"&count+integer=42/count +``` + +1. **Base:** `/` - The base URL of the HyperBEAM node. +2. **Root Device:** [`~message@1.0`](../devices/message-at-1-0.md) +3. **Query Params:** `greeting="Hello"` (binary) and `count+integer=42` (integer), forming the message `#{ <<"greeting">> => <<"Hello">>, <<"count">> => 42 }`. +4. **Path:** `/count` tells `~message@1.0` to retrieve the value associated with the key `count`. + +**Response:** The integer `42`. + +### Example 4: Using the `~message@1.0` Device with Type Casting + +The [`~message@1.0`](../devices/message-at-1-0.md) device can be used to construct and query transient messages, utilizing type casting in query parameters. + +Consider the following URL: + +``` +GET /~message@1.0&name="Alice"&age+integer=30&items+list="apple",1,"banana"&config+map=key1="val1";key2=true/[PATH] +``` + +HyperBEAM processes this as follows: + +1. **Base:** `/` - The base URL of the HyperBEAM node. +2. **Root Device:** [`~message@1.0`](../devices/message-at-1-0.md) +3. **Query Parameters (with type casting):** + * `name="Alice"` -> `#{ <<"name">> => <<"Alice">> }` (binary) + * `age+integer=30` -> `#{ <<"age">> => 30 }` (integer) + * `items+list="apple",1,"banana"` -> `#{ <<"items">> => [<<"apple">>, 1, <<"banana">>] }` (list) + * `config+map=key1="val1";key2=true` -> `#{ <<"config">> => #{<<"key1">> => <<"val1">>, <<"key2">> => true} }` (map) +4. **Initial Message Map:** A combination of the above key-value pairs. +5. **Path Evaluation:** + * If `[PATH]` is `/items/1`, the response is the integer `1`. + * If `[PATH]` is `/config/key1`, the response is the binary `<<"val1">>`. + +## Best Practices + +1. Always verify cryptographic signatures on responses +2. Use appropriate caching strategies for frequently accessed data +3. Implement proper error handling for network requests +4. Consider rate limits and performance implications +5. Keep sensitive data secure and use appropriate authentication methods +--- END OF FILE: docs/introduction/pathing-in-ao-core.md --- + +--- START OF FILE: docs/introduction/what-is-ao-core.md --- +# What is AO-Core? + +AO-Core is the foundational protocol underpinning the [AO Computer](https://ao.arweave.net). It defines a minimal, generalized model for decentralized computation built around standard web technologies like HTTP. Think of it as a way to interpret the Arweave permaweb not just as static storage, but as a dynamic, programmable, and infinitely scalable computing environment. + +## Core Concepts + +AO-Core revolves around three fundamental components: + +1. **Messages:** The smallest units of data and computation. Messages can be simple data blobs or maps of named functions. They are the primary means of communication and triggering execution within the system. Messages are cryptographically linked, forming a verifiable computation graph. +2. **Devices:** Modules responsible for interpreting and processing messages. Each device defines specific logic for how messages are handled (e.g., executing WASM, storing data, relaying information). This modular design allows nodes to specialize and the system to be highly extensible. +3. **Paths:** Structures that link messages over time, creating a verifiable history of computations. Paths allow users to navigate the computation graph and access specific states or results. They leverage `HashPaths`, cryptographic fingerprints representing the sequence of operations leading to a specific message state, ensuring traceability and integrity. + +## Key Principles + +* **Minimalism:** AO-Core provides the simplest possible representation of data and computation, avoiding prescriptive consensus mechanisms or specific VM requirements. +* **HTTP Native:** Designed for compatibility with HTTP protocols, making it accessible via standard web tools and infrastructure. +* **Scalability:** By allowing parallel message processing and modular device execution, AO-Core enables hyper-parallel computing, overcoming the limitations of traditional sequential blockchains. +* **Permissionlessness & Trustlessness:** While AO-Core itself is minimal, it provides the framework upon which higher-level protocols like AO can build systems that allow anyone to participate (`permissionlessness`) without needing to trust intermediaries (`trustlessness`). Users can choose their desired security and performance trade-offs. + +AO-Core transforms the permanent data storage of Arweave into a global, shared computation space, enabling the creation of complex, autonomous, and scalable decentralized applications. + + +--- END OF FILE: docs/introduction/what-is-ao-core.md --- + +--- START OF FILE: docs/introduction/what-is-hyperbeam.md --- +# What is HyperBEAM? + +HyperBEAM is the primary, production-ready implementation of the [AO-Core protocol](./what-is-ao-core.md), built on the robust Erlang/OTP framework. It serves as a decentralized operating system, powering the AO Computer—a scalable, trust-minimized, distributed supercomputer built on permanent storage. HyperBEAM provides the runtime environment and essential services to execute AO-Core computations across a network of distributed nodes. + +## Why HyperBEAM Matters + +HyperBEAM transforms the abstract concepts of AO-Core—such as [Messages](./what-is-ao-core.md#core-concepts), [Devices](./what-is-ao-core.md#core-concepts), and [Paths](./what-is-ao-core.md#core-concepts)—into a concrete, operational system. Here's why it's pivotal to the AO ecosystem: + +- **Modularity via Devices:** HyperBEAM introduces a uniquely modular architecture centered around [Devices](./ao-devices.md). These pluggable components define specific computational logic (like running WASM, managing state, or relaying data), allowing for unprecedented flexibility, specialization, and extensibility in a decentralized system. +- **Decentralized OS:** It equips nodes with the infrastructure to join the AO network, manage resources, execute computations, and communicate seamlessly. +- **Erlang/OTP Powerhouse:** Leveraging the BEAM virtual machine, HyperBEAM inherits Erlang's concurrency, fault tolerance, and scalability—perfect for distributed systems with lightweight processes and message passing. +- **Hardware Independence:** It abstracts underlying hardware, allowing diverse nodes to contribute resources without compatibility issues. +- **Node Coordination:** It governs how nodes join the network, offer services through specific Devices, and interact with one another. +- **Verifiable Computation:** Through hashpaths and the Converge Protocol, HyperBEAM ensures computation results are cryptographically verified and trustworthy. + +In essence, HyperBEAM is the engine that drives the AO Computer, enabling a vision of decentralized, verifiable computing at scale. + +## Core Components & Features + +- **Pluggable Devices:** The heart of HyperBEAM's extensibility. It includes essential built-in devices like [`~meta`](../devices/meta-at-1-0.md), [`~relay`](../devices/relay-at-1-0.md), [`~process`](../devices/process-at-1-0.md), [`~scheduler`](../devices/scheduler-at-1-0.md), and [`~wasm64`](../devices/wasm64-at-1-0.md) for core functionality, but the system is designed for easy addition of new custom devices. +- **Message System:** Everything in HyperBEAM is a "Message"—a map of named functions or binary data that can be processed, transformed, and cryptographically verified. +- **HTTP Interface:** Nodes expose an HTTP server for interaction via standard web requests and HyperPATHs, structured URLs that represent computation paths. +- **Modularity:** Its design supports easy extension, allowing new devices and functionalities to be added effortlessly. + +## Architecture + +* **Initialization Flow:** When a HyperBEAM node starts, it initializes the name service, scheduler registry, timestamp server, and HTTP server, establishing core services for process management, timing, communication, and storage. +* **Compute Model:** Computation follows the pattern \`Message1(Message2) => Message3\`, where messages are resolved through their devices and [paths](./pathing-in-ao-core.md). The integrity and history of these computations are ensured by **hashpaths**, which serves as a cryptographic audit trail. +* **Scheduler System:** The scheduler component manages execution order using ["slots"](../devices/scheduler-at-1-0.md#slot-system) — sequential positions that guarantee deterministic computation. +* **Process Slots:** Each process has numbered slots starting from 0 that track message execution order, ensuring consistent computation even across distributed nodes. + +## HTTP API and Pathing + +HyperBEAM exposes a powerful HTTP API that allows for interacting with processes and accessing data through structured URL patterns. We call URLs that represent computation paths "HyperPATHs". The URL bar effectively functions as a command-line interface for AO's trustless and verifiable compute. + +For a comprehensive guide on constructing and interpreting paths in AO-Core, including detailed examples and best practices, see [Pathing in AO-Core](./pathing-in-ao-core.md). + +In essence, HyperBEAM is the engine that powers the AO Computer, enabling the vision of a scalable, trust-minimized, decentralized supercomputer built on permanent storage. + +*See also: [HyperBEAM GitHub Repository](https://github.com/permaweb/HyperBEAM)* + +--- END OF FILE: docs/introduction/what-is-hyperbeam.md --- + +--- START OF FILE: docs/resources/llms.md --- +# LLM Context Files + +This section provides access to specially formatted files intended for consumption by Large Language Models (LLMs) to provide context about the HyperBEAM documentation. + +1. **[LLM Summary (llms.txt)](../llms.txt)** + * **Content**: Contains a brief summary of the HyperBEAM documentation structure and a list of relative file paths for all markdown documents included in the build. + * **Usage**: Useful for providing an LLM with a high-level overview and the available navigation routes within the documentation. + +2. **[LLM Full Content (llms-full.txt)](../llms-full.txt)** + * **Content**: A single text file containing the complete, concatenated content of all markdown documents from the specified documentation directories (`begin`, `run`, `guides`, `devices`, `resources`). Each file's content is clearly demarcated. + * **Usage**: Ideal for feeding the entire documentation content into an LLM for comprehensive context, analysis, or question-answering based on the full documentation set. + +!!! note "Generation Process" + These files are automatically generated by the `docs/build-all.sh` script during the documentation build process. They consolidate information from the following directories: + + * `docs/begin` + * `docs/run` + * `docs/guides` + * `docs/devices` + * `docs/resources` + +--- END OF FILE: docs/resources/llms.md --- + +--- START OF FILE: docs/resources/reference/faq.md --- +# Frequently Asked Questions + +This page answers common questions about HyperBEAM, its components, and how to use them effectively. + +## General Questions + +### What is HyperBEAM? + +HyperBEAM is a client implementation of the AO-Core protocol written in Erlang. It serves as the node software for a decentralized operating system that allows operators to offer computational resources to users in the AO network. + +### How does HyperBEAM differ from other distributed systems? + +HyperBEAM focuses on true decentralization with asynchronous message passing between isolated processes. Unlike many distributed systems that rely on central coordination, HyperBEAM nodes can operate independently while still forming a cohesive network. Additionally, its Erlang foundation provides robust fault tolerance and concurrency capabilities. + +### What can I build with HyperBEAM? + +You can build a wide range of applications, including: + +- Decentralized applications (dApps) +- Distributed computation systems +- Peer-to-peer services +- Resilient microservices +- IoT device networks +- Decentralized storage solutions + +### Is HyperBEAM open source? + +Yes, HyperBEAM is open-source software licensed under the Business Source License License. + +### What is the current focus or phase of HyperBEAM development? + +The initial development phase focuses on integrating AO processes more deeply with HyperBEAM. A key part of this is phasing out the reliance on traditional "dryrun" simulations for reading process state. Instead, processes are encouraged to use the [~patch@1.0 device](../../resources/source-code/dev_patch.md) to expose specific parts of their state directly via HyperPATH GET requests. This allows for more efficient and direct state access, particularly for web interfaces and external integrations. You can learn more about this mechanism in the [Exposing Process State with the Patch Device](../../build/exposing-process-state.md) guide. + +## Installation and Setup + +### What are the system requirements for running HyperBEAM? + +Currently, HyperBEAM is primarily tested and documented for Ubuntu 22.04. Support for macOS and other platforms will be added in future updates. For detailed requirements, see the [System Requirements](../getting-started/requirements.md) page. + +### Can I run HyperBEAM in a container? + +While technically possible, running HyperBEAM in Docker containers or other containerization technologies is currently not recommended. The containerization approach may introduce additional complexity and potential performance issues. We recommend running HyperBEAM directly on the host system until container support is more thoroughly tested and optimized. + +### How do I update HyperBEAM to the latest version? + +To update HyperBEAM: + +1. Pull the latest code from the repository +2. Rebuild the application +3. Restart the HyperBEAM service + +Specific update instructions will vary depending on your installation method. + +### Can I run multiple HyperBEAM nodes on a single machine? + +Yes, you can run multiple HyperBEAM nodes on a single machine, but you'll need to configure them to use different ports and data directories to avoid conflicts. However, this is not recommended for production environments as each node should ideally have a unique IP address to properly participate in the network. Running multiple nodes on a single machine is primarily useful for development and testing purposes. + +## Architecture and Components + +### What is the difference between HyperBEAM and Compute Unit? + +- **HyperBEAM**: The Erlang-based node software that handles message routing, process management, and device coordination. +- **Compute Unit (CU)**: A NodeJS implementation that executes WebAssembly modules and handles computational tasks. + +Together, these components form a complete execution environment for AO processes. + +## Development and Usage + +### What programming languages can I use with HyperBEAM? + +You can use any programming language that compiles to WebAssembly (WASM) for creating modules that run on the Compute Unit. This includes languages like: + +- Lua +- Rust +- C/C++ +- And many others with WebAssembly support + +### How do I debug processes running in HyperBEAM? + +Debugging processes in HyperBEAM can be done through: + +1. Logging messages to the system log +2. Monitoring process state and message flow +3. Inspecting memory usage and performance metrics + +### Is there a limit to how many processes can run on a node? + +The practical limit depends on your hardware resources. Erlang is designed to handle millions of lightweight processes efficiently, but the actual number will be determined by: + +- Available memory +- CPU capacity +- Network bandwidth +- Storage speed +- The complexity of your processes + + +## Troubleshooting + +### What should I do if a node becomes unresponsive? + +If a node becomes unresponsive: + +1. Check the node's logs for error messages +2. Verify network connectivity +3. Ensure sufficient system resources +4. Restart the node if necessary +5. Check for configuration issues + +For persistent problems, consult the [Troubleshooting](troubleshooting.md) page. + +### Where can I get help if I encounter issues? + +If you encounter issues: + +- Check the [Troubleshooting](troubleshooting.md) guide +- Search or ask questions on [GitHub Issues](https://github.com/permaweb/HyperBEAM/issues) +- Join the community on [Discord](https://discord.gg/V3yjzrBxPM) +--- END OF FILE: docs/resources/reference/faq.md --- + +--- START OF FILE: docs/resources/reference/glossary.md --- +# Glossary + +This glossary provides definitions for terms and concepts used throughout the HyperBEAM documentation. For a comprehensive glossary of permaweb-specific terminology, check out the [permaweb glossary](#permaweb-glossary) section below. + +## AO-Core Protocol +The underlying protocol that HyperBEAM implements, enabling decentralized computing and communication between nodes. AO-Core provides a framework into which any number of different computational models, encapsulated as primitive devices, can be attached. + +## Asynchronous Message Passing +A communication paradigm where senders don't wait for receivers to be ready, allowing for non-blocking operations and better scalability. + +## Checkpoint +A saved state of a process that can be used to resume execution from a known point, used for persistence and recovery. + +## Compute Unit (CU) +The NodeJS component of HyperBEAM that executes WebAssembly modules and handles computational tasks. + +## Decentralized Execution +The ability to run processes across a distributed network without centralized control or coordination. + +## Device +A functional unit in HyperBEAM that provides specific capabilities to the system, such as storage, networking, or computational resources. + +## Erlang +The programming language used to implement the HyperBEAM core, known for its robustness and support for building distributed, fault-tolerant applications. + +## ~flat@1.0 +A format used for encoding settings files in HyperBEAM configuration, using HTTP header styling. + +## Hashpaths +A mechanism for referencing locations in a program's state-space prior to execution. These state-space links are represented as Merklized lists of programs inputs and initial states. + +## HyperBEAM +The Erlang-based node software that handles message routing, process management, and device coordination in the HyperBEAM ecosystem. + +## Message +A data structure used for communication between processes in the HyperBEAM system. Messages can be interpreted as a binary term or as a collection of named functions (a Map of functions). + +## Module +A unit of code that can be loaded and executed by the Compute Unit, typically in WebAssembly format. + +## Node +An instance of HyperBEAM running on a physical or virtual machine that participates in the distributed network. + +## ~p4@1.0 +A device that runs as a pre-processor and post-processor in HyperBEAM, enabling a framework for node operators to sell usage of their machine's hardware to execute AO-Core devices. + +## Process +An independent unit of computation in HyperBEAM with its own state and execution context. + +## Process ID +A unique identifier assigned to a process within the HyperBEAM system. + +## ~scheduler@1.0 +A device used to assign a linear hashpath to an execution, such that all users may access it with a deterministic ordering. + +## ~compute-lite@1.0 +A lightweight device wrapping a local WASM executor, used for executing legacynet AO processes inside HyperBEAM. + +## ~json-iface@1.0 +A device that offers a translation layer between the JSON-encoded message format used by legacy versions and HyperBEAM's native HTTP message format. + +## ~meta@1.0 +A device used to configure the node's hardware, supported devices, metering and payments information, amongst other configuration options. + +## ~process@1.0 +A device that enables users to create persistent, shared executions that can be accessed by any number of users, each of whom may add additional inputs to its hashpath. + +## ~relay@1.0 +A device used to relay messages between nodes and the wider HTTP network. It offers an interface for sending and receiving messages using a variety of execution strategies. + +## ~simple-pay@1.0 +A simple, flexible pricing device that can be used in conjunction with p4@1.0 to offer flat-fees for the execution of AO-Core messages. + +## ~snp@1.0 +A device used to generate and validate proofs that a node is executing inside a Trusted Execution Environment (TEE). + +## ~wasm64@1.0 +A device used to execute WebAssembly code, using the Web Assembly Micro-Runtime (WAMR) under-the-hood. + +## ~stack@1.0 +A device used to execute an ordered set of devices over the same inputs, allowing users to create complex combinations of other devices. + +## Trusted Execution Environment (TEE) +A secure area inside a processor that ensures the confidentiality and integrity of code and data loaded within it. Used in HyperBEAM for trust-minimized computation. + +## WebAssembly (WASM) +A binary instruction format that serves as a portable compilation target for programming languages, enabling deployment on the web and other environments. + +## Permaweb Glossary + +For a more comprehensive glossary of terms used in the permaweb, try the [Permaweb Glossary](https://glossary.arweave.net). Or use it below: + + + + +
+
+ +
+
+ +
+
+--- END OF FILE: docs/resources/reference/glossary.md --- + +--- START OF FILE: docs/resources/reference/troubleshooting.md --- +# Troubleshooting Guide + +This guide addresses common issues you might encounter when working with HyperBEAM and the Compute Unit. + +## Installation Issues + +### Erlang Installation Fails + +**Symptoms**: Errors during Erlang compilation or installation + +**Solutions**: + +- Ensure all required dependencies are installed: `sudo apt-get install -y libssl-dev ncurses-dev make cmake gcc g++` +- Try configuring with fewer options: `./configure --without-wx --without-debugger --without-observer --without-et` +- Check disk space, as compilation requires several GB of free space + +### Rebar3 Bootstrap Fails + +**Symptoms**: Errors when running `./bootstrap` for Rebar3 + +**Solutions**: + +- Verify Erlang is correctly installed: `erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().'` +- Ensure you have the latest version of the repository: `git fetch && git reset --hard origin/master` +- Try manually downloading a precompiled Rebar3 binary + +## HyperBEAM Issues + +### HyperBEAM Won't Start + +**Symptoms**: Errors when running `rebar3 shell` or the HyperBEAM startup command + +**Solutions**: + +- Check for port conflicts: Another service might be using the configured port +- Verify the wallet key file exists and is accessible +- Examine Erlang crash dumps for detailed error information +- Ensure all required dependencies are installed + +### HyperBEAM Crashes During Operation + +**Symptoms**: Unexpected termination of the HyperBEAM process + +**Solutions**: + +- Check system resources (memory, disk space) +- Examine Erlang crash dumps for details +- Reduce memory limits if the system is resource-constrained +- Check for network connectivity issues if connecting to external services + +## Compute Unit Issues + +### Compute Unit Won't Start + +**Symptoms**: Errors when running `npm start` in the CU directory + +**Solutions**: + +- Verify Node.js is installed correctly: `node -v` +- Ensure all dependencies are installed: `npm i` +- Check that the wallet file exists and is correctly formatted +- Verify the `.env` file has all required settings + +### Memory Errors in Compute Unit + +**Symptoms**: Out of memory errors or excessive memory usage + +**Solutions**: + +- Adjust the `PROCESS_WASM_MEMORY_MAX_LIMIT` environment variable +- Enable garbage collection by setting an appropriate `GC_INTERVAL_MS` +- Monitor memory usage and adjust limits as needed +- If on a low-memory system, reduce concurrent process execution + +## Integration Issues + +### HyperBEAM Can't Connect to Compute Unit + +**Symptoms**: Connection errors in HyperBEAM logs when trying to reach the CU + +**Solutions**: + +- Verify the CU is running: `curl http://localhost:6363` +- Ensure there are no firewall rules blocking the connection +- Verify network configuration if components are on different machines + +### Process Execution Fails + +**Symptoms**: Errors when deploying or executing processes + +**Solutions**: + +- Check both HyperBEAM and CU logs for specific error messages +- Verify that the WASM module is correctly compiled and valid +- Test with a simple example process to isolate the issue +- Adjust memory limits if the process requires more resources + +## Getting Help + +If you're still experiencing issues after trying these troubleshooting steps: + +1. Check the [GitHub repository](https://github.com/permaweb/HyperBEAM) for known issues +2. Join the [Discord community](https://discord.gg/V3yjzrBxPM) for support +3. Open an issue on GitHub with detailed information about your problem +--- END OF FILE: docs/resources/reference/troubleshooting.md --- + +--- START OF FILE: docs/resources/source-code/ar_bundles.md --- +# [Module ar_bundles.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_bundles.erl) + + + + + + +## Function Index ## + + +
add_bundle_tags/1*
add_list_tags/1*
add_manifest_tags/2*
ar_bundles_test_/0*
assert_data_item/7*
check_size/2*Force that a binary is either empty or the given number of bytes.
check_type/2*Ensure that a value is of the given type.
data_item_signature_data/1Generate the data segment to be signed for a data item.
data_item_signature_data/2*
decode_avro_name/3*
decode_avro_tags/2*Decode Avro blocks (for tags) from binary.
decode_avro_value/4*
decode_bundle_header/2*
decode_bundle_header/3*
decode_bundle_items/2*
decode_optional_field/1*
decode_signature/1*Decode the signature from a binary format.
decode_tags/1Decode tags from a binary format using Apache Avro.
decode_vint/3*
decode_zigzag/1*Decode a VInt encoded ZigZag integer from binary.
deserialize/1Convert binary data back to a #tx record.
deserialize/2
encode_avro_string/1*Encode a string for Avro using ZigZag and VInt encoding.
encode_optional_field/1*Encode an optional field (target, anchor) with a presence byte.
encode_signature_type/1*Only RSA 4096 is currently supported.
encode_tags/1Encode tags into a binary format using Apache Avro.
encode_tags_size/2*
encode_vint/1*Encode a ZigZag integer to VInt binary format.
encode_vint/2*
encode_zigzag/1*Encode an integer using ZigZag encoding.
enforce_valid_tx/1*Take an item and ensure that it is of valid form.
finalize_bundle_data/1*
find/2Find an item in a bundle-map/list and return it.
find_single_layer/2*An internal helper for finding an item in a single-layer of a bundle.
format/1
format/2
format_binary/1*
format_data/2*
format_line/2*
format_line/3*
hd/1Return the first item in a bundle-map/list.
id/1Return the ID of an item -- either signed or unsigned as specified.
id/2
is_signed/1Check if an item is signed.
manifest/1
manifest_item/1Return the manifest item in a bundle-map/list.
map/1Convert an item containing a map or list into an Erlang map.
maybe_map_to_list/1*
maybe_unbundle/1*
maybe_unbundle_map/1*
member/2Check if an item exists in a bundle-map/list.
new_item/4Create a new data item.
new_manifest/1*
normalize/1
normalize_data/1*Ensure that a data item (potentially containing a map or list) has a standard, serialized form.
normalize_data_size/1*Reset the data size of a data item.
ok_or_throw/3*Throw an error if the given value is not ok.
parse_manifest/1
print/1
reset_ids/1Re-calculate both of the IDs for an item.
run_test/0*
serialize/1Convert a #tx record to its binary representation.
serialize/2
serialize_bundle_data/2*
sign_item/2Sign a data item.
signer/1Return the address of the signer of an item, if it is signed.
test_basic_member_id/0*
test_bundle_map/0*
test_bundle_with_one_item/0*
test_bundle_with_two_items/0*
test_deep_member/0*
test_empty_bundle/0*
test_extremely_large_bundle/0*
test_no_tags/0*
test_recursive_bundle/0*
test_serialize_deserialize_deep_signed_bundle/0*
test_unsigned_data_item_id/0*
test_unsigned_data_item_normalization/0*
test_with_tags/0*
test_with_zero_length_tag/0*
to_serialized_pair/1*
type/1
unbundle/1*
unbundle_list/1*
update_ids/1*Take an item and ensure that both the unsigned and signed IDs are +appropriately set.
utf8_encoded/1*Encode a UTF-8 string to binary.
verify_data_item_id/1*Verify the data item's ID matches the signature.
verify_data_item_signature/1*Verify the data item's signature.
verify_data_item_tags/1*Verify the validity of the data item's tags.
verify_item/1Verify the validity of a data item.
+ + + + +## Function Details ## + + + +### add_bundle_tags/1 * ### + +`add_bundle_tags(Tags) -> any()` + + + +### add_list_tags/1 * ### + +`add_list_tags(Tags) -> any()` + + + +### add_manifest_tags/2 * ### + +`add_manifest_tags(Tags, ManifestID) -> any()` + + + +### ar_bundles_test_/0 * ### + +`ar_bundles_test_() -> any()` + + + +### assert_data_item/7 * ### + +`assert_data_item(KeyType, Owner, Target, Anchor, Tags, Data, DataItem) -> any()` + + + +### check_size/2 * ### + +`check_size(Bin, Sizes) -> any()` + +Force that a binary is either empty or the given number of bytes. + + + +### check_type/2 * ### + +`check_type(Value, X2) -> any()` + +Ensure that a value is of the given type. + + + +### data_item_signature_data/1 ### + +`data_item_signature_data(RawItem) -> any()` + +Generate the data segment to be signed for a data item. + + + +### data_item_signature_data/2 * ### + +`data_item_signature_data(RawItem, X2) -> any()` + + + +### decode_avro_name/3 * ### + +`decode_avro_name(NameSize, Rest, Count) -> any()` + + + +### decode_avro_tags/2 * ### + +`decode_avro_tags(Binary, Count) -> any()` + +Decode Avro blocks (for tags) from binary. + + + +### decode_avro_value/4 * ### + +`decode_avro_value(ValueSize, Name, Rest, Count) -> any()` + + + +### decode_bundle_header/2 * ### + +`decode_bundle_header(Count, Bin) -> any()` + + + +### decode_bundle_header/3 * ### + +`decode_bundle_header(Count, ItemsBin, Header) -> any()` + + + +### decode_bundle_items/2 * ### + +`decode_bundle_items(RestItems, ItemsBin) -> any()` + + + +### decode_optional_field/1 * ### + +`decode_optional_field(X1) -> any()` + + + +### decode_signature/1 * ### + +`decode_signature(Other) -> any()` + +Decode the signature from a binary format. Only RSA 4096 is currently supported. +Note: the signature type '1' corresponds to RSA 4096 - but it is is written in +little-endian format which is why we match on `<<1, 0>>`. + + + +### decode_tags/1 ### + +`decode_tags(X1) -> any()` + +Decode tags from a binary format using Apache Avro. + + + +### decode_vint/3 * ### + +`decode_vint(X1, Result, Shift) -> any()` + + + +### decode_zigzag/1 * ### + +`decode_zigzag(Binary) -> any()` + +Decode a VInt encoded ZigZag integer from binary. + + + +### deserialize/1 ### + +`deserialize(Binary) -> any()` + +Convert binary data back to a #tx record. + + + +### deserialize/2 ### + +`deserialize(Item, X2) -> any()` + + + +### encode_avro_string/1 * ### + +`encode_avro_string(String) -> any()` + +Encode a string for Avro using ZigZag and VInt encoding. + + + +### encode_optional_field/1 * ### + +`encode_optional_field(Field) -> any()` + +Encode an optional field (target, anchor) with a presence byte. + + + +### encode_signature_type/1 * ### + +`encode_signature_type(X1) -> any()` + +Only RSA 4096 is currently supported. +Note: the signature type '1' corresponds to RSA 4096 -- but it is is written in +little-endian format which is why we encode to `<<1, 0>>`. + + + +### encode_tags/1 ### + +`encode_tags(Tags) -> any()` + +Encode tags into a binary format using Apache Avro. + + + +### encode_tags_size/2 * ### + +`encode_tags_size(Tags, EncodedTags) -> any()` + + + +### encode_vint/1 * ### + +`encode_vint(ZigZag) -> any()` + +Encode a ZigZag integer to VInt binary format. + + + +### encode_vint/2 * ### + +`encode_vint(ZigZag, Acc) -> any()` + + + +### encode_zigzag/1 * ### + +`encode_zigzag(Int) -> any()` + +Encode an integer using ZigZag encoding. + + + +### enforce_valid_tx/1 * ### + +`enforce_valid_tx(List) -> any()` + +Take an item and ensure that it is of valid form. Useful for ensuring +that a message is viable for serialization/deserialization before execution. +This function should throw simple, easy to follow errors to aid devs in +debugging issues. + + + +### finalize_bundle_data/1 * ### + +`finalize_bundle_data(Processed) -> any()` + + + +### find/2 ### + +`find(Key, Map) -> any()` + +Find an item in a bundle-map/list and return it. + + + +### find_single_layer/2 * ### + +`find_single_layer(UnsignedID, TX) -> any()` + +An internal helper for finding an item in a single-layer of a bundle. +Does not recurse! You probably want `find/2` in most cases. + + + +### format/1 ### + +`format(Item) -> any()` + + + +### format/2 ### + +`format(Item, Indent) -> any()` + + + +### format_binary/1 * ### + +`format_binary(Bin) -> any()` + + + +### format_data/2 * ### + +`format_data(Item, Indent) -> any()` + + + +### format_line/2 * ### + +`format_line(Str, Indent) -> any()` + + + +### format_line/3 * ### + +`format_line(RawStr, Fmt, Ind) -> any()` + + + +### hd/1 ### + +`hd(Tx) -> any()` + +Return the first item in a bundle-map/list. + + + +### id/1 ### + +`id(Item) -> any()` + +Return the ID of an item -- either signed or unsigned as specified. +If the item is unsigned and the user requests the signed ID, we return +the atom `not_signed`. In all other cases, we return the ID of the item. + + + +### id/2 ### + +`id(Item, Type) -> any()` + + + +### is_signed/1 ### + +`is_signed(Item) -> any()` + +Check if an item is signed. + + + +### manifest/1 ### + +`manifest(Map) -> any()` + + + +### manifest_item/1 ### + +`manifest_item(Tx) -> any()` + +Return the manifest item in a bundle-map/list. + + + +### map/1 ### + +`map(Tx) -> any()` + +Convert an item containing a map or list into an Erlang map. + + + +### maybe_map_to_list/1 * ### + +`maybe_map_to_list(Item) -> any()` + + + +### maybe_unbundle/1 * ### + +`maybe_unbundle(Item) -> any()` + + + +### maybe_unbundle_map/1 * ### + +`maybe_unbundle_map(Bundle) -> any()` + + + +### member/2 ### + +`member(Key, Item) -> any()` + +Check if an item exists in a bundle-map/list. + + + +### new_item/4 ### + +`new_item(Target, Anchor, Tags, Data) -> any()` + +Create a new data item. Should only be used for testing. + + + +### new_manifest/1 * ### + +`new_manifest(Index) -> any()` + + + +### normalize/1 ### + +`normalize(Item) -> any()` + + + +### normalize_data/1 * ### + +`normalize_data(Bundle) -> any()` + +Ensure that a data item (potentially containing a map or list) has a standard, serialized form. + + + +### normalize_data_size/1 * ### + +`normalize_data_size(Item) -> any()` + +Reset the data size of a data item. Assumes that the data is already normalized. + + + +### ok_or_throw/3 * ### + +`ok_or_throw(TX, X2, Error) -> any()` + +Throw an error if the given value is not ok. + + + +### parse_manifest/1 ### + +`parse_manifest(Item) -> any()` + + + +### print/1 ### + +`print(Item) -> any()` + + + +### reset_ids/1 ### + +`reset_ids(Item) -> any()` + +Re-calculate both of the IDs for an item. This is a wrapper +function around `update_id/1` that ensures both IDs are set from +scratch. + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### serialize/1 ### + +`serialize(TX) -> any()` + +Convert a #tx record to its binary representation. + + + +### serialize/2 ### + +`serialize(TX, X2) -> any()` + + + +### serialize_bundle_data/2 * ### + +`serialize_bundle_data(Map, Manifest) -> any()` + + + +### sign_item/2 ### + +`sign_item(RawItem, X2) -> any()` + +Sign a data item. + + + +### signer/1 ### + +`signer(Tx) -> any()` + +Return the address of the signer of an item, if it is signed. + + + +### test_basic_member_id/0 * ### + +`test_basic_member_id() -> any()` + + + +### test_bundle_map/0 * ### + +`test_bundle_map() -> any()` + + + +### test_bundle_with_one_item/0 * ### + +`test_bundle_with_one_item() -> any()` + + + +### test_bundle_with_two_items/0 * ### + +`test_bundle_with_two_items() -> any()` + + + +### test_deep_member/0 * ### + +`test_deep_member() -> any()` + + + +### test_empty_bundle/0 * ### + +`test_empty_bundle() -> any()` + + + +### test_extremely_large_bundle/0 * ### + +`test_extremely_large_bundle() -> any()` + + + +### test_no_tags/0 * ### + +`test_no_tags() -> any()` + + + +### test_recursive_bundle/0 * ### + +`test_recursive_bundle() -> any()` + + + +### test_serialize_deserialize_deep_signed_bundle/0 * ### + +`test_serialize_deserialize_deep_signed_bundle() -> any()` + + + +### test_unsigned_data_item_id/0 * ### + +`test_unsigned_data_item_id() -> any()` + + + +### test_unsigned_data_item_normalization/0 * ### + +`test_unsigned_data_item_normalization() -> any()` + + + +### test_with_tags/0 * ### + +`test_with_tags() -> any()` + + + +### test_with_zero_length_tag/0 * ### + +`test_with_zero_length_tag() -> any()` + + + +### to_serialized_pair/1 * ### + +`to_serialized_pair(Item) -> any()` + + + +### type/1 ### + +`type(Item) -> any()` + + + +### unbundle/1 * ### + +`unbundle(Item) -> any()` + + + +### unbundle_list/1 * ### + +`unbundle_list(Item) -> any()` + + + +### update_ids/1 * ### + +`update_ids(Item) -> any()` + +Take an item and ensure that both the unsigned and signed IDs are +appropriately set. This function is structured to fall through all cases +of poorly formed items, recursively ensuring its correctness for each case +until the item has a coherent set of IDs. +The cases in turn are: +- The item has no unsigned_id. This is never valid. +- The item has the default signature and ID. This is valid. +- The item has the default signature but a non-default ID. Reset the ID. +- The item has a signature. We calculate the ID from the signature. +- Valid: The item is fully formed and has both an unsigned and signed ID. + + + +### utf8_encoded/1 * ### + +`utf8_encoded(String) -> any()` + +Encode a UTF-8 string to binary. + + + +### verify_data_item_id/1 * ### + +`verify_data_item_id(DataItem) -> any()` + +Verify the data item's ID matches the signature. + + + +### verify_data_item_signature/1 * ### + +`verify_data_item_signature(DataItem) -> any()` + +Verify the data item's signature. + + + +### verify_data_item_tags/1 * ### + +`verify_data_item_tags(DataItem) -> any()` + +Verify the validity of the data item's tags. + + + +### verify_item/1 ### + +`verify_item(DataItem) -> any()` + +Verify the validity of a data item. + + +--- END OF FILE: docs/resources/source-code/ar_bundles.md --- + +--- START OF FILE: docs/resources/source-code/ar_deep_hash.md --- +# [Module ar_deep_hash.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_deep_hash.erl) + + + + + + +## Function Index ## + + +
hash/1
hash_bin/1*
hash_bin_or_list/1*
hash_list/2*
+ + + + +## Function Details ## + + + +### hash/1 ### + +`hash(List) -> any()` + + + +### hash_bin/1 * ### + +`hash_bin(Bin) -> any()` + + + +### hash_bin_or_list/1 * ### + +`hash_bin_or_list(Bin) -> any()` + + + +### hash_list/2 * ### + +`hash_list(List, Acc) -> any()` + + +--- END OF FILE: docs/resources/source-code/ar_deep_hash.md --- + +--- START OF FILE: docs/resources/source-code/ar_rate_limiter.md --- +# [Module ar_rate_limiter.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_rate_limiter.erl) + + + + +__Behaviours:__ [`gen_server`](gen_server.md). + + + +## Function Index ## + + +
cut_trace/4*
handle_call/3
handle_cast/2
handle_info/2
init/1
off/0Turn rate limiting off.
on/0Turn rate limiting on.
start_link/1
terminate/2
throttle/3Hang until it is safe to make another request to the given Peer with the +given Path.
throttle2/3*
+ + + + +## Function Details ## + + + +### cut_trace/4 * ### + +`cut_trace(N, Trace, Now, Opts) -> any()` + + + +### handle_call/3 ### + +`handle_call(Request, From, State) -> any()` + + + +### handle_cast/2 ### + +`handle_cast(Cast, State) -> any()` + + + +### handle_info/2 ### + +`handle_info(Message, State) -> any()` + + + +### init/1 ### + +`init(Opts) -> any()` + + + +### off/0 ### + +`off() -> any()` + +Turn rate limiting off. + + + +### on/0 ### + +`on() -> any()` + +Turn rate limiting on. + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + + + +### terminate/2 ### + +`terminate(Reason, State) -> any()` + + + +### throttle/3 ### + +`throttle(Peer, Path, Opts) -> any()` + +Hang until it is safe to make another request to the given Peer with the +given Path. The limits are configured in include/ar_blacklist_middleware.hrl. + + + +### throttle2/3 * ### + +`throttle2(Peer, Path, Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/ar_rate_limiter.md --- + +--- START OF FILE: docs/resources/source-code/ar_timestamp.md --- +# [Module ar_timestamp.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_timestamp.erl) + + + + + + +## Function Index ## + + +
cache/1*Cache the current timestamp from Arweave.
get/0Get the current timestamp from the server, starting the server if it +isn't already running.
refresher/1*Refresh the timestamp cache periodically.
spawn_server/0*Spawn a new server and its refresher.
start/0Check if the server is already running, and if not, start it.
+ + + + +## Function Details ## + + + +### cache/1 * ### + +`cache(Current) -> any()` + +Cache the current timestamp from Arweave. + + + +### get/0 ### + +`get() -> any()` + +Get the current timestamp from the server, starting the server if it +isn't already running. + + + +### refresher/1 * ### + +`refresher(TSServer) -> any()` + +Refresh the timestamp cache periodically. + + + +### spawn_server/0 * ### + +`spawn_server() -> any()` + +Spawn a new server and its refresher. + + + +### start/0 ### + +`start() -> any()` + +Check if the server is already running, and if not, start it. + + +--- END OF FILE: docs/resources/source-code/ar_timestamp.md --- + +--- START OF FILE: docs/resources/source-code/ar_tx.md --- +# [Module ar_tx.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_tx.erl) + + + + +The module with utilities for transaction creation, signing, and verification. + + + +## Function Index ## + + +
collect_validation_results/2*
do_verify/2*Verify transaction.
json_struct_to_tx/1
new/4Create a new transaction.
new/5
sign/2Cryptographically sign (claim ownership of) a transaction.
signature_data_segment/1*Generate the data segment to be signed for a given TX.
tx_to_json_struct/1
verify/1Verify whether a transaction is valid.
verify_hash/1*Verify that the transaction's ID is a hash of its signature.
verify_signature/2*Verify the transaction's signature.
verify_tx_id/2Verify the given transaction actually has the given identifier.
+ + + + +## Function Details ## + + + +### collect_validation_results/2 * ### + +`collect_validation_results(TXID, Checks) -> any()` + + + +### do_verify/2 * ### + +`do_verify(TX, VerifySignature) -> any()` + +Verify transaction. + + + +### json_struct_to_tx/1 ### + +`json_struct_to_tx(TXStruct) -> any()` + + + +### new/4 ### + +`new(Dest, Reward, Qty, Last) -> any()` + +Create a new transaction. + + + +### new/5 ### + +`new(Dest, Reward, Qty, Last, SigType) -> any()` + + + +### sign/2 ### + +`sign(TX, X2) -> any()` + +Cryptographically sign (claim ownership of) a transaction. + + + +### signature_data_segment/1 * ### + +`signature_data_segment(TX) -> any()` + +Generate the data segment to be signed for a given TX. + + + +### tx_to_json_struct/1 ### + +`tx_to_json_struct(Tx) -> any()` + + + +### verify/1 ### + +`verify(TX) -> any()` + +Verify whether a transaction is valid. + + + +### verify_hash/1 * ### + +`verify_hash(Tx) -> any()` + +Verify that the transaction's ID is a hash of its signature. + + + +### verify_signature/2 * ### + +`verify_signature(TX, X2) -> any()` + +Verify the transaction's signature. + + + +### verify_tx_id/2 ### + +`verify_tx_id(ExpectedID, Tx) -> any()` + +Verify the given transaction actually has the given identifier. + + +--- END OF FILE: docs/resources/source-code/ar_tx.md --- + +--- START OF FILE: docs/resources/source-code/ar_wallet.md --- +# [Module ar_wallet.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_wallet.erl) + + + + + + +## Function Index ## + + +
compress_ecdsa_pubkey/1*
hash_address/1*
hmac/1
hmac/2
load_key/1Read the keyfile for the key with the given address from disk.
load_keyfile/1Extract the public and private key from a keyfile.
new/0
new/1
new_keyfile/2Generate a new wallet public and private key, with a corresponding keyfile.
sign/2Sign some data with a private key.
sign/3sign some data, hashed using the provided DigestType.
to_address/1Generate an address from a public key.
to_address/2
to_ecdsa_address/1*
to_rsa_address/1*
verify/3Verify that a signature is correct.
verify/4
wallet_filepath/1*
wallet_filepath/3*
wallet_filepath2/1*
wallet_name/3*
+ + + + +## Function Details ## + + + +### compress_ecdsa_pubkey/1 * ### + +`compress_ecdsa_pubkey(X1) -> any()` + + + +### hash_address/1 * ### + +`hash_address(PubKey) -> any()` + + + +### hmac/1 ### + +`hmac(Data) -> any()` + + + +### hmac/2 ### + +`hmac(Data, DigestType) -> any()` + + + +### load_key/1 ### + +`load_key(Addr) -> any()` + +Read the keyfile for the key with the given address from disk. +Return not_found if arweave_keyfile_[addr].json or [addr].json is not found +in [data_dir]/?WALLET_DIR. + + + +### load_keyfile/1 ### + +`load_keyfile(File) -> any()` + +Extract the public and private key from a keyfile. + + + +### new/0 ### + +`new() -> any()` + + + +### new/1 ### + +`new(KeyType) -> any()` + + + +### new_keyfile/2 ### + +`new_keyfile(KeyType, WalletName) -> any()` + +Generate a new wallet public and private key, with a corresponding keyfile. +The provided key is used as part of the file name. + + + +### sign/2 ### + +`sign(Key, Data) -> any()` + +Sign some data with a private key. + + + +### sign/3 ### + +`sign(X1, Data, DigestType) -> any()` + +sign some data, hashed using the provided DigestType. + + + +### to_address/1 ### + +`to_address(Pubkey) -> any()` + +Generate an address from a public key. + + + +### to_address/2 ### + +`to_address(PubKey, X2) -> any()` + + + +### to_ecdsa_address/1 * ### + +`to_ecdsa_address(PubKey) -> any()` + + + +### to_rsa_address/1 * ### + +`to_rsa_address(PubKey) -> any()` + + + +### verify/3 ### + +`verify(Key, Data, Sig) -> any()` + +Verify that a signature is correct. + + + +### verify/4 ### + +`verify(X1, Data, Sig, DigestType) -> any()` + + + +### wallet_filepath/1 * ### + +`wallet_filepath(Wallet) -> any()` + + + +### wallet_filepath/3 * ### + +`wallet_filepath(WalletName, PubKey, KeyType) -> any()` + + + +### wallet_filepath2/1 * ### + +`wallet_filepath2(Wallet) -> any()` + + + +### wallet_name/3 * ### + +`wallet_name(WalletName, PubKey, KeyType) -> any()` + + +--- END OF FILE: docs/resources/source-code/ar_wallet.md --- + +--- START OF FILE: docs/resources/source-code/dev_cache.md --- +# [Module dev_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cache.erl) + + + + +A device that looks up an ID from a local store and returns it, +honoring the `accept` key to return the correct format. + + + +## Description ## +The cache also +supports writing messages to the store, if the node message has the +writer's address in its `cache_writers` key. + +## Function Index ## + + +
cache_write_binary_test/0*Ensure that we can write direct binaries to the cache.
cache_write_message_test/0*Test that the cache can be written to and read from using the hb_cache +API.
is_trusted_writer/2*Verify that the request originates from a trusted writer.
link/3Link a source to a destination in the cache.
read/3Read data from the cache.
read_from_cache/2*Read data from the cache via HTTP.
setup_test_env/0*Create a test environment with a local store and node.
write/3Write data to the cache.
write_single/2*Helper function to write a single data item to the cache.
write_to_cache/3*Write data to the cache via HTTP.
+ + + + +## Function Details ## + + + +### cache_write_binary_test/0 * ### + +`cache_write_binary_test() -> any()` + +Ensure that we can write direct binaries to the cache. + + + +### cache_write_message_test/0 * ### + +`cache_write_message_test() -> any()` + +Test that the cache can be written to and read from using the hb_cache +API. + + + +### is_trusted_writer/2 * ### + +`is_trusted_writer(Req, Opts) -> any()` + +Verify that the request originates from a trusted writer. +Checks that the single signer of the request is present in the list +of trusted cache writer addresses specified in the options. + + + +### link/3 ### + +`link(Base, Req, Opts) -> any()` + +Link a source to a destination in the cache. + + + +### read/3 ### + +`read(M1, M2, Opts) -> any()` + +Read data from the cache. +Retrieves data corresponding to a key from a local store. +The key is extracted from the incoming message under <<"target">>. +The options map may include store configuration. +If the "accept" header is set to <<"application/aos-2">>, the result is +converted to a JSON structure and encoded. + + + +### read_from_cache/2 * ### + +`read_from_cache(Node, Path) -> any()` + +Read data from the cache via HTTP. +Constructs a GET request using the provided path, sends it to the node, +and returns the response. + + + +### setup_test_env/0 * ### + +`setup_test_env() -> any()` + +Create a test environment with a local store and node. +Ensures that the required application is started, configures a local +file-system store, resets the store for a clean state, creates a wallet +for signing requests, and starts a node with the store and trusted cache +writer configuration. + + + +### write/3 ### + +`write(M1, M2, Opts) -> any()` + +Write data to the cache. +Processes a write request by first verifying that the request comes from a +trusted writer (as defined by the `cache_writers` configuration in the +options). The write type is determined from the message ("single" or "batch") +and the data is stored accordingly. + + + +### write_single/2 * ### + +`write_single(Msg, Opts) -> any()` + +Helper function to write a single data item to the cache. +Extracts the body, location, and operation from the message. +Depending on the type of data (map or binary) or if a link operation is +requested, it writes the data to the store using the appropriate function. + + + +### write_to_cache/3 * ### + +`write_to_cache(Node, Data, Wallet) -> any()` + +Write data to the cache via HTTP. +Constructs a write request message with the provided data, signs it with the +given wallet, sends it to the node, and verifies that the response indicates +a successful write. + + +--- END OF FILE: docs/resources/source-code/dev_cache.md --- + +--- START OF FILE: docs/resources/source-code/dev_cacheviz.md --- +# [Module dev_cacheviz.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cacheviz.erl) + + + + +A device that generates renders (or renderable dot output) of a node's +cache. + + + +## Function Index ## + + +
dot/3Output the dot representation of the cache, or a specific path within +the cache set by the target key in the request.
svg/3Output the SVG representation of the cache, or a specific path within +the cache set by the target key in the request.
+ + + + +## Function Details ## + + + +### dot/3 ### + +`dot(X1, Req, Opts) -> any()` + +Output the dot representation of the cache, or a specific path within +the cache set by the `target` key in the request. + + + +### svg/3 ### + +`svg(Base, Req, Opts) -> any()` + +Output the SVG representation of the cache, or a specific path within +the cache set by the `target` key in the request. + + +--- END OF FILE: docs/resources/source-code/dev_cacheviz.md --- + +--- START OF FILE: docs/resources/source-code/dev_codec_ans104.md --- +# [Module dev_codec_ans104.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_ans104.erl) + + + + +Codec for managing transformations from `ar_bundles`-style Arweave TX +records to and from TABMs. + + + +## Function Index ## + + +
commit/3Sign a message using the priv_wallet key in the options.
committed/3Return a list of committed keys from an ANS-104 message.
committed_from_trusted_keys/3*
content_type/1Return the content type for the codec.
deduplicating_from_list/1*Deduplicate a list of key-value pairs by key, generating a list of +values for each normalized key if there are duplicates.
deserialize/1Deserialize a binary ans104 message to a TABM.
do_from/1*
duplicated_tag_name_test/0*
encoded_tags_to_map/1*Convert an ANS-104 encoded tag list into a HyperBEAM-compatible map.
from/1Convert a #tx record into a message map recursively.
from_maintains_tag_name_case_test/0*
id/1Return the ID of a message.
normal_tags/1*Check whether a list of key-value pairs contains only normalized keys.
normal_tags_test/0*
only_committed_maintains_target_test/0*
quantity_field_is_ignored_in_from_test/0*
quantity_key_encoded_as_tag_test/0*
restore_tag_name_case_from_cache_test/0*
serialize/1Serialize a message or TX to a binary.
signed_duplicated_tag_name_test/0*
simple_to_conversion_test/0*
tag_map_to_encoded_tags/1*Convert a HyperBEAM-compatible map into an ANS-104 encoded tag list, +recreating the original order of the tags.
to/1Internal helper to translate a message to its #tx record representation, +which can then be used by ar_bundles to serialize the message.
verify/3Verify an ANS-104 commitment.
+ + + + +## Function Details ## + + + +### commit/3 ### + +`commit(Msg, Req, Opts) -> any()` + +Sign a message using the `priv_wallet` key in the options. + + + +### committed/3 ### + +`committed(Msg, Req, Opts) -> any()` + +Return a list of committed keys from an ANS-104 message. + + + +### committed_from_trusted_keys/3 * ### + +`committed_from_trusted_keys(Msg, TrustedKeys, Opts) -> any()` + + + +### content_type/1 ### + +`content_type(X1) -> any()` + +Return the content type for the codec. + + + +### deduplicating_from_list/1 * ### + +`deduplicating_from_list(Tags) -> any()` + +Deduplicate a list of key-value pairs by key, generating a list of +values for each normalized key if there are duplicates. + + + +### deserialize/1 ### + +`deserialize(Binary) -> any()` + +Deserialize a binary ans104 message to a TABM. + + + +### do_from/1 * ### + +`do_from(RawTX) -> any()` + + + +### duplicated_tag_name_test/0 * ### + +`duplicated_tag_name_test() -> any()` + + + +### encoded_tags_to_map/1 * ### + +`encoded_tags_to_map(Tags) -> any()` + +Convert an ANS-104 encoded tag list into a HyperBEAM-compatible map. + + + +### from/1 ### + +`from(Binary) -> any()` + +Convert a #tx record into a message map recursively. + + + +### from_maintains_tag_name_case_test/0 * ### + +`from_maintains_tag_name_case_test() -> any()` + + + +### id/1 ### + +`id(Msg) -> any()` + +Return the ID of a message. + + + +### normal_tags/1 * ### + +`normal_tags(Tags) -> any()` + +Check whether a list of key-value pairs contains only normalized keys. + + + +### normal_tags_test/0 * ### + +`normal_tags_test() -> any()` + + + +### only_committed_maintains_target_test/0 * ### + +`only_committed_maintains_target_test() -> any()` + + + +### quantity_field_is_ignored_in_from_test/0 * ### + +`quantity_field_is_ignored_in_from_test() -> any()` + + + +### quantity_key_encoded_as_tag_test/0 * ### + +`quantity_key_encoded_as_tag_test() -> any()` + + + +### restore_tag_name_case_from_cache_test/0 * ### + +`restore_tag_name_case_from_cache_test() -> any()` + + + +### serialize/1 ### + +`serialize(Msg) -> any()` + +Serialize a message or TX to a binary. + + + +### signed_duplicated_tag_name_test/0 * ### + +`signed_duplicated_tag_name_test() -> any()` + + + +### simple_to_conversion_test/0 * ### + +`simple_to_conversion_test() -> any()` + + + +### tag_map_to_encoded_tags/1 * ### + +`tag_map_to_encoded_tags(TagMap) -> any()` + +Convert a HyperBEAM-compatible map into an ANS-104 encoded tag list, +recreating the original order of the tags. + + + +### to/1 ### + +`to(Binary) -> any()` + +Internal helper to translate a message to its #tx record representation, +which can then be used by ar_bundles to serialize the message. We call the +message's device in order to get the keys that we will be checkpointing. We +do this recursively to handle nested messages. The base case is that we hit +a binary, which we return as is. + + + +### verify/3 ### + +`verify(Msg, Req, Opts) -> any()` + +Verify an ANS-104 commitment. + + +--- END OF FILE: docs/resources/source-code/dev_codec_ans104.md --- + +--- START OF FILE: docs/resources/source-code/dev_codec_flat.md --- +# [Module dev_codec_flat.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_flat.erl) + + + + +A codec for turning TABMs into/from flat Erlang maps that have +(potentially multi-layer) paths as their keys, and a normal TABM binary as +their value. + + + +## Function Index ## + + +
binary_passthrough_test/0*
commit/3
committed/3
deep_nesting_test/0*
deserialize/1
empty_map_test/0*
from/1Convert a flat map to a TABM.
inject_at_path/3*
multiple_paths_test/0*
nested_conversion_test/0*
path_list_test/0*
serialize/1
simple_conversion_test/0*
to/1Convert a TABM to a flat map.
verify/3
+ + + + +## Function Details ## + + + +### binary_passthrough_test/0 * ### + +`binary_passthrough_test() -> any()` + + + +### commit/3 ### + +`commit(Msg, Req, Opts) -> any()` + + + +### committed/3 ### + +`committed(Msg, Req, Opts) -> any()` + + + +### deep_nesting_test/0 * ### + +`deep_nesting_test() -> any()` + + + +### deserialize/1 ### + +`deserialize(Bin) -> any()` + + + +### empty_map_test/0 * ### + +`empty_map_test() -> any()` + + + +### from/1 ### + +`from(Bin) -> any()` + +Convert a flat map to a TABM. + + + +### inject_at_path/3 * ### + +`inject_at_path(Rest, Value, Map) -> any()` + + + +### multiple_paths_test/0 * ### + +`multiple_paths_test() -> any()` + + + +### nested_conversion_test/0 * ### + +`nested_conversion_test() -> any()` + + + +### path_list_test/0 * ### + +`path_list_test() -> any()` + + + +### serialize/1 ### + +`serialize(Map) -> any()` + + + +### simple_conversion_test/0 * ### + +`simple_conversion_test() -> any()` + + + +### to/1 ### + +`to(Bin) -> any()` + +Convert a TABM to a flat map. + + + +### verify/3 ### + +`verify(Msg, Req, Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_codec_flat.md --- + +--- START OF FILE: docs/resources/source-code/dev_codec_httpsig_conv.md --- +# [Module dev_codec_httpsig_conv.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_httpsig_conv.erl) + + + + +A codec for the that marshals TABM encoded messages to and from the +"HTTP" message structure. + + + +## Description ## + +Every HTTP message is an HTTP multipart message. +See https://datatracker.ietf.org/doc/html/rfc7578 + +For each TABM Key: + +The Key/Value Pair will be encoded according to the following rules: +"signatures" -> {SignatureInput, Signature} header Tuples, each encoded +as a Structured Field Dictionary +"body" -> +- if a map, then recursively encode as its own HyperBEAM message +- otherwise encode as a normal field +_ -> encode as a normal field + +Each field will be mapped to the HTTP Message according to the following +rules: +"body" -> always encoded part of the body as with Content-Disposition +type of "inline" +_ -> +- If the byte size of the value is less than the ?MAX_TAG_VALUE, +then encode as a header, also attempting to encode as a +structured field. +- Otherwise encode the value as a part in the multipart response + + +## Function Index ## + + +
boundary_from_parts/1*Generate a unique, reproducible boundary for the +multipart body, however we cannot use the id of the message as +the boundary, as the id is not known until the message is +encoded.
commitments_from_signature/4*Populate the /commitments key on the TABM with the dictionary of +signatures and their corresponding inputs.
do_to/2*
encode_body_keys/1*Encode a list of body parts into a binary.
encode_body_part/3*Encode a multipart body part to a flat binary.
encode_http_msg/1*Encode a HTTP message into a binary.
extract_hashpaths/1*Extract all keys labelled hashpath* from the commitments, and add them +to the HTTP message as hashpath* keys.
field_to_http/3*All maps are encoded into the body of the HTTP message +to be further encoded later.
from/1Convert a HTTP Message into a TABM.
from_body/4*
from_body_parts/3*
group_ids/1*Group all elements with: +1.
group_maps/1*Merge maps at the same level, if possible.
group_maps/3*
group_maps_flat_compatible_test/0*The grouped maps encoding is a subset of the flat encoding, +where on keys with maps values are flattened.
group_maps_test/0*
hashpaths_from_message/1*
inline_key/1*given a message, returns a binary tuple: +- A list of pairs to add to the msg, if any +- the field name for the inlined key.
to/1Convert a TABM into an HTTP Message.
to/2*
ungroup_ids/1*Decode the ao-ids key into a map.
+ + + + +## Function Details ## + + + +### boundary_from_parts/1 * ### + +`boundary_from_parts(PartList) -> any()` + +Generate a unique, reproducible boundary for the +multipart body, however we cannot use the id of the message as +the boundary, as the id is not known until the message is +encoded. Subsequently, we generate each body part individually, +concatenate them, and apply a SHA2-256 hash to the result. +This ensures that the boundary is unique, reproducible, and +secure. + + + +### commitments_from_signature/4 * ### + +`commitments_from_signature(Map, HPs, RawSig, RawSigInput) -> any()` + +Populate the `/commitments` key on the TABM with the dictionary of +signatures and their corresponding inputs. + + + +### do_to/2 * ### + +`do_to(Binary, Opts) -> any()` + + + +### encode_body_keys/1 * ### + +`encode_body_keys(PartList) -> any()` + +Encode a list of body parts into a binary. + + + +### encode_body_part/3 * ### + +`encode_body_part(PartName, BodyPart, InlineKey) -> any()` + +Encode a multipart body part to a flat binary. + + + +### encode_http_msg/1 * ### + +`encode_http_msg(Httpsig) -> any()` + +Encode a HTTP message into a binary. + + + +### extract_hashpaths/1 * ### + +`extract_hashpaths(Map) -> any()` + +Extract all keys labelled `hashpath*` from the commitments, and add them +to the HTTP message as `hashpath*` keys. + + + +### field_to_http/3 * ### + +`field_to_http(Httpsig, X2, Opts) -> any()` + +All maps are encoded into the body of the HTTP message +to be further encoded later. + + + +### from/1 ### + +`from(Bin) -> any()` + +Convert a HTTP Message into a TABM. +HTTP Structured Field is encoded into it's equivalent TABM encoding. + + + +### from_body/4 * ### + +`from_body(TABM, InlinedKey, ContentType, Body) -> any()` + + + +### from_body_parts/3 * ### + +`from_body_parts(TABM, InlinedKey, Rest) -> any()` + + + +### group_ids/1 * ### + +`group_ids(Map) -> any()` + +Group all elements with: +1. A key that ?IS_ID returns true for, and +2. A value that is immediate +into a combined SF dict-_like_ structure. If not encoded, these keys would +be sent as headers and lower-cased, losing their comparability against the +original keys. The structure follows all SF dict rules, except that it allows +for keys to contain capitals. The HyperBEAM SF parser will accept these keys, +but standard RFC 8741 parsers will not. Subsequently, the resulting `ao-cased` +key is not added to the `ao-types` map. + + + +### group_maps/1 * ### + +`group_maps(Map) -> any()` + +Merge maps at the same level, if possible. + + + +### group_maps/3 * ### + +`group_maps(Map, Parent, Top) -> any()` + + + +### group_maps_flat_compatible_test/0 * ### + +`group_maps_flat_compatible_test() -> any()` + +The grouped maps encoding is a subset of the flat encoding, +where on keys with maps values are flattened. + +So despite needing a special encoder to produce it +We can simply apply the flat encoder to it to get back +the original message. + +The test asserts that is indeed the case. + + + +### group_maps_test/0 * ### + +`group_maps_test() -> any()` + + + +### hashpaths_from_message/1 * ### + +`hashpaths_from_message(Msg) -> any()` + + + +### inline_key/1 * ### + +`inline_key(Msg) -> any()` + +given a message, returns a binary tuple: +- A list of pairs to add to the msg, if any +- the field name for the inlined key + +In order to preserve the field name of the inlined +part, an additional field may need to be added + + + +### to/1 ### + +`to(Bin) -> any()` + +Convert a TABM into an HTTP Message. The HTTP Message is a simple Erlang Map +that can translated to a given web server Response API + + + +### to/2 * ### + +`to(TABM, Opts) -> any()` + + + +### ungroup_ids/1 * ### + +`ungroup_ids(Msg) -> any()` + +Decode the `ao-ids` key into a map. + + +--- END OF FILE: docs/resources/source-code/dev_codec_httpsig_conv.md --- + +--- START OF FILE: docs/resources/source-code/dev_codec_httpsig.md --- +# [Module dev_codec_httpsig.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_httpsig.erl) + + + + +This module implements HTTP Message Signatures as described in RFC-9421 +(https://datatracker.ietf.org/doc/html/rfc9421), as an AO-Core device. + + + +## Description ## +It implements the codec standard (from/1, to/1), as well as the optional +commitment functions (id/3, sign/3, verify/3). The commitment functions +are found in this module, while the codec functions are relayed to the +`dev_codec_httpsig_conv` module. + + +## Data Types ## + + + + +### authority_state() ### + + +

+authority_state() = #{component_identifiers => [component_identifier()], sig_params => signature_params(), key => binary()}
+
+ + + + +### component_identifier() ### + + +

+component_identifier() = {item, {string, binary()}, {binary(), integer() | boolean() | {string | token | binary, binary()}}}
+
+ + + + +### fields() ### + + +

+fields() = #{binary() | atom() | string() => binary() | atom() | string()}
+
+ + + + +### request_message() ### + + +

+request_message() = #{url => binary(), method => binary(), headers => fields(), trailers => fields(), is_absolute_form => boolean()}
+
+ + + + +### response_message() ### + + +

+response_message() = #{status => integer(), headers => fields(), trailers => fields()}
+
+ + + + +### signature_params() ### + + +

+signature_params() = #{atom() | binary() | string() => binary() | integer()}
+
+ + + +## Function Index ## + + +
add_content_digest/1If the body key is present, replace it with a content-digest.
add_derived_specifiers/1Normalize key parameters to ensure their names are correct.
add_sig_params/2*Add the signature parameters to the authority state.
address_to_sig_name/1*Convert an address to a signature name that is short, unique to the +address, and lowercase.
authority/3*A helper to validate and produce an "Authority" State.
bin/1*
commit/3Main entrypoint for signing a HTTP Message, using the standardized format.
committed/3Return the list of committed keys from a message.
committed_from_body/1*Return the list of committed keys from a message that are derived from +the body components.
committed_id_test/0*
derive_component/3*Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, specifically a "Derived" Component within the Messages Context, +and return the normalized form of the identifier, along with the extracted +encoded value.
derive_component/4*
derive_component_error_query_param_no_name_test/0*
derive_component_error_req_param_on_request_target_test/0*
derive_component_error_status_req_target_test/0*
do_committed/4*
extract_dictionary_field_value/2*Extract a value from a Structured Field, and return the normalized field, +along with the encoded value.
extract_field/3*Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, specifically a field on a Message within the Messages Context, +and return the normalized form of the identifier, along with the extracted +encoded value.
extract_field_value/2*Extract values from the field and return the normalized field, +along with encoded value.
find_byte_sequence_param/1*
find_id/1*Find the ID of the message, which is the hmac of the fields referenced in +the signature and signature input.
find_key_param/1*
find_name_param/1*
find_request_param/1*
find_sf_param/3*Given a parameter Name, extract the Parameter value from the HTTP +Structured Field data structure.
find_strict_format_param/1*
find_trailer_param/1*
from/1
hmac/1*Generate the ID of the message, with the current signature and signature +input as the components for the hmac.
id/3
identifier_to_component/3*Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, and return the normalized form of the identifier, along with the +extracted encoded value.
join_signature_base/2*
join_signature_base_test/0*
lower_bin/1*
multicommitted_id_test/0*
normalize_component_identifiers/1*Takes a list of keys that will be used in the signature inputs and +ensures that they have deterministic sorting, as well as the coorect +component identifiers if applicable.
public_keys/1
remove_derived_specifiers/1Remove derived specifiers from a list of component identifiers.
reset_hmac/1Ensure that the commitments and hmac are properly encoded.
sf_encode/1*Attempt to encode the data structure into an HTTP Structured Field.
sf_encode/2*
sf_item/1*Attempt to parse the provided value into an HTTP Structured Field Item.
sf_parse/1*Attempt to parse the binary into a data structure that represents +an HTTP Structured Field.
sf_parse/2*
sf_signature_param/1*construct the structured field Parameter for the signature parameter, +checking whether the parameter name is valid according RFC-9421.
sf_signature_params/2*construct the structured field List for the +"signature-params-line" part of the signature base.
sig_name_from_dict/1*
sign_auth/3*using the provided Authority and Request/Response Messages Context, +create a Name, Signature and SignatureInput that can be used to additional +signatures to a corresponding HTTP Message.
signature_base/3*create the signature base that will be signed in order to create the +Signature and SignatureInput.
signature_components_line/3*Given a list of Component Identifiers and a Request/Response Message +context, create the "signature-base-line" portion of the signature base.
signature_params_line/2*construct the "signature-params-line" part of the signature base.
signature_params_line_test/0*
to/1
trim_and_normalize/1*
trim_ws/1*Recursively trim space characters from the beginning of the binary.
trim_ws_end/2*
trim_ws_test/0*
upper_bin/1*
validate_large_message_from_http_test/0*Ensure that we can validate a signature on an extremely large and complex +message that is sent over HTTP, signed with the codec.
verify/3Verify different forms of httpsig committed messages.
verify_auth/2*same verify/3, but with an empty Request Message Context.
verify_auth/3*Given the signature name, and the Request/Response Message Context +verify the named signature by constructing the signature base and comparing.
+ + + + +## Function Details ## + + + +### add_content_digest/1 ### + +`add_content_digest(Msg) -> any()` + +If the `body` key is present, replace it with a content-digest. + + + +### add_derived_specifiers/1 ### + +`add_derived_specifiers(ComponentIdentifiers) -> any()` + +Normalize key parameters to ensure their names are correct. + + + +### add_sig_params/2 * ### + +`add_sig_params(Authority, X2) -> any()` + +Add the signature parameters to the authority state + + + +### address_to_sig_name/1 * ### + +

+address_to_sig_name(Address::binary()) -> binary()
+
+
+ +Convert an address to a signature name that is short, unique to the +address, and lowercase. + + + +### authority/3 * ### + +

+authority(ComponentIdentifiers::[binary() | component_identifier()], SigParams::#{binary() => binary() | integer()}, PubKey::{}) -> authority_state()
+
+
+ +A helper to validate and produce an "Authority" State + + + +### bin/1 * ### + +`bin(Item) -> any()` + + + +### commit/3 ### + +`commit(MsgToSign, Req, Opts) -> any()` + +Main entrypoint for signing a HTTP Message, using the standardized format. + + + +### committed/3 ### + +`committed(RawMsg, Req, Opts) -> any()` + +Return the list of committed keys from a message. The message will have +had the `commitments` key removed and the signature inputs added to the +root. Subsequently, we can parse that to get the list of committed keys. + + + +### committed_from_body/1 * ### + +`committed_from_body(Msg) -> any()` + +Return the list of committed keys from a message that are derived from +the body components. + + + +### committed_id_test/0 * ### + +`committed_id_test() -> any()` + + + +### derive_component/3 * ### + +`derive_component(Identifier, Req, Res) -> any()` + +Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, specifically a "Derived" Component within the Messages Context, +and return the normalized form of the identifier, along with the extracted +encoded value. + +This implements a portion of RFC-9421 +See https://datatracker.ietf.org/doc/html/rfc9421#name-derived-components + + + +### derive_component/4 * ### + +`derive_component(X1, Req, Res, Subject) -> any()` + + + +### derive_component_error_query_param_no_name_test/0 * ### + +`derive_component_error_query_param_no_name_test() -> any()` + + + +### derive_component_error_req_param_on_request_target_test/0 * ### + +`derive_component_error_req_param_on_request_target_test() -> any()` + + + +### derive_component_error_status_req_target_test/0 * ### + +`derive_component_error_status_req_target_test() -> any()` + + + +### do_committed/4 * ### + +`do_committed(SigInputStr, Msg, Req, Opts) -> any()` + + + +### extract_dictionary_field_value/2 * ### + +`extract_dictionary_field_value(StructuredField, Key) -> any()` + +Extract a value from a Structured Field, and return the normalized field, +along with the encoded value + + + +### extract_field/3 * ### + +`extract_field(X1, Req, Res) -> any()` + +Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, specifically a field on a Message within the Messages Context, +and return the normalized form of the identifier, along with the extracted +encoded value. + +This implements a portion of RFC-9421 +See https://datatracker.ietf.org/doc/html/rfc9421#name-http-fields + + + +### extract_field_value/2 * ### + +`extract_field_value(RawFields, X2) -> any()` + +Extract values from the field and return the normalized field, +along with encoded value + + + +### find_byte_sequence_param/1 * ### + +`find_byte_sequence_param(Params) -> any()` + + + +### find_id/1 * ### + +`find_id(Msg) -> any()` + +Find the ID of the message, which is the hmac of the fields referenced in +the signature and signature input. If the message already has a signature-input, +directly, it is treated differently: We relabel it as `x-signature-input` to +avoid key collisions. + + + +### find_key_param/1 * ### + +`find_key_param(Params) -> any()` + + + +### find_name_param/1 * ### + +`find_name_param(Params) -> any()` + + + +### find_request_param/1 * ### + +`find_request_param(Params) -> any()` + + + +### find_sf_param/3 * ### + +`find_sf_param(Name, Params, Default) -> any()` + +Given a parameter Name, extract the Parameter value from the HTTP +Structured Field data structure. + +If no value is found, then false is returned + + + +### find_strict_format_param/1 * ### + +`find_strict_format_param(Params) -> any()` + + + +### find_trailer_param/1 * ### + +`find_trailer_param(Params) -> any()` + + + +### from/1 ### + +`from(Msg) -> any()` + + + +### hmac/1 * ### + +`hmac(Msg) -> any()` + +Generate the ID of the message, with the current signature and signature +input as the components for the hmac. + + + +### id/3 ### + +`id(Msg, Params, Opts) -> any()` + + + +### identifier_to_component/3 * ### + +`identifier_to_component(Identifier, Req, Res) -> any()` + +Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, and return the normalized form of the identifier, along with the +extracted encoded value. + +Generally speaking, a Component Identifier may reference a "Derived" Component, +a Message Field, or a sub-component of a Message Field. + +Since a Component Identifier is itself a Structured Field, it may also specify +parameters, which are used to describe behavior such as which Message to +derive a field or sub-component of the field, and how to encode the value as +part of the signature base. + + + +### join_signature_base/2 * ### + +`join_signature_base(ComponentsLine, ParamsLine) -> any()` + + + +### join_signature_base_test/0 * ### + +`join_signature_base_test() -> any()` + + + +### lower_bin/1 * ### + +`lower_bin(Item) -> any()` + + + +### multicommitted_id_test/0 * ### + +`multicommitted_id_test() -> any()` + + + +### normalize_component_identifiers/1 * ### + +`normalize_component_identifiers(ComponentIdentifiers) -> any()` + +Takes a list of keys that will be used in the signature inputs and +ensures that they have deterministic sorting, as well as the coorect +component identifiers if applicable. + + + +### public_keys/1 ### + +`public_keys(Commitment) -> any()` + + + +### remove_derived_specifiers/1 ### + +`remove_derived_specifiers(ComponentIdentifiers) -> any()` + +Remove derived specifiers from a list of component identifiers. + + + +### reset_hmac/1 ### + +`reset_hmac(RawMsg) -> any()` + +Ensure that the commitments and hmac are properly encoded + + + +### sf_encode/1 * ### + +`sf_encode(StructuredField) -> any()` + +Attempt to encode the data structure into an HTTP Structured Field. +This is the inverse of sf_parse. + + + +### sf_encode/2 * ### + +`sf_encode(Serializer, StructuredField) -> any()` + + + +### sf_item/1 * ### + +`sf_item(SfItem) -> any()` + +Attempt to parse the provided value into an HTTP Structured Field Item + + + +### sf_parse/1 * ### + +`sf_parse(Raw) -> any()` + +Attempt to parse the binary into a data structure that represents +an HTTP Structured Field. + +Lacking some sort of "hint", there isn't a way to know which "kind" of +Structured Field the binary is, apriori. So we simply try each parser, +and return the first invocation that doesn't result in an error. + +If no parser is successful, then we return an error tuple + + + +### sf_parse/2 * ### + +`sf_parse(Rest, Raw) -> any()` + + + +### sf_signature_param/1 * ### + +`sf_signature_param(X1) -> any()` + +construct the structured field Parameter for the signature parameter, +checking whether the parameter name is valid according RFC-9421 + +See https://datatracker.ietf.org/doc/html/rfc9421#section-2.3-3 + + + +### sf_signature_params/2 * ### + +`sf_signature_params(ComponentIdentifiers, SigParams) -> any()` + +construct the structured field List for the +"signature-params-line" part of the signature base. + +Can be parsed into a binary by simply passing to hb_structured_fields:list/1 + +See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.3.2.4 + + + +### sig_name_from_dict/1 * ### + +`sig_name_from_dict(DictBin) -> any()` + + + +### sign_auth/3 * ### + +

+sign_auth(Authority::authority_state(), Req::request_message(), Res::response_message()) -> {ok, {binary(), binary(), binary()}}
+
+
+ +using the provided Authority and Request/Response Messages Context, +create a Name, Signature and SignatureInput that can be used to additional +signatures to a corresponding HTTP Message + + + +### signature_base/3 * ### + +`signature_base(Authority, Req, Res) -> any()` + +create the signature base that will be signed in order to create the +Signature and SignatureInput. + +This implements a portion of RFC-9421 see: +https://datatracker.ietf.org/doc/html/rfc9421#name-creating-the-signature-base + + + +### signature_components_line/3 * ### + +`signature_components_line(ComponentIdentifiers, Req, Res) -> any()` + +Given a list of Component Identifiers and a Request/Response Message +context, create the "signature-base-line" portion of the signature base + + + +### signature_params_line/2 * ### + +`signature_params_line(ComponentIdentifiers, SigParams) -> any()` + +construct the "signature-params-line" part of the signature base. + +See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.3.2.4 + + + +### signature_params_line_test/0 * ### + +`signature_params_line_test() -> any()` + + + +### to/1 ### + +`to(Msg) -> any()` + + + +### trim_and_normalize/1 * ### + +`trim_and_normalize(Bin) -> any()` + + + +### trim_ws/1 * ### + +`trim_ws(Bin) -> any()` + +Recursively trim space characters from the beginning of the binary + + + +### trim_ws_end/2 * ### + +`trim_ws_end(Value, N) -> any()` + + + +### trim_ws_test/0 * ### + +`trim_ws_test() -> any()` + + + +### upper_bin/1 * ### + +`upper_bin(Item) -> any()` + + + +### validate_large_message_from_http_test/0 * ### + +`validate_large_message_from_http_test() -> any()` + +Ensure that we can validate a signature on an extremely large and complex +message that is sent over HTTP, signed with the codec. + + + +### verify/3 ### + +`verify(MsgToVerify, Req, Opts) -> any()` + +Verify different forms of httpsig committed messages. `dev_message:verify` +already places the keys from the commitment message into the root of the +message. + + + +### verify_auth/2 * ### + +`verify_auth(Verifier, Msg) -> any()` + +same verify/3, but with an empty Request Message Context + + + +### verify_auth/3 * ### + +`verify_auth(X1, Req, Res) -> any()` + +Given the signature name, and the Request/Response Message Context +verify the named signature by constructing the signature base and comparing + + +--- END OF FILE: docs/resources/source-code/dev_codec_httpsig.md --- + +--- START OF FILE: docs/resources/source-code/dev_codec_json.md --- +# [Module dev_codec_json.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_json.erl) + + + + +A simple JSON codec for HyperBEAM's message format. + + + +## Description ## +Takes a +message as TABM and returns an encoded JSON string representation. +This codec utilizes the httpsig@1.0 codec for signing and verifying. + +## Function Index ## + + +
commit/3
committed/1
content_type/1Return the content type for the codec.
deserialize/3Deserialize the JSON string found at the given path.
from/1Decode a JSON string to a message.
serialize/3Serialize a message to a JSON string.
to/1Encode a message to a JSON string.
verify/3
+ + + + +## Function Details ## + + + +### commit/3 ### + +`commit(Msg, Req, Opts) -> any()` + + + +### committed/1 ### + +`committed(Msg) -> any()` + + + +### content_type/1 ### + +`content_type(X1) -> any()` + +Return the content type for the codec. + + + +### deserialize/3 ### + +`deserialize(Base, Req, Opts) -> any()` + +Deserialize the JSON string found at the given path. + + + +### from/1 ### + +`from(Map) -> any()` + +Decode a JSON string to a message. + + + +### serialize/3 ### + +`serialize(Base, Msg, Opts) -> any()` + +Serialize a message to a JSON string. + + + +### to/1 ### + +`to(Msg) -> any()` + +Encode a message to a JSON string. + + + +### verify/3 ### + +`verify(Msg, Req, Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_codec_json.md --- + +--- START OF FILE: docs/resources/source-code/dev_codec_structured.md --- +# [Module dev_codec_structured.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_structured.erl) + + + + +A device implementing the codec interface (to/1, from/1) for +HyperBEAM's internal, richly typed message format. + + + +## Description ## + +This format mirrors HTTP Structured Fields, aside from its limitations of +compound type depths, as well as limited floating point representations. + +As with all AO-Core codecs, its target format (the format it expects to +receive in the `to/1` function, and give in `from/1`) is TABM. + +For more details, see the HTTP Structured Fields (RFC-9651) specification. + +## Function Index ## + + +
commit/3
committed/3
decode_value/2Convert non-binary values to binary for serialization.
encode_value/1Convert a term to a binary representation, emitting its type for +serialization as a separate tag.
from/1Convert a rich message into a 'Type-Annotated-Binary-Message' (TABM).
implicit_keys/1Find the implicit keys of a TABM.
list_encoding_test/0*
parse_ao_types/1*Parse the ao-types field of a TABM and return a map of keys and their +types.
to/1Convert a TABM into a native HyperBEAM message.
verify/3
+ + + + +## Function Details ## + + + +### commit/3 ### + +`commit(Msg, Req, Opts) -> any()` + + + +### committed/3 ### + +`committed(Msg, Req, Opts) -> any()` + + + +### decode_value/2 ### + +`decode_value(Type, Value) -> any()` + +Convert non-binary values to binary for serialization. + + + +### encode_value/1 ### + +`encode_value(Value) -> any()` + +Convert a term to a binary representation, emitting its type for +serialization as a separate tag. + + + +### from/1 ### + +`from(Bin) -> any()` + +Convert a rich message into a 'Type-Annotated-Binary-Message' (TABM). + + + +### implicit_keys/1 ### + +`implicit_keys(Req) -> any()` + +Find the implicit keys of a TABM. + + + +### list_encoding_test/0 * ### + +`list_encoding_test() -> any()` + + + +### parse_ao_types/1 * ### + +`parse_ao_types(Msg) -> any()` + +Parse the `ao-types` field of a TABM and return a map of keys and their +types + + + +### to/1 ### + +`to(Bin) -> any()` + +Convert a TABM into a native HyperBEAM message. + + + +### verify/3 ### + +`verify(Msg, Req, Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_codec_structured.md --- + +--- START OF FILE: docs/resources/source-code/dev_cron.md --- +# [Module dev_cron.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cron.erl) + + + + +A device that inserts new messages into the schedule to allow processes +to passively 'call' themselves without user interaction. + + + +## Function Index ## + + +
every/3Exported function for scheduling a recurring message.
every_worker_loop/4*
every_worker_loop_test/0*This test verifies that a recurring task can be scheduled and executed.
info/1Exported function for getting device info.
info/3
once/3Exported function for scheduling a one-time message.
once_executed_test/0*This test verifies that a one-time task can be scheduled and executed.
once_worker/3*Internal function for scheduling a one-time message.
parse_time/1*Parse a time string into milliseconds.
stop/3Exported function for stopping a scheduled task.
stop_every_test/0*This test verifies that a recurring task can be stopped by +calling the stop function with the task ID.
stop_once_test/0*
test_worker/0*This is a helper function that is used to test the cron device.
test_worker/1*
+ + + + +## Function Details ## + + + +### every/3 ### + +`every(Msg1, Msg2, Opts) -> any()` + +Exported function for scheduling a recurring message. + + + +### every_worker_loop/4 * ### + +`every_worker_loop(CronPath, Req, Opts, IntervalMillis) -> any()` + + + +### every_worker_loop_test/0 * ### + +`every_worker_loop_test() -> any()` + +This test verifies that a recurring task can be scheduled and executed. + + + +### info/1 ### + +`info(X1) -> any()` + +Exported function for getting device info. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + + + +### once/3 ### + +`once(Msg1, Msg2, Opts) -> any()` + +Exported function for scheduling a one-time message. + + + +### once_executed_test/0 * ### + +`once_executed_test() -> any()` + +This test verifies that a one-time task can be scheduled and executed. + + + +### once_worker/3 * ### + +`once_worker(Path, Req, Opts) -> any()` + +Internal function for scheduling a one-time message. + + + +### parse_time/1 * ### + +`parse_time(BinString) -> any()` + +Parse a time string into milliseconds. + + + +### stop/3 ### + +`stop(Msg1, Msg2, Opts) -> any()` + +Exported function for stopping a scheduled task. + + + +### stop_every_test/0 * ### + +`stop_every_test() -> any()` + +This test verifies that a recurring task can be stopped by +calling the stop function with the task ID. + + + +### stop_once_test/0 * ### + +`stop_once_test() -> any()` + + + +### test_worker/0 * ### + +`test_worker() -> any()` + +This is a helper function that is used to test the cron device. +It is used to increment a counter and update the state of the worker. + + + +### test_worker/1 * ### + +`test_worker(State) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_cron.md --- + +--- START OF FILE: docs/resources/source-code/dev_cu.md --- +# [Module dev_cu.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cu.erl) + + + + + + +## Function Index ## + + +
execute/2
push/2
+ + + + +## Function Details ## + + + +### execute/2 ### + +`execute(CarrierMsg, S) -> any()` + + + +### push/2 ### + +`push(Msg, S) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_cu.md --- + +--- START OF FILE: docs/resources/source-code/dev_dedup.md --- +# [Module dev_dedup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_dedup.erl) + + + + +A device that deduplicates messages send to a process. + + + +## Description ## +Only runs on the first pass of the `compute` key call if executed +in a stack. Currently the device stores its list of already seen +items in memory, but at some point it will likely make sense to +drop them in the cache. + +## Function Index ## + + +
dedup_test/0*
dedup_with_multipass_test/0*
handle/4*Forward the keys function to the message device, handle all others +with deduplication.
info/1
+ + + + +## Function Details ## + + + +### dedup_test/0 * ### + +`dedup_test() -> any()` + + + +### dedup_with_multipass_test/0 * ### + +`dedup_with_multipass_test() -> any()` + + + +### handle/4 * ### + +`handle(Key, M1, M2, Opts) -> any()` + +Forward the keys function to the message device, handle all others +with deduplication. We only act on the first pass. + + + +### info/1 ### + +`info(M1) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_dedup.md --- + +--- START OF FILE: docs/resources/source-code/dev_delegated_compute.md --- +# [Module dev_delegated_compute.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_delegated_compute.erl) + + + + +Simple wrapper module that enables compute on remote machines, +implementing the JSON-Iface. + + + +## Description ## +This can be used either as a standalone, to +bring trusted results into the local node, or as the `Execution-Device` of +an AO process. + +## Function Index ## + + +
compute/3
do_compute/3*Execute computation on a remote machine via relay and the JSON-Iface.
init/3Initialize or normalize the compute-lite device.
normalize/3
snapshot/3
+ + + + +## Function Details ## + + + +### compute/3 ### + +`compute(Msg1, Msg2, Opts) -> any()` + + + +### do_compute/3 * ### + +`do_compute(ProcID, Msg2, Opts) -> any()` + +Execute computation on a remote machine via relay and the JSON-Iface. + + + +### init/3 ### + +`init(Msg1, Msg2, Opts) -> any()` + +Initialize or normalize the compute-lite device. For now, we don't +need to do anything special here. + + + +### normalize/3 ### + +`normalize(Msg1, Msg2, Opts) -> any()` + + + +### snapshot/3 ### + +`snapshot(Msg1, Msg2, Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_delegated_compute.md --- + +--- START OF FILE: docs/resources/source-code/dev_faff.md --- +# [Module dev_faff.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_faff.erl) + + + + +A module that implements a 'friends and family' pricing policy. + + + +## Description ## + +It will allow users to process requests only if their addresses are +in the allow-list for the node. + +Fundamentally against the spirit of permissionlessness, but it is useful if +you are running a node for your own purposes and would not like to allow +others to make use of it -- even for a fee. It also serves as a useful +example of how to implement a custom pricing policy, as it implements stubs +for both the pricing and ledger P4 APIs. + +## Function Index ## + + +
debit/3Debit the user's account if the request is allowed.
estimate/3Decide whether or not to service a request from a given address.
is_admissible/2*Check whether all of the signers of the request are in the allow-list.
+ + + + +## Function Details ## + + + +### debit/3 ### + +`debit(X1, Req, NodeMsg) -> any()` + +Debit the user's account if the request is allowed. + + + +### estimate/3 ### + +`estimate(X1, Msg, NodeMsg) -> any()` + +Decide whether or not to service a request from a given address. + + + +### is_admissible/2 * ### + +`is_admissible(Msg, NodeMsg) -> any()` + +Check whether all of the signers of the request are in the allow-list. + + +--- END OF FILE: docs/resources/source-code/dev_faff.md --- + +--- START OF FILE: docs/resources/source-code/dev_genesis_wasm.md --- +# [Module dev_genesis_wasm.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_genesis_wasm.erl) + + + + +A device that mimics an environment suitable for `legacynet` AO +processes, using HyperBEAM infrastructure. + + + +## Description ## +This allows existing `legacynet` +AO process definitions to be used in HyperBEAM. + +## Function Index ## + + +
collect_events/1*Collect events from the port and log them.
collect_events/2*
compute/3All the delegated-compute@1.0 device to execute the request.
ensure_started/1*Ensure the local genesis-wasm@1.0 is live.
init/3Initialize the device.
is_genesis_wasm_server_running/1*Check if the genesis-wasm server is running, using the cached process ID +if available.
log_server_events/1*Log lines of output from the genesis-wasm server.
normalize/3Normalize the device.
snapshot/3Snapshot the device.
status/1*Check if the genesis-wasm server is running by requesting its status +endpoint.
+ + + + +## Function Details ## + + + +### collect_events/1 * ### + +`collect_events(Port) -> any()` + +Collect events from the port and log them. + + + +### collect_events/2 * ### + +`collect_events(Port, Acc) -> any()` + + + +### compute/3 ### + +`compute(Msg, Msg2, Opts) -> any()` + +All the `delegated-compute@1.0` device to execute the request. We then apply +the `patch@1.0` device, applying any state patches that the AO process may have +requested. + + + +### ensure_started/1 * ### + +`ensure_started(Opts) -> any()` + +Ensure the local `genesis-wasm@1.0` is live. If it not, start it. + + + +### init/3 ### + +`init(Msg, Msg2, Opts) -> any()` + +Initialize the device. + + + +### is_genesis_wasm_server_running/1 * ### + +`is_genesis_wasm_server_running(Opts) -> any()` + +Check if the genesis-wasm server is running, using the cached process ID +if available. + + + +### log_server_events/1 * ### + +`log_server_events(Bin) -> any()` + +Log lines of output from the genesis-wasm server. + + + +### normalize/3 ### + +`normalize(Msg, Msg2, Opts) -> any()` + +Normalize the device. + + + +### snapshot/3 ### + +`snapshot(Msg, Msg2, Opts) -> any()` + +Snapshot the device. + + + +### status/1 * ### + +`status(Opts) -> any()` + +Check if the genesis-wasm server is running by requesting its status +endpoint. + + +--- END OF FILE: docs/resources/source-code/dev_genesis_wasm.md --- + +--- START OF FILE: docs/resources/source-code/dev_green_zone.md --- +# [Module dev_green_zone.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_green_zone.erl) + + + + +The green zone device, which provides secure communication and identity +management between trusted nodes. + + + +## Description ## +It handles node initialization, joining existing green zones, key exchange, +and node identity cloning. All operations are protected by hardware +commitment and encryption. + +## Function Index ## + + +
add_trusted_node/4*Adds a node to the trusted nodes list with its commitment report.
become/3Clones the identity of a target node in the green zone.
calculate_node_message/3*Generate the node message that should be set prior to joining +a green zone.
decrypt_zone_key/2*Decrypts an AES key using the node's RSA private key.
default_zone_required_opts/1*Provides the default required options for a green zone.
encrypt_payload/2*Encrypts an AES key with a node's RSA public key.
finalize_become/5*
info/1Controls which functions are exposed via the device API.
info/3Provides information about the green zone device and its API.
init/3Initialize the green zone for a node.
join/3Initiates the join process for a node to enter an existing green zone.
join_peer/5*Processes a join request to a specific peer node.
key/3Encrypts and provides the node's private key for secure sharing.
maybe_set_zone_opts/4*Adopts configuration from a peer when joining a green zone.
rsa_wallet_integration_test/0*Test RSA operations with the existing wallet structure.
try_mount_encrypted_volume/2*Attempts to mount an encrypted volume using the green zone AES key.
validate_join/3*Validates an incoming join request from another node.
validate_peer_opts/2*Validates that a peer's configuration matches required options.
+ + + + +## Function Details ## + + + +### add_trusted_node/4 * ### + +

+add_trusted_node(NodeAddr::binary(), Report::map(), RequesterPubKey::term(), Opts::map()) -> ok
+
+
+ +`NodeAddr`: The joining node's address
`Report`: The commitment report provided by the joining node
`RequesterPubKey`: The joining node's public key
`Opts`: A map of configuration options
+ +returns: ok + +Adds a node to the trusted nodes list with its commitment report. + +This function updates the trusted nodes configuration: +1. Retrieves the current trusted nodes map +2. Adds the new node with its report and public key +3. Updates the node configuration with the new trusted nodes list + + + +### become/3 ### + +

+become(M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options
+ +returns: `{ok, Map}` on success with confirmation details, or +`{error, Binary}` if the node is not part of a green zone or +identity adoption fails. + +Clones the identity of a target node in the green zone. + +This function performs the following operations: +1. Retrieves target node location and ID from the configuration +2. Verifies that the local node has a valid shared AES key +3. Requests the target node's encrypted key via its key endpoint +4. Verifies the response is from the expected peer +5. Decrypts the target node's private key using the shared AES key +6. Updates the local node's wallet with the target node's identity + +Required configuration in Opts map: +- green_zone_peer_location: Target node's address +- green_zone_peer_id: Target node's unique identifier +- priv_green_zone_aes: The shared AES key for the green zone + + + +### calculate_node_message/3 * ### + +`calculate_node_message(RequiredOpts, Req, List) -> any()` + +Generate the node message that should be set prior to joining +a green zone. + +This function takes a required opts message, a request message, and an +`adopt-config` value. The `adopt-config` value can be a boolean, a list of +fields that should be included in the node message from the request, or a +binary string of fields to include, separated by commas. + + + +### decrypt_zone_key/2 * ### + +

+decrypt_zone_key(EncZoneKey::binary(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`EncZoneKey`: The encrypted zone AES key (Base64 encoded or binary)
`Opts`: A map of configuration options
+ +returns: {ok, DecryptedKey} on success with the decrypted AES key + +Decrypts an AES key using the node's RSA private key. + +This function handles decryption of the zone key: +1. Decodes the encrypted key if it's in Base64 format +2. Extracts the RSA private key components from the wallet +3. Creates an RSA private key record +4. Performs private key decryption on the encrypted key + + + +### default_zone_required_opts/1 * ### + +

+default_zone_required_opts(Opts::map()) -> map()
+
+
+ +`Opts`: A map of configuration options from which to derive defaults
+ +returns: A map of required configuration options for the green zone + +Provides the default required options for a green zone. + +This function defines the baseline security requirements for nodes in a green zone: +1. Restricts loading of remote devices and only allows trusted signers +2. Limits to preloaded devices from the initiating machine +3. Enforces specific store configuration +4. Prevents route changes from the defaults +5. Requires matching hooks across all peers +6. Disables message scheduling to prevent conflicts +7. Enforces a permanent state to prevent further configuration changes + + + +### encrypt_payload/2 * ### + +

+encrypt_payload(AESKey::binary(), RequesterPubKey::term()) -> binary()
+
+
+ +`AESKey`: The shared AES key (256-bit binary)
`RequesterPubKey`: The node's public RSA key
+ +returns: The encrypted AES key + +Encrypts an AES key with a node's RSA public key. + +This function securely encrypts the shared key for transmission: +1. Extracts the RSA public key components +2. Creates an RSA public key record +3. Performs public key encryption on the AES key + + + +### finalize_become/5 * ### + +`finalize_become(KeyResp, NodeLocation, NodeID, GreenZoneAES, Opts) -> any()` + + + +### info/1 ### + +`info(X1) -> any()` + +Controls which functions are exposed via the device API. + +This function defines the security boundary for the green zone device by +explicitly listing which functions are available through the API. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + +Provides information about the green zone device and its API. + +This function returns detailed documentation about the device, including: +1. A high-level description of the device's purpose +2. Version information +3. Available API endpoints with their parameters and descriptions + + + +### init/3 ### + +

+init(M1::term(), M2::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options
+ +returns: `{ok, Binary}` on success with confirmation message, or +`{error, Binary}` on failure with error message. + +Initialize the green zone for a node. + +This function performs the following operations: +1. Validates the node's history to ensure this is a valid initialization +2. Retrieves or creates a required configuration for the green zone +3. Ensures a wallet (keypair) exists or creates a new one +4. Generates a new 256-bit AES key for secure communication +5. Updates the node's configuration with these cryptographic identities + +Config options in Opts map: +- green_zone_required_config: (Optional) Custom configuration requirements +- priv_wallet: (Optional) Existing wallet to use instead of creating a new one +- priv_green_zone_aes: (Optional) Existing AES key, if already part of a zone + + + +### join/3 ### + +

+join(M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`M1`: The join request message with target peer information
`M2`: Additional request details, may include adoption preferences
`Opts`: A map of configuration options for join operations
+ +returns: `{ok, Map}` on success with join response details, or +`{error, Binary}` on failure with error message. + +Initiates the join process for a node to enter an existing green zone. + +This function performs the following operations depending on the state: +1. Validates the node's history to ensure proper initialization +2. Checks for target peer information (location and ID) +3. If target peer is specified: +a. Generates a commitment report for the peer +b. Prepares and sends a POST request to the target peer +c. Verifies the response and decrypts the returned zone key +d. Updates local configuration with the shared AES key +4. If no peer is specified, processes the join request locally + +Config options in Opts map: +- green_zone_peer_location: Target peer's address +- green_zone_peer_id: Target peer's unique identifier +- green_zone_adopt_config: +(Optional) Whether to adopt peer's configuration (default: true) + + + +### join_peer/5 * ### + +

+join_peer(PeerLocation::binary(), PeerID::binary(), M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, map() | binary()}
+
+
+ +`PeerLocation`: The target peer's address
`PeerID`: The target peer's unique identifier
`M2`: May contain ShouldMount flag to enable encrypted volume mounting
+ +returns: `{ok, Map}` on success with confirmation message, or +`{error, Map|Binary}` on failure with error details + +Processes a join request to a specific peer node. + +This function handles the client-side join flow when connecting to a peer: +1. Verifies the node is not already in a green zone +2. Optionally adopts configuration from the target peer +3. Generates a hardware-backed commitment report +4. Sends a POST request to the peer's join endpoint +5. Verifies the response signature +6. Decrypts the returned AES key +7. Updates local configuration with the shared key +8. Optionally mounts an encrypted volume using the shared key + + + +### key/3 ### + +

+key(M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options
+ +returns: `{ok, Map}` containing the encrypted key and IV on success, or +`{error, Binary}` if the node is not part of a green zone + +Encrypts and provides the node's private key for secure sharing. + +This function performs the following operations: +1. Retrieves the shared AES key and the node's wallet +2. Verifies that the node is part of a green zone (has a shared AES key) +3. Generates a random initialization vector (IV) for encryption +4. Encrypts the node's private key using AES-256-GCM with the shared key +5. Returns the encrypted key and IV for secure transmission + +Required configuration in Opts map: +- priv_green_zone_aes: The shared AES key for the green zone +- priv_wallet: The node's wallet containing the private key to encrypt + + + +### maybe_set_zone_opts/4 * ### + +

+maybe_set_zone_opts(PeerLocation::binary(), PeerID::binary(), Req::map(), InitOpts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`PeerLocation`: The location of the peer node to join
`PeerID`: The ID of the peer node to join
`Req`: The request message with adoption preferences
`InitOpts`: A map of initial configuration options
+ +returns: `{ok, Map}` with updated configuration on success, or +`{error, Binary}` if configuration retrieval fails + +Adopts configuration from a peer when joining a green zone. + +This function handles the conditional adoption of peer configuration: +1. Checks if adoption is enabled (default: true) +2. Requests required configuration from the peer +3. Verifies the authenticity of the configuration +4. Creates a node message with appropriate settings +5. Updates the local node configuration + +Config options: +- green_zone_adopt_config: Controls configuration adoption (boolean, list, or binary) + + + +### rsa_wallet_integration_test/0 * ### + +`rsa_wallet_integration_test() -> any()` + +Test RSA operations with the existing wallet structure. + +This test function verifies that encryption and decryption using the RSA keys +from the wallet work correctly. It creates a new wallet, encrypts a test +message with the RSA public key, and then decrypts it with the RSA private +key, asserting that the decrypted message matches the original. + + + +### try_mount_encrypted_volume/2 * ### + +`try_mount_encrypted_volume(AESKey, Opts) -> any()` + +Attempts to mount an encrypted volume using the green zone AES key. + +This function handles the complete process of secure storage setup by +delegating to the dev_volume module, which provides a unified interface +for volume management. + +The encryption key used for the volume is the same AES key used for green zone +communication, ensuring that only nodes in the green zone can access the data. + + + +### validate_join/3 * ### + +

+validate_join(M1::term(), Req::map(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`M1`: Ignored parameter
`Req`: The join request containing commitment report and public key
`Opts`: A map of configuration options
+ +returns: `{ok, Map}` on success with encrypted AES key, or +`{error, Binary}` on failure with error message + +Validates an incoming join request from another node. + +This function handles the server-side join flow when receiving a connection +request: +1. Validates the peer's configuration meets required standards +2. Extracts the commitment report and public key from the request +3. Verifies the hardware-backed commitment report +4. Adds the joining node to the trusted nodes list +5. Encrypts the shared AES key with the peer's public key +6. Returns the encrypted key to the requesting node + + + +### validate_peer_opts/2 * ### + +

+validate_peer_opts(Req::map(), Opts::map()) -> boolean()
+
+
+ +`Req`: The request message containing the peer's configuration
`Opts`: A map of the local node's configuration options
+ +returns: true if the peer's configuration is valid, false otherwise + +Validates that a peer's configuration matches required options. + +This function ensures the peer node meets configuration requirements: +1. Retrieves the local node's required configuration +2. Gets the peer's options from its message +3. Adds required configuration to peer's required options list +4. Verifies the peer's node history is valid +5. Checks that the peer's options match the required configuration + + +--- END OF FILE: docs/resources/source-code/dev_green_zone.md --- + +--- START OF FILE: docs/resources/source-code/dev_hook.md --- +# [Module dev_hook.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_hook.erl) + + + + +A generalized interface for `hooking` into HyperBEAM nodes. + + + +## Description ## + +This module allows users to define `hooks` that are executed at various +points in the lifecycle of nodes and message evaluations. + +Hooks are maintained in the `node message` options, under the key `on` +key. Each `hook` may have zero or many `handlers` which their request is +executed against. A new `handler` of a hook can be registered by simply +adding a new key to that message. If multiple hooks need to be executed for +a single event, the key's value can be set to a list of hooks. + +`hook`s themselves do not need to be added explicitly. Any device can add +a hook by simply executing `dev_hook:on(HookName, Req, Opts)`. This +function is does not affect the hashpath of a message and is not exported on +the device`s API, such that it is not possible to call it directly with +AO-Core resolution. + +All handlers are expressed in the form of a message, upon which the hook's +request is evaluated: + +AO(HookMsg, Req, Opts) => {Status, Result} + +The `Status` and `Result` of the evaluation can be used at the `hook` caller's +discretion. If multiple handlers are to be executed for a single `hook`, the +result of each is used as the input to the next, on the assumption that the +status of the previous is `ok`. If a non-`ok` status is encountered, the +evaluation is halted and the result is returned to the caller. This means +that in most cases, hooks take the form of chainable pipelines of functions, +passing the most pertinent data in the `body` key of both the request and +result. Hook definitions can also set the `hook/result` key to `ignore`, if +the result of the execution should be discarded and the prior value (the +input to the hook) should be used instead. The `hook/commit-request` key can +also be set to `true` if the request should be committed by the node before +execution of the hook. + +The default HyperBEAM node implements several useful hooks. They include: + +start: Executed when the node starts. +Req/body: The node's initial configuration. +Result/body: The node's possibly updated configuration. +request: Executed when a request is received via the HTTP API. +Req/body: The sequence of messages that the node will evaluate. +Req/request: The raw, unparsed singleton request. +Result/body: The sequence of messages that the node will evaluate. +step: Executed after each message in a sequence has been evaluated. +Req/body: The result of the evaluation. +Result/body: The result of the evaluation. +response: Executed when a response is sent via the HTTP API. +Req/body: The result of the evaluation. +Req/request: The raw, unparsed singleton request that was used to +generate the response. +Result/body: The message to be sent in response to the request. + +Additionally, this module implements a traditional device API, allowing the +node operator to register hooks to the node and find those that are +currently active. + +## Function Index ## + + +
execute_handler/4*Execute a single handler +Handlers are expressed as messages that can be resolved via AO.
execute_handlers/4*Execute a list of handlers in sequence.
find/2Get all handlers for a specific hook from the node message options.
find/3
halt_on_error_test/0*Test that pipeline execution halts on error.
info/1Device API information.
multiple_handlers_test/0*Test that multiple handlers form a pipeline.
no_handlers_test/0*Test that hooks with no handlers return the original request.
on/3Execute a named hook with the provided request and options +This function finds all handlers for the hook and evaluates them in sequence.
single_handler_test/0*Test that a single handler is executed correctly.
+ + + + +## Function Details ## + + + +### execute_handler/4 * ### + +`execute_handler(HookName, Handler, Req, Opts) -> any()` + +Execute a single handler +Handlers are expressed as messages that can be resolved via AO. + + + +### execute_handlers/4 * ### + +`execute_handlers(HookName, Rest, Req, Opts) -> any()` + +Execute a list of handlers in sequence. +The result of each handler is used as input to the next handler. +If a handler returns a non-ok status, execution is halted. + + + +### find/2 ### + +`find(HookName, Opts) -> any()` + +Get all handlers for a specific hook from the node message options. +Handlers are stored in the `on` key of this message. The `find/2` variant of +this function only takes a hook name and node message, and is not called +directly via the device API. Instead it is used by `on/3` and other internal +functionality to find handlers when necessary. The `find/3` variant can, +however, be called directly via the device API. + + + +### find/3 ### + +`find(Base, Req, Opts) -> any()` + + + +### halt_on_error_test/0 * ### + +`halt_on_error_test() -> any()` + +Test that pipeline execution halts on error + + + +### info/1 ### + +`info(X1) -> any()` + +Device API information + + + +### multiple_handlers_test/0 * ### + +`multiple_handlers_test() -> any()` + +Test that multiple handlers form a pipeline + + + +### no_handlers_test/0 * ### + +`no_handlers_test() -> any()` + +Test that hooks with no handlers return the original request + + + +### on/3 ### + +`on(HookName, Req, Opts) -> any()` + +Execute a named hook with the provided request and options +This function finds all handlers for the hook and evaluates them in sequence. +The result of each handler is used as input to the next handler. + + + +### single_handler_test/0 * ### + +`single_handler_test() -> any()` + +Test that a single handler is executed correctly + + +--- END OF FILE: docs/resources/source-code/dev_hook.md --- + +--- START OF FILE: docs/resources/source-code/dev_hyperbuddy.md --- +# [Module dev_hyperbuddy.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_hyperbuddy.erl) + + + + +A device that renders a REPL-like interface for AO-Core via HTML. + + + +## Function Index ## + + +
format/3Employ HyperBEAM's internal pretty printer to format a message.
info/0Export an explicit list of files via http.
metrics/3The main HTML page for the REPL device.
return_file/1*Read a file from disk and serve it as a static HTML page.
serve/4*Serve a file from the priv directory.
+ + + + +## Function Details ## + + + +### format/3 ### + +`format(Base, X2, X3) -> any()` + +Employ HyperBEAM's internal pretty printer to format a message. + + + +### info/0 ### + +`info() -> any()` + +Export an explicit list of files via http. + + + +### metrics/3 ### + +`metrics(X1, Req, Opts) -> any()` + +The main HTML page for the REPL device. + + + +### return_file/1 * ### + +`return_file(Name) -> any()` + +Read a file from disk and serve it as a static HTML page. + + + +### serve/4 * ### + +`serve(Key, M1, M2, Opts) -> any()` + +Serve a file from the priv directory. Only serves files that are explicitly +listed in the `routes` field of the `info/0` return value. + + +--- END OF FILE: docs/resources/source-code/dev_hyperbuddy.md --- + +--- START OF FILE: docs/resources/source-code/dev_json_iface.md --- +# [Module dev_json_iface.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_json_iface.erl) + + + + +A device that provides a way for WASM execution to interact with +the HyperBEAM (and AO) systems, using JSON as a shared data representation. + + + +## Description ## + +The interface is easy to use. It works as follows: + +1. The device is given a message that contains a process definition, WASM +environment, and a message that contains the data to be processed, +including the image to be used in part of `execute{pass=1}`. +2. The device is called with `execute{pass=2}`, which reads the result of +the process execution from the WASM environment and adds it to the +message. + +The device has the following requirements and interface: + +``` + + M1/Computed when /Pass == 1 -> + Assumes: + M1/priv/wasm/instance + M1/Process + M2/Message + M2/Assignment/Block-Height + Generates: + /wasm/handler + /wasm/params + Side-effects: + Writes the process and message as JSON representations into the + WASM environment. + M1/Computed when M2/Pass == 2 -> + Assumes: + M1/priv/wasm/instance + M2/Results + M2/Process + Generates: + /Results/Outbox + /Results/Data +``` + + +## Function Index ## + + +
aos_stack_benchmark_test_/0*
basic_aos_call_test_/0*
compute/3On first pass prepare the call, on second pass get the results.
denormalize_message/1*Normalize a message for AOS-compatibility.
env_read/3*Read the results out of the execution environment.
env_write/5*Write the message and process into the execution environment.
generate_aos_msg/2
generate_stack/1
generate_stack/2
header_case_string/1*
init/3Initialize the device.
json_to_message/2Translates a compute result -- either from a WASM execution using the +JSON-Iface, or from a Legacy CU -- and transforms it into a result message.
maybe_list_to_binary/1*
message_to_json_struct/1
message_to_json_struct/2*
normalize_results/1*Normalize the results of an evaluation.
postprocess_outbox/3*Post-process messages in the outbox to add the correct from-process +and from-image tags.
prep_call/3*Prepare the WASM environment for execution by writing the process string and +the message as JSON representations into the WASM environment.
prepare_header_case_tags/1*Convert a message without an original-tags field into a list of +key-value pairs, with the keys in HTTP header-case.
prepare_tags/1*Prepare the tags of a message as a key-value list, for use in the +construction of the JSON-Struct message.
preprocess_results/2*After the process returns messages from an evaluation, the +signing node needs to add some tags to each message and spawn such that +the target process knows these messages are created by a process.
results/3*Read the computed results out of the WASM environment, assuming that +the environment has been set up by prep_call/3 and that the WASM executor +has been called with computed{pass=1}.
safe_to_id/1*
tags_to_map/1*Convert a message with tags into a map of their key-value pairs.
test_init/0*
+ + + + +## Function Details ## + + + +### aos_stack_benchmark_test_/0 * ### + +`aos_stack_benchmark_test_() -> any()` + + + +### basic_aos_call_test_/0 * ### + +`basic_aos_call_test_() -> any()` + + + +### compute/3 ### + +`compute(M1, M2, Opts) -> any()` + +On first pass prepare the call, on second pass get the results. + + + +### denormalize_message/1 * ### + +`denormalize_message(Message) -> any()` + +Normalize a message for AOS-compatibility. + + + +### env_read/3 * ### + +`env_read(M1, M2, Opts) -> any()` + +Read the results out of the execution environment. + + + +### env_write/5 * ### + +`env_write(ProcessStr, MsgStr, Base, Req, Opts) -> any()` + +Write the message and process into the execution environment. + + + +### generate_aos_msg/2 ### + +`generate_aos_msg(ProcID, Code) -> any()` + + + +### generate_stack/1 ### + +`generate_stack(File) -> any()` + + + +### generate_stack/2 ### + +`generate_stack(File, Mode) -> any()` + + + +### header_case_string/1 * ### + +`header_case_string(Key) -> any()` + + + +### init/3 ### + +`init(M1, M2, Opts) -> any()` + +Initialize the device. + + + +### json_to_message/2 ### + +`json_to_message(JSON, Opts) -> any()` + +Translates a compute result -- either from a WASM execution using the +JSON-Iface, or from a `Legacy` CU -- and transforms it into a result message. + + + +### maybe_list_to_binary/1 * ### + +`maybe_list_to_binary(List) -> any()` + + + +### message_to_json_struct/1 ### + +`message_to_json_struct(RawMsg) -> any()` + + + +### message_to_json_struct/2 * ### + +`message_to_json_struct(RawMsg, Features) -> any()` + + + +### normalize_results/1 * ### + +`normalize_results(Msg) -> any()` + +Normalize the results of an evaluation. + + + +### postprocess_outbox/3 * ### + +`postprocess_outbox(Msg, Proc, Opts) -> any()` + +Post-process messages in the outbox to add the correct `from-process` +and `from-image` tags. + + + +### prep_call/3 * ### + +`prep_call(M1, M2, Opts) -> any()` + +Prepare the WASM environment for execution by writing the process string and +the message as JSON representations into the WASM environment. + + + +### prepare_header_case_tags/1 * ### + +`prepare_header_case_tags(TABM) -> any()` + +Convert a message without an `original-tags` field into a list of +key-value pairs, with the keys in HTTP header-case. + + + +### prepare_tags/1 * ### + +`prepare_tags(Msg) -> any()` + +Prepare the tags of a message as a key-value list, for use in the +construction of the JSON-Struct message. + + + +### preprocess_results/2 * ### + +`preprocess_results(Msg, Opts) -> any()` + +After the process returns messages from an evaluation, the +signing node needs to add some tags to each message and spawn such that +the target process knows these messages are created by a process. + + + +### results/3 * ### + +`results(M1, M2, Opts) -> any()` + +Read the computed results out of the WASM environment, assuming that +the environment has been set up by `prep_call/3` and that the WASM executor +has been called with `computed{pass=1}`. + + + +### safe_to_id/1 * ### + +`safe_to_id(ID) -> any()` + + + +### tags_to_map/1 * ### + +`tags_to_map(Msg) -> any()` + +Convert a message with tags into a map of their key-value pairs. + + + +### test_init/0 * ### + +`test_init() -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_json_iface.md --- + +--- START OF FILE: docs/resources/source-code/dev_local_name.md --- +# [Module dev_local_name.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_local_name.erl) + + + + +A device for registering and looking up local names. + + + +## Description ## +This device uses +the node message to store a local cache of its known names, and the typical +non-volatile storage of the node message to store the names long-term. + +## Function Index ## + + +
default_lookup/4*Handle all other requests by delegating to the lookup function.
direct_register/2Register a name without checking if the caller is an operator.
find_names/1*Returns a message containing all known names.
generate_test_opts/0*
http_test/0*
info/1Export only the lookup and register functions.
load_names/1*Loads all known names from the cache and returns the new node message +with those names loaded into it.
lookup/3Takes a key argument and returns the value of the name, if it exists.
lookup_opts_name_test/0*
no_names_test/0*
register/3Takes a key and value argument and registers the name.
register_test/0*
unauthorized_test/0*
update_names/2*Updates the node message with the new names.
+ + + + +## Function Details ## + + + +### default_lookup/4 * ### + +`default_lookup(Key, X2, Req, Opts) -> any()` + +Handle all other requests by delegating to the lookup function. + + + +### direct_register/2 ### + +`direct_register(Req, Opts) -> any()` + +Register a name without checking if the caller is an operator. Exported +for use by other devices, but not publicly available. + + + +### find_names/1 * ### + +`find_names(Opts) -> any()` + +Returns a message containing all known names. + + + +### generate_test_opts/0 * ### + +`generate_test_opts() -> any()` + + + +### http_test/0 * ### + +`http_test() -> any()` + + + +### info/1 ### + +`info(Opts) -> any()` + +Export only the `lookup` and `register` functions. + + + +### load_names/1 * ### + +`load_names(Opts) -> any()` + +Loads all known names from the cache and returns the new `node message` +with those names loaded into it. + + + +### lookup/3 ### + +`lookup(X1, Req, Opts) -> any()` + +Takes a `key` argument and returns the value of the name, if it exists. + + + +### lookup_opts_name_test/0 * ### + +`lookup_opts_name_test() -> any()` + + + +### no_names_test/0 * ### + +`no_names_test() -> any()` + + + +### register/3 ### + +`register(X1, Req, Opts) -> any()` + +Takes a `key` and `value` argument and registers the name. The caller +must be the node operator in order to register a name. + + + +### register_test/0 * ### + +`register_test() -> any()` + + + +### unauthorized_test/0 * ### + +`unauthorized_test() -> any()` + + + +### update_names/2 * ### + +`update_names(LocalNames, Opts) -> any()` + +Updates the node message with the new names. Further HTTP requests will +use this new message, removing the need to look up the names from non-volatile +storage. + + +--- END OF FILE: docs/resources/source-code/dev_local_name.md --- + +--- START OF FILE: docs/resources/source-code/dev_lookup.md --- +# [Module dev_lookup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lookup.erl) + + + + +A device that looks up an ID from a local store and returns it, honoring +the `accept` key to return the correct format. + + + +## Function Index ## + + +
aos2_message_lookup_test/0*
binary_lookup_test/0*
http_lookup_test/0*
message_lookup_test/0*
read/3Fetch a resource from the cache using "target" ID extracted from the message.
+ + + + +## Function Details ## + + + +### aos2_message_lookup_test/0 * ### + +`aos2_message_lookup_test() -> any()` + + + +### binary_lookup_test/0 * ### + +`binary_lookup_test() -> any()` + + + +### http_lookup_test/0 * ### + +`http_lookup_test() -> any()` + + + +### message_lookup_test/0 * ### + +`message_lookup_test() -> any()` + + + +### read/3 ### + +`read(M1, M2, Opts) -> any()` + +Fetch a resource from the cache using "target" ID extracted from the message + + +--- END OF FILE: docs/resources/source-code/dev_lookup.md --- + +--- START OF FILE: docs/resources/source-code/dev_lua_lib.md --- +# [Module dev_lua_lib.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lua_lib.erl) + + + + +A module for providing AO library functions to the Lua environment. + + + +## Description ## + +This module contains the implementation of the functions, each by the name +that should be used in the `ao` table in the Lua environment. Every export +is imported into the Lua environment. + +Each function adheres closely to the Luerl calling convention, adding the +appropriate node message as a third argument: + +fun(Args, State, NodeMsg) -> {ResultTerms, NewState} + +As Lua allows for multiple return values, each function returns a list of +terms to grant to the caller. Matching the tuple convention used by AO-Core, +the first term is typically the status, and the second term is the result. + +## Function Index ## + + +
convert_as/1*Converts any as terms from Lua to their HyperBEAM equivalents.
event/3Allows Lua scripts to signal events using the HyperBEAM hosts internal +event system.
install/3Install the library into the given Lua environment.
resolve/3A wrapper function for performing AO-Core resolutions.
return/2*Helper function for returning a result from a Lua function.
set/3Wrapper for hb_ao's set functionality.
+ + + + +## Function Details ## + + + +### convert_as/1 * ### + +`convert_as(Other) -> any()` + +Converts any `as` terms from Lua to their HyperBEAM equivalents. + + + +### event/3 ### + +`event(X1, ExecState, Opts) -> any()` + +Allows Lua scripts to signal events using the HyperBEAM hosts internal +event system. + + + +### install/3 ### + +`install(Base, State, Opts) -> any()` + +Install the library into the given Lua environment. + + + +### resolve/3 ### + +`resolve(Msgs, ExecState, ExecOpts) -> any()` + +A wrapper function for performing AO-Core resolutions. Offers both the +single-message (using `hb_singleton:from/1` to parse) and multiple-message +(using `hb_ao:resolve_many/2`) variants. + + + +### return/2 * ### + +`return(Result, ExecState) -> any()` + +Helper function for returning a result from a Lua function. + + + +### set/3 ### + +`set(X1, ExecState, ExecOpts) -> any()` + +Wrapper for `hb_ao`'s `set` functionality. + + +--- END OF FILE: docs/resources/source-code/dev_lua_lib.md --- + +--- START OF FILE: docs/resources/source-code/dev_lua_test.md --- +# [Module dev_lua_test.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lua_test.erl) + + + + + + +## Function Index ## + + +
exec_test/2*Generate an EUnit test for a given function.
exec_test_/0*Main entrypoint for Lua tests.
new_state/1*Create a new Lua environment for a given script.
parse_spec/1Parse a string representation of test descriptions received from the +command line via the LUA_TESTS environment variable.
suite/2*Generate an EUnit test suite for a given Lua script.
terminates_with/2*Check if a string terminates with a given suffix.
+ + + + +## Function Details ## + + + +### exec_test/2 * ### + +`exec_test(State, Function) -> any()` + +Generate an EUnit test for a given function. + + + +### exec_test_/0 * ### + +`exec_test_() -> any()` + +Main entrypoint for Lua tests. + + + +### new_state/1 * ### + +`new_state(File) -> any()` + +Create a new Lua environment for a given script. + + + +### parse_spec/1 ### + +`parse_spec(Str) -> any()` + +Parse a string representation of test descriptions received from the +command line via the `LUA_TESTS` environment variable. + +Supported syntax in loose BNF/RegEx: + +Definitions := (ModDef,)+ +ModDef := ModName(TestDefs)? +ModName := ModuleInLUA_SCRIPTS|(FileName[.lua])? +TestDefs := (:TestDef)+ +TestDef := TestName + +File names ending in `.lua` are assumed to be relative paths from the current +working directory. Module names lacking the `.lua` extension are assumed to +be modules found in the `LUA_SCRIPTS` environment variable (defaulting to +`scripts/`). + +For example, to run a single test one could call the following: + +LUA_TESTS=~/src/LuaScripts/test.yourTest rebar3 lua-tests + +To specify that one would like to run all of the tests in the +`scripts/test.lua` file and two tests from the `scripts/test2.lua` file, the +user could provide the following test definition: + +LUA_TESTS="test,scripts/test2.userTest1|userTest2" rebar3 lua-tests + + + +### suite/2 * ### + +`suite(File, Funcs) -> any()` + +Generate an EUnit test suite for a given Lua script. If the `Funcs` is +the atom `tests` we find all of the global functions in the script, then +filter for those ending in `_test` in a similar fashion to Eunit. + + + +### terminates_with/2 * ### + +`terminates_with(String, Suffix) -> any()` + +Check if a string terminates with a given suffix. + + +--- END OF FILE: docs/resources/source-code/dev_lua_test.md --- + +--- START OF FILE: docs/resources/source-code/dev_lua.md --- +# [Module dev_lua.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lua.erl) + + + + +A device that calls a Lua module upon a request and returns the result. + + + +## Function Index ## + + +
ao_core_resolution_from_lua_test/0*Run an AO-Core resolution from the Lua environment.
ao_core_sandbox_test/0*Run an AO-Core resolution from the Lua environment.
aos_authority_not_trusted_test/0*
aos_process_benchmark_test_/0*Benchmark the performance of Lua executions.
compute/4*Call the Lua script with the given arguments.
decode/1Decode a Lua result into a HyperBEAM structured@1.0 message.
decode_params/2*Decode a list of Lua references, as found in a stack trace, into a +list of Erlang terms.
decode_stacktrace/2*Parse a Lua stack trace into a list of messages.
decode_stacktrace/3*
direct_benchmark_test/0*Benchmark the performance of Lua executions.
encode/1Encode a HyperBEAM structured@1.0 message into a Lua term.
ensure_initialized/3*Initialize the Lua VM if it is not already initialized.
error_response_test/0*
find_modules/2*Find the script in the base message, either by ID or by string.
functions/3Return a list of all functions in the Lua environment.
generate_lua_process/1*Generate a Lua process message.
generate_stack/1*Generate a stack message for the Lua process.
generate_test_message/1*Generate a test message for a Lua process.
info/1All keys that are not directly available in the base message are +resolved by calling the Lua function in the module of the same name.
init/3Initialize the device state, loading the script into memory if it is +a reference.
initialize/3*Initialize a new Lua state with a given base message and module.
invoke_aos_test/0*
invoke_non_compute_key_test/0*Call a non-compute key on a Lua device message and ensure that the +function of the same name in the script is called.
load_modules/2*Load a list of modules for installation into the Lua VM.
load_modules/3*
load_modules_by_id_test/0*
lua_http_hook_test/0*Use a Lua module as a hook on the HTTP server via ~meta@1.0.
multiple_modules_test/0*
normalize/3Restore the Lua state from a snapshot, if it exists.
process_response/2*Process a response to a Luerl invocation.
pure_lua_process_benchmark_test_/0*
pure_lua_process_test/0*Call a process whose execution-device is set to lua@5.3a.
sandbox/3*Sandbox (render inoperable) a set of Lua functions.
sandboxed_failure_test/0*
simple_invocation_test/0*
snapshot/3Snapshot the Lua state from a live computation.
+ + + + +## Function Details ## + + + +### ao_core_resolution_from_lua_test/0 * ### + +`ao_core_resolution_from_lua_test() -> any()` + +Run an AO-Core resolution from the Lua environment. + + + +### ao_core_sandbox_test/0 * ### + +`ao_core_sandbox_test() -> any()` + +Run an AO-Core resolution from the Lua environment. + + + +### aos_authority_not_trusted_test/0 * ### + +`aos_authority_not_trusted_test() -> any()` + + + +### aos_process_benchmark_test_/0 * ### + +`aos_process_benchmark_test_() -> any()` + +Benchmark the performance of Lua executions. + + + +### compute/4 * ### + +`compute(Key, RawBase, Req, Opts) -> any()` + +Call the Lua script with the given arguments. + + + +### decode/1 ### + +`decode(EncMsg) -> any()` + +Decode a Lua result into a HyperBEAM `structured@1.0` message. + + + +### decode_params/2 * ### + +`decode_params(Rest, State) -> any()` + +Decode a list of Lua references, as found in a stack trace, into a +list of Erlang terms. + + + +### decode_stacktrace/2 * ### + +`decode_stacktrace(StackTrace, State0) -> any()` + +Parse a Lua stack trace into a list of messages. + + + +### decode_stacktrace/3 * ### + +`decode_stacktrace(Rest, State, Acc) -> any()` + + + +### direct_benchmark_test/0 * ### + +`direct_benchmark_test() -> any()` + +Benchmark the performance of Lua executions. + + + +### encode/1 ### + +`encode(Map) -> any()` + +Encode a HyperBEAM `structured@1.0` message into a Lua term. + + + +### ensure_initialized/3 * ### + +`ensure_initialized(Base, Req, Opts) -> any()` + +Initialize the Lua VM if it is not already initialized. Optionally takes +the script as a Binary string. If not provided, the module will be loaded +from the base message. + + + +### error_response_test/0 * ### + +`error_response_test() -> any()` + + + +### find_modules/2 * ### + +`find_modules(Base, Opts) -> any()` + +Find the script in the base message, either by ID or by string. + + + +### functions/3 ### + +`functions(Base, Req, Opts) -> any()` + +Return a list of all functions in the Lua environment. + + + +### generate_lua_process/1 * ### + +`generate_lua_process(File) -> any()` + +Generate a Lua process message. + + + +### generate_stack/1 * ### + +`generate_stack(File) -> any()` + +Generate a stack message for the Lua process. + + + +### generate_test_message/1 * ### + +`generate_test_message(Process) -> any()` + +Generate a test message for a Lua process. + + + +### info/1 ### + +`info(Base) -> any()` + +All keys that are not directly available in the base message are +resolved by calling the Lua function in the module of the same name. +Additionally, we exclude the `keys`, `set`, `encode` and `decode` functions +which are `message@1.0` core functions, and Lua public utility functions. + + + +### init/3 ### + +`init(Base, Req, Opts) -> any()` + +Initialize the device state, loading the script into memory if it is +a reference. + + + +### initialize/3 * ### + +`initialize(Base, Modules, Opts) -> any()` + +Initialize a new Lua state with a given base message and module. + + + +### invoke_aos_test/0 * ### + +`invoke_aos_test() -> any()` + + + +### invoke_non_compute_key_test/0 * ### + +`invoke_non_compute_key_test() -> any()` + +Call a non-compute key on a Lua device message and ensure that the +function of the same name in the script is called. + + + +### load_modules/2 * ### + +`load_modules(Modules, Opts) -> any()` + +Load a list of modules for installation into the Lua VM. + + + +### load_modules/3 * ### + +`load_modules(Rest, Opts, Acc) -> any()` + + + +### load_modules_by_id_test/0 * ### + +`load_modules_by_id_test() -> any()` + + + +### lua_http_hook_test/0 * ### + +`lua_http_hook_test() -> any()` + +Use a Lua module as a hook on the HTTP server via `~meta@1.0`. + + + +### multiple_modules_test/0 * ### + +`multiple_modules_test() -> any()` + + + +### normalize/3 ### + +`normalize(Base, Req, RawOpts) -> any()` + +Restore the Lua state from a snapshot, if it exists. + + + +### process_response/2 * ### + +`process_response(X1, Priv) -> any()` + +Process a response to a Luerl invocation. Returns the typical AO-Core +HyperBEAM response format. + + + +### pure_lua_process_benchmark_test_/0 * ### + +`pure_lua_process_benchmark_test_() -> any()` + + + +### pure_lua_process_test/0 * ### + +`pure_lua_process_test() -> any()` + +Call a process whose `execution-device` is set to `lua@5.3a`. + + + +### sandbox/3 * ### + +`sandbox(State, Map, Opts) -> any()` + +Sandbox (render inoperable) a set of Lua functions. Each function is +referred to as if it is a path in AO-Core, with its value being what to +return to the caller. For example, 'os.exit' would be referred to as +referred to as `os/exit`. If preferred, a list rather than a map may be +provided, in which case the functions all return `sandboxed`. + + + +### sandboxed_failure_test/0 * ### + +`sandboxed_failure_test() -> any()` + + + +### simple_invocation_test/0 * ### + +`simple_invocation_test() -> any()` + + + +### snapshot/3 ### + +`snapshot(Base, Req, Opts) -> any()` + +Snapshot the Lua state from a live computation. Normalizes its `priv` +state element, then serializes the state to a binary. + + +--- END OF FILE: docs/resources/source-code/dev_lua.md --- + +--- START OF FILE: docs/resources/source-code/dev_manifest.md --- +# [Module dev_manifest.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_manifest.erl) + + + + +An Arweave path manifest resolution device. + + + +## Description ## +Follows the v1 schema: +https://specs.ar.io/?tx=lXLd0OPwo-dJLB_Amz5jgIeDhiOkjXuM3-r0H_aiNj0 + +## Function Index ## + + +
info/0Use the route/4 function as the handler for all requests, aside +from keys and set, which are handled by the default resolver.
manifest/3*Find and deserialize a manifest from the given base.
route/4*Route a request to the associated data via its manifest.
+ + + + +## Function Details ## + + + +### info/0 ### + +`info() -> any()` + +Use the `route/4` function as the handler for all requests, aside +from `keys` and `set`, which are handled by the default resolver. + + + +### manifest/3 * ### + +`manifest(Base, Req, Opts) -> any()` + +Find and deserialize a manifest from the given base. + + + +### route/4 * ### + +`route(Key, M1, M2, Opts) -> any()` + +Route a request to the associated data via its manifest. + + +--- END OF FILE: docs/resources/source-code/dev_manifest.md --- + +--- START OF FILE: docs/resources/source-code/dev_message.md --- +# [Module dev_message.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_message.erl) + + + + +The identity device: For non-reserved keys, it simply returns a key +from the message as it is found in the message's underlying Erlang map. + + + +## Description ## +Private keys (`priv[.*]`) are not included. +Reserved keys are: `id`, `commitments`, `committers`, `keys`, `path`, +`set`, `remove`, `get`, and `verify`. Their function comments describe the +behaviour of the device when these keys are set. + +## Function Index ## + + +
calculate_ids/3*
cannot_get_private_keys_test/0*
case_insensitive_get/2*Key matching should be case insensitive, following RFC-9110, so we +implement a case-insensitive key lookup rather than delegating to +maps:get/2.
case_insensitive_get_test/0*
commit/3Commit to a message, using the commitment-device key to specify the +device that should be used to commit to the message.
commitment_ids_from_committers/2*Returns a list of commitment IDs in a commitments map that are relevant +for a list of given committer addresses.
commitment_ids_from_request/3*Implements a standardized form of specifying commitment IDs for a +message request.
committed/3Return the list of committed keys from a message.
committers/1Return the committers of a message that are present in the given request.
committers/2
committers/3
deep_unset_test/0*
exec_for_commitment/5*Execute a function for a single commitment in the context of its +parent message.
get/2Return the value associated with the key as it exists in the message's +underlying Erlang map.
get/3
get_keys_mod_test/0*
id/1Return the ID of a message, using the committers list if it exists.
id/2
id/3
id_device/1*Locate the ID device of a message.
info/0Return the info for the identity device.
is_private_mod_test/0*
key_from_device_test/0*
keys/1Get the public keys of a message.
keys_from_device_test/0*
private_keys_are_filtered_test/0*
remove/2Remove a key or keys from a message.
remove_test/0*
run_test/0*
set/3Deep merge keys in a message.
set_conflicting_keys_test/0*
set_ignore_undefined_test/0*
set_path/3Special case of set/3 for setting the path key.
unset_with_set_test/0*
verify/3Verify a message.
verify_test/0*
with_relevant_commitments/3*Return a message with only the relevant commitments for a given request.
+ + + + +## Function Details ## + + + +### calculate_ids/3 * ### + +`calculate_ids(Base, Req, NodeOpts) -> any()` + + + +### cannot_get_private_keys_test/0 * ### + +`cannot_get_private_keys_test() -> any()` + + + +### case_insensitive_get/2 * ### + +`case_insensitive_get(Key, Msg) -> any()` + +Key matching should be case insensitive, following RFC-9110, so we +implement a case-insensitive key lookup rather than delegating to +`maps:get/2`. Encode the key to a binary if it is not already. + + + +### case_insensitive_get_test/0 * ### + +`case_insensitive_get_test() -> any()` + + + +### commit/3 ### + +`commit(Self, Req, Opts) -> any()` + +Commit to a message, using the `commitment-device` key to specify the +device that should be used to commit to the message. If the key is not set, +the default device (`httpsig@1.0`) is used. + + + +### commitment_ids_from_committers/2 * ### + +`commitment_ids_from_committers(CommitterAddrs, Commitments) -> any()` + +Returns a list of commitment IDs in a commitments map that are relevant +for a list of given committer addresses. + + + +### commitment_ids_from_request/3 * ### + +`commitment_ids_from_request(Base, Req, Opts) -> any()` + +Implements a standardized form of specifying commitment IDs for a +message request. The caller may specify a list of committers (by address) +or a list of commitment IDs directly. They may specify both, in which case +the returned list will be the union of the two lists. In each case, they +may specify `all` or `none` for each group. If no specifiers are provided, +the default is `all` for commitments -- also implying `all` for committers. + + + +### committed/3 ### + +`committed(Self, Req, Opts) -> any()` + +Return the list of committed keys from a message. + + + +### committers/1 ### + +`committers(Base) -> any()` + +Return the committers of a message that are present in the given request. + + + +### committers/2 ### + +`committers(Base, Req) -> any()` + + + +### committers/3 ### + +`committers(X1, X2, NodeOpts) -> any()` + + + +### deep_unset_test/0 * ### + +`deep_unset_test() -> any()` + + + +### exec_for_commitment/5 * ### + +`exec_for_commitment(Func, Base, Commitment, Req, Opts) -> any()` + +Execute a function for a single commitment in the context of its +parent message. +Note: Assumes that the `commitments` key has already been removed from the +message if applicable. + + + +### get/2 ### + +`get(Key, Msg) -> any()` + +Return the value associated with the key as it exists in the message's +underlying Erlang map. First check the public keys, then check case- +insensitively if the key is a binary. + + + +### get/3 ### + +`get(Key, Msg, Msg2) -> any()` + + + +### get_keys_mod_test/0 * ### + +`get_keys_mod_test() -> any()` + + + +### id/1 ### + +`id(Base) -> any()` + +Return the ID of a message, using the `committers` list if it exists. +If the `committers` key is `all`, return the ID including all known +commitments -- `none` yields the ID without any commitments. If the +`committers` key is a list/map, return the ID including only the specified +commitments. + +The `id-device` key in the message can be used to specify the device that +should be used to calculate the ID. If it is not set, the default device +(`httpsig@1.0`) is used. + +Note: This function _does not_ use AO-Core's `get/3` function, as it +as it would require significant computation. We may want to change this +if/when non-map message structures are created. + + + +### id/2 ### + +`id(Base, Req) -> any()` + + + +### id/3 ### + +`id(Base, Req, NodeOpts) -> any()` + + + +### id_device/1 * ### + +`id_device(X1) -> any()` + +Locate the ID device of a message. The ID device is determined the +`device` set in _all_ of the commitments. If no commitments are present, +the default device (`httpsig@1.0`) is used. + + + +### info/0 ### + +`info() -> any()` + +Return the info for the identity device. + + + +### is_private_mod_test/0 * ### + +`is_private_mod_test() -> any()` + + + +### key_from_device_test/0 * ### + +`key_from_device_test() -> any()` + + + +### keys/1 ### + +`keys(Msg) -> any()` + +Get the public keys of a message. + + + +### keys_from_device_test/0 * ### + +`keys_from_device_test() -> any()` + + + +### private_keys_are_filtered_test/0 * ### + +`private_keys_are_filtered_test() -> any()` + + + +### remove/2 ### + +`remove(Message1, X2) -> any()` + +Remove a key or keys from a message. + + + +### remove_test/0 * ### + +`remove_test() -> any()` + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### set/3 ### + +`set(Message1, NewValuesMsg, Opts) -> any()` + +Deep merge keys in a message. Takes a map of key-value pairs and sets +them in the message, overwriting any existing values. + + + +### set_conflicting_keys_test/0 * ### + +`set_conflicting_keys_test() -> any()` + + + +### set_ignore_undefined_test/0 * ### + +`set_ignore_undefined_test() -> any()` + + + +### set_path/3 ### + +`set_path(Message1, X2, Opts) -> any()` + +Special case of `set/3` for setting the `path` key. This cannot be set +using the normal `set` function, as the `path` is a reserved key, necessary +for AO-Core to know the key to evaluate in requests. + + + +### unset_with_set_test/0 * ### + +`unset_with_set_test() -> any()` + + + +### verify/3 ### + +`verify(Self, Req, Opts) -> any()` + +Verify a message. By default, all commitments are verified. The +`committers` key in the request can be used to specify that only the +commitments from specific committers should be verified. Similarly, specific +commitments can be specified using the `commitments` key. + + + +### verify_test/0 * ### + +`verify_test() -> any()` + + + +### with_relevant_commitments/3 * ### + +`with_relevant_commitments(Base, Req, Opts) -> any()` + +Return a message with only the relevant commitments for a given request. +See `commitment_ids_from_request/3` for more information on the request format. + + +--- END OF FILE: docs/resources/source-code/dev_message.md --- + +--- START OF FILE: docs/resources/source-code/dev_meta.md --- +# [Module dev_meta.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_meta.erl) + + + + +The hyperbeam meta device, which is the default entry point +for all messages processed by the machine. + + + +## Description ## +This device executes a +AO-Core singleton request, after first applying the node's +pre-processor, if set. The pre-processor can halt the request by +returning an error, or return a modified version if it deems necessary -- +the result of the pre-processor is used as the request for the AO-Core +resolver. Additionally, a post-processor can be set, which is executed after +the AO-Core resolver has returned a result. + +## Function Index ## + + +
add_dynamic_keys/1*Add dynamic keys to the node message.
adopt_node_message/2Attempt to adopt changes to a node message.
authorized_set_node_msg_succeeds_test/0*Test that we can set the node message if the request is signed by the +owner of the node.
build/3Emits the version number and commit hash of the HyperBEAM node source, +if available.
buildinfo_test/0*Test that version information is available and returned correctly.
claim_node_test/0*Test that we can claim the node correctly and set the node message after.
config_test/0*Test that we can get the node message.
embed_status/1*Wrap the result of a device call in a status.
filter_node_msg/1*Remove items from the node message that are not encodable into a +message.
halt_request_test/0*Test that we can halt a request if the hook returns an error.
handle/2Normalize and route messages downstream based on their path.
handle_initialize/2*
handle_resolve/3*Handle an AO-Core request, which is a list of messages.
info/1Ensure that the helper function adopt_node_message/2 is not exported.
info/3Get/set the node message.
is/2Check if the request in question is signed by a given role on the node.
is/3
maybe_sign/2*Sign the result of a device call if the node is configured to do so.
message_to_status/1*Get the HTTP status code from a transaction (if it exists).
modify_request_test/0*Test that a hook can modify a request.
permanent_node_message_test/0*Test that a permanent node message cannot be changed.
priv_inaccessible_test/0*Test that we can't get the node message if the requested key is private.
request_response_hooks_test/0*
resolve_hook/4*Execute a hook from the node message upon the user's request.
status_code/1*Calculate the appropriate HTTP status code for an AO-Core result.
unauthorized_set_node_msg_fails_test/0*Test that we can't set the node message if the request is not signed by +the owner of the node.
uninitialized_node_test/0*Test that an uninitialized node will not run computation.
update_node_message/2*Validate that the request is signed by the operator of the node, then +allow them to update the node message.
+ + + + +## Function Details ## + + + +### add_dynamic_keys/1 * ### + +`add_dynamic_keys(NodeMsg) -> any()` + +Add dynamic keys to the node message. + + + +### adopt_node_message/2 ### + +`adopt_node_message(Request, NodeMsg) -> any()` + +Attempt to adopt changes to a node message. + + + +### authorized_set_node_msg_succeeds_test/0 * ### + +`authorized_set_node_msg_succeeds_test() -> any()` + +Test that we can set the node message if the request is signed by the +owner of the node. + + + +### build/3 ### + +`build(X1, X2, NodeMsg) -> any()` + +Emits the version number and commit hash of the HyperBEAM node source, +if available. + +We include the short hash separately, as the length of this hash may change in +the future, depending on the git version/config used to build the node. +Subsequently, rather than embedding the `git-short-hash-length`, for the +avoidance of doubt, we include the short hash separately, as well as its long +hash. + + + +### buildinfo_test/0 * ### + +`buildinfo_test() -> any()` + +Test that version information is available and returned correctly. + + + +### claim_node_test/0 * ### + +`claim_node_test() -> any()` + +Test that we can claim the node correctly and set the node message after. + + + +### config_test/0 * ### + +`config_test() -> any()` + +Test that we can get the node message. + + + +### embed_status/1 * ### + +`embed_status(X1) -> any()` + +Wrap the result of a device call in a status. + + + +### filter_node_msg/1 * ### + +`filter_node_msg(Msg) -> any()` + +Remove items from the node message that are not encodable into a +message. + + + +### halt_request_test/0 * ### + +`halt_request_test() -> any()` + +Test that we can halt a request if the hook returns an error. + + + +### handle/2 ### + +`handle(NodeMsg, RawRequest) -> any()` + +Normalize and route messages downstream based on their path. Messages +with a `Meta` key are routed to the `handle_meta/2` function, while all +other messages are routed to the `handle_resolve/2` function. + + + +### handle_initialize/2 * ### + +`handle_initialize(Rest, NodeMsg) -> any()` + + + +### handle_resolve/3 * ### + +`handle_resolve(Req, Msgs, NodeMsg) -> any()` + +Handle an AO-Core request, which is a list of messages. We apply +the node's pre-processor to the request first, and then resolve the request +using the node's AO-Core implementation if its response was `ok`. +After execution, we run the node's `response` hook on the result of +the request before returning the result it grants back to the user. + + + +### info/1 ### + +`info(X1) -> any()` + +Ensure that the helper function `adopt_node_message/2` is not exported. +The naming of this method carefully avoids a clash with the exported `info/3` +function. We would like the node information to be easily accessible via the +`info` endpoint, but AO-Core also uses `info` as the name of the function +that grants device information. The device call takes two or fewer arguments, +so we are safe to use the name for both purposes in this case, as the user +info call will match the three-argument version of the function. If in the +future the `request` is added as an argument to AO-Core's internal `info` +function, we will need to find a different approach. + + + +### info/3 ### + +`info(X1, Request, NodeMsg) -> any()` + +Get/set the node message. If the request is a `POST`, we check that the +request is signed by the owner of the node. If not, we return the node message +as-is, aside all keys that are private (according to `hb_private`). + + + +### is/2 ### + +`is(Request, NodeMsg) -> any()` + +Check if the request in question is signed by a given `role` on the node. +The `role` can be one of `operator` or `initiator`. + + + +### is/3 ### + +`is(X1, Request, NodeMsg) -> any()` + + + +### maybe_sign/2 * ### + +`maybe_sign(Res, NodeMsg) -> any()` + +Sign the result of a device call if the node is configured to do so. + + + +### message_to_status/1 * ### + +`message_to_status(Item) -> any()` + +Get the HTTP status code from a transaction (if it exists). + + + +### modify_request_test/0 * ### + +`modify_request_test() -> any()` + +Test that a hook can modify a request. + + + +### permanent_node_message_test/0 * ### + +`permanent_node_message_test() -> any()` + +Test that a permanent node message cannot be changed. + + + +### priv_inaccessible_test/0 * ### + +`priv_inaccessible_test() -> any()` + +Test that we can't get the node message if the requested key is private. + + + +### request_response_hooks_test/0 * ### + +`request_response_hooks_test() -> any()` + + + +### resolve_hook/4 * ### + +`resolve_hook(HookName, InitiatingRequest, Body, NodeMsg) -> any()` + +Execute a hook from the node message upon the user's request. The +invocation of the hook provides a request of the following form: + +``` + + /path => request | response + /request => the original request singleton + /body => parsed sequence of messages to process | the execution result +``` + + + +### status_code/1 * ### + +`status_code(X1) -> any()` + +Calculate the appropriate HTTP status code for an AO-Core result. +The order of precedence is: +1. The status code from the message. +2. The HTTP representation of the status code. +3. The default status code. + + + +### unauthorized_set_node_msg_fails_test/0 * ### + +`unauthorized_set_node_msg_fails_test() -> any()` + +Test that we can't set the node message if the request is not signed by +the owner of the node. + + + +### uninitialized_node_test/0 * ### + +`uninitialized_node_test() -> any()` + +Test that an uninitialized node will not run computation. + + + +### update_node_message/2 * ### + +`update_node_message(Request, NodeMsg) -> any()` + +Validate that the request is signed by the operator of the node, then +allow them to update the node message. + + +--- END OF FILE: docs/resources/source-code/dev_meta.md --- + +--- START OF FILE: docs/resources/source-code/dev_monitor.md --- +# [Module dev_monitor.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_monitor.erl) + + + + + + +## Function Index ## + + +
add_monitor/2
end_of_schedule/1
execute/2
init/3
signal/2*
uses/0
+ + + + +## Function Details ## + + + +### add_monitor/2 ### + +`add_monitor(Mon, State) -> any()` + + + +### end_of_schedule/1 ### + +`end_of_schedule(State) -> any()` + + + +### execute/2 ### + +`execute(Message, State) -> any()` + + + +### init/3 ### + +`init(State, X2, InitState) -> any()` + + + +### signal/2 * ### + +`signal(State, Signal) -> any()` + + + +### uses/0 ### + +`uses() -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_monitor.md --- + +--- START OF FILE: docs/resources/source-code/dev_multipass.md --- +# [Module dev_multipass.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_multipass.erl) + + + + +A device that triggers repass events until a certain counter has been +reached. + + + +## Description ## +This is useful for certain types of stacks that need various +execution passes to be completed in sequence across devices. + +## Function Index ## + + +
basic_multipass_test/0*
handle/4*Forward the keys function to the message device, handle all others +with deduplication.
info/1
+ + + + +## Function Details ## + + + +### basic_multipass_test/0 * ### + +`basic_multipass_test() -> any()` + + + +### handle/4 * ### + +`handle(Key, M1, M2, Opts) -> any()` + +Forward the keys function to the message device, handle all others +with deduplication. We only act on the first pass. + + + +### info/1 ### + +`info(M1) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_multipass.md --- + +--- START OF FILE: docs/resources/source-code/dev_name.md --- +# [Module dev_name.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_name.erl) + + + + +A device for resolving names to their corresponding values, through the +use of a `resolver` interface. + + + +## Description ## +Each `resolver` is a message that can be +given a `key` and returns an associated value. The device will attempt to +match the key against each resolver in turn, and return the value of the +first resolver that matches. + +## Function Index ## + + +
execute_resolver/3*Execute a resolver with the given key and return its value.
info/1Configure the default key to proxy to the resolver/4 function.
load_and_execute_test/0*Test that we can resolve messages from a name loaded with the device.
match_resolver/3*Find the first resolver that matches the key and return its value.
message_lookup_device_resolver/1*
multiple_resolvers_test/0*
no_resolvers_test/0*
resolve/4*Resolve a name to its corresponding value.
single_resolver_test/0*
+ + + + +## Function Details ## + + + +### execute_resolver/3 * ### + +`execute_resolver(Key, Resolver, Opts) -> any()` + +Execute a resolver with the given key and return its value. + + + +### info/1 ### + +`info(X1) -> any()` + +Configure the `default` key to proxy to the `resolver/4` function. +Exclude the `keys` and `set` keys from being processed by this device, as +these are needed to modify the base message itself. + + + +### load_and_execute_test/0 * ### + +`load_and_execute_test() -> any()` + +Test that we can resolve messages from a name loaded with the device. + + + +### match_resolver/3 * ### + +`match_resolver(Key, Resolvers, Opts) -> any()` + +Find the first resolver that matches the key and return its value. + + + +### message_lookup_device_resolver/1 * ### + +`message_lookup_device_resolver(Msg) -> any()` + + + +### multiple_resolvers_test/0 * ### + +`multiple_resolvers_test() -> any()` + + + +### no_resolvers_test/0 * ### + +`no_resolvers_test() -> any()` + + + +### resolve/4 * ### + +`resolve(Key, X2, Req, Opts) -> any()` + +Resolve a name to its corresponding value. The name is given by the key +called. For example, `GET /~name@1.0/hello&load=false` grants the value of +`hello`. If the `load` key is set to `true`, the value is treated as a +pointer and its contents is loaded from the cache. For example, +`GET /~name@1.0/reference` yields the message at the path specified by the +`reference` key. + + + +### single_resolver_test/0 * ### + +`single_resolver_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_name.md --- + +--- START OF FILE: docs/resources/source-code/dev_node_process.md --- +# [Module dev_node_process.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_node_process.erl) + + + + +A device that implements the singleton pattern for processes specific +to an individual node. + + + +## Description ## + +This device uses the `local-name@1.0` device to +register processes with names locally, persistenting them across reboots. + +Definitions of singleton processes are expected to be found with their +names in the `node_processes` section of the node message. + +## Function Index ## + + +
augment_definition/2*Augment the given process definition with the node's address.
generate_test_opts/0*Helper function to generate a test environment and its options.
generate_test_opts/1*
info/1Register a default handler for the device.
lookup/4*Lookup a process by name.
lookup_execute_test/0*Test that a process can be spawned, executed upon, and its result retrieved.
lookup_no_spawn_test/0*
lookup_spawn_test/0*
spawn_register/2*Spawn a new process according to the process definition found in the +node message, and register it with the given name.
+ + + + +## Function Details ## + + + +### augment_definition/2 * ### + +`augment_definition(BaseDef, Opts) -> any()` + +Augment the given process definition with the node's address. + + + +### generate_test_opts/0 * ### + +`generate_test_opts() -> any()` + +Helper function to generate a test environment and its options. + + + +### generate_test_opts/1 * ### + +`generate_test_opts(Defs) -> any()` + + + +### info/1 ### + +`info(Opts) -> any()` + +Register a default handler for the device. Inherits `keys` and `set` +from the default device. + + + +### lookup/4 * ### + +`lookup(Name, Base, Req, Opts) -> any()` + +Lookup a process by name. + + + +### lookup_execute_test/0 * ### + +`lookup_execute_test() -> any()` + +Test that a process can be spawned, executed upon, and its result retrieved. + + + +### lookup_no_spawn_test/0 * ### + +`lookup_no_spawn_test() -> any()` + + + +### lookup_spawn_test/0 * ### + +`lookup_spawn_test() -> any()` + + + +### spawn_register/2 * ### + +`spawn_register(Name, Opts) -> any()` + +Spawn a new process according to the process definition found in the +node message, and register it with the given name. + + +--- END OF FILE: docs/resources/source-code/dev_node_process.md --- + +--- START OF FILE: docs/resources/source-code/dev_p4.md --- +# [Module dev_p4.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_p4.erl) + + + + +The HyperBEAM core payment ledger. + + + +## Description ## + +This module allows the operator to +specify another device that can act as a pricing mechanism for transactions +on the node, as well as orchestrating a payment ledger to calculate whether +the node should fulfil services for users. + +The device requires the following node message settings in order to function: + +- `p4_pricing-device`: The device that will estimate the cost of a request. +- `p4_ledger-device`: The device that will act as a payment ledger. + +The pricing device should implement the following keys: + +``` +GET /estimate?type=pre|post&body=[...]&request=RequestMessageGET /price?type=pre|post&body=[...]&request=RequestMessage +``` + +The `body` key is used to pass either the request or response messages to the +device. The `type` key is used to specify whether the inquiry is for a request +(pre) or a response (post) object. Requests carry lists of messages that will +be executed, while responses carry the results of the execution. The `price` +key may return `infinity` if the node will not serve a user under any +circumstances. Else, the value returned by the `price` key will be passed to +the ledger device as the `amount` key. + +A ledger device should implement the following keys: + +``` +POST /credit?message=PaymentMessage&request=RequestMessagePOST /debit?amount=PriceMessage&request=RequestMessageGET /balance?request=RequestMessage +``` + +The `type` key is optional and defaults to `pre`. If `type` is set to `post`, +the debit must be applied to the ledger, whereas the `pre` type is used to +check whether the debit would succeed before execution. + +## Function Index ## + + +
balance/3Get the balance of a user in the ledger.
faff_test/0*Simple test of p4's capabilities with the faff@1.0 device.
is_chargable_req/2*The node operator may elect to make certain routes non-chargable, using +the routes syntax also used to declare routes in router@1.0.
lua_pricing_test/0*Ensure that Lua modules can be used as pricing and ledger devices.
non_chargable_route_test/0*Test that a non-chargable route is not charged for.
request/3Estimate the cost of a transaction and decide whether to proceed with +a request.
response/3Postprocess the request after it has been fulfilled.
test_opts/1*
test_opts/2*
test_opts/3*
+ + + + +## Function Details ## + + + +### balance/3 ### + +`balance(X1, Req, NodeMsg) -> any()` + +Get the balance of a user in the ledger. + + + +### faff_test/0 * ### + +`faff_test() -> any()` + +Simple test of p4's capabilities with the `faff@1.0` device. + + + +### is_chargable_req/2 * ### + +`is_chargable_req(Req, NodeMsg) -> any()` + +The node operator may elect to make certain routes non-chargable, using +the `routes` syntax also used to declare routes in `router@1.0`. + + + +### lua_pricing_test/0 * ### + +`lua_pricing_test() -> any()` + +Ensure that Lua modules can be used as pricing and ledger devices. Our +modules come in two parts: +- A `process` module which is executed as a persistent `local-process` on the +node, and which maintains the state of the ledger. +- A `client` module, which is executed as a `p4@1.0` device, marshalling +requests to the `process` module. + + + +### non_chargable_route_test/0 * ### + +`non_chargable_route_test() -> any()` + +Test that a non-chargable route is not charged for. + + + +### request/3 ### + +`request(State, Raw, NodeMsg) -> any()` + +Estimate the cost of a transaction and decide whether to proceed with +a request. The default behavior if `pricing-device` or `p4_balances` are +not set is to proceed, so it is important that a user initialize them. + + + +### response/3 ### + +`response(State, RawResponse, NodeMsg) -> any()` + +Postprocess the request after it has been fulfilled. + + + +### test_opts/1 * ### + +`test_opts(Opts) -> any()` + + + +### test_opts/2 * ### + +`test_opts(Opts, PricingDev) -> any()` + + + +### test_opts/3 * ### + +`test_opts(Opts, PricingDev, LedgerDev) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_p4.md --- + +--- START OF FILE: docs/resources/source-code/dev_patch.md --- +# [Module dev_patch.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_patch.erl) + + + + +A device that can be used to reorganize a message: Moving data from +one path inside it to another. + + + +## Description ## + +This device's function runs in two modes: + +1. When using `all` to move all data at the path given in `from` to the +path given in `to`. +2. When using `patches` to move all submessages in the source to the target, +_if_ they have a `method` key of `PATCH` or a `device` key of `patch@1.0`. + +Source and destination paths may be prepended by `base:` or `req:` keys to +indicate that they are relative to either of the message`s that the +computation is being performed on. + +The search order for finding the source and destination keys is as follows, +where `X` is either `from` or `to`: + +1. The `patch-X` key of the execution message. +2. The `X` key of the execution message. +3. The `patch-X` key of the request message. +4. The `X` key of the request message. + +Additionally, this device implements the standard computation device keys, +allowing it to be used as an element of an execution stack pipeline, etc. + +## Function Index ## + + +
all/3Get the value found at the patch-from key of the message, or the +from key if the former is not present.
all_mode_test/0*
compute/3
init/3Necessary hooks for compliance with the execution-device standard.
move/4*Unified executor for the all and patches modes.
normalize/3
patch_to_submessage_test/0*
patches/3Find relevant PATCH messages in the given source key of the execution +and request messages, and apply them to the given destination key of the +request.
req_prefix_test/0*
snapshot/3
uninitialized_patch_test/0*
+ + + + +## Function Details ## + + + +### all/3 ### + +`all(Msg1, Msg2, Opts) -> any()` + +Get the value found at the `patch-from` key of the message, or the +`from` key if the former is not present. Remove it from the message and set +the new source to the value found. + + + +### all_mode_test/0 * ### + +`all_mode_test() -> any()` + + + +### compute/3 ### + +`compute(Msg1, Msg2, Opts) -> any()` + + + +### init/3 ### + +`init(Msg1, Msg2, Opts) -> any()` + +Necessary hooks for compliance with the `execution-device` standard. + + + +### move/4 * ### + +`move(Mode, Msg1, Msg2, Opts) -> any()` + +Unified executor for the `all` and `patches` modes. + + + +### normalize/3 ### + +`normalize(Msg1, Msg2, Opts) -> any()` + + + +### patch_to_submessage_test/0 * ### + +`patch_to_submessage_test() -> any()` + + + +### patches/3 ### + +`patches(Msg1, Msg2, Opts) -> any()` + +Find relevant `PATCH` messages in the given source key of the execution +and request messages, and apply them to the given destination key of the +request. + + + +### req_prefix_test/0 * ### + +`req_prefix_test() -> any()` + + + +### snapshot/3 ### + +`snapshot(Msg1, Msg2, Opts) -> any()` + + + +### uninitialized_patch_test/0 * ### + +`uninitialized_patch_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_patch.md --- + +--- START OF FILE: docs/resources/source-code/dev_poda.md --- +# [Module dev_poda.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_poda.erl) + + + + +A simple exemplar decentralized proof of authority consensus algorithm +for AO processes. + + + +## Description ## + +This device is split into two flows, spanning three +actions. + +Execution flow: +1. Initialization. +2. Validation of incoming messages before execution. +Commitment flow: +1. Adding commitments to results, either on a CU or MU. + +## Function Index ## + + +
add_commitments/2*
commit_to_results/2*
execute/3
extract_opts/1*
find_process/2*Find the process that this message is targeting, in order to +determine which commitments to add.
init/2
is_user_signed/1Determines if a user committed.
pfiltermap/2*Helper function for parallel execution of commitment +gathering.
push/2Hook used by the MU pathway (currently) to add commitments to an +outbound message if the computation requests it.
return_error/2*
validate/2*
validate_commitment/3*
validate_stage/3*
validate_stage/4*
+ + + + +## Function Details ## + + + +### add_commitments/2 * ### + +`add_commitments(NewMsg, S) -> any()` + + + +### commit_to_results/2 * ### + +`commit_to_results(Msg, S) -> any()` + + + +### execute/3 ### + +`execute(Outer, S, Opts) -> any()` + + + +### extract_opts/1 * ### + +`extract_opts(Params) -> any()` + + + +### find_process/2 * ### + +`find_process(Item, X2) -> any()` + +Find the process that this message is targeting, in order to +determine which commitments to add. + + + +### init/2 ### + +`init(S, Params) -> any()` + + + +### is_user_signed/1 ### + +`is_user_signed(Tx) -> any()` + +Determines if a user committed + + + +### pfiltermap/2 * ### + +`pfiltermap(Pred, List) -> any()` + +Helper function for parallel execution of commitment +gathering. + + + +### push/2 ### + +`push(Item, S) -> any()` + +Hook used by the MU pathway (currently) to add commitments to an +outbound message if the computation requests it. + + + +### return_error/2 * ### + +`return_error(S, Reason) -> any()` + + + +### validate/2 * ### + +`validate(Msg, Opts) -> any()` + + + +### validate_commitment/3 * ### + +`validate_commitment(Msg, Comm, Opts) -> any()` + + + +### validate_stage/3 * ### + +`validate_stage(X1, Msg, Opts) -> any()` + + + +### validate_stage/4 * ### + +`validate_stage(X1, Tx, Content, Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_poda.md --- + +--- START OF FILE: docs/resources/source-code/dev_process_cache.md --- +# [Module dev_process_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_process_cache.erl) + + + + +A wrapper around the hb_cache module that provides a more +convenient interface for reading the result of a process at a given slot or +message ID. + + + +## Function Index ## + + +
find_latest_outputs/1*Test for retrieving the latest computed output for a process.
first_with_path/4*Find the latest assignment with the requested path suffix.
first_with_path/5*
latest/2Retrieve the latest slot for a given process.
latest/3
latest/4
path/3*Calculate the path of a result, given a process ID and a slot.
path/4*
process_cache_suite_test_/0*
read/2Read the result of a process at a given slot.
read/3
test_write_and_read_output/1*Test for writing multiple computed outputs, then getting them by +their slot number and by their signed and unsigned IDs.
write/4Write a process computation result to the cache.
+ + + + +## Function Details ## + + + +### find_latest_outputs/1 * ### + +`find_latest_outputs(Opts) -> any()` + +Test for retrieving the latest computed output for a process. + + + +### first_with_path/4 * ### + +`first_with_path(ProcID, RequiredPath, Slots, Opts) -> any()` + +Find the latest assignment with the requested path suffix. + + + +### first_with_path/5 * ### + +`first_with_path(ProcID, Required, Rest, Opts, Store) -> any()` + + + +### latest/2 ### + +`latest(ProcID, Opts) -> any()` + +Retrieve the latest slot for a given process. Optionally state a limit +on the slot number to search for, as well as a required path that the slot +must have. + + + +### latest/3 ### + +`latest(ProcID, RequiredPath, Opts) -> any()` + + + +### latest/4 ### + +`latest(ProcID, RawRequiredPath, Limit, Opts) -> any()` + + + +### path/3 * ### + +`path(ProcID, Ref, Opts) -> any()` + +Calculate the path of a result, given a process ID and a slot. + + + +### path/4 * ### + +`path(ProcID, Ref, PathSuffix, Opts) -> any()` + + + +### process_cache_suite_test_/0 * ### + +`process_cache_suite_test_() -> any()` + + + +### read/2 ### + +`read(ProcID, Opts) -> any()` + +Read the result of a process at a given slot. + + + +### read/3 ### + +`read(ProcID, SlotRef, Opts) -> any()` + + + +### test_write_and_read_output/1 * ### + +`test_write_and_read_output(Opts) -> any()` + +Test for writing multiple computed outputs, then getting them by +their slot number and by their signed and unsigned IDs. + + + +### write/4 ### + +`write(ProcID, Slot, Msg, Opts) -> any()` + +Write a process computation result to the cache. + + +--- END OF FILE: docs/resources/source-code/dev_process_cache.md --- + +--- START OF FILE: docs/resources/source-code/dev_process_worker.md --- +# [Module dev_process_worker.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_process_worker.erl) + + + + +A long-lived process worker that keeps state in memory between +calls. + + + +## Description ## +Implements the interface of `hb_ao` to receive and respond +to computation requests regarding a process as a singleton. + +## Function Index ## + + +
await/5Await a resolution from a worker executing the process@1.0 device.
group/3Returns a group name for a request.
grouper_test/0*
info_test/0*
notify_compute/4Notify any waiters for a specific slot of the computed results.
notify_compute/5*
process_to_group_name/2*
send_notification/4*
server/3Spawn a new worker process.
stop/1Stop a worker process.
test_init/0*
+ + + + +## Function Details ## + + + +### await/5 ### + +`await(Worker, GroupName, Msg1, Msg2, Opts) -> any()` + +Await a resolution from a worker executing the `process@1.0` device. + + + +### group/3 ### + +`group(Msg1, Msg2, Opts) -> any()` + +Returns a group name for a request. The worker is responsible for all +computation work on the same process on a single node, so we use the +process ID as the group name. + + + +### grouper_test/0 * ### + +`grouper_test() -> any()` + + + +### info_test/0 * ### + +`info_test() -> any()` + + + +### notify_compute/4 ### + +`notify_compute(GroupName, SlotToNotify, Msg3, Opts) -> any()` + +Notify any waiters for a specific slot of the computed results. + + + +### notify_compute/5 * ### + +`notify_compute(GroupName, SlotToNotify, Msg3, Opts, Count) -> any()` + + + +### process_to_group_name/2 * ### + +`process_to_group_name(Msg1, Opts) -> any()` + + + +### send_notification/4 * ### + +`send_notification(Listener, GroupName, SlotToNotify, Msg3) -> any()` + + + +### server/3 ### + +`server(GroupName, Msg1, Opts) -> any()` + +Spawn a new worker process. This is called after the end of the first +execution of `hb_ao:resolve/3`, so the state we are given is the +already current. + + + +### stop/1 ### + +`stop(Worker) -> any()` + +Stop a worker process. + + + +### test_init/0 * ### + +`test_init() -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_process_worker.md --- + +--- START OF FILE: docs/resources/source-code/dev_process.md --- +# [Module dev_process.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_process.erl) + + + + +This module contains the device implementation of AO processes +in AO-Core. + + + +## Description ## + +The core functionality of the module is in 'routing' requests +for different functionality (scheduling, computing, and pushing messages) +to the appropriate device. This is achieved by swapping out the device +of the process message with the necessary component in order to run the +execution, then swapping it back before returning. Computation is supported +as a stack of devices, customizable by the user, while the scheduling +device is (by default) a single device. + +This allows the devices to share state as needed. Additionally, after each +computation step the device caches the result at a path relative to the +process definition itself, such that the process message's ID can act as an +immutable reference to the process's growing list of interactions. See +`dev_process_cache` for details. + +The external API of the device is as follows: + +``` + + GET /ID/Schedule: Returns the messages in the schedule + POST /ID/Schedule: Adds a message to the schedule + GET /ID/Compute/[IDorSlotNum]: Returns the state of the process after + applying a message + GET /ID/Now: Returns the /Results key of the latest + computed message +``` + +An example process definition will look like this: + +``` + + Device: Process/1.0 + Scheduler-Device: Scheduler/1.0 + Execution-Device: Stack/1.0 + Execution-Stack: "Scheduler/1.0", "Cron/1.0", "WASM/1.0", "PoDA/1.0" + Cron-Frequency: 10-Minutes + WASM-Image: WASMImageID + PoDA: + Device: PoDA/1.0 + Authority: A + Authority: B + Authority: C + Quorum: 2 +``` + +Runtime options: +Cache-Frequency: The number of assignments that will be computed +before the full (restorable) state should be cached. +Cache-Keys: A list of the keys that should be cached for all +assignments, in addition to `/Results`. + +## Function Index ## + + +
aos_browsable_state_test_/0*
aos_compute_test_/0*
aos_persistent_worker_benchmark_test_/0*
aos_state_access_via_http_test_/0*
aos_state_patch_test_/0*
as_process/2Change the message to for that has the device set as this module.
compute/3Compute the result of an assignment applied to the process state, if it +is the next message.
compute_slot/5*Compute a single slot for a process, given an initialized state.
compute_to_slot/5*Continually get and apply the next assignment from the scheduler until +we reach the target slot that the user has requested.
default_device/3*Returns the default device for a given piece of functionality.
default_device_index/1*
dev_test_process/0Generate a device that has a stack of two dev_tests for +execution.
do_test_restore/0
ensure_loaded/3*Ensure that the process message we have in memory is live and +up-to-date.
ensure_process_key/2Helper function to store a copy of the process key in the message.
get_scheduler_slot_test/0*
http_wasm_process_by_id_test/0*
info/1When the info key is called, we should return the process exports.
init/0
init/3*Before computation begins, a boot phase is required.
next/3*
now/3Returns the known state of the process at either the current slot, or +the latest slot in the cache depending on the process_now_from_cache option.
now_results_test_/0*
persistent_process_test/0*
prior_results_accessible_test_/0*
process_id/3Returns the process ID of the current process.
push/3Recursively push messages to the scheduler until we find a message +that does not lead to any further messages being scheduled.
recursive_path_resolution_test/0*
restore_test_/0*Manually test state restoration without using the cache.
run_as/4*Run a message against Msg1, with the device being swapped out for +the device found at Key.
schedule/3Wraps functions in the Scheduler device.
schedule_aos_call/2
schedule_aos_call/3
schedule_on_process_test/0*
schedule_test_message/2*
schedule_test_message/3*
schedule_wasm_call/3*
schedule_wasm_call/4*
simple_wasm_persistent_worker_benchmark_test/0*
slot/3
snapshot/3
store_result/5*Store the resulting state in the cache, potentially with the snapshot +key.
test_aos_process/0Generate a process message with a random number, and the +dev_wasm device for execution.
test_aos_process/1
test_aos_process/2*
test_base_process/0*Generate a process message with a random number, and no +executor.
test_base_process/1*
test_device_compute_test/0*
test_wasm_process/1
test_wasm_process/2*
wasm_compute_from_id_test/0*
wasm_compute_test/0*
+ + + + +## Function Details ## + + + +### aos_browsable_state_test_/0 * ### + +`aos_browsable_state_test_() -> any()` + + + +### aos_compute_test_/0 * ### + +`aos_compute_test_() -> any()` + + + +### aos_persistent_worker_benchmark_test_/0 * ### + +`aos_persistent_worker_benchmark_test_() -> any()` + + + +### aos_state_access_via_http_test_/0 * ### + +`aos_state_access_via_http_test_() -> any()` + + + +### aos_state_patch_test_/0 * ### + +`aos_state_patch_test_() -> any()` + + + +### as_process/2 ### + +`as_process(Msg1, Opts) -> any()` + +Change the message to for that has the device set as this module. +In situations where the key that is `run_as` returns a message with a +transformed device, this is useful. + + + +### compute/3 ### + +`compute(Msg1, Msg2, Opts) -> any()` + +Compute the result of an assignment applied to the process state, if it +is the next message. + + + +### compute_slot/5 * ### + +`compute_slot(ProcID, State, RawInputMsg, ReqMsg, Opts) -> any()` + +Compute a single slot for a process, given an initialized state. + + + +### compute_to_slot/5 * ### + +`compute_to_slot(ProcID, Msg1, Msg2, TargetSlot, Opts) -> any()` + +Continually get and apply the next assignment from the scheduler until +we reach the target slot that the user has requested. + + + +### default_device/3 * ### + +`default_device(Msg1, Key, Opts) -> any()` + +Returns the default device for a given piece of functionality. Expects +the `process/variant` key to be set in the message. The `execution-device` +_must_ be set in all processes aside those marked with `ao.TN.1` variant. +This is in order to ensure that post-mainnet processes do not default to +using infrastructure that should not be present on nodes in the future. + + + +### default_device_index/1 * ### + +`default_device_index(X1) -> any()` + + + +### dev_test_process/0 ### + +`dev_test_process() -> any()` + +Generate a device that has a stack of two `dev_test`s for +execution. This should generate a message state has doubled +`Already-Seen` elements for each assigned slot. + + + +### do_test_restore/0 ### + +`do_test_restore() -> any()` + + + +### ensure_loaded/3 * ### + +`ensure_loaded(Msg1, Msg2, Opts) -> any()` + +Ensure that the process message we have in memory is live and +up-to-date. + + + +### ensure_process_key/2 ### + +`ensure_process_key(Msg1, Opts) -> any()` + +Helper function to store a copy of the `process` key in the message. + + + +### get_scheduler_slot_test/0 * ### + +`get_scheduler_slot_test() -> any()` + + + +### http_wasm_process_by_id_test/0 * ### + +`http_wasm_process_by_id_test() -> any()` + + + +### info/1 ### + +`info(Msg1) -> any()` + +When the info key is called, we should return the process exports. + + + +### init/0 ### + +`init() -> any()` + + + +### init/3 * ### + +`init(Msg1, Msg2, Opts) -> any()` + +Before computation begins, a boot phase is required. This phase +allows devices on the execution stack to initialize themselves. We set the +`Initialized` key to `True` to indicate that the process has been +initialized. + + + +### next/3 * ### + +`next(Msg1, Msg2, Opts) -> any()` + + + +### now/3 ### + +`now(RawMsg1, Msg2, Opts) -> any()` + +Returns the known state of the process at either the current slot, or +the latest slot in the cache depending on the `process_now_from_cache` option. + + + +### now_results_test_/0 * ### + +`now_results_test_() -> any()` + + + +### persistent_process_test/0 * ### + +`persistent_process_test() -> any()` + + + +### prior_results_accessible_test_/0 * ### + +`prior_results_accessible_test_() -> any()` + + + +### process_id/3 ### + +`process_id(Msg1, Msg2, Opts) -> any()` + +Returns the process ID of the current process. + + + +### push/3 ### + +`push(Msg1, Msg2, Opts) -> any()` + +Recursively push messages to the scheduler until we find a message +that does not lead to any further messages being scheduled. + + + +### recursive_path_resolution_test/0 * ### + +`recursive_path_resolution_test() -> any()` + + + +### restore_test_/0 * ### + +`restore_test_() -> any()` + +Manually test state restoration without using the cache. + + + +### run_as/4 * ### + +`run_as(Key, Msg1, Msg2, Opts) -> any()` + +Run a message against Msg1, with the device being swapped out for +the device found at `Key`. After execution, the device is swapped back +to the original device if the device is the same as we left it. + + + +### schedule/3 ### + +`schedule(Msg1, Msg2, Opts) -> any()` + +Wraps functions in the Scheduler device. + + + +### schedule_aos_call/2 ### + +`schedule_aos_call(Msg1, Code) -> any()` + + + +### schedule_aos_call/3 ### + +`schedule_aos_call(Msg1, Code, Opts) -> any()` + + + +### schedule_on_process_test/0 * ### + +`schedule_on_process_test() -> any()` + + + +### schedule_test_message/2 * ### + +`schedule_test_message(Msg1, Text) -> any()` + + + +### schedule_test_message/3 * ### + +`schedule_test_message(Msg1, Text, MsgBase) -> any()` + + + +### schedule_wasm_call/3 * ### + +`schedule_wasm_call(Msg1, FuncName, Params) -> any()` + + + +### schedule_wasm_call/4 * ### + +`schedule_wasm_call(Msg1, FuncName, Params, Opts) -> any()` + + + +### simple_wasm_persistent_worker_benchmark_test/0 * ### + +`simple_wasm_persistent_worker_benchmark_test() -> any()` + + + +### slot/3 ### + +`slot(Msg1, Msg2, Opts) -> any()` + + + +### snapshot/3 ### + +`snapshot(RawMsg1, Msg2, Opts) -> any()` + + + +### store_result/5 * ### + +`store_result(ProcID, Slot, Msg3, Msg2, Opts) -> any()` + +Store the resulting state in the cache, potentially with the snapshot +key. + + + +### test_aos_process/0 ### + +`test_aos_process() -> any()` + +Generate a process message with a random number, and the +`dev_wasm` device for execution. + + + +### test_aos_process/1 ### + +`test_aos_process(Opts) -> any()` + + + +### test_aos_process/2 * ### + +`test_aos_process(Opts, Stack) -> any()` + + + +### test_base_process/0 * ### + +`test_base_process() -> any()` + +Generate a process message with a random number, and no +executor. + + + +### test_base_process/1 * ### + +`test_base_process(Opts) -> any()` + + + +### test_device_compute_test/0 * ### + +`test_device_compute_test() -> any()` + + + +### test_wasm_process/1 ### + +`test_wasm_process(WASMImage) -> any()` + + + +### test_wasm_process/2 * ### + +`test_wasm_process(WASMImage, Opts) -> any()` + + + +### wasm_compute_from_id_test/0 * ### + +`wasm_compute_from_id_test() -> any()` + + + +### wasm_compute_test/0 * ### + +`wasm_compute_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_process.md --- + +--- START OF FILE: docs/resources/source-code/dev_push.md --- +# [Module dev_push.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_push.erl) + + + + +`push@1.0` takes a message or slot number, evaluates it, and recursively +pushes the resulting messages to other processes. + + + +## Description ## +The `push`ing mechanism +continues until the there are no remaining messages to push. + +## Function Index ## + + +
additional_keys/3*Set the necessary keys in order for the recipient to know where the +message came from.
do_push/3*Push a message or slot number, including its downstream results.
extract/2*Return either the target or the hint.
find_type/2*
full_push_test_/0*
is_async/3*Determine if the push is asynchronous.
multi_process_push_test_/0*
normalize_message/2*Augment the message with from-* keys, if it doesn't already have them.
parse_redirect/1*
ping_pong_script/1*
push/3Push either a message or an assigned slot number.
push_prompts_encoding_change_test/0*
push_result_message/4*Push a downstream message result.
push_with_mode/3*
push_with_redirect_hint_test_disabled/0*
remote_schedule_result/3*
reply_script/0*
schedule_initial_message/3*Push a message or a process, prior to pushing the resulting slot number.
schedule_result/4*Add the necessary keys to the message to be scheduled, then schedule it.
schedule_result/5*
split_target/1*Split the target into the process ID and the optional query string.
target_process/2*Find the target process ID for a message to push.
+ + + + +## Function Details ## + + + +### additional_keys/3 * ### + +`additional_keys(Origin, ToSched, Opts) -> any()` + +Set the necessary keys in order for the recipient to know where the +message came from. + + + +### do_push/3 * ### + +`do_push(Process, Assignment, Opts) -> any()` + +Push a message or slot number, including its downstream results. + + + +### extract/2 * ### + +`extract(X1, Raw) -> any()` + +Return either the `target` or the `hint`. + + + +### find_type/2 * ### + +`find_type(Req, Opts) -> any()` + + + +### full_push_test_/0 * ### + +`full_push_test_() -> any()` + + + +### is_async/3 * ### + +`is_async(Process, Req, Opts) -> any()` + +Determine if the push is asynchronous. + + + +### multi_process_push_test_/0 * ### + +`multi_process_push_test_() -> any()` + + + +### normalize_message/2 * ### + +`normalize_message(MsgToPush, Opts) -> any()` + +Augment the message with from-* keys, if it doesn't already have them. + + + +### parse_redirect/1 * ### + +`parse_redirect(Location) -> any()` + + + +### ping_pong_script/1 * ### + +`ping_pong_script(Limit) -> any()` + + + +### push/3 ### + +`push(Base, Req, Opts) -> any()` + +Push either a message or an assigned slot number. If a `Process` is +provided in the `body` of the request, it will be scheduled (initializing +it if it does not exist). Otherwise, the message specified by the given +`slot` key will be pushed. + +Optional parameters: +`/result-depth`: The depth to which the full contents of the result +will be included in the response. Default: 1, returning +the full result of the first message, but only the 'tree' +of downstream messages. +`/push-mode`: Whether or not the push should be done asynchronously. +Default: `sync`, pushing synchronously. + + + +### push_prompts_encoding_change_test/0 * ### + +`push_prompts_encoding_change_test() -> any()` + + + +### push_result_message/4 * ### + +`push_result_message(TargetProcess, MsgToPush, Origin, Opts) -> any()` + +Push a downstream message result. The `Origin` map contains information +about the origin of the message: The process that originated the message, +the slot number from which it was sent, and the outbox key of the message, +and the depth to which downstream results should be included in the message. + + + +### push_with_mode/3 * ### + +`push_with_mode(Process, Req, Opts) -> any()` + + + +### push_with_redirect_hint_test_disabled/0 * ### + +`push_with_redirect_hint_test_disabled() -> any()` + + + +### remote_schedule_result/3 * ### + +`remote_schedule_result(Location, SignedReq, Opts) -> any()` + + + +### reply_script/0 * ### + +`reply_script() -> any()` + + + +### schedule_initial_message/3 * ### + +`schedule_initial_message(Base, Req, Opts) -> any()` + +Push a message or a process, prior to pushing the resulting slot number. + + + +### schedule_result/4 * ### + +`schedule_result(TargetProcess, MsgToPush, Origin, Opts) -> any()` + +Add the necessary keys to the message to be scheduled, then schedule it. +If the remote scheduler does not support the given codec, it will be +downgraded and re-signed. + + + +### schedule_result/5 * ### + +`schedule_result(TargetProcess, MsgToPush, Codec, Origin, Opts) -> any()` + + + +### split_target/1 * ### + +`split_target(RawTarget) -> any()` + +Split the target into the process ID and the optional query string. + + + +### target_process/2 * ### + +`target_process(MsgToPush, Opts) -> any()` + +Find the target process ID for a message to push. + + +--- END OF FILE: docs/resources/source-code/dev_push.md --- + +--- START OF FILE: docs/resources/source-code/dev_relay.md --- +# [Module dev_relay.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_relay.erl) + + + + +This module implements the relay device, which is responsible for +relaying messages between nodes and other HTTP(S) endpoints. + + + +## Description ## + +It can be called in either `call` or `cast` mode. In `call` mode, it +returns a `{ok, Result}` tuple, where `Result` is the response from the +remote peer to the message sent. In `cast` mode, the invocation returns +immediately, and the message is relayed asynchronously. No response is given +and the device returns `{ok, <<"OK">>}`. + +Example usage: + +``` + + curl /~relay@.1.0/call?method=GET?0.path=https://www.arweave.net/ +``` + + +## Function Index ## + + +
call/3Execute a call request using a node's routes.
call_get_test/0*
cast/3Execute a request in the same way as call/3, but asynchronously.
request/3Preprocess a request to check if it should be relayed to a different node.
request_hook_reroute_to_nearest_test/0*Test that the preprocess/3 function re-routes a request to remote +peers, according to the node's routing table.
+ + + + +## Function Details ## + + + +### call/3 ### + +`call(M1, RawM2, Opts) -> any()` + +Execute a `call` request using a node's routes. + +Supports the following options: +- `target`: The target message to relay. Defaults to the original message. +- `relay-path`: The path to relay the message to. Defaults to the original path. +- `method`: The method to use for the request. Defaults to the original method. +- `requires-sign`: Whether the request requires signing before dispatching. +Defaults to `false`. + + + +### call_get_test/0 * ### + +`call_get_test() -> any()` + + + +### cast/3 ### + +`cast(M1, M2, Opts) -> any()` + +Execute a request in the same way as `call/3`, but asynchronously. Always +returns `<<"OK">>`. + + + +### request/3 ### + +`request(Msg1, Msg2, Opts) -> any()` + +Preprocess a request to check if it should be relayed to a different node. + + + +### request_hook_reroute_to_nearest_test/0 * ### + +`request_hook_reroute_to_nearest_test() -> any()` + +Test that the `preprocess/3` function re-routes a request to remote +peers, according to the node's routing table. + + +--- END OF FILE: docs/resources/source-code/dev_relay.md --- + +--- START OF FILE: docs/resources/source-code/dev_router.md --- +# [Module dev_router.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_router.erl) + + + + +A device that routes outbound messages from the node to their +appropriate network recipients via HTTP. + + + +## Description ## + +All messages are initially +routed to a single process per node, which then load-balances them +between downstream workers that perform the actual requests. + +The routes for the router are defined in the `routes` key of the `Opts`, +as a precidence-ordered list of maps. The first map that matches the +message will be used to determine the route. + +Multiple nodes can be specified as viable for a single route, with the +`Choose` key determining how many nodes to choose from the list (defaulting +to 1). The `Strategy` key determines the load distribution strategy, +which can be one of `Random`, `By-Base`, or `Nearest`. The route may also +define additional parallel execution parameters, which are used by the +`hb_http` module to manage control of requests. + +The structure of the routes should be as follows: + +``` + + Node?: The node to route the message to. + Nodes?: A list of nodes to route the message to. + Strategy?: The load distribution strategy to use. + Choose?: The number of nodes to choose from the list. + Template?: A message template to match the message against, either as a + map or a path regex. +``` + + +## Function Index ## + + +
add_route_test/0*
apply_route/2*Apply a node map's rules for transforming the path of the message.
apply_routes/3*Generate a uri key for each node in a route.
binary_to_bignum/1*Cast a human-readable or native-encoded ID to a big integer.
by_base_determinism_test/0*Ensure that By-Base always chooses the same node for the same +hashpath.
choose/5*Implements the load distribution strategies if given a cluster.
choose_1_test/1*
choose_n_test/1*
device_call_from_singleton_test/0*
dynamic_route_provider_test/0*
dynamic_router_test/0*Example of a Lua module being used as the route_provider for a +HyperBEAM node.
dynamic_routing_by_performance/0*
dynamic_routing_by_performance_test_/0*Demonstrates routing tables being dynamically created and adjusted +according to the real-time performance of nodes.
explicit_route_test/0*
extract_base/2*Extract the base message ID from a request message.
field_distance/2*Calculate the minimum distance between two numbers +(either progressing backwards or forwards), assuming a +256-bit field.
find_target_path/2*Find the target path to route for a request message.
generate_hashpaths/1*
generate_nodes/1*
get_routes_test/0*
info/1Exported function for getting device info, controls which functions are +exposed via the device API.
info/3HTTP info response providing information about this device.
load_routes/1*Load the current routes for the node.
local_dynamic_router_test/0*Example of a Lua module being used as the route_provider for a +HyperBEAM node.
local_process_route_provider_test/0*
lowest_distance/1*Find the node with the lowest distance to the given hashpath.
lowest_distance/2*
match/3Find the first matching template in a list of known routes.
match_routes/3*
match_routes/4*
preprocess/3Preprocess a request to check if it should be relayed to a different node.
register/3
relay_nearest_test/0*
route/2Find the appropriate route for the given message.
route/3
route_provider_test/0*
route_regex_matches_test/0*
route_template_message_matches_test/0*
routes/3Device function that returns all known routes.
simulate/4*
simulation_distribution/2*
simulation_occurences/2*
strategy_suite_test_/0*
template_matches/3*Check if a message matches a message template or path regex.
unique_nodes/1*
unique_test/1*
weighted_random_strategy_test/0*
within_norms/3*
+ + + + +## Function Details ## + + + +### add_route_test/0 * ### + +`add_route_test() -> any()` + + + +### apply_route/2 * ### + +`apply_route(Msg, Route) -> any()` + +Apply a node map's rules for transforming the path of the message. +Supports the following keys: +- `opts`: A map of options to pass to the request. +- `prefix`: The prefix to add to the path. +- `suffix`: The suffix to add to the path. +- `replace`: A regex to replace in the path. + + + +### apply_routes/3 * ### + +`apply_routes(Msg, R, Opts) -> any()` + +Generate a `uri` key for each node in a route. + + + +### binary_to_bignum/1 * ### + +`binary_to_bignum(Bin) -> any()` + +Cast a human-readable or native-encoded ID to a big integer. + + + +### by_base_determinism_test/0 * ### + +`by_base_determinism_test() -> any()` + +Ensure that `By-Base` always chooses the same node for the same +hashpath. + + + +### choose/5 * ### + +`choose(N, X2, Hashpath, Nodes, Opts) -> any()` + +Implements the load distribution strategies if given a cluster. + + + +### choose_1_test/1 * ### + +`choose_1_test(Strategy) -> any()` + + + +### choose_n_test/1 * ### + +`choose_n_test(Strategy) -> any()` + + + +### device_call_from_singleton_test/0 * ### + +`device_call_from_singleton_test() -> any()` + + + +### dynamic_route_provider_test/0 * ### + +`dynamic_route_provider_test() -> any()` + + + +### dynamic_router_test/0 * ### + +`dynamic_router_test() -> any()` + +Example of a Lua module being used as the `route_provider` for a +HyperBEAM node. The module utilized in this example dynamically adjusts the +likelihood of routing to a given node, depending upon price and performance. +also include preprocessing support for routing + + + +### dynamic_routing_by_performance/0 * ### + +`dynamic_routing_by_performance() -> any()` + + + +### dynamic_routing_by_performance_test_/0 * ### + +`dynamic_routing_by_performance_test_() -> any()` + +Demonstrates routing tables being dynamically created and adjusted +according to the real-time performance of nodes. This test utilizes the +`dynamic-router` script to manage routes and recalculate weights based on the +reported performance. + + + +### explicit_route_test/0 * ### + +`explicit_route_test() -> any()` + + + +### extract_base/2 * ### + +`extract_base(RawPath, Opts) -> any()` + +Extract the base message ID from a request message. Produces a single +binary ID that can be used for routing decisions. + + + +### field_distance/2 * ### + +`field_distance(A, B) -> any()` + +Calculate the minimum distance between two numbers +(either progressing backwards or forwards), assuming a +256-bit field. + + + +### find_target_path/2 * ### + +`find_target_path(Msg, Opts) -> any()` + +Find the target path to route for a request message. + + + +### generate_hashpaths/1 * ### + +`generate_hashpaths(Runs) -> any()` + + + +### generate_nodes/1 * ### + +`generate_nodes(N) -> any()` + + + +### get_routes_test/0 * ### + +`get_routes_test() -> any()` + + + +### info/1 ### + +`info(X1) -> any()` + +Exported function for getting device info, controls which functions are +exposed via the device API. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + +HTTP info response providing information about this device + + + +### load_routes/1 * ### + +`load_routes(Opts) -> any()` + +Load the current routes for the node. Allows either explicit routes from +the node message's `routes` key, or dynamic routes generated by resolving the +`route_provider` message. + + + +### local_dynamic_router_test/0 * ### + +`local_dynamic_router_test() -> any()` + +Example of a Lua module being used as the `route_provider` for a +HyperBEAM node. The module utilized in this example dynamically adjusts the +likelihood of routing to a given node, depending upon price and performance. + + + +### local_process_route_provider_test/0 * ### + +`local_process_route_provider_test() -> any()` + + + +### lowest_distance/1 * ### + +`lowest_distance(Nodes) -> any()` + +Find the node with the lowest distance to the given hashpath. + + + +### lowest_distance/2 * ### + +`lowest_distance(Nodes, X) -> any()` + + + +### match/3 ### + +`match(Base, Req, Opts) -> any()` + +Find the first matching template in a list of known routes. Allows the +path to be specified by either the explicit `path` (for internal use by this +module), or `route-path` for use by external devices and users. + + + +### match_routes/3 * ### + +`match_routes(ToMatch, Routes, Opts) -> any()` + + + +### match_routes/4 * ### + +`match_routes(ToMatch, Routes, Keys, Opts) -> any()` + + + +### preprocess/3 ### + +`preprocess(Msg1, Msg2, Opts) -> any()` + +Preprocess a request to check if it should be relayed to a different node. + + + +### register/3 ### + +`register(M1, M2, Opts) -> any()` + + + +### relay_nearest_test/0 * ### + +`relay_nearest_test() -> any()` + + + +### route/2 ### + +`route(Msg, Opts) -> any()` + +Find the appropriate route for the given message. If we are able to +resolve to a single host+path, we return that directly. Otherwise, we return +the matching route (including a list of nodes under `nodes`) from the list of +routes. + +If we have a route that has multiple resolving nodes, check +the load distribution strategy and choose a node. Supported strategies: + +``` + + All: Return all nodes (default). + Random: Distribute load evenly across all nodes, non-deterministically. + By-Base: According to the base message's hashpath. + By-Weight: According to the node's weight key. + Nearest: According to the distance of the node's wallet address to the + base message's hashpath. +``` + +`By-Base` will ensure that all traffic for the same hashpath is routed to the +same node, minimizing work duplication, while `Random` ensures a more even +distribution of the requests. + +Can operate as a `~router@1.0` device, which will ignore the base message, +routing based on the Opts and request message provided, or as a standalone +function, taking only the request message and the `Opts` map. + + + +### route/3 ### + +`route(X1, Msg, Opts) -> any()` + + + +### route_provider_test/0 * ### + +`route_provider_test() -> any()` + + + +### route_regex_matches_test/0 * ### + +`route_regex_matches_test() -> any()` + + + +### route_template_message_matches_test/0 * ### + +`route_template_message_matches_test() -> any()` + + + +### routes/3 ### + +`routes(M1, M2, Opts) -> any()` + +Device function that returns all known routes. + + + +### simulate/4 * ### + +`simulate(Runs, ChooseN, Nodes, Strategy) -> any()` + + + +### simulation_distribution/2 * ### + +`simulation_distribution(SimRes, Nodes) -> any()` + + + +### simulation_occurences/2 * ### + +`simulation_occurences(SimRes, Nodes) -> any()` + + + +### strategy_suite_test_/0 * ### + +`strategy_suite_test_() -> any()` + + + +### template_matches/3 * ### + +`template_matches(ToMatch, Template, Opts) -> any()` + +Check if a message matches a message template or path regex. + + + +### unique_nodes/1 * ### + +`unique_nodes(Simulation) -> any()` + + + +### unique_test/1 * ### + +`unique_test(Strategy) -> any()` + + + +### weighted_random_strategy_test/0 * ### + +`weighted_random_strategy_test() -> any()` + + + +### within_norms/3 * ### + +`within_norms(SimRes, Nodes, TestSize) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_router.md --- + +--- START OF FILE: docs/resources/source-code/dev_scheduler_cache.md --- +# [Module dev_scheduler_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_cache.erl) + + + + + + +## Function Index ## + + +
latest/2Get the latest assignment from the cache.
list/2Get the assignments for a process.
read/3Get an assignment message from the cache.
read_location/2Read the latest known scheduler location for an address.
write/2Write an assignment message into the cache.
write_location/2Write the latest known scheduler location for an address.
+ + + + +## Function Details ## + + + +### latest/2 ### + +`latest(ProcID, Opts) -> any()` + +Get the latest assignment from the cache. + + + +### list/2 ### + +`list(ProcID, Opts) -> any()` + +Get the assignments for a process. + + + +### read/3 ### + +`read(ProcID, Slot, Opts) -> any()` + +Get an assignment message from the cache. + + + +### read_location/2 ### + +`read_location(Address, Opts) -> any()` + +Read the latest known scheduler location for an address. + + + +### write/2 ### + +`write(Assignment, Opts) -> any()` + +Write an assignment message into the cache. + + + +### write_location/2 ### + +`write_location(LocMsg, Opts) -> any()` + +Write the latest known scheduler location for an address. + + +--- END OF FILE: docs/resources/source-code/dev_scheduler_cache.md --- + +--- START OF FILE: docs/resources/source-code/dev_scheduler_formats.md --- +# [Module dev_scheduler_formats.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_formats.erl) + + + + +This module is used by dev_scheduler in order to produce outputs that +are compatible with various forms of AO clients. + + + +## Description ## + +It features two main formats: + +- `application/json` +- `application/http` + +The `application/json` format is a legacy format that is not recommended for +new integrations of the AO protocol. + +## Function Index ## + + +
aos2_normalize_data/1*The hb_gateway_client module expects all JSON structures to at least +have a data field.
aos2_normalize_types/1Normalize an AOS2 formatted message to ensure that all field NAMES and +types are correct.
aos2_to_assignment/2Create and normalize an assignment from an AOS2-style JSON structure.
aos2_to_assignments/3Convert an AOS2-style JSON structure to a normalized HyperBEAM +assignments response.
assignment_to_aos2/2*Convert an assignment to an AOS2-compatible JSON structure.
assignments_to_aos2/4
assignments_to_bundle/4Generate a GET /schedule response for a process as HTTP-sig bundles.
assignments_to_bundle/5*
cursor/2*Generate a cursor for an assignment.
format_opts/1*For all scheduler format operations, we do not calculate hashpaths, +perform cache lookups, or await inprogress results.
+ + + + +## Function Details ## + + + +### aos2_normalize_data/1 * ### + +`aos2_normalize_data(JSONStruct) -> any()` + +The `hb_gateway_client` module expects all JSON structures to at least +have a `data` field. This function ensures that. + + + +### aos2_normalize_types/1 ### + +`aos2_normalize_types(Msg) -> any()` + +Normalize an AOS2 formatted message to ensure that all field NAMES and +types are correct. This involves converting field names to integers and +specific field names to their canonical form. +NOTE: This will result in a message that is not verifiable! It is, however, +necessary for gaining compatibility with the AOS2-style scheduling API. + + + +### aos2_to_assignment/2 ### + +`aos2_to_assignment(A, RawOpts) -> any()` + +Create and normalize an assignment from an AOS2-style JSON structure. +NOTE: This method is destructive to the verifiability of the assignment. + + + +### aos2_to_assignments/3 ### + +`aos2_to_assignments(ProcID, Body, RawOpts) -> any()` + +Convert an AOS2-style JSON structure to a normalized HyperBEAM +assignments response. + + + +### assignment_to_aos2/2 * ### + +`assignment_to_aos2(Assignment, RawOpts) -> any()` + +Convert an assignment to an AOS2-compatible JSON structure. + + + +### assignments_to_aos2/4 ### + +`assignments_to_aos2(ProcID, Assignments, More, RawOpts) -> any()` + + + +### assignments_to_bundle/4 ### + +`assignments_to_bundle(ProcID, Assignments, More, Opts) -> any()` + +Generate a `GET /schedule` response for a process as HTTP-sig bundles. + + + +### assignments_to_bundle/5 * ### + +`assignments_to_bundle(ProcID, Assignments, More, TimeInfo, RawOpts) -> any()` + + + +### cursor/2 * ### + +`cursor(Assignment, RawOpts) -> any()` + +Generate a cursor for an assignment. This should be the slot number, at +least in the case of mainnet `ao.N.1` assignments. In the case of legacynet +(`ao.TN.1`) assignments, we may want to use the assignment ID. + + + +### format_opts/1 * ### + +`format_opts(Opts) -> any()` + +For all scheduler format operations, we do not calculate hashpaths, +perform cache lookups, or await inprogress results. + + +--- END OF FILE: docs/resources/source-code/dev_scheduler_formats.md --- + +--- START OF FILE: docs/resources/source-code/dev_scheduler_registry.md --- +# [Module dev_scheduler_registry.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_registry.erl) + + + + + + +## Function Index ## + + +
create_and_find_process_test/0*
create_multiple_processes_test/0*
find/1Find a process associated with the processor ID in the local registry +If the process is not found, it will not create a new one.
find/2Find a process associated with the processor ID in the local registry +If the process is not found and GenIfNotHosted is true, it attemps to create a new one.
find/3Same as find/2 but with additional options passed when spawning a new process (if needed).
find_non_existent_process_test/0*
get_all_processes_test/0*
get_processes/0Return a list of all currently registered ProcID.
get_wallet/0
maybe_new_proc/3*
start/0
+ + + + +## Function Details ## + + + +### create_and_find_process_test/0 * ### + +`create_and_find_process_test() -> any()` + + + +### create_multiple_processes_test/0 * ### + +`create_multiple_processes_test() -> any()` + + + +### find/1 ### + +`find(ProcID) -> any()` + +Find a process associated with the processor ID in the local registry +If the process is not found, it will not create a new one + + + +### find/2 ### + +`find(ProcID, GenIfNotHosted) -> any()` + +Find a process associated with the processor ID in the local registry +If the process is not found and `GenIfNotHosted` is true, it attemps to create a new one + + + +### find/3 ### + +`find(ProcID, GenIfNotHosted, Opts) -> any()` + +Same as `find/2` but with additional options passed when spawning a new process (if needed) + + + +### find_non_existent_process_test/0 * ### + +`find_non_existent_process_test() -> any()` + + + +### get_all_processes_test/0 * ### + +`get_all_processes_test() -> any()` + + + +### get_processes/0 ### + +`get_processes() -> any()` + +Return a list of all currently registered ProcID. + + + +### get_wallet/0 ### + +`get_wallet() -> any()` + + + +### maybe_new_proc/3 * ### + +`maybe_new_proc(ProcID, GenIfNotHosted, Opts) -> any()` + + + +### start/0 ### + +`start() -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_scheduler_registry.md --- + +--- START OF FILE: docs/resources/source-code/dev_scheduler_server.md --- +# [Module dev_scheduler_server.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_server.erl) + + + + +A long-lived server that schedules messages for a process. + + + +## Description ## +It acts as a deliberate 'bottleneck' to prevent the server accidentally +assigning multiple messages to the same slot. + +## Function Index ## + + +
assign/3*Assign a message to the next slot.
do_assign/3*Generate and store the actual assignment message.
info/1Get the current slot from the scheduling server.
maybe_inform_recipient/5*
new_proc_test_/0*Test the basic functionality of the server.
next_hashchain/2*Create the next element in a chain of hashes that links this and prior +assignments.
schedule/2Call the appropriate scheduling server to assign a message.
server/1*The main loop of the server.
start/2Start a scheduling server for a given computation.
stop/1
+ + + + +## Function Details ## + + + +### assign/3 * ### + +`assign(State, Message, ReplyPID) -> any()` + +Assign a message to the next slot. + + + +### do_assign/3 * ### + +`do_assign(State, Message, ReplyPID) -> any()` + +Generate and store the actual assignment message. + + + +### info/1 ### + +`info(ProcID) -> any()` + +Get the current slot from the scheduling server. + + + +### maybe_inform_recipient/5 * ### + +`maybe_inform_recipient(Mode, ReplyPID, Message, Assignment, State) -> any()` + + + +### new_proc_test_/0 * ### + +`new_proc_test_() -> any()` + +Test the basic functionality of the server. + + + +### next_hashchain/2 * ### + +`next_hashchain(HashChain, Message) -> any()` + +Create the next element in a chain of hashes that links this and prior +assignments. + + + +### schedule/2 ### + +`schedule(AOProcID, Message) -> any()` + +Call the appropriate scheduling server to assign a message. + + + +### server/1 * ### + +`server(State) -> any()` + +The main loop of the server. Simply waits for messages to assign and +returns the current slot. + + + +### start/2 ### + +`start(ProcID, Opts) -> any()` + +Start a scheduling server for a given computation. + + + +### stop/1 ### + +`stop(ProcID) -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_scheduler_server.md --- + +--- START OF FILE: docs/resources/source-code/dev_scheduler.md --- +# [Module dev_scheduler.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler.erl) + + + + +A simple scheduler scheme for AO. + + + +## Description ## +This device expects a message of the form: +Process: `#{ id, Scheduler: #{ Authority } }` + +``` + + It exposes the following keys for scheduling:#{ method: GET, path: <<"/info">> } -> + Returns information about the scheduler.#{ method: GET, path: <<"/slot">> } -> slot(Msg1, Msg2, Opts) + Returns the current slot for a process.#{ method: GET, path: <<"/schedule">> } -> get_schedule(Msg1, Msg2, Opts) + Returns the schedule for a process in a cursor-traversable format.#{ method: POST, path: <<"/schedule">> } -> post_schedule(Msg1, Msg2, Opts) + Schedules a new message for a process, or starts a new scheduler + for the given message. +``` + + +## Function Index ## + + +
benchmark_suite/2*
benchmark_suite_test_/0*
cache_remote_schedule/2*Cache a schedule received from a remote scheduler.
check_lookahead_and_local_cache/4*Check if we have a result from a lookahead worker or from our local +cache.
checkpoint/1Returns the current state of the scheduler.
do_get_remote_schedule/6*Get a schedule from a remote scheduler, unless we already have already +read all of the assignments from the local cache.
do_post_schedule/4*Post schedule the message.
filter_json_assignments/3*Filter JSON assignment results from a remote legacy scheduler.
find_message_to_schedule/3*Search the given base and request message pair to find the message to +schedule.
find_remote_scheduler/3*Use the SchedulerLocation to the remote path and return a redirect.
find_server/3*Locate the correct scheduling server for a given process.
find_server/4*
find_target_id/3*Find the schedule ID from a given request.
generate_local_schedule/5*Generate a GET /schedule response for a process.
generate_redirect/3*Generate a redirect message to a scheduler.
get_hint/2*If a hint is present in the string, return it.
get_local_assignments/4*Get the assignments for a process, and whether the request was truncated.
get_local_schedule_test/0*
get_location/3*Search for the location of the scheduler in the scheduler-location +cache.
get_remote_schedule/5*Get a schedule from a remote scheduler, but first read all of the +assignments from the local cache that we already know about.
get_schedule/3*Generate and return a schedule for a process, optionally between +two slots -- labelled as from and to.
http_get_json_schedule_test_/0*
http_get_legacy_schedule_as_aos2_test_/0*
http_get_legacy_schedule_slot_range_test_/0*
http_get_legacy_schedule_test_/0*
http_get_legacy_slot_test_/0*
http_get_schedule/4*
http_get_schedule/5*
http_get_schedule_redirect_test/0*
http_get_schedule_test_/0*
http_get_slot/2*
http_init/0*
http_init/1*
http_post_legacy_schedule_test_/0*
http_post_schedule_sign/4*
http_post_schedule_test/0*
info/0This device uses a default_handler to route requests to the correct +function.
location/3Router for record requests.
many_clients/1*
message_cached_assignments/2*Non-device exported helper to get the cached assignments held in a +process.
next/3Load the schedule for a process into the cache, then return the next +assignment.
node_from_redirect/2*Get the node URL from a redirect.
post_legacy_schedule/4*
post_location/3*Generate a new scheduler location record and register it.
post_remote_schedule/4*
post_schedule/3*Schedules a new message on the SU.
read_local_assignments/4*Get the assignments for a process.
redirect_from_graphql_test/0*
redirect_to_hint_test/0*
register_location_on_boot_test/0*Test that a scheduler location is registered on boot.
register_new_process_test/0*
register_scheduler_test/0*
remote_slot/3*Get the current slot from a remote scheduler.
remote_slot/4*Get the current slot from a remote scheduler, based on the variant of +the process's scheduler.
router/4The default handler for the scheduler device.
schedule/3A router for choosing between getting the existing schedule, or +scheduling a new message.
schedule_message_and_get_slot_test/0*
single_resolution/1*
slot/3Returns information about the current slot for a process.
spawn_lookahead_worker/3*Spawn a new Erlang process to fetch the next assignments from the local +cache, if we have them available.
start/0Helper to ensure that the environment is started.
status/3Returns information about the entire scheduler.
status_test/0*
test_process/0Generate a _transformed_ process message, not as they are generated +by users.
test_process/1*
without_hint/1*Take a process ID or target with a potential hint and return just the +process ID.
+ + + + +## Function Details ## + + + +### benchmark_suite/2 * ### + +`benchmark_suite(Port, Base) -> any()` + + + +### benchmark_suite_test_/0 * ### + +`benchmark_suite_test_() -> any()` + + + +### cache_remote_schedule/2 * ### + +`cache_remote_schedule(Schedule, Opts) -> any()` + +Cache a schedule received from a remote scheduler. + + + +### check_lookahead_and_local_cache/4 * ### + +`check_lookahead_and_local_cache(Msg1, ProcID, TargetSlot, Opts) -> any()` + +Check if we have a result from a lookahead worker or from our local +cache. If we have a result in the local cache, we may also start a new +lookahead worker to fetch the next assignments if we have them locally, +ahead of time. This can be enabled/disabled with the `scheduler_lookahead` +option. + + + +### checkpoint/1 ### + +`checkpoint(State) -> any()` + +Returns the current state of the scheduler. + + + +### do_get_remote_schedule/6 * ### + +`do_get_remote_schedule(ProcID, LocalAssignments, From, To, Redirect, Opts) -> any()` + +Get a schedule from a remote scheduler, unless we already have already +read all of the assignments from the local cache. + + + +### do_post_schedule/4 * ### + +`do_post_schedule(ProcID, PID, Msg2, Opts) -> any()` + +Post schedule the message. `Msg2` by this point has been refined to only +committed keys, and to only include the `target` message that is to be +scheduled. + + + +### filter_json_assignments/3 * ### + +`filter_json_assignments(JSONRes, To, From) -> any()` + +Filter JSON assignment results from a remote legacy scheduler. + + + +### find_message_to_schedule/3 * ### + +`find_message_to_schedule(Msg1, Msg2, Opts) -> any()` + +Search the given base and request message pair to find the message to +schedule. The precidence order for search is as follows: +1. `Msg2/body` +2. `Msg2` + + + +### find_remote_scheduler/3 * ### + +`find_remote_scheduler(ProcID, Scheduler, Opts) -> any()` + +Use the SchedulerLocation to the remote path and return a redirect. + + + +### find_server/3 * ### + +`find_server(ProcID, Msg1, Opts) -> any()` + +Locate the correct scheduling server for a given process. + + + +### find_server/4 * ### + +`find_server(ProcID, Msg1, ToSched, Opts) -> any()` + + + +### find_target_id/3 * ### + +`find_target_id(Msg1, Msg2, Opts) -> any()` + +Find the schedule ID from a given request. The precidence order for +search is as follows: +[1. `ToSched/id` -- in the case of `POST schedule`, handled locally] +2. `Msg2/target` +3. `Msg2/id` when `Msg2` has `type: Process` +4. `Msg1/process/id` +5. `Msg1/id` when `Msg1` has `type: Process` +6. `Msg2/id` + + + +### generate_local_schedule/5 * ### + +`generate_local_schedule(Format, ProcID, From, To, Opts) -> any()` + +Generate a `GET /schedule` response for a process. + + + +### generate_redirect/3 * ### + +`generate_redirect(ProcID, SchedulerLocation, Opts) -> any()` + +Generate a redirect message to a scheduler. + + + +### get_hint/2 * ### + +`get_hint(Str, Opts) -> any()` + +If a hint is present in the string, return it. Else, return not_found. + + + +### get_local_assignments/4 * ### + +`get_local_assignments(ProcID, From, RequestedTo, Opts) -> any()` + +Get the assignments for a process, and whether the request was truncated. + + + +### get_local_schedule_test/0 * ### + +`get_local_schedule_test() -> any()` + + + +### get_location/3 * ### + +`get_location(Msg1, Req, Opts) -> any()` + +Search for the location of the scheduler in the scheduler-location +cache. If an address is provided, we search for the location of that +specific scheduler. Otherwise, we return the location record for the current +node's scheduler, if it has been established. + + + +### get_remote_schedule/5 * ### + +`get_remote_schedule(RawProcID, From, To, Redirect, Opts) -> any()` + +Get a schedule from a remote scheduler, but first read all of the +assignments from the local cache that we already know about. + + + +### get_schedule/3 * ### + +`get_schedule(Msg1, Msg2, Opts) -> any()` + +Generate and return a schedule for a process, optionally between +two slots -- labelled as `from` and `to`. If the schedule is not local, +we redirect to the remote scheduler or proxy based on the node opts. + + + +### http_get_json_schedule_test_/0 * ### + +`http_get_json_schedule_test_() -> any()` + + + +### http_get_legacy_schedule_as_aos2_test_/0 * ### + +`http_get_legacy_schedule_as_aos2_test_() -> any()` + + + +### http_get_legacy_schedule_slot_range_test_/0 * ### + +`http_get_legacy_schedule_slot_range_test_() -> any()` + + + +### http_get_legacy_schedule_test_/0 * ### + +`http_get_legacy_schedule_test_() -> any()` + + + +### http_get_legacy_slot_test_/0 * ### + +`http_get_legacy_slot_test_() -> any()` + + + +### http_get_schedule/4 * ### + +`http_get_schedule(N, PMsg, From, To) -> any()` + + + +### http_get_schedule/5 * ### + +`http_get_schedule(N, PMsg, From, To, Format) -> any()` + + + +### http_get_schedule_redirect_test/0 * ### + +`http_get_schedule_redirect_test() -> any()` + + + +### http_get_schedule_test_/0 * ### + +`http_get_schedule_test_() -> any()` + + + +### http_get_slot/2 * ### + +`http_get_slot(N, PMsg) -> any()` + + + +### http_init/0 * ### + +`http_init() -> any()` + + + +### http_init/1 * ### + +`http_init(Opts) -> any()` + + + +### http_post_legacy_schedule_test_/0 * ### + +`http_post_legacy_schedule_test_() -> any()` + + + +### http_post_schedule_sign/4 * ### + +`http_post_schedule_sign(Node, Msg, ProcessMsg, Wallet) -> any()` + + + +### http_post_schedule_test/0 * ### + +`http_post_schedule_test() -> any()` + + + +### info/0 ### + +`info() -> any()` + +This device uses a default_handler to route requests to the correct +function. + + + +### location/3 ### + +`location(Msg1, Msg2, Opts) -> any()` + +Router for `record` requests. Expects either a `POST` or `GET` request. + + + +### many_clients/1 * ### + +`many_clients(Opts) -> any()` + + + +### message_cached_assignments/2 * ### + +`message_cached_assignments(Msg, Opts) -> any()` + +Non-device exported helper to get the cached assignments held in a +process. + + + +### next/3 ### + +`next(Msg1, Msg2, Opts) -> any()` + +Load the schedule for a process into the cache, then return the next +assignment. Assumes that Msg1 is a `dev_process` or similar message, having +a `Current-Slot` key. It stores a local cache of the schedule in the +`priv/To-Process` key. + + + +### node_from_redirect/2 * ### + +`node_from_redirect(Redirect, Opts) -> any()` + +Get the node URL from a redirect. + + + +### post_legacy_schedule/4 * ### + +`post_legacy_schedule(ProcID, OnlyCommitted, Node, Opts) -> any()` + + + +### post_location/3 * ### + +`post_location(Msg1, RawReq, Opts) -> any()` + +Generate a new scheduler location record and register it. We both send +the new scheduler-location to the given registry, and return it to the caller. + + + +### post_remote_schedule/4 * ### + +`post_remote_schedule(RawProcID, Redirect, OnlyCommitted, Opts) -> any()` + + + +### post_schedule/3 * ### + +`post_schedule(Msg1, Msg2, Opts) -> any()` + +Schedules a new message on the SU. Searches Msg1 for the appropriate ID, +then uses the wallet address of the scheduler to determine if the message is +for this scheduler. If so, it schedules the message and returns the assignment. + + + +### read_local_assignments/4 * ### + +`read_local_assignments(ProcID, From, To, Opts) -> any()` + +Get the assignments for a process. + + + +### redirect_from_graphql_test/0 * ### + +`redirect_from_graphql_test() -> any()` + + + +### redirect_to_hint_test/0 * ### + +`redirect_to_hint_test() -> any()` + + + +### register_location_on_boot_test/0 * ### + +`register_location_on_boot_test() -> any()` + +Test that a scheduler location is registered on boot. + + + +### register_new_process_test/0 * ### + +`register_new_process_test() -> any()` + + + +### register_scheduler_test/0 * ### + +`register_scheduler_test() -> any()` + + + +### remote_slot/3 * ### + +`remote_slot(ProcID, Redirect, Opts) -> any()` + +Get the current slot from a remote scheduler. + + + +### remote_slot/4 * ### + +`remote_slot(X1, ProcID, Node, Opts) -> any()` + +Get the current slot from a remote scheduler, based on the variant of +the process's scheduler. + + + +### router/4 ### + +`router(X1, Msg1, Msg2, Opts) -> any()` + +The default handler for the scheduler device. + + + +### schedule/3 ### + +`schedule(Msg1, Msg2, Opts) -> any()` + +A router for choosing between getting the existing schedule, or +scheduling a new message. + + + +### schedule_message_and_get_slot_test/0 * ### + +`schedule_message_and_get_slot_test() -> any()` + + + +### single_resolution/1 * ### + +`single_resolution(Opts) -> any()` + + + +### slot/3 ### + +`slot(M1, M2, Opts) -> any()` + +Returns information about the current slot for a process. + + + +### spawn_lookahead_worker/3 * ### + +`spawn_lookahead_worker(ProcID, Slot, Opts) -> any()` + +Spawn a new Erlang process to fetch the next assignments from the local +cache, if we have them available. + + + +### start/0 ### + +`start() -> any()` + +Helper to ensure that the environment is started. + + + +### status/3 ### + +`status(M1, M2, Opts) -> any()` + +Returns information about the entire scheduler. + + + +### status_test/0 * ### + +`status_test() -> any()` + + + +### test_process/0 ### + +`test_process() -> any()` + +Generate a _transformed_ process message, not as they are generated +by users. See `dev_process` for examples of AO process messages. + + + +### test_process/1 * ### + +`test_process(Wallet) -> any()` + + + +### without_hint/1 * ### + +`without_hint(Target) -> any()` + +Take a process ID or target with a potential hint and return just the +process ID. + + +--- END OF FILE: docs/resources/source-code/dev_scheduler.md --- + +--- START OF FILE: docs/resources/source-code/dev_simple_pay.md --- +# [Module dev_simple_pay.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_simple_pay.erl) + + + + +A simple device that allows the operator to specify a price for a +request and then charge the user for it, on a per message basis. + + + +## Description ## +The device's ledger is stored in the node message at `simple_pay_ledger`, +and can be topped-up by either the operator, or an external device. The +price is specified in the node message at `simple_pay_price`. +This device acts as both a pricing device and a ledger device, by p4's +definition. + +## Function Index ## + + +
balance/3Get the balance of a user in the ledger.
debit/3Preprocess a request by checking the ledger and charging the user.
estimate/3Estimate the cost of a request by counting the number of messages in +the request, then multiplying by the per-message price.
get_balance/2*Get the balance of a user in the ledger.
get_balance_and_top_up_test/0*
is_operator/2*Check if the request is from the operator.
set_balance/3*Adjust a user's balance, normalizing their wallet ID first.
test_opts/1*
topup/3Top up the user's balance in the ledger.
+ + + + +## Function Details ## + + + +### balance/3 ### + +`balance(X1, RawReq, NodeMsg) -> any()` + +Get the balance of a user in the ledger. + + + +### debit/3 ### + +`debit(X1, RawReq, NodeMsg) -> any()` + +Preprocess a request by checking the ledger and charging the user. We +can charge the user at this stage because we know statically what the price +will be + + + +### estimate/3 ### + +`estimate(X1, EstimateReq, NodeMsg) -> any()` + +Estimate the cost of a request by counting the number of messages in +the request, then multiplying by the per-message price. The operator does +not pay for their own requests. + + + +### get_balance/2 * ### + +`get_balance(Signer, NodeMsg) -> any()` + +Get the balance of a user in the ledger. + + + +### get_balance_and_top_up_test/0 * ### + +`get_balance_and_top_up_test() -> any()` + + + +### is_operator/2 * ### + +`is_operator(Req, NodeMsg) -> any()` + +Check if the request is from the operator. + + + +### set_balance/3 * ### + +`set_balance(Signer, Amount, NodeMsg) -> any()` + +Adjust a user's balance, normalizing their wallet ID first. + + + +### test_opts/1 * ### + +`test_opts(Ledger) -> any()` + + + +### topup/3 ### + +`topup(X1, Req, NodeMsg) -> any()` + +Top up the user's balance in the ledger. + + +--- END OF FILE: docs/resources/source-code/dev_simple_pay.md --- + +--- START OF FILE: docs/resources/source-code/dev_snp_nif.md --- +# [Module dev_snp_nif.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_snp_nif.erl) + + + + + + +## Function Index ## + + +
check_snp_support/0
compute_launch_digest/1
compute_launch_digest_test/0*
generate_attestation_report/2
generate_attestation_report_test/0*
init/0*
not_loaded/1*
verify_measurement/2
verify_measurement_test/0*
verify_signature/1
verify_signature_test/0*
+ + + + +## Function Details ## + + + +### check_snp_support/0 ### + +`check_snp_support() -> any()` + + + +### compute_launch_digest/1 ### + +`compute_launch_digest(Args) -> any()` + + + +### compute_launch_digest_test/0 * ### + +`compute_launch_digest_test() -> any()` + + + +### generate_attestation_report/2 ### + +`generate_attestation_report(UniqueData, VMPL) -> any()` + + + +### generate_attestation_report_test/0 * ### + +`generate_attestation_report_test() -> any()` + + + +### init/0 * ### + +`init() -> any()` + + + +### not_loaded/1 * ### + +`not_loaded(Line) -> any()` + + + +### verify_measurement/2 ### + +`verify_measurement(Report, Expected) -> any()` + + + +### verify_measurement_test/0 * ### + +`verify_measurement_test() -> any()` + + + +### verify_signature/1 ### + +`verify_signature(Report) -> any()` + + + +### verify_signature_test/0 * ### + +`verify_signature_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_snp_nif.md --- + +--- START OF FILE: docs/resources/source-code/dev_snp.md --- +# [Module dev_snp.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_snp.erl) + + + + +This device offers an interface for validating AMD SEV-SNP commitments, +as well as generating them, if called in an appropriate environment. + + + +## Function Index ## + + +
execute_is_trusted/3*Ensure that all of the software hashes are trusted.
generate/3Generate an commitment report and emit it as a message, including all of +the necessary data to generate the nonce (ephemeral node address + node +message ID), as well as the expected measurement (firmware, kernel, and VMSAs +hashes).
generate_nonce/2*Generate the nonce to use in the commitment report.
is_debug/1*Ensure that the node's debug policy is disabled.
real_node_test/0*
report_data_matches/3*Ensure that the report data matches the expected report data.
trusted/3Validates if a given message parameter matches a trusted value from the SNP trusted list +Returns {ok, true} if the message is trusted, {ok, false} otherwise.
verify/3Verify an commitment report message; validating the identity of a +remote node, its ephemeral private address, and the integrity of the report.
+ + + + +## Function Details ## + + + +### execute_is_trusted/3 * ### + +`execute_is_trusted(M1, Msg, NodeOpts) -> any()` + +Ensure that all of the software hashes are trusted. The caller may set +a specific device to use for the `is-trusted` key. The device must then +implement the `trusted` resolver. + + + +### generate/3 ### + +`generate(M1, M2, Opts) -> any()` + +Generate an commitment report and emit it as a message, including all of +the necessary data to generate the nonce (ephemeral node address + node +message ID), as well as the expected measurement (firmware, kernel, and VMSAs +hashes). + + + +### generate_nonce/2 * ### + +`generate_nonce(RawAddress, RawNodeMsgID) -> any()` + +Generate the nonce to use in the commitment report. + + + +### is_debug/1 * ### + +`is_debug(Report) -> any()` + +Ensure that the node's debug policy is disabled. + + + +### real_node_test/0 * ### + +`real_node_test() -> any()` + + + +### report_data_matches/3 * ### + +`report_data_matches(Address, NodeMsgID, ReportData) -> any()` + +Ensure that the report data matches the expected report data. + + + +### trusted/3 ### + +`trusted(Msg1, Msg2, NodeOpts) -> any()` + +Validates if a given message parameter matches a trusted value from the SNP trusted list +Returns {ok, true} if the message is trusted, {ok, false} otherwise + + + +### verify/3 ### + +`verify(M1, M2, NodeOpts) -> any()` + +Verify an commitment report message; validating the identity of a +remote node, its ephemeral private address, and the integrity of the report. +The checks that must be performed to validate the report are: +1. Verify the address and the node message ID are the same as the ones +used to generate the nonce. +2. Verify the address that signed the message is the same as the one used +to generate the nonce. +3. Verify that the debug flag is disabled. +4. Verify that the firmware, kernel, and OS (VMSAs) hashes, part of the +measurement, are trusted. +5. Verify the measurement is valid. +6. Verify the report's certificate chain to hardware root of trust. + + +--- END OF FILE: docs/resources/source-code/dev_snp.md --- + +--- START OF FILE: docs/resources/source-code/dev_stack.md --- +# [Module dev_stack.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_stack.erl) + + + + +A device that contains a stack of other devices, and manages their +execution. + + + +## Description ## + +It can run in two modes: fold (the default), and map. + +In fold mode, it runs upon input messages in the order of their keys. A +stack maintains and passes forward a state (expressed as a message) as it +progresses through devices. + +For example, a stack of devices as follows: + +``` + + Device -> Stack + Device-Stack/1/Name -> Add-One-Device + Device-Stack/2/Name -> Add-Two-Device +``` + +When called with the message: + +``` + + #{ Path = "FuncName", binary => <<"0">> } +``` + +Will produce the output: + +``` + + #{ Path = "FuncName", binary => <<"3">> } + {ok, #{ bin => <<"3">> }} +``` + +In map mode, the stack will run over all the devices in the stack, and +combine their results into a single message. Each of the devices' +output values have a key that is the device's name in the `Device-Stack` +(its number if the stack is a list). + +You can switch between fold and map modes by setting the `Mode` key in the +`Msg2` to either `Fold` or `Map`, or set it globally for the stack by +setting the `Mode` key in the `Msg1` message. The key in `Msg2` takes +precedence over the key in `Msg1`. + +The key that is called upon the device stack is the same key that is used +upon the devices that are contained within it. For example, in the above +scenario we resolve FuncName on the stack, leading FuncName to be called on +Add-One-Device and Add-Two-Device. + +A device stack responds to special statuses upon responses as follows: + +`skip`: Skips the rest of the device stack for the current pass. + +`pass`: Causes the stack to increment its pass number and re-execute +the stack from the first device, maintaining the state +accumulated so far. Only available in fold mode. + +In all cases, the device stack will return the accumulated state to the +caller as the result of the call to the stack. + +The dev_stack adds additional metadata to the message in order to track +the state of its execution as it progresses through devices. These keys +are as follows: + +`Stack-Pass`: The number of times the stack has reset and re-executed +from the first device for the current message. + +`Input-Prefix`: The prefix that the device should use for its outputs +and inputs. + +`Output-Prefix`: The device that was previously executed. + +All counters used by the stack are initialized to 1. + +Additionally, as implemented in HyperBEAM, the device stack will honor a +number of options that are passed to it as keys in the message. Each of +these options is also passed through to the devices contained within the +stack during execution. These options include: + +`Error-Strategy`: Determines how the stack handles errors from devices. +See `maybe_error/5` for more information. + +`Allow-Multipass`: Determines whether the stack is allowed to automatically +re-execute from the first device when the `pass` tag is returned. See +`maybe_pass/3` for more information. + +Under-the-hood, dev_stack uses a `default` handler to resolve all calls to +devices, aside `set/2` which it calls itself to mutate the message's `device` +key in order to change which device is currently being executed. This method +allows dev_stack to ensure that the message's HashPath is always correct, +even as it delegates calls to other devices. An example flow for a `dev_stack` +execution is as follows: + +``` + + /Msg1/AlicesExcitingKey -> + dev_stack:execute -> + /Msg1/Set?device=/Device-Stack/1 -> + /Msg2/AlicesExcitingKey -> + /Msg3/Set?device=/Device-Stack/2 -> + /Msg4/AlicesExcitingKey + ... -> + /MsgN/Set?device=[This-Device] -> + returns {ok, /MsgN+1} -> + /MsgN+1 +``` + +In this example, the `device` key is mutated a number of times, but the +resulting HashPath remains correct and verifiable. + +## Function Index ## + + +
benchmark_test/0*
example_device_for_stack_test/0*
generate_append_device/1
generate_append_device/2*
increment_pass/2*Helper to increment the pass number.
info/1
input_and_output_prefixes_test/0*
input_output_prefixes_passthrough_test/0*
input_prefix/3Return the input prefix for the stack.
many_devices_test/0*
maybe_error/5*
no_prefix_test/0*
not_found_test/0*
output_prefix/3Return the output prefix for the stack.
output_prefix_test/0*
pass_test/0*
prefix/3Return the default prefix for the stack.
reinvocation_test/0*
resolve_fold/3*The main device stack execution engine.
resolve_fold/4*
resolve_map/3*Map over the devices in the stack, accumulating the output in a single +message of keys and values, where keys are the same as the keys in the +original message (typically a number).
router/3*
router/4The device stack key router.
simple_map_test/0*
simple_stack_execute_test/0*
skip_test/0*
test_prefix_msg/0*
transform/3*Return Message1, transformed such that the device named Key from the +Device-Stack key in the message takes the place of the original Device +key.
transform_external_call_device_test/0*Ensure we can generate a transformer message that can be called to +return a version of msg1 with only that device attached.
transform_internal_call_device_test/0*Test that the transform function can be called correctly internally +by other functions in the module.
transformer_message/2*Return a message which, when given a key, will transform the message +such that the device named Key from the Device-Stack key in the message +takes the place of the original Device key.
+ + + + +## Function Details ## + + + +### benchmark_test/0 * ### + +`benchmark_test() -> any()` + + + +### example_device_for_stack_test/0 * ### + +`example_device_for_stack_test() -> any()` + + + +### generate_append_device/1 ### + +`generate_append_device(Separator) -> any()` + + + +### generate_append_device/2 * ### + +`generate_append_device(Separator, Status) -> any()` + + + +### increment_pass/2 * ### + +`increment_pass(Message, Opts) -> any()` + +Helper to increment the pass number. + + + +### info/1 ### + +`info(Msg) -> any()` + + + +### input_and_output_prefixes_test/0 * ### + +`input_and_output_prefixes_test() -> any()` + + + +### input_output_prefixes_passthrough_test/0 * ### + +`input_output_prefixes_passthrough_test() -> any()` + + + +### input_prefix/3 ### + +`input_prefix(Msg1, Msg2, Opts) -> any()` + +Return the input prefix for the stack. + + + +### many_devices_test/0 * ### + +`many_devices_test() -> any()` + + + +### maybe_error/5 * ### + +`maybe_error(Message1, Message2, DevNum, Info, Opts) -> any()` + + + +### no_prefix_test/0 * ### + +`no_prefix_test() -> any()` + + + +### not_found_test/0 * ### + +`not_found_test() -> any()` + + + +### output_prefix/3 ### + +`output_prefix(Msg1, Msg2, Opts) -> any()` + +Return the output prefix for the stack. + + + +### output_prefix_test/0 * ### + +`output_prefix_test() -> any()` + + + +### pass_test/0 * ### + +`pass_test() -> any()` + + + +### prefix/3 ### + +`prefix(Msg1, Msg2, Opts) -> any()` + +Return the default prefix for the stack. + + + +### reinvocation_test/0 * ### + +`reinvocation_test() -> any()` + + + +### resolve_fold/3 * ### + +`resolve_fold(Message1, Message2, Opts) -> any()` + +The main device stack execution engine. See the moduledoc for more +information. + + + +### resolve_fold/4 * ### + +`resolve_fold(Message1, Message2, DevNum, Opts) -> any()` + + + +### resolve_map/3 * ### + +`resolve_map(Message1, Message2, Opts) -> any()` + +Map over the devices in the stack, accumulating the output in a single +message of keys and values, where keys are the same as the keys in the +original message (typically a number). + + + +### router/3 * ### + +`router(Message1, Message2, Opts) -> any()` + + + +### router/4 ### + +`router(Key, Message1, Message2, Opts) -> any()` + +The device stack key router. Sends the request to `resolve_stack`, +except for `set/2` which is handled by the default implementation in +`dev_message`. + + + +### simple_map_test/0 * ### + +`simple_map_test() -> any()` + + + +### simple_stack_execute_test/0 * ### + +`simple_stack_execute_test() -> any()` + + + +### skip_test/0 * ### + +`skip_test() -> any()` + + + +### test_prefix_msg/0 * ### + +`test_prefix_msg() -> any()` + + + +### transform/3 * ### + +`transform(Msg1, Key, Opts) -> any()` + +Return Message1, transformed such that the device named `Key` from the +`Device-Stack` key in the message takes the place of the original `Device` +key. This transformation allows dev_stack to correctly track the HashPath +of the message as it delegates execution to devices contained within it. + + + +### transform_external_call_device_test/0 * ### + +`transform_external_call_device_test() -> any()` + +Ensure we can generate a transformer message that can be called to +return a version of msg1 with only that device attached. + + + +### transform_internal_call_device_test/0 * ### + +`transform_internal_call_device_test() -> any()` + +Test that the transform function can be called correctly internally +by other functions in the module. + + + +### transformer_message/2 * ### + +`transformer_message(Msg1, Opts) -> any()` + +Return a message which, when given a key, will transform the message +such that the device named `Key` from the `Device-Stack` key in the message +takes the place of the original `Device` key. This allows users to call +a single device from the stack: + +/Msg1/Transform/DeviceName/keyInDevice -> +keyInDevice executed on DeviceName against Msg1. + + +--- END OF FILE: docs/resources/source-code/dev_stack.md --- + +--- START OF FILE: docs/resources/source-code/dev_test.md --- +# [Module dev_test.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_test.erl) + + + + + + +## Function Index ## + + +
compute/3Example implementation of a compute handler.
compute_test/0*
delay/3Does nothing, just sleeps Req/duration or 750 ms and returns the +appropriate form in order to be used as a hook.
device_with_function_key_module_test/0*Tests the resolution of a default function.
increment_counter/3Find a test worker's PID and send it an increment message.
info/1Exports a default_handler function that can be used to test the +handler resolution mechanism.
info/3Exports a default_handler function that can be used to test the +handler resolution mechanism.
init/3Example init/3 handler.
mul/2Example implementation of an imported function for a WASM +executor.
restore/3Example restore/3 handler.
restore_test/0*
snapshot/3Do nothing when asked to snapshot.
test_func/1
update_state/3Find a test worker's PID and send it an update message.
+ + + + +## Function Details ## + + + +### compute/3 ### + +`compute(Msg1, Msg2, Opts) -> any()` + +Example implementation of a `compute` handler. Makes a running list of +the slots that have been computed in the state message and places the new +slot number in the results key. + + + +### compute_test/0 * ### + +`compute_test() -> any()` + + + +### delay/3 ### + +`delay(Msg1, Req, Opts) -> any()` + +Does nothing, just sleeps `Req/duration or 750` ms and returns the +appropriate form in order to be used as a hook. + + + +### device_with_function_key_module_test/0 * ### + +`device_with_function_key_module_test() -> any()` + +Tests the resolution of a default function. + + + +### increment_counter/3 ### + +`increment_counter(Msg1, Msg2, Opts) -> any()` + +Find a test worker's PID and send it an increment message. + + + +### info/1 ### + +`info(X1) -> any()` + +Exports a default_handler function that can be used to test the +handler resolution mechanism. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + +Exports a default_handler function that can be used to test the +handler resolution mechanism. + + + +### init/3 ### + +`init(Msg, Msg2, Opts) -> any()` + +Example `init/3` handler. Sets the `Already-Seen` key to an empty list. + + + +### mul/2 ### + +`mul(Msg1, Msg2) -> any()` + +Example implementation of an `imported` function for a WASM +executor. + + + +### restore/3 ### + +`restore(Msg, Msg2, Opts) -> any()` + +Example `restore/3` handler. Sets the hidden key `Test/Started` to the +value of `Current-Slot` and checks whether the `Already-Seen` key is valid. + + + +### restore_test/0 * ### + +`restore_test() -> any()` + + + +### snapshot/3 ### + +`snapshot(Msg1, Msg2, Opts) -> any()` + +Do nothing when asked to snapshot. + + + +### test_func/1 ### + +`test_func(X1) -> any()` + + + +### update_state/3 ### + +`update_state(Msg, Msg2, Opts) -> any()` + +Find a test worker's PID and send it an update message. + + +--- END OF FILE: docs/resources/source-code/dev_test.md --- + +--- START OF FILE: docs/resources/source-code/dev_volume.md --- +# [Module dev_volume.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_volume.erl) + + + + +Secure Volume Management for HyperBEAM Nodes. + + + +## Description ## + +This module handles encrypted storage operations for HyperBEAM, providing +a robust and secure approach to data persistence. It manages the complete +lifecycle of encrypted volumes from detection to creation, formatting, and +mounting. + +Key responsibilities: +- Volume detection and initialization +- Encrypted partition creation and formatting +- Secure mounting using cryptographic keys +- Store path reconfiguration to use mounted volumes +- Automatic handling of various system states +(new device, existing partition, etc.) + +The primary entry point is the `mount/3` function, which orchestrates the +entire process based on the provided configuration parameters. This module +works alongside `hb_volume` which provides the low-level operations for +device manipulation. + +Security considerations: +- Ensures data at rest is protected through LUKS encryption +- Provides proper volume sanitization and secure mounting +- IMPORTANT: This module only applies configuration set in node options and +does NOT accept disk operations via HTTP requests. It cannot format arbitrary +disks as all operations are safeguarded by host operating system permissions +enforced upon the HyperBEAM environment. + +## Function Index ## + + +
check_base_device/8*Check if the base device exists and if it does, check if the partition exists.
check_partition/8*Check if the partition exists.
create_and_mount_partition/8*Create, format and mount a new partition.
decrypt_volume_key/2*Decrypts an encrypted volume key using the node's private key.
format_and_mount/6*Format and mount a newly created partition.
info/1Exported function for getting device info, controls which functions are +exposed via the device API.
info/3HTTP info response providing information about this device.
mount/3Handles the complete process of secure encrypted volume mounting.
mount_existing_partition/6*Mount an existing partition.
mount_formatted_partition/6*Mount a newly formatted partition.
public_key/3Returns the node's public key for secure key exchange.
update_node_config/2*Update the node's configuration with the new store.
update_store_path/2*Update the store path to use the mounted volume.
+ + + + +## Function Details ## + + + +### check_base_device/8 * ### + +

+check_base_device(Device::term(), Partition::term(), PartitionType::term(), VolumeName::term(), MountPoint::term(), StorePath::term(), Key::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Device`: The base device to check.
`Partition`: The partition to check.
`PartitionType`: The type of partition to check.
`VolumeName`: The name of the volume to check.
`MountPoint`: The mount point to check.
`StorePath`: The store path to check.
`Key`: The key to check.
`Opts`: The options to check.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Check if the base device exists and if it does, check if the partition exists. + + + +### check_partition/8 * ### + +

+check_partition(Device::term(), Partition::term(), PartitionType::term(), VolumeName::term(), MountPoint::term(), StorePath::term(), Key::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Device`: The base device to check.
`Partition`: The partition to check.
`PartitionType`: The type of partition to check.
`VolumeName`: The name of the volume to check.
`MountPoint`: The mount point to check.
`StorePath`: The store path to check.
`Key`: The key to check.
`Opts`: The options to check.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Check if the partition exists. If it does, attempt to mount it. +If it doesn't exist, create it, format it with encryption and mount it. + + + +### create_and_mount_partition/8 * ### + +

+create_and_mount_partition(Device::term(), Partition::term(), PartitionType::term(), Key::term(), MountPoint::term(), VolumeName::term(), StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Device`: The device to create the partition on.
`Partition`: The partition to create.
`PartitionType`: The type of partition to create.
`Key`: The key to create the partition with.
`MountPoint`: The mount point to mount the partition to.
`VolumeName`: The name of the volume to mount.
`StorePath`: The store path to mount.
`Opts`: The options to mount.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Create, format and mount a new partition. + + + +### decrypt_volume_key/2 * ### + +

+decrypt_volume_key(EncryptedKeyBase64::binary(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options.
+ +returns: `{ok, DecryptedKey}` on successful decryption, or +`{error, Binary}` if decryption fails. + +Decrypts an encrypted volume key using the node's private key. + +This function takes an encrypted key (typically sent by a client who encrypted +it with the node's public key) and decrypts it using the node's private RSA key. + + + +### format_and_mount/6 * ### + +

+format_and_mount(Partition::term(), Key::term(), MountPoint::term(), VolumeName::term(), StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Partition`: The partition to format and mount.
`Key`: The key to format and mount the partition with.
`MountPoint`: The mount point to mount the partition to.
`VolumeName`: The name of the volume to mount.
`StorePath`: The store path to mount.
`Opts`: The options to mount.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Format and mount a newly created partition. + + + +### info/1 ### + +`info(X1) -> any()` + +Exported function for getting device info, controls which functions are +exposed via the device API. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + +HTTP info response providing information about this device + + + +### mount/3 ### + +

+mount(M1::term(), M2::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`M1`: Base message for context.
`M2`: Request message with operation details.
`Opts`: A map of configuration options for volume operations.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Handles the complete process of secure encrypted volume mounting. + +This function performs the following operations depending on the state: +1. Validates the encryption key is present +2. Checks if the base device exists +3. Checks if the partition exists on the device +4. If the partition exists, attempts to mount it +5. If the partition doesn't exist, creates it, formats it with encryption +and mounts it +6. Updates the node's store configuration to use the mounted volume + +Config options in Opts map: +- volume_key: (Required) The encryption key +- volume_device: Base device path +- volume_partition: Partition path +- volume_partition_type: Filesystem type +- volume_name: Name for encrypted volume +- volume_mount_point: Where to mount +- volume_store_path: Store path on volume + + + +### mount_existing_partition/6 * ### + +

+mount_existing_partition(Partition::term(), Key::term(), MountPoint::term(), VolumeName::term(), StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Partition`: The partition to mount.
`Key`: The key to mount.
`MountPoint`: The mount point to mount.
`VolumeName`: The name of the volume to mount.
`StorePath`: The store path to mount.
`Opts`: The options to mount.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Mount an existing partition. + + + +### mount_formatted_partition/6 * ### + +

+mount_formatted_partition(Partition::term(), Key::term(), MountPoint::term(), VolumeName::term(), StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Partition`: The partition to mount.
`Key`: The key to mount the partition with.
`MountPoint`: The mount point to mount the partition to.
`VolumeName`: The name of the volume to mount.
`StorePath`: The store path to mount.
`Opts`: The options to mount.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Mount a newly formatted partition. + + + +### public_key/3 ### + +

+public_key(M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options.
+ +returns: `{ok, Map}` containing the node's public key on success, or +`{error, Binary}` if the node's wallet is not available. + +Returns the node's public key for secure key exchange. + +This function retrieves the node's wallet and extracts the public key +for encryption purposes. It allows users to securely exchange encryption keys +by first encrypting their volume key with the node's public key. + +The process ensures that sensitive keys are never transmitted in plaintext. +The encrypted key can then be securely sent to the node, which will decrypt it +using its private key before using it for volume encryption. + + + +### update_node_config/2 * ### + +

+update_node_config(NewStore::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`NewStore`: The new store to update the node's configuration with.
`Opts`: The options to update the node's configuration with.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Update the node's configuration with the new store. + + + +### update_store_path/2 * ### + +

+update_store_path(StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`StorePath`: The store path to update.
`Opts`: The options to update.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Update the store path to use the mounted volume. + + +--- END OF FILE: docs/resources/source-code/dev_volume.md --- + +--- START OF FILE: docs/resources/source-code/dev_wasi.md --- +# [Module dev_wasi.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_wasi.erl) + + + + +A virtual filesystem device. + + + +## Description ## +Implements a file-system-as-map structure, which is traversible externally. +Each file is a binary and each directory is an AO-Core message. +Additionally, this module adds a series of WASI-preview-1 compatible +functions for accessing the filesystem as imported functions by WASM +modules. + +## Function Index ## + + +
basic_aos_exec_test/0*
clock_time_get/3
compute/1
fd_read/3Read from a file using the WASI-p1 standard interface.
fd_read/5*
fd_write/3WASM stdlib implementation of fd_write, using the WASI-p1 standard +interface.
fd_write/5*
gen_test_aos_msg/1*
gen_test_env/0*
generate_wasi_stack/3*
init/0*
init/3On-boot, initialize the virtual file system with: +- Empty stdio files +- WASI-preview-1 compatible functions for accessing the filesystem +- File descriptors for those files.
parse_iovec/2*Parse an iovec in WASI-preview-1 format.
path_open/3Adds a file descriptor to the state message.
stdout/1Return the stdout buffer from a state message.
vfs_is_serializable_test/0*
wasi_stack_is_serializable_test/0*
+ + + + +## Function Details ## + + + +### basic_aos_exec_test/0 * ### + +`basic_aos_exec_test() -> any()` + + + +### clock_time_get/3 ### + +`clock_time_get(Msg1, Msg2, Opts) -> any()` + + + +### compute/1 ### + +`compute(Msg1) -> any()` + + + +### fd_read/3 ### + +`fd_read(Msg1, Msg2, Opts) -> any()` + +Read from a file using the WASI-p1 standard interface. + + + +### fd_read/5 * ### + +`fd_read(S, Instance, X3, BytesRead, Opts) -> any()` + + + +### fd_write/3 ### + +`fd_write(Msg1, Msg2, Opts) -> any()` + +WASM stdlib implementation of `fd_write`, using the WASI-p1 standard +interface. + + + +### fd_write/5 * ### + +`fd_write(S, Instance, X3, BytesWritten, Opts) -> any()` + + + +### gen_test_aos_msg/1 * ### + +`gen_test_aos_msg(Command) -> any()` + + + +### gen_test_env/0 * ### + +`gen_test_env() -> any()` + + + +### generate_wasi_stack/3 * ### + +`generate_wasi_stack(File, Func, Params) -> any()` + + + +### init/0 * ### + +`init() -> any()` + + + +### init/3 ### + +`init(M1, M2, Opts) -> any()` + +On-boot, initialize the virtual file system with: +- Empty stdio files +- WASI-preview-1 compatible functions for accessing the filesystem +- File descriptors for those files. + + + +### parse_iovec/2 * ### + +`parse_iovec(Instance, Ptr) -> any()` + +Parse an iovec in WASI-preview-1 format. + + + +### path_open/3 ### + +`path_open(Msg1, Msg2, Opts) -> any()` + +Adds a file descriptor to the state message. +path_open(M, Instance, [FDPtr, LookupFlag, PathPtr|_]) -> + + + +### stdout/1 ### + +`stdout(M) -> any()` + +Return the stdout buffer from a state message. + + + +### vfs_is_serializable_test/0 * ### + +`vfs_is_serializable_test() -> any()` + + + +### wasi_stack_is_serializable_test/0 * ### + +`wasi_stack_is_serializable_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/dev_wasi.md --- + +--- START OF FILE: docs/resources/source-code/dev_wasm.md --- +# [Module dev_wasm.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_wasm.erl) + + + + +A device that executes a WASM image on messages using the Memory-64 +preview standard. + + + +## Description ## + +In the backend, this device uses `beamr`: An Erlang wrapper +for WAMR, the WebAssembly Micro Runtime. + +The device has the following requirements and interface: + +``` + + M1/Init -> + Assumes: + M1/process + M1/[Prefix]/image + Generates: + /priv/[Prefix]/instance + /priv/[Prefix]/import-resolver + Side-effects: + Creates a WASM executor loaded in memory of the HyperBEAM node. + M1/Compute -> + Assumes: + M1/priv/[Prefix]/instance + M1/priv/[Prefix]/import-resolver + M1/process + M2/message + M2/message/function OR M1/function + M2/message/parameters OR M1/parameters + Generates: + /results/[Prefix]/type + /results/[Prefix]/output + Side-effects: + Calls the WASM executor with the message and process. + M1/[Prefix]/state -> + Assumes: + M1/priv/[Prefix]/instance + Generates: + Raw binary WASM state +``` + + +## Function Index ## + + +
basic_execution_64_test/0*
basic_execution_test/0*
benchmark_test/0*
cache_wasm_image/1
cache_wasm_image/2
compute/3Call the WASM executor with a message that has been prepared by a prior +pass.
default_import_resolver/3*Take a BEAMR import call and resolve it using hb_ao.
import/3Handle standard library calls by: +1.
imported_function_test/0*
info/2Export all functions aside the instance/3 function.
init/0*
init/3Boot a WASM image on the image stated in the process/image field of +the message.
init_test/0*
input_prefix_test/0*
instance/3Get the WASM instance from the message.
normalize/3Normalize the message to have an open WASM instance, but no literal +State key.
process_prefixes_test/0*Test that realistic prefixing for a dev_process works -- +including both inputs (from Process/) and outputs (to the +Device-Key) work.
snapshot/3Serialize the WASM state to a binary.
state_export_and_restore_test/0*
terminate/3Tear down the WASM executor.
test_run_wasm/4*
undefined_import_stub/3*Log the call to the standard library as an event, and write the +call details into the message.
+ + + + +## Function Details ## + + + +### basic_execution_64_test/0 * ### + +`basic_execution_64_test() -> any()` + + + +### basic_execution_test/0 * ### + +`basic_execution_test() -> any()` + + + +### benchmark_test/0 * ### + +`benchmark_test() -> any()` + + + +### cache_wasm_image/1 ### + +`cache_wasm_image(Image) -> any()` + + + +### cache_wasm_image/2 ### + +`cache_wasm_image(Image, Opts) -> any()` + + + +### compute/3 ### + +`compute(RawM1, M2, Opts) -> any()` + +Call the WASM executor with a message that has been prepared by a prior +pass. + + + +### default_import_resolver/3 * ### + +`default_import_resolver(Msg1, Msg2, Opts) -> any()` + +Take a BEAMR import call and resolve it using `hb_ao`. + + + +### import/3 ### + +`import(Msg1, Msg2, Opts) -> any()` + +Handle standard library calls by: +1. Adding the right prefix to the path from BEAMR. +2. Adding the state to the message at the stdlib path. +3. Resolving the adjusted-path-Msg2 against the added-state-Msg1. +4. If it succeeds, return the new state from the message. +5. If it fails with `not_found`, call the stub handler. + + + +### imported_function_test/0 * ### + +`imported_function_test() -> any()` + + + +### info/2 ### + +`info(Msg1, Opts) -> any()` + +Export all functions aside the `instance/3` function. + + + +### init/0 * ### + +`init() -> any()` + + + +### init/3 ### + +`init(M1, M2, Opts) -> any()` + +Boot a WASM image on the image stated in the `process/image` field of +the message. + + + +### init_test/0 * ### + +`init_test() -> any()` + + + +### input_prefix_test/0 * ### + +`input_prefix_test() -> any()` + + + +### instance/3 ### + +`instance(M1, M2, Opts) -> any()` + +Get the WASM instance from the message. Note that this function is exported +such that other devices can use it, but it is excluded from calls from AO-Core +resolution directly. + + + +### normalize/3 ### + +`normalize(RawM1, M2, Opts) -> any()` + +Normalize the message to have an open WASM instance, but no literal +`State` key. Ensure that we do not change the hashpath during this process. + + + +### process_prefixes_test/0 * ### + +`process_prefixes_test() -> any()` + +Test that realistic prefixing for a `dev_process` works -- +including both inputs (from `Process/`) and outputs (to the +Device-Key) work + + + +### snapshot/3 ### + +`snapshot(M1, M2, Opts) -> any()` + +Serialize the WASM state to a binary. + + + +### state_export_and_restore_test/0 * ### + +`state_export_and_restore_test() -> any()` + + + +### terminate/3 ### + +`terminate(M1, M2, Opts) -> any()` + +Tear down the WASM executor. + + + +### test_run_wasm/4 * ### + +`test_run_wasm(File, Func, Params, AdditionalMsg) -> any()` + + + +### undefined_import_stub/3 * ### + +`undefined_import_stub(Msg1, Msg2, Opts) -> any()` + +Log the call to the standard library as an event, and write the +call details into the message. + + +--- END OF FILE: docs/resources/source-code/dev_wasm.md --- + +--- START OF FILE: docs/resources/source-code/hb_ao_test_vectors.md --- +# [Module hb_ao_test_vectors.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_ao_test_vectors.erl) + + + + +Uses a series of different `Opts` values to test the resolution engine's +execution under different circumstances. + + + +## Function Index ## + + +
as_path_test/1*
basic_get_test/1*
basic_set_test/1*
continue_as_test/1*
deep_recursive_get_test/1*
deep_set_new_messages_test/0*
deep_set_test/1*
deep_set_with_device_test/1*
denormalized_device_key_test/1*
device_excludes_test/1*
device_exports_test/1*
device_with_default_handler_function_test/1*
device_with_handler_function_test/1*
exec_dummy_device/2*Ensure that we can read a device from the cache then execute it.
gen_default_device/0*Create a simple test device that implements the default handler.
gen_handler_device/0*Create a simple test device that implements the handler key.
generate_device_with_keys_using_args/0*Generates a test device with three keys, each of which uses +progressively more of the arguments that can be passed to a device key.
get_as_with_device_test/1*
get_with_device_test/1*
key_from_id_device_with_args_test/1*Test that arguments are passed to a device key as expected.
key_to_binary_test/1*
list_transform_test/1*
load_as_test/1*
load_device_test/0*
recursive_get_test/1*
resolve_binary_key_test/1*
resolve_from_multiple_keys_test/1*
resolve_id_test/1*
resolve_key_twice_test/1*
resolve_path_element_test/1*
resolve_simple_test/1*
run_all_test_/0*Run each test in the file with each set of options.
run_test/0*
set_with_device_test/1*
start_as_test/1*
start_as_with_parameters_test/1*
step_hook_test/1*
test_opts/0*
test_suite/0*
untrusted_load_device_test/0*
+ + + + +## Function Details ## + + + +### as_path_test/1 * ### + +`as_path_test(Opts) -> any()` + + + +### basic_get_test/1 * ### + +`basic_get_test(Opts) -> any()` + + + +### basic_set_test/1 * ### + +`basic_set_test(Opts) -> any()` + + + +### continue_as_test/1 * ### + +`continue_as_test(Opts) -> any()` + + + +### deep_recursive_get_test/1 * ### + +`deep_recursive_get_test(Opts) -> any()` + + + +### deep_set_new_messages_test/0 * ### + +`deep_set_new_messages_test() -> any()` + + + +### deep_set_test/1 * ### + +`deep_set_test(Opts) -> any()` + + + +### deep_set_with_device_test/1 * ### + +`deep_set_with_device_test(Opts) -> any()` + + + +### denormalized_device_key_test/1 * ### + +`denormalized_device_key_test(Opts) -> any()` + + + +### device_excludes_test/1 * ### + +`device_excludes_test(Opts) -> any()` + + + +### device_exports_test/1 * ### + +`device_exports_test(Opts) -> any()` + + + +### device_with_default_handler_function_test/1 * ### + +`device_with_default_handler_function_test(Opts) -> any()` + + + +### device_with_handler_function_test/1 * ### + +`device_with_handler_function_test(Opts) -> any()` + + + +### exec_dummy_device/2 * ### + +`exec_dummy_device(SigningWallet, Opts) -> any()` + +Ensure that we can read a device from the cache then execute it. By +extension, this will also allow us to load a device from Arweave due to the +remote store implementations. + + + +### gen_default_device/0 * ### + +`gen_default_device() -> any()` + +Create a simple test device that implements the default handler. + + + +### gen_handler_device/0 * ### + +`gen_handler_device() -> any()` + +Create a simple test device that implements the handler key. + + + +### generate_device_with_keys_using_args/0 * ### + +`generate_device_with_keys_using_args() -> any()` + +Generates a test device with three keys, each of which uses +progressively more of the arguments that can be passed to a device key. + + + +### get_as_with_device_test/1 * ### + +`get_as_with_device_test(Opts) -> any()` + + + +### get_with_device_test/1 * ### + +`get_with_device_test(Opts) -> any()` + + + +### key_from_id_device_with_args_test/1 * ### + +`key_from_id_device_with_args_test(Opts) -> any()` + +Test that arguments are passed to a device key as expected. +Particularly, we need to ensure that the key function in the device can +specify any arity (1 through 3) and the call is handled correctly. + + + +### key_to_binary_test/1 * ### + +`key_to_binary_test(Opts) -> any()` + + + +### list_transform_test/1 * ### + +`list_transform_test(Opts) -> any()` + + + +### load_as_test/1 * ### + +`load_as_test(Opts) -> any()` + + + +### load_device_test/0 * ### + +`load_device_test() -> any()` + + + +### recursive_get_test/1 * ### + +`recursive_get_test(Opts) -> any()` + + + +### resolve_binary_key_test/1 * ### + +`resolve_binary_key_test(Opts) -> any()` + + + +### resolve_from_multiple_keys_test/1 * ### + +`resolve_from_multiple_keys_test(Opts) -> any()` + + + +### resolve_id_test/1 * ### + +`resolve_id_test(Opts) -> any()` + + + +### resolve_key_twice_test/1 * ### + +`resolve_key_twice_test(Opts) -> any()` + + + +### resolve_path_element_test/1 * ### + +`resolve_path_element_test(Opts) -> any()` + + + +### resolve_simple_test/1 * ### + +`resolve_simple_test(Opts) -> any()` + + + +### run_all_test_/0 * ### + +`run_all_test_() -> any()` + +Run each test in the file with each set of options. Start and reset +the store for each test. + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### set_with_device_test/1 * ### + +`set_with_device_test(Opts) -> any()` + + + +### start_as_test/1 * ### + +`start_as_test(Opts) -> any()` + + + +### start_as_with_parameters_test/1 * ### + +`start_as_with_parameters_test(Opts) -> any()` + + + +### step_hook_test/1 * ### + +`step_hook_test(InitOpts) -> any()` + + + +### test_opts/0 * ### + +`test_opts() -> any()` + + + +### test_suite/0 * ### + +`test_suite() -> any()` + + + +### untrusted_load_device_test/0 * ### + +`untrusted_load_device_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_ao_test_vectors.md --- + +--- START OF FILE: docs/resources/source-code/hb_ao.md --- +# [Module hb_ao.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_ao.erl) + + + + +This module is the root of the device call logic of the +AO-Core protocol in HyperBEAM. + + + +## Description ## + +At the implementation level, every message is simply a collection of keys, +dictated by its `Device`, that can be resolved in order to yield their +values. Each key may return another message or a raw value: + +`ao(Message1, Message2) -> {Status, Message3}` + +Under-the-hood, `AO-Core(Message1, Message2)` leads to the evaluation of +`DeviceMod:PathPart(Message1, Message2)`, which defines the user compute +to be performed. If `Message1` does not specify a device, `dev_message` is +assumed. The key to resolve is specified by the `Path` field of the message. + +After each output, the `HashPath` is updated to include the `Message2` +that was executed upon it. + +Because each message implies a device that can resolve its keys, as well +as generating a merkle tree of the computation that led to the result, +you can see AO-Core protocol as a system for cryptographically chaining +the execution of `combinators`. See `docs/ao-core-protocol.md` for more +information about AO-Core. + +The `Fun(Message1, Message2)` pattern is repeated throughout the HyperBEAM +codebase, sometimes with `MessageX` replaced with `MX` or `MsgX` for brevity. + +Message3 can be either a new message or a raw output value (a binary, integer, +float, atom, or list of such values). + +Devices can be expressed as either modules or maps. They can also be +referenced by an Arweave ID, which can be used to load a device from +the network (depending on the value of the `load_remote_devices` and +`trusted_device_signers` environment settings). + +HyperBEAM device implementations are defined as follows: + +``` + + DevMod:ExportedFunc : Key resolution functions. All are assumed to be + device keys (thus, present in every message that + uses it) unless specified by DevMod:info(). + Each function takes a set of parameters + of the form DevMod:KeyHandler(Msg1, Msg2, Opts). + Each of these arguments can be ommitted if not + needed. Non-exported functions are not assumed + to be device keys. + DevMod:info : Optional. Returns a map of options for the device. All + options are optional and assumed to be the defaults if + not specified. This function can accept a Message1 as + an argument, allowing it to specify its functionality + based on a specific message if appropriate. + info/exports : Overrides the export list of the Erlang module, such that + only the functions in this list are assumed to be device + keys. Defaults to all of the functions that DevMod + exports in the Erlang environment. + info/excludes : A list of keys that should not be resolved by the device, + despite being present in the Erlang module exports list. + info/handler : A function that should be used to handle _all_ keys for + messages using the device. + info/default : A function that should be used to handle all keys that + are not explicitly implemented by the device. Defaults to + the dev_message device, which contains general keys for + interacting with messages. + info/default_mod : A different device module that should be used to + handle all keys that are not explicitly implemented + by the device. Defaults to the dev_message device. + info/grouper : A function that returns the concurrency 'group' name for + an execution. Executions with the same group name will + be executed by sending a message to the associated process + and waiting for a response. This allows you to control + concurrency of execution and to allow executions to share + in-memory state as applicable. Default: A derivation of + Msg1+Msg2. This means that concurrent calls for the same + output will lead to only a single execution. + info/worker : A function that should be run as the 'server' loop of + the executor for interactions using the device. + The HyperBEAM resolver also takes a number of runtime options that change + the way that the environment operates:update_hashpath: Whether to add the Msg2 to HashPath for the Msg3. + Default: true.add_key: Whether to add the key to the start of the arguments. + Default: . +``` + + +## Function Index ## + + +
deep_set/4Recursively search a map, resolving keys, and set the value of the key +at the given path.
default_module/0*The default device is the identity device, which simply returns the +value associated with any key as it exists in its Erlang map.
device_set/4*Call the device's set function.
device_set/5*
do_resolve_many/2*
ensure_loaded/2*Ensure that the message is loaded from the cache if it is an ID.
error_execution/5*Handle an error in a device call.
error_infinite/3*Catch all return if we are in an infinite loop.
error_invalid_intermediate_status/5*
error_invalid_message/3*Catch all return if the message is invalid.
find_exported_function/5Find the function with the highest arity that has the given name, if it +exists.
force_message/2
get/2Shortcut for resolving a key in a message without its status if it is +ok.
get/3
get/4
get_first/2take a sequence of base messages and paths, then return the value of the +first message that can be resolved using a path.
get_first/3
info/2Get the info map for a device, optionally giving it a message if the +device's info function is parameterized by one.
info/3*
info_handler_to_fun/4*Parse a handler key given by a device's info.
internal_opts/1*The execution options that are used internally by this module +when calling itself.
is_exported/2*
is_exported/4Check if a device is guarding a key via its exports list.
keys/1Shortcut to get the list of keys from a message.
keys/2
keys/3
load_device/2Load a device module from its name or a message ID.
maybe_force_message/2*Force the result of a device call into a message if the result is not +requested by the Opts.
message_to_device/2Extract the device module from a message.
message_to_fun/3Calculate the Erlang function that should be called to get a value for +a given key from a device.
normalize_key/1Convert a key to a binary in normalized form.
normalize_key/2
normalize_keys/1Ensure that a message is processable by the AO-Core resolver: No lists.
remove/2Remove a key from a message, using its underlying device.
remove/3
resolve/2Get the value of a message's key by running its associated device +function.
resolve/3
resolve_many/2Resolve a list of messages in sequence.
resolve_stage/4*
resolve_stage/5*
resolve_stage/6*
set/2Shortcut for setting a key in the message using its underlying device.
set/3
set/4
subresolve/4*Execute a sub-resolution.
truncate_args/2Truncate the arguments of a function to the number of arguments it +actually takes.
verify_device_compatibility/2*Verify that a device is compatible with the current machine.
+ + + + +## Function Details ## + + + +### deep_set/4 ### + +`deep_set(Msg, Rest, Value, Opts) -> any()` + +Recursively search a map, resolving keys, and set the value of the key +at the given path. This function has special cases for handling `set` calls +where the path is an empty list (`/`). In this case, if the value is an +immediate, non-complex term, we can set it directly. Otherwise, we use the +device's `set` function to set the value. + + + +### default_module/0 * ### + +`default_module() -> any()` + +The default device is the identity device, which simply returns the +value associated with any key as it exists in its Erlang map. It should also +implement the `set` key, which returns a `Message3` with the values changed +according to the `Message2` passed to it. + + + +### device_set/4 * ### + +`device_set(Msg, Key, Value, Opts) -> any()` + +Call the device's `set` function. + + + +### device_set/5 * ### + +`device_set(Msg, Key, Value, Mode, Opts) -> any()` + + + +### do_resolve_many/2 * ### + +`do_resolve_many(MsgList, Opts) -> any()` + + + +### ensure_loaded/2 * ### + +`ensure_loaded(MsgID, Opts) -> any()` + +Ensure that the message is loaded from the cache if it is an ID. If is +not loadable or already present, we raise an error. + + + +### error_execution/5 * ### + +`error_execution(ExecGroup, Msg2, Whence, X4, Opts) -> any()` + +Handle an error in a device call. + + + +### error_infinite/3 * ### + +`error_infinite(Msg1, Msg2, Opts) -> any()` + +Catch all return if we are in an infinite loop. + + + +### error_invalid_intermediate_status/5 * ### + +`error_invalid_intermediate_status(Msg1, Msg2, Msg3, RemainingPath, Opts) -> any()` + + + +### error_invalid_message/3 * ### + +`error_invalid_message(Msg1, Msg2, Opts) -> any()` + +Catch all return if the message is invalid. + + + +### find_exported_function/5 ### + +`find_exported_function(Msg, Dev, Key, MaxArity, Opts) -> any()` + +Find the function with the highest arity that has the given name, if it +exists. + +If the device is a module, we look for a function with the given name. + +If the device is a map, we look for a key in the map. First we try to find +the key using its literal value. If that fails, we cast the key to an atom +and try again. + + + +### force_message/2 ### + +`force_message(X1, Opts) -> any()` + + + +### get/2 ### + +`get(Path, Msg) -> any()` + +Shortcut for resolving a key in a message without its status if it is +`ok`. This makes it easier to write complex logic on top of messages while +maintaining a functional style. + +Additionally, this function supports the `{as, Device, Msg}` syntax, which +allows the key to be resolved using another device to resolve the key, +while maintaining the tracability of the `HashPath` of the output message. + +Returns the value of the key if it is found, otherwise returns the default +provided by the user, or `not_found` if no default is provided. + + + +### get/3 ### + +`get(Path, Msg, Opts) -> any()` + + + +### get/4 ### + +`get(Path, Msg, Default, Opts) -> any()` + + + +### get_first/2 ### + +`get_first(Paths, Opts) -> any()` + +take a sequence of base messages and paths, then return the value of the +first message that can be resolved using a path. + + + +### get_first/3 ### + +`get_first(Msgs, Default, Opts) -> any()` + + + +### info/2 ### + +`info(Msg, Opts) -> any()` + +Get the info map for a device, optionally giving it a message if the +device's info function is parameterized by one. + + + +### info/3 * ### + +`info(DevMod, Msg, Opts) -> any()` + + + +### info_handler_to_fun/4 * ### + +`info_handler_to_fun(Handler, Msg, Key, Opts) -> any()` + +Parse a handler key given by a device's `info`. + + + +### internal_opts/1 * ### + +`internal_opts(Opts) -> any()` + +The execution options that are used internally by this module +when calling itself. + + + +### is_exported/2 * ### + +`is_exported(Info, Key) -> any()` + + + +### is_exported/4 ### + +`is_exported(Msg, Dev, Key, Opts) -> any()` + +Check if a device is guarding a key via its `exports` list. Defaults to +true if the device does not specify an `exports` list. The `info` function is +always exported, if it exists. Elements of the `exludes` list are not +exported. Note that we check for info _twice_ -- once when the device is +given but the info result is not, and once when the info result is given. +The reason for this is that `info/3` calls other functions that may need to +check if a key is exported, so we must avoid infinite loops. We must, however, +also return a consistent result in the case that only the info result is +given, so we check for it in both cases. + + + +### keys/1 ### + +`keys(Msg) -> any()` + +Shortcut to get the list of keys from a message. + + + +### keys/2 ### + +`keys(Msg, Opts) -> any()` + + + +### keys/3 ### + +`keys(Msg, Opts, X3) -> any()` + + + +### load_device/2 ### + +`load_device(Map, Opts) -> any()` + +Load a device module from its name or a message ID. +Returns {ok, Executable} where Executable is the device module. On error, +a tuple of the form {error, Reason} is returned. + + + +### maybe_force_message/2 * ### + +`maybe_force_message(X1, Opts) -> any()` + +Force the result of a device call into a message if the result is not +requested by the `Opts`. If the result is a literal, we wrap it in a message +and signal the location of the result inside. We also similarly handle ao-result +when the result is a single value and an explicit status code. + + + +### message_to_device/2 ### + +`message_to_device(Msg, Opts) -> any()` + +Extract the device module from a message. + + + +### message_to_fun/3 ### + +`message_to_fun(Msg, Key, Opts) -> any()` + +Calculate the Erlang function that should be called to get a value for +a given key from a device. + +This comes in 7 forms: +1. The message does not specify a device, so we use the default device. +2. The device has a `handler` key in its `Dev:info()` map, which is a +function that takes a key and returns a function to handle that key. We pass +the key as an additional argument to this function. +3. The device has a function of the name `Key`, which should be called +directly. +4. The device does not implement the key, but does have a default handler +for us to call. We pass it the key as an additional argument. +5. The device does not implement the key, and has no default handler. We use +the default device to handle the key. +Error: If the device is specified, but not loadable, we raise an error. + +Returns {ok | add_key, Fun} where Fun is the function to call, and add_key +indicates that the key should be added to the start of the call's arguments. + + + +### normalize_key/1 ### + +`normalize_key(Key) -> any()` + +Convert a key to a binary in normalized form. + + + +### normalize_key/2 ### + +`normalize_key(Key, Opts) -> any()` + + + +### normalize_keys/1 ### + +`normalize_keys(Msg1) -> any()` + +Ensure that a message is processable by the AO-Core resolver: No lists. + + + +### remove/2 ### + +`remove(Msg, Key) -> any()` + +Remove a key from a message, using its underlying device. + + + +### remove/3 ### + +`remove(Msg, Key, Opts) -> any()` + + + +### resolve/2 ### + +`resolve(SingletonMsg, Opts) -> any()` + +Get the value of a message's key by running its associated device +function. Optionally, takes options that control the runtime environment. +This function returns the raw result of the device function call: +`{ok | error, NewMessage}.` +The resolver is composed of a series of discrete phases: +1: Normalization. +2: Cache lookup. +3: Validation check. +4: Persistent-resolver lookup. +5: Device lookup. +6: Execution. +7: Execution of the `step` hook. +8: Subresolution. +9: Cryptographic linking. +10: Result caching. +11: Notify waiters. +12: Fork worker. +13: Recurse or terminate. + + + +### resolve/3 ### + +`resolve(Msg1, Path, Opts) -> any()` + + + +### resolve_many/2 ### + +`resolve_many(ListMsg, Opts) -> any()` + +Resolve a list of messages in sequence. Take the output of the first +message as the input for the next message. Once the last message is resolved, +return the result. +A `resolve_many` call with only a single ID will attempt to read the message +directly from the store. No execution is performed. + + + +### resolve_stage/4 * ### + +`resolve_stage(X1, Raw, Msg2, Opts) -> any()` + + + +### resolve_stage/5 * ### + +`resolve_stage(X1, Msg1, Msg2, ExecName, Opts) -> any()` + + + +### resolve_stage/6 * ### + +`resolve_stage(X1, Func, Msg1, Msg2, ExecName, Opts) -> any()` + + + +### set/2 ### + +`set(Msg1, Msg2) -> any()` + +Shortcut for setting a key in the message using its underlying device. +Like the `get/3` function, this function honors the `error_strategy` option. +`set` works with maps and recursive paths while maintaining the appropriate +`HashPath` for each step. + + + +### set/3 ### + +`set(RawMsg1, RawMsg2, Opts) -> any()` + + + +### set/4 ### + +`set(Msg1, Key, Value, Opts) -> any()` + + + +### subresolve/4 * ### + +`subresolve(RawMsg1, DevID, ReqPath, Opts) -> any()` + +Execute a sub-resolution. + + + +### truncate_args/2 ### + +`truncate_args(Fun, Args) -> any()` + +Truncate the arguments of a function to the number of arguments it +actually takes. + + + +### verify_device_compatibility/2 * ### + +`verify_device_compatibility(Msg, Opts) -> any()` + +Verify that a device is compatible with the current machine. + + +--- END OF FILE: docs/resources/source-code/hb_ao.md --- + +--- START OF FILE: docs/resources/source-code/hb_app.md --- +# [Module hb_app.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_app.erl) + + + + +The main HyperBEAM application module. + +__Behaviours:__ [`application`](application.md). + + + +## Function Index ## + + +
start/2
stop/1
+ + + + +## Function Details ## + + + +### start/2 ### + +`start(StartType, StartArgs) -> any()` + + + +### stop/1 ### + +`stop(State) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_app.md --- + +--- START OF FILE: docs/resources/source-code/hb_beamr_io.md --- +# [Module hb_beamr_io.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_beamr_io.erl) + + + + +Simple interface for memory management for Beamr instances. + + + +## Description ## + +It allows for reading and writing to memory, as well as allocating and +freeing memory by calling the WASM module's exported malloc and free +functions. + +Unlike the majority of HyperBEAM modules, this module takes a defensive +approach to type checking, breaking from the conventional Erlang style, +such that failures are caught in the Erlang-side of functions rather than +in the C/WASM-side. + +## Function Index ## + + +
do_read_string/3*
free/2Free space allocated in the Beamr instance's native memory via a +call to the exported free function from the WASM.
malloc/2Allocate space for (via an exported malloc function from the WASM) in +the Beamr instance's native memory.
malloc_test/0*Test allocating and freeing memory.
read/3Read a binary from the Beamr instance's native memory at a given offset +and of a given size.
read_string/2Simple helper function to read a string from the Beamr instance's native +memory at a given offset.
read_string/3*
read_test/0*Test reading memory in and out of bounds.
size/1Get the size (in bytes) of the native memory allocated in the Beamr +instance.
size_test/0*
string_write_and_read_test/0*Write and read strings to memory.
write/3Write a binary to the Beamr instance's native memory at a given offset.
write_string/2Simple helper function to allocate space for (via malloc) and write a +string to the Beamr instance's native memory.
write_test/0*Test writing memory in and out of bounds.
+ + + + +## Function Details ## + + + +### do_read_string/3 * ### + +`do_read_string(WASM, Offset, ChunkSize) -> any()` + + + +### free/2 ### + +`free(WASM, Ptr) -> any()` + +Free space allocated in the Beamr instance's native memory via a +call to the exported free function from the WASM. + + + +### malloc/2 ### + +`malloc(WASM, Size) -> any()` + +Allocate space for (via an exported malloc function from the WASM) in +the Beamr instance's native memory. + + + +### malloc_test/0 * ### + +`malloc_test() -> any()` + +Test allocating and freeing memory. + + + +### read/3 ### + +`read(WASM, Offset, Size) -> any()` + +Read a binary from the Beamr instance's native memory at a given offset +and of a given size. + + + +### read_string/2 ### + +`read_string(Port, Offset) -> any()` + +Simple helper function to read a string from the Beamr instance's native +memory at a given offset. Memory is read by default in chunks of 8 bytes, +but this can be overridden by passing a different chunk size. Strings are +assumed to be null-terminated. + + + +### read_string/3 * ### + +`read_string(WASM, Offset, ChunkSize) -> any()` + + + +### read_test/0 * ### + +`read_test() -> any()` + +Test reading memory in and out of bounds. + + + +### size/1 ### + +`size(WASM) -> any()` + +Get the size (in bytes) of the native memory allocated in the Beamr +instance. Note that WASM memory can never be reduced once granted to an +instance (although it can, of course, be reallocated _inside_ the +environment). + + + +### size_test/0 * ### + +`size_test() -> any()` + + + +### string_write_and_read_test/0 * ### + +`string_write_and_read_test() -> any()` + +Write and read strings to memory. + + + +### write/3 ### + +`write(WASM, Offset, Data) -> any()` + +Write a binary to the Beamr instance's native memory at a given offset. + + + +### write_string/2 ### + +`write_string(WASM, Data) -> any()` + +Simple helper function to allocate space for (via malloc) and write a +string to the Beamr instance's native memory. This can be helpful for easily +pushing a string into the instance, such that the resulting pointer can be +passed to exported functions from the instance. +Assumes that the input is either an iolist or a binary, adding a null byte +to the end of the string. + + + +### write_test/0 * ### + +`write_test() -> any()` + +Test writing memory in and out of bounds. + + +--- END OF FILE: docs/resources/source-code/hb_beamr_io.md --- + +--- START OF FILE: docs/resources/source-code/hb_beamr.md --- +# [Module hb_beamr.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_beamr.erl) + + + + +BEAMR: A WAMR wrapper for BEAM. + + + +## Description ## + +Beamr is a library that allows you to run WASM modules in BEAM, using the +Webassembly Micro Runtime (WAMR) as its engine. Each WASM module is +executed using a Linked-In Driver (LID) that is loaded into BEAM. It is +designed with a focus on supporting long-running WASM executions that +interact with Erlang functions and processes easily. + +Because each WASM module runs as an independent async worker, if you plan +to run many instances in parallel, you should be sure to configure the +BEAM to have enough async worker threads enabled (see `erl +A N` in the +Erlang manuals). + +The core API is simple: + +``` + + start(WasmBinary) -> {ok, Port, Imports, Exports} + Where: + WasmBinary is the WASM binary to load. + Port is the port to the LID. + Imports is a list of tuples of the form {Module, Function, + Args, Signature}. + Exports is a list of tuples of the form {Function, Args, + Signature}. + stop(Port) -> ok + call(Port, FunctionName, Args) -> {ok, Result} + Where: + FunctionName is the name of the function to call. + Args is a list of Erlang terms (converted to WASM values by + BEAMR) that match the signature of the function. + Result is a list of Erlang terms (converted from WASM values). + call(Port, FunName, Args[, Import, State, Opts]) -> {ok, Res, NewState} + Where: + ImportFun is a function that will be called upon each import. + ImportFun must have an arity of 2: Taking an arbitrary state + term, and a map containing the port, module, func, args,signature, and the options map of the import. + It must return a tuple of the form {ok, Response, NewState}. + serialize(Port) -> {ok, Mem} + Where: + Port is the port to the LID. + Mem is a binary representing the full WASM state. + deserialize(Port, Mem) -> ok + Where: + Port is the port to the LID. + Mem is a binary output of a previous serialize/1 call. +``` + +BEAMR was designed for use in the HyperBEAM project, but is suitable for +deployment in other Erlang applications that need to run WASM modules. PRs +are welcome. + +## Function Index ## + + +
benchmark_test/0*
call/3Call a function in the WASM executor (see moduledoc for more details).
call/4
call/5
call/6
deserialize/2Deserialize a WASM state from a binary.
dispatch_response/2*Check the type of an import response and dispatch it to a Beamr port.
driver_loads_test/0*
imported_function_test/0*Test that imported functions can be called from the WASM module.
is_valid_arg_list/1*Check that a list of arguments is valid for a WASM function call.
load_driver/0*Load the driver for the WASM executor.
monitor_call/4*Synchonously monitor the WASM executor for a call result and any +imports that need to be handled.
multiclient_test/0*Ensure that processes outside of the initial one can interact with +the WASM executor.
serialize/1Serialize the WASM state to a binary.
simple_wasm_test/0*Test standalone hb_beamr correctly after loading a WASM module.
start/1Start a WASM executor context.
start/2
stop/1Stop a WASM executor context.
stub/3Stub import function for the WASM executor.
wasm64_test/0*Test that WASM Memory64 modules load and execute correctly.
wasm_send/2
worker/2*A worker process that is responsible for handling a WASM instance.
+ + + + +## Function Details ## + + + +### benchmark_test/0 * ### + +`benchmark_test() -> any()` + + + +### call/3 ### + +`call(PID, FuncRef, Args) -> any()` + +Call a function in the WASM executor (see moduledoc for more details). + + + +### call/4 ### + +`call(PID, FuncRef, Args, ImportFun) -> any()` + + + +### call/5 ### + +`call(PID, FuncRef, Args, ImportFun, StateMsg) -> any()` + + + +### call/6 ### + +`call(PID, FuncRef, Args, ImportFun, StateMsg, Opts) -> any()` + + + +### deserialize/2 ### + +`deserialize(WASM, Bin) -> any()` + +Deserialize a WASM state from a binary. + + + +### dispatch_response/2 * ### + +`dispatch_response(WASM, Term) -> any()` + +Check the type of an import response and dispatch it to a Beamr port. + + + +### driver_loads_test/0 * ### + +`driver_loads_test() -> any()` + + + +### imported_function_test/0 * ### + +`imported_function_test() -> any()` + +Test that imported functions can be called from the WASM module. + + + +### is_valid_arg_list/1 * ### + +`is_valid_arg_list(Args) -> any()` + +Check that a list of arguments is valid for a WASM function call. + + + +### load_driver/0 * ### + +`load_driver() -> any()` + +Load the driver for the WASM executor. + + + +### monitor_call/4 * ### + +`monitor_call(WASM, ImportFun, StateMsg, Opts) -> any()` + +Synchonously monitor the WASM executor for a call result and any +imports that need to be handled. + + + +### multiclient_test/0 * ### + +`multiclient_test() -> any()` + +Ensure that processes outside of the initial one can interact with +the WASM executor. + + + +### serialize/1 ### + +`serialize(WASM) -> any()` + +Serialize the WASM state to a binary. + + + +### simple_wasm_test/0 * ### + +`simple_wasm_test() -> any()` + +Test standalone `hb_beamr` correctly after loading a WASM module. + + + +### start/1 ### + +`start(WasmBinary) -> any()` + +Start a WASM executor context. Yields a port to the LID, and the +imports and exports of the WASM module. Optionally, specify a mode +(wasm or aot) to indicate the type of WASM module being loaded. + + + +### start/2 ### + +`start(WasmBinary, Mode) -> any()` + + + +### stop/1 ### + +`stop(WASM) -> any()` + +Stop a WASM executor context. + + + +### stub/3 ### + +`stub(Msg1, Msg2, Opts) -> any()` + +Stub import function for the WASM executor. + + + +### wasm64_test/0 * ### + +`wasm64_test() -> any()` + +Test that WASM Memory64 modules load and execute correctly. + + + +### wasm_send/2 ### + +`wasm_send(WASM, Message) -> any()` + + + +### worker/2 * ### + +`worker(Port, Listener) -> any()` + +A worker process that is responsible for handling a WASM instance. +It wraps the WASM port, handling inputs and outputs from the WASM module. +The last sender to the port is always the recipient of its messages, so +be careful to ensure that there is only one active sender to the port at +any time. + + +--- END OF FILE: docs/resources/source-code/hb_beamr.md --- + +--- START OF FILE: docs/resources/source-code/hb_cache_control.md --- +# [Module hb_cache_control.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_cache_control.erl) + + + + +Cache control logic for the AO-Core resolver. + + + +## Description ## +It derives cache settings +from request, response, execution-local node Opts, as well as the global +node Opts. It applies these settings when asked to maybe store/lookup in +response to a request. + +## Function Index ## + + +
cache_binary_result_test/0*
cache_message_result_test/0*
cache_source_to_cache_settings/1*Convert a cache source to a cache setting.
derive_cache_settings/2*Derive cache settings from a series of option sources and the opts, +honoring precidence order.
dispatch_cache_write/4*Dispatch the cache write to a worker process if requested.
empty_message_list_test/0*
exec_likely_faster_heuristic/3*Determine whether we are likely to be faster looking up the result in +our cache (hoping we have it), or executing it directly.
hashpath_ignore_prevents_storage_test/0*
is_explicit_lookup/3*
lookup/3*
maybe_lookup/3Handles cache lookup, modulated by the caching options requested by +the user.
maybe_set/2*Takes a key and two maps, returning the first map with the key set to +the value of the second map _if_ the value is not undefined.
maybe_store/4Write a resulting M3 message to the cache if requested.
message_source_cache_control_test/0*
message_without_cache_control_test/0*
msg_precidence_overrides_test/0*
msg_with_cc/1*
multiple_directives_test/0*
necessary_messages_not_found_error/3*Generate a message to return when the necessary messages to execute a +cache lookup are not found in the cache.
no_cache_directive_test/0*
no_store_directive_test/0*
only_if_cached_directive_test/0*
only_if_cached_not_found_error/3*Generate a message to return when only_if_cached was specified, and +we don't have a cached result.
opts_override_message_settings_test/0*
opts_source_cache_control_test/0*
opts_with_cc/1*
specifiers_to_cache_settings/1*Convert a cache control list as received via HTTP headers into a +normalized map of simply whether we should store and/or lookup the result.
+ + + + +## Function Details ## + + + +### cache_binary_result_test/0 * ### + +`cache_binary_result_test() -> any()` + + + +### cache_message_result_test/0 * ### + +`cache_message_result_test() -> any()` + + + +### cache_source_to_cache_settings/1 * ### + +`cache_source_to_cache_settings(Msg) -> any()` + +Convert a cache source to a cache setting. The setting _must_ always be +directly in the source, not an AO-Core-derivable value. The +`to_cache_control_map` function is used as the source of settings in all +cases, except where an `Opts` specifies that hashpaths should not be updated, +which leads to the result not being cached (as it may be stored with an +incorrect hashpath). + + + +### derive_cache_settings/2 * ### + +`derive_cache_settings(SourceList, Opts) -> any()` + +Derive cache settings from a series of option sources and the opts, +honoring precidence order. The Opts is used as the first source. Returns a +map with `store` and `lookup` keys, each of which is a boolean. + +For example, if the last source has a `no_store`, the first expresses no +preference, but the Opts has `cache_control => [always]`, then the result +will contain a `store => true` entry. + + + +### dispatch_cache_write/4 * ### + +`dispatch_cache_write(Msg1, Msg2, Msg3, Opts) -> any()` + +Dispatch the cache write to a worker process if requested. +Invoke the appropriate cache write function based on the type of the message. + + + +### empty_message_list_test/0 * ### + +`empty_message_list_test() -> any()` + + + +### exec_likely_faster_heuristic/3 * ### + +`exec_likely_faster_heuristic(Msg1, Msg2, Opts) -> any()` + +Determine whether we are likely to be faster looking up the result in +our cache (hoping we have it), or executing it directly. + + + +### hashpath_ignore_prevents_storage_test/0 * ### + +`hashpath_ignore_prevents_storage_test() -> any()` + + + +### is_explicit_lookup/3 * ### + +`is_explicit_lookup(Msg1, X2, Opts) -> any()` + + + +### lookup/3 * ### + +`lookup(Msg1, Msg2, Opts) -> any()` + + + +### maybe_lookup/3 ### + +`maybe_lookup(Msg1, Msg2, Opts) -> any()` + +Handles cache lookup, modulated by the caching options requested by +the user. Honors the following `Opts` cache keys: +`only_if_cached`: If set and we do not find a result in the cache, +return an error with a `Cache-Status` of `miss` and +a 504 `Status`. +`no_cache`: If set, the cached values are never used. Returns +`continue` to the caller. + + + +### maybe_set/2 * ### + +`maybe_set(Map1, Map2) -> any()` + +Takes a key and two maps, returning the first map with the key set to +the value of the second map _if_ the value is not undefined. + + + +### maybe_store/4 ### + +`maybe_store(Msg1, Msg2, Msg3, Opts) -> any()` + +Write a resulting M3 message to the cache if requested. The precedence +order of cache control sources is as follows: +1. The `Opts` map (letting the node operator have the final say). +2. The `Msg3` results message (granted by Msg1's device). +3. The `Msg2` message (the user's request). +Msg1 is not used, such that it can specify cache control information about +itself, without affecting its outputs. + + + +### message_source_cache_control_test/0 * ### + +`message_source_cache_control_test() -> any()` + + + +### message_without_cache_control_test/0 * ### + +`message_without_cache_control_test() -> any()` + + + +### msg_precidence_overrides_test/0 * ### + +`msg_precidence_overrides_test() -> any()` + + + +### msg_with_cc/1 * ### + +`msg_with_cc(CC) -> any()` + + + +### multiple_directives_test/0 * ### + +`multiple_directives_test() -> any()` + + + +### necessary_messages_not_found_error/3 * ### + +`necessary_messages_not_found_error(Msg1, Msg2, Opts) -> any()` + +Generate a message to return when the necessary messages to execute a +cache lookup are not found in the cache. + + + +### no_cache_directive_test/0 * ### + +`no_cache_directive_test() -> any()` + + + +### no_store_directive_test/0 * ### + +`no_store_directive_test() -> any()` + + + +### only_if_cached_directive_test/0 * ### + +`only_if_cached_directive_test() -> any()` + + + +### only_if_cached_not_found_error/3 * ### + +`only_if_cached_not_found_error(Msg1, Msg2, Opts) -> any()` + +Generate a message to return when `only_if_cached` was specified, and +we don't have a cached result. + + + +### opts_override_message_settings_test/0 * ### + +`opts_override_message_settings_test() -> any()` + + + +### opts_source_cache_control_test/0 * ### + +`opts_source_cache_control_test() -> any()` + + + +### opts_with_cc/1 * ### + +`opts_with_cc(CC) -> any()` + + + +### specifiers_to_cache_settings/1 * ### + +`specifiers_to_cache_settings(CCSpecifier) -> any()` + +Convert a cache control list as received via HTTP headers into a +normalized map of simply whether we should store and/or lookup the result. + + +--- END OF FILE: docs/resources/source-code/hb_cache_control.md --- + +--- START OF FILE: docs/resources/source-code/hb_cache_render.md --- +# [Module hb_cache_render.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_cache_render.erl) + + + + +A module that helps to render given Key graphs into the .dot files. + + + +## Function Index ## + + +
add_arc/4*Add an arc to the graph.
add_node/3*Add a node to the graph.
cache_path_to_dot/2Generate a dot file from a cache path and options/store.
cache_path_to_dot/3
cache_path_to_graph/3Main function to collect graph elements.
collect_output/2*Helper function to collect output from port.
dot_to_svg/1Convert a dot graph to SVG format.
extract_label/1*Extract a label from a path.
get_graph_data/1Get graph data for the Three.js visualization.
get_label/1*Extract a readable label from a path.
get_node_type/1*Convert node color from hb_cache_render to node type for visualization.
graph_to_dot/1*Generate the DOT file from the graph.
prepare_deeply_nested_complex_message/0
prepare_signed_data/0
prepare_unsigned_data/0
process_composite_node/6*Process a composite (directory) node.
process_simple_node/6*Process a simple (leaf) node.
render/1Render the given Key into svg.
render/2
test_signed/2*
test_unsigned/1*
traverse_store/4*Traverse the store recursively to build the graph.
+ + + + +## Function Details ## + + + +### add_arc/4 * ### + +`add_arc(Graph, From, To, Label) -> any()` + +Add an arc to the graph + + + +### add_node/3 * ### + +`add_node(Graph, ID, Color) -> any()` + +Add a node to the graph + + + +### cache_path_to_dot/2 ### + +`cache_path_to_dot(ToRender, StoreOrOpts) -> any()` + +Generate a dot file from a cache path and options/store + + + +### cache_path_to_dot/3 ### + +`cache_path_to_dot(ToRender, RenderOpts, StoreOrOpts) -> any()` + + + +### cache_path_to_graph/3 ### + +`cache_path_to_graph(ToRender, GraphOpts, StoreOrOpts) -> any()` + +Main function to collect graph elements + + + +### collect_output/2 * ### + +`collect_output(Port, Acc) -> any()` + +Helper function to collect output from port + + + +### dot_to_svg/1 ### + +`dot_to_svg(DotInput) -> any()` + +Convert a dot graph to SVG format + + + +### extract_label/1 * ### + +`extract_label(Path) -> any()` + +Extract a label from a path + + + +### get_graph_data/1 ### + +`get_graph_data(Opts) -> any()` + +Get graph data for the Three.js visualization + + + +### get_label/1 * ### + +`get_label(Path) -> any()` + +Extract a readable label from a path + + + +### get_node_type/1 * ### + +`get_node_type(Color) -> any()` + +Convert node color from hb_cache_render to node type for visualization + + + +### graph_to_dot/1 * ### + +`graph_to_dot(Graph) -> any()` + +Generate the DOT file from the graph + + + +### prepare_deeply_nested_complex_message/0 ### + +`prepare_deeply_nested_complex_message() -> any()` + + + +### prepare_signed_data/0 ### + +`prepare_signed_data() -> any()` + + + +### prepare_unsigned_data/0 ### + +`prepare_unsigned_data() -> any()` + + + +### process_composite_node/6 * ### + +`process_composite_node(Store, Key, Parent, ResolvedPath, JoinedPath, Graph) -> any()` + +Process a composite (directory) node + + + +### process_simple_node/6 * ### + +`process_simple_node(Store, Key, Parent, ResolvedPath, JoinedPath, Graph) -> any()` + +Process a simple (leaf) node + + + +### render/1 ### + +`render(StoreOrOpts) -> any()` + +Render the given Key into svg + + + +### render/2 ### + +`render(ToRender, StoreOrOpts) -> any()` + + + +### test_signed/2 * ### + +`test_signed(Data, Wallet) -> any()` + + + +### test_unsigned/1 * ### + +`test_unsigned(Data) -> any()` + + + +### traverse_store/4 * ### + +`traverse_store(Store, Key, Parent, Graph) -> any()` + +Traverse the store recursively to build the graph + + +--- END OF FILE: docs/resources/source-code/hb_cache_render.md --- + +--- START OF FILE: docs/resources/source-code/hb_cache.md --- +# [Module hb_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_cache.erl) + + + + +A cache of AO-Core protocol messages and compute results. + + + +## Description ## + +HyperBEAM stores all paths in key value stores, abstracted by the `hb_store` +module. Each store has its own storage backend, but each works with simple +key-value pairs. Each store can write binary keys at paths, and link between +paths. + +There are three layers to HyperBEAMs internal data representation on-disk: + +1. The raw binary data, written to the store at the hash of the content. +Storing binary paths in this way effectively deduplicates the data. +2. The hashpath-graph of all content, stored as a set of links between +hashpaths, their keys, and the data that underlies them. This allows +all messages to share the same hashpath space, such that all requests +from users additively fill-in the hashpath space, minimizing duplicated +compute. +3. Messages, referrable by their IDs (committed or uncommitted). These are +stored as a set of links commitment IDs and the uncommitted message. + +Before writing a message to the store, we convert it to Type-Annotated +Binary Messages (TABMs), such that each of the keys in the message is +either a map or a direct binary. + +## Function Index ## + + +
cache_suite_test_/0*
calculate_all_ids/2*Calculate the IDs for a message.
do_read/4*Read a path from the store.
do_write_message/4*
link/3Make a link from one path to another in the store.
list/2List all items under a given path.
list_numbered/2List all items in a directory, assuming they are numbered.
read/2Read the message at a path.
read_resolved/3Read the output of a prior computation, given Msg1, Msg2, and some +options.
run_test/0*
store_read/3*List all of the subpaths of a given path, read each in turn, returning a +flat map.
store_read/4*
test_deeply_nested_complex_message/1*Test deeply nested item storage and retrieval.
test_device_map_cannot_be_written_test/0*Test that message whose device is #{} cannot be written.
test_message_with_message/1*
test_signed/1
test_signed/2*
test_store_ans104_message/1*
test_store_binary/1*
test_store_simple_signed_message/1*Test storing and retrieving a simple unsigned item.
test_store_simple_unsigned_message/1*Test storing and retrieving a simple unsigned item.
test_store_unsigned_empty_message/1*
test_unsigned/1
to_integer/1*
write/2Write a message to the cache.
write_binary/3Write a raw binary keys into the store and link it at a given hashpath.
write_binary/4*
write_hashpath/2Write a hashpath and its message to the store and link it.
write_hashpath/3*
+ + + + +## Function Details ## + + + +### cache_suite_test_/0 * ### + +`cache_suite_test_() -> any()` + + + +### calculate_all_ids/2 * ### + +`calculate_all_ids(Bin, Opts) -> any()` + +Calculate the IDs for a message. + + + +### do_read/4 * ### + +`do_read(Path, Store, Opts, AlreadyRead) -> any()` + +Read a path from the store. Unsafe: May recurse indefinitely if circular +links are present. + + + +### do_write_message/4 * ### + +`do_write_message(Bin, AllIDs, Store, Opts) -> any()` + + + +### link/3 ### + +`link(Existing, New, Opts) -> any()` + +Make a link from one path to another in the store. +Note: Argument order is `link(Src, Dst, Opts)`. + + + +### list/2 ### + +`list(Path, Opts) -> any()` + +List all items under a given path. + + + +### list_numbered/2 ### + +`list_numbered(Path, Opts) -> any()` + +List all items in a directory, assuming they are numbered. + + + +### read/2 ### + +`read(Path, Opts) -> any()` + +Read the message at a path. Returns in `structured@1.0` format: Either a +richly typed map or a direct binary. + + + +### read_resolved/3 ### + +`read_resolved(MsgID1, MsgID2, Opts) -> any()` + +Read the output of a prior computation, given Msg1, Msg2, and some +options. + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### store_read/3 * ### + +`store_read(Path, Store, Opts) -> any()` + +List all of the subpaths of a given path, read each in turn, returning a +flat map. We track the paths that we have already read to avoid circular +links. + + + +### store_read/4 * ### + +`store_read(Path, Store, Opts, AlreadyRead) -> any()` + + + +### test_deeply_nested_complex_message/1 * ### + +`test_deeply_nested_complex_message(Opts) -> any()` + +Test deeply nested item storage and retrieval + + + +### test_device_map_cannot_be_written_test/0 * ### + +`test_device_map_cannot_be_written_test() -> any()` + +Test that message whose device is `#{}` cannot be written. If it were to +be written, it would cause an infinite loop. + + + +### test_message_with_message/1 * ### + +`test_message_with_message(Opts) -> any()` + + + +### test_signed/1 ### + +`test_signed(Data) -> any()` + + + +### test_signed/2 * ### + +`test_signed(Data, Wallet) -> any()` + + + +### test_store_ans104_message/1 * ### + +`test_store_ans104_message(Opts) -> any()` + + + +### test_store_binary/1 * ### + +`test_store_binary(Opts) -> any()` + + + +### test_store_simple_signed_message/1 * ### + +`test_store_simple_signed_message(Opts) -> any()` + +Test storing and retrieving a simple unsigned item + + + +### test_store_simple_unsigned_message/1 * ### + +`test_store_simple_unsigned_message(Opts) -> any()` + +Test storing and retrieving a simple unsigned item + + + +### test_store_unsigned_empty_message/1 * ### + +`test_store_unsigned_empty_message(Opts) -> any()` + + + +### test_unsigned/1 ### + +`test_unsigned(Data) -> any()` + + + +### to_integer/1 * ### + +`to_integer(Value) -> any()` + + + +### write/2 ### + +`write(RawMsg, Opts) -> any()` + +Write a message to the cache. For raw binaries, we write the data at +the hashpath of the data (by default the SHA2-256 hash of the data). We link +the unattended ID's hashpath for the keys (including `/commitments`) on the +message to the underlying data and recurse. We then link each commitment ID +to the uncommitted message, such that any of the committed or uncommitted IDs +can be read, and once in memory all of the commitments are available. For +deep messages, the commitments will also be read, such that the ID of the +outer message (which does not include its commitments) will be built upon +the commitments of the inner messages. We do not, however, store the IDs from +commitments on signed _inner_ messages. We may wish to revisit this. + + + +### write_binary/3 ### + +`write_binary(Hashpath, Bin, Opts) -> any()` + +Write a raw binary keys into the store and link it at a given hashpath. + + + +### write_binary/4 * ### + +`write_binary(Hashpath, Bin, Store, Opts) -> any()` + + + +### write_hashpath/2 ### + +`write_hashpath(Msg, Opts) -> any()` + +Write a hashpath and its message to the store and link it. + + + +### write_hashpath/3 * ### + +`write_hashpath(HP, Msg, Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_cache.md --- + +--- START OF FILE: docs/resources/source-code/hb_client.md --- +# [Module hb_client.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_client.erl) + + + + + + +## Function Index ## + + +
add_route/3
arweave_timestamp/0Grab the latest block information from the Arweave gateway node.
prefix_keys/3*
resolve/4Resolve a message pair on a remote node.
routes/2
upload/2Upload a data item to the bundler node.
upload/3*
upload_empty_message_test/0*
upload_empty_raw_ans104_test/0*
upload_raw_ans104_test/0*
upload_raw_ans104_with_anchor_test/0*
upload_single_layer_message_test/0*
+ + + + +## Function Details ## + + + +### add_route/3 ### + +`add_route(Node, Route, Opts) -> any()` + + + +### arweave_timestamp/0 ### + +`arweave_timestamp() -> any()` + +Grab the latest block information from the Arweave gateway node. + + + +### prefix_keys/3 * ### + +`prefix_keys(Prefix, Message, Opts) -> any()` + + + +### resolve/4 ### + +`resolve(Node, Msg1, Msg2, Opts) -> any()` + +Resolve a message pair on a remote node. +The message pair is first transformed into a singleton request, by +prefixing the keys in both messages for the path segment that they relate to, +and then adjusting the "Path" field from the second message. + + + +### routes/2 ### + +`routes(Node, Opts) -> any()` + + + +### upload/2 ### + +`upload(Msg, Opts) -> any()` + +Upload a data item to the bundler node. + + + +### upload/3 * ### + +`upload(Msg, Opts, X3) -> any()` + + + +### upload_empty_message_test/0 * ### + +`upload_empty_message_test() -> any()` + + + +### upload_empty_raw_ans104_test/0 * ### + +`upload_empty_raw_ans104_test() -> any()` + + + +### upload_raw_ans104_test/0 * ### + +`upload_raw_ans104_test() -> any()` + + + +### upload_raw_ans104_with_anchor_test/0 * ### + +`upload_raw_ans104_with_anchor_test() -> any()` + + + +### upload_single_layer_message_test/0 * ### + +`upload_single_layer_message_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_client.md --- + +--- START OF FILE: docs/resources/source-code/hb_crypto.md --- +# [Module hb_crypto.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_crypto.erl) + + + + +Implements the cryptographic functions and wraps the primitives +used in HyperBEAM. + + + +## Description ## + +Abstracted such that this (extremely!) dangerous code +can be carefully managed. + +HyperBEAM currently implements two hashpath algorithms: + +* `sha-256-chain`: A simple chained SHA-256 hash. + +* `accumulate-256`: A SHA-256 hash that chains the given IDs and accumulates +their values into a single commitment. + +The accumulate algorithm is experimental and at this point only exists to +allow us to test multiple HashPath algorithms in HyperBEAM. + +## Function Index ## + + +
accumulate/2Accumulate two IDs into a single commitment.
count_zeroes/1*Count the number of leading zeroes in a bitstring.
sha256/1Wrap Erlang's crypto:hash/2 to provide a standard interface.
sha256_chain/2Add a new ID to the end of a SHA-256 hash chain.
sha256_chain_test/0*Check that sha-256-chain correctly produces a hash matching +the machine's OpenSSL lib's output.
+ + + + +## Function Details ## + + + +### accumulate/2 ### + +`accumulate(ID1, ID2) -> any()` + +Accumulate two IDs into a single commitment. +Experimental! This is not necessarily a cryptographically-secure operation. + + + +### count_zeroes/1 * ### + +`count_zeroes(X1) -> any()` + +Count the number of leading zeroes in a bitstring. + + + +### sha256/1 ### + +`sha256(Data) -> any()` + +Wrap Erlang's `crypto:hash/2` to provide a standard interface. +Under-the-hood, this uses OpenSSL. + + + +### sha256_chain/2 ### + +`sha256_chain(ID1, ID2) -> any()` + +Add a new ID to the end of a SHA-256 hash chain. + + + +### sha256_chain_test/0 * ### + +`sha256_chain_test() -> any()` + +Check that `sha-256-chain` correctly produces a hash matching +the machine's OpenSSL lib's output. Further (in case of a bug in our +or Erlang's usage of OpenSSL), check that the output has at least has +a high level of entropy. + + +--- END OF FILE: docs/resources/source-code/hb_crypto.md --- + +--- START OF FILE: docs/resources/source-code/hb_debugger.md --- +# [Module hb_debugger.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_debugger.erl) + + + + +A module that provides bootstrapping interfaces for external debuggers +to connect to HyperBEAM. + + + +## Description ## + +The simplest way to utilize an external graphical debugger is to use the +`erlang-ls` extension for VS Code, Emacs, or other Language Server Protocol +(LSP) compatible editors. This repository contains a `launch.json` +configuration file for VS Code that can be used to spawn a new HyperBEAM, +attach the debugger to it, and execute the specified `Module:Function(Args)`. +Additionally, the node can be started with `rebar3 debugging` in order to +allow access to the console while also allowing the debugger to attach. + +Boot time is approximately 10 seconds. + +## Function Index ## + + +
await_breakpoint/0Await a new breakpoint being set by the debugger.
await_breakpoint/1*
await_debugger/0*Await a debugger to be attached to the node.
await_debugger/1*
interpret/1*Attempt to interpret a specified module to load it into the debugger.
is_debugging_node_connected/0*Is another Distributed Erlang node connected to us?.
start/0
start_and_break/2A bootstrapping function to wait for an external debugger to be attached, +then add a breakpoint on the specified Module:Function(Args), then call it.
start_and_break/3
+ + + + +## Function Details ## + + + +### await_breakpoint/0 ### + +`await_breakpoint() -> any()` + +Await a new breakpoint being set by the debugger. + + + +### await_breakpoint/1 * ### + +`await_breakpoint(N) -> any()` + + + +### await_debugger/0 * ### + +`await_debugger() -> any()` + +Await a debugger to be attached to the node. + + + +### await_debugger/1 * ### + +`await_debugger(N) -> any()` + + + +### interpret/1 * ### + +`interpret(Module) -> any()` + +Attempt to interpret a specified module to load it into the debugger. +`int:i/1` seems to have an issue that will cause it to fail sporadically +with `error:undef` on some modules. This error appears not to be catchable +through the normal means. Subsequently, we attempt the load in a separate +process and wait for it to complete. If we do not receive a response in a +reasonable amount of time, we assume that the module failed to load and +return `false`. + + + +### is_debugging_node_connected/0 * ### + +`is_debugging_node_connected() -> any()` + +Is another Distributed Erlang node connected to us? + + + +### start/0 ### + +`start() -> any()` + + + +### start_and_break/2 ### + +`start_and_break(Module, Function) -> any()` + +A bootstrapping function to wait for an external debugger to be attached, +then add a breakpoint on the specified `Module:Function(Args)`, then call it. + + + +### start_and_break/3 ### + +`start_and_break(Module, Function, Args) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_debugger.md --- + +--- START OF FILE: docs/resources/source-code/hb_escape.md --- +# [Module hb_escape.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_escape.erl) + + + + +Escape and unescape mixed case values for use in HTTP headers. + + + +## Description ## +This is necessary for encodings of AO-Core messages for transmission in +HTTP/2 and HTTP/3, because uppercase header keys are explicitly disallowed. +While most map keys in HyperBEAM are normalized to lowercase, IDs are not. +Subsequently, we encode all header keys to lowercase %-encoded URI-style +strings because transmission. + +## Function Index ## + + +
decode/1Decode a URI-encoded string back to a binary.
decode_keys/1Return a message with all of its keys decoded.
encode/1Encode a binary as a URI-encoded string.
encode_keys/1URI encode keys in the base layer of a message.
escape_byte/1*Escape a single byte as a URI-encoded string.
escape_unescape_identity_test/0*
escape_unescape_special_chars_test/0*
hex_digit/1*
hex_value/1*
percent_escape/1*Escape a list of characters as a URI-encoded string.
percent_unescape/1*Unescape a URI-encoded string.
unescape_specific_test/0*
uppercase_test/0*
+ + + + +## Function Details ## + + + +### decode/1 ### + +`decode(Bin) -> any()` + +Decode a URI-encoded string back to a binary. + + + +### decode_keys/1 ### + +`decode_keys(Msg) -> any()` + +Return a message with all of its keys decoded. + + + +### encode/1 ### + +`encode(Bin) -> any()` + +Encode a binary as a URI-encoded string. + + + +### encode_keys/1 ### + +`encode_keys(Msg) -> any()` + +URI encode keys in the base layer of a message. Does not recurse. + + + +### escape_byte/1 * ### + +`escape_byte(C) -> any()` + +Escape a single byte as a URI-encoded string. + + + +### escape_unescape_identity_test/0 * ### + +`escape_unescape_identity_test() -> any()` + + + +### escape_unescape_special_chars_test/0 * ### + +`escape_unescape_special_chars_test() -> any()` + + + +### hex_digit/1 * ### + +`hex_digit(N) -> any()` + + + +### hex_value/1 * ### + +`hex_value(C) -> any()` + + + +### percent_escape/1 * ### + +`percent_escape(Cs) -> any()` + +Escape a list of characters as a URI-encoded string. + + + +### percent_unescape/1 * ### + +`percent_unescape(Cs) -> any()` + +Unescape a URI-encoded string. + + + +### unescape_specific_test/0 * ### + +`unescape_specific_test() -> any()` + + + +### uppercase_test/0 * ### + +`uppercase_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_escape.md --- + +--- START OF FILE: docs/resources/source-code/hb_event.md --- +# [Module hb_event.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_event.erl) + + + + +Wrapper for incrementing prometheus counters. + + + +## Function Index ## + + +
await_prometheus_started/0*Delay the event server until prometheus is started.
handle_events/0*
handle_tracer/3*
increment/3Increment the counter for the given topic and message.
log/1Debugging log logging function.
log/2
log/3
log/4
log/5
log/6
parse_name/1*
server/0*
+ + + + +## Function Details ## + + + +### await_prometheus_started/0 * ### + +`await_prometheus_started() -> any()` + +Delay the event server until prometheus is started. + + + +### handle_events/0 * ### + +`handle_events() -> any()` + + + +### handle_tracer/3 * ### + +`handle_tracer(Topic, X, Opts) -> any()` + + + +### increment/3 ### + +`increment(Topic, Message, Opts) -> any()` + +Increment the counter for the given topic and message. Registers the +counter if it doesn't exist. If the topic is `global`, the message is ignored. +This means that events must specify a topic if they want to be counted, +filtering debug messages. Similarly, events with a topic that begins with +`debug` are ignored. + + + +### log/1 ### + +`log(X) -> any()` + +Debugging log logging function. For now, it just prints to standard +error. + + + +### log/2 ### + +`log(Topic, X) -> any()` + + + +### log/3 ### + +`log(Topic, X, Mod) -> any()` + + + +### log/4 ### + +`log(Topic, X, Mod, Func) -> any()` + + + +### log/5 ### + +`log(Topic, X, Mod, Func, Line) -> any()` + + + +### log/6 ### + +`log(Topic, X, Mod, Func, Line, Opts) -> any()` + + + +### parse_name/1 * ### + +`parse_name(Name) -> any()` + + + +### server/0 * ### + +`server() -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_event.md --- + +--- START OF FILE: docs/resources/source-code/hb_examples.md --- +# [Module hb_examples.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_examples.erl) + + + + +This module contains end-to-end tests for Hyperbeam, accessing through +the HTTP interface. + + + +## Description ## +As well as testing the system, you can use these tests +as examples of how to interact with HyperBEAM nodes. + +## Function Index ## + + +
create_schedule_aos2_test_disabled/0*
paid_wasm_test/0*Gain signed WASM responses from a node and verify them.
relay_with_payments_test/0*Start a node running the simple pay meta device, and use it to relay +a message for a client.
schedule/2*
schedule/3*
schedule/4*
+ + + + +## Function Details ## + + + +### create_schedule_aos2_test_disabled/0 * ### + +`create_schedule_aos2_test_disabled() -> any()` + + + +### paid_wasm_test/0 * ### + +`paid_wasm_test() -> any()` + +Gain signed WASM responses from a node and verify them. +1. Start the client with a small balance. +2. Execute a simple WASM function on the host node. +3. Verify the response is correct and signed by the host node. +4. Get the balance of the client and verify it has been deducted. + + + +### relay_with_payments_test/0 * ### + +`relay_with_payments_test() -> any()` + +Start a node running the simple pay meta device, and use it to relay +a message for a client. We must ensure: +1. When the client has no balance, the relay fails. +2. The operator is able to topup for the client. +3. The client has the correct balance after the topup. +4. The relay succeeds when the client has enough balance. +5. The received message is signed by the host using http-sig and validates +correctly. + + + +### schedule/2 * ### + +`schedule(ProcMsg, Target) -> any()` + + + +### schedule/3 * ### + +`schedule(ProcMsg, Target, Wallet) -> any()` + + + +### schedule/4 * ### + +`schedule(ProcMsg, Target, Wallet, Node) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_examples.md --- + +--- START OF FILE: docs/resources/source-code/hb_features.md --- +# [Module hb_features.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_features.erl) + + + + +A module that exports a list of feature flags that the node supports +using the `-ifdef` macro. + + + +## Description ## +As a consequence, this module acts as a proxy of information between the +build system and the runtime execution environment. + +## Function Index ## + + +
all/0Returns a list of all feature flags that the node supports.
enabled/1Returns true if the feature flag is enabled.
genesis_wasm/0
http3/0
rocksdb/0
test/0
+ + + + +## Function Details ## + + + +### all/0 ### + +`all() -> any()` + +Returns a list of all feature flags that the node supports. + + + +### enabled/1 ### + +`enabled(Feature) -> any()` + +Returns true if the feature flag is enabled. + + + +### genesis_wasm/0 ### + +`genesis_wasm() -> any()` + + + +### http3/0 ### + +`http3() -> any()` + + + +### rocksdb/0 ### + +`rocksdb() -> any()` + + + +### test/0 ### + +`test() -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_features.md --- + +--- START OF FILE: docs/resources/source-code/hb_gateway_client.md --- +# [Module hb_gateway_client.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_gateway_client.erl) + + + + +Implementation of Arweave's GraphQL API to gain access to specific +items of data stored on the network. + + + +## Description ## +This module must be used to get full HyperBEAM `structured@1.0` form messages +from data items stored on the network, as Arweave gateways do not presently +expose all necessary fields to retrieve this information outside of the +GraphQL API. When gateways integrate serving in `httpsig@1.0` form, this +module will be deprecated. + +## Function Index ## + + +
ans104_no_data_item_test/0*
ao_dataitem_test/0*Test optimistic index.
data/2Get the data associated with a transaction by its ID, using the node's +Arweave gateway peers.
decode_id_or_null/1*
decode_or_null/1*
item_spec/0*Gives the fields of a transaction that are needed to construct an +ANS-104 message.
l1_transaction_test/0*Test l1 message from graphql.
l2_dataitem_test/0*Test l2 message from graphql.
normalize_null/1*
query/2*Run a GraphQL request encoded as a binary.
read/2Get a data item (including data and tags) by its ID, using the node's +GraphQL peers.
result_to_message/2Takes a GraphQL item node, matches it with the appropriate data from a +gateway, then returns {ok, ParsedMsg}.
result_to_message/3*
scheduler_location/2Find the location of the scheduler based on its ID, through GraphQL.
scheduler_location_test/0*Test that we can get the scheduler location.
subindex_to_tags/1*Takes a list of messages with name and value fields, and formats +them as a GraphQL tags argument.
+ + + + +## Function Details ## + + + +### ans104_no_data_item_test/0 * ### + +`ans104_no_data_item_test() -> any()` + + + +### ao_dataitem_test/0 * ### + +`ao_dataitem_test() -> any()` + +Test optimistic index + + + +### data/2 ### + +`data(ID, Opts) -> any()` + +Get the data associated with a transaction by its ID, using the node's +Arweave `gateway` peers. The item is expected to be available in its +unmodified (by caches or other proxies) form at the following location: +https:///raw/ +where `<id>` is the base64-url-encoded transaction ID. + + + +### decode_id_or_null/1 * ### + +`decode_id_or_null(Bin) -> any()` + + + +### decode_or_null/1 * ### + +`decode_or_null(Bin) -> any()` + + + +### item_spec/0 * ### + +`item_spec() -> any()` + +Gives the fields of a transaction that are needed to construct an +ANS-104 message. + + + +### l1_transaction_test/0 * ### + +`l1_transaction_test() -> any()` + +Test l1 message from graphql + + + +### l2_dataitem_test/0 * ### + +`l2_dataitem_test() -> any()` + +Test l2 message from graphql + + + +### normalize_null/1 * ### + +`normalize_null(Bin) -> any()` + + + +### query/2 * ### + +`query(Query, Opts) -> any()` + +Run a GraphQL request encoded as a binary. The node message may contain +a list of URLs to use, optionally as a tuple with an additional map of options +to use for the request. + + + +### read/2 ### + +`read(ID, Opts) -> any()` + +Get a data item (including data and tags) by its ID, using the node's +GraphQL peers. +It uses the following GraphQL schema: +type Transaction { +id: ID! +anchor: String! +signature: String! +recipient: String! +owner: Owner { address: String! key: String! }! +fee: Amount! +quantity: Amount! +data: MetaData! +tags: [Tag { name: String! value: String! }!]! +} +type Amount { +winston: String! +ar: String! +} + + + +### result_to_message/2 ### + +`result_to_message(Item, Opts) -> any()` + +Takes a GraphQL item node, matches it with the appropriate data from a +gateway, then returns `{ok, ParsedMsg}`. + + + +### result_to_message/3 * ### + +`result_to_message(ExpectedID, Item, Opts) -> any()` + + + +### scheduler_location/2 ### + +`scheduler_location(Address, Opts) -> any()` + +Find the location of the scheduler based on its ID, through GraphQL. + + + +### scheduler_location_test/0 * ### + +`scheduler_location_test() -> any()` + +Test that we can get the scheduler location. + + + +### subindex_to_tags/1 * ### + +`subindex_to_tags(Subindex) -> any()` + +Takes a list of messages with `name` and `value` fields, and formats +them as a GraphQL `tags` argument. + + +--- END OF FILE: docs/resources/source-code/hb_gateway_client.md --- + +--- START OF FILE: docs/resources/source-code/hb_http_benchmark_tests.md --- +# [Module hb_http_benchmark_tests.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_benchmark_tests.erl) + + + + + +--- END OF FILE: docs/resources/source-code/hb_http_benchmark_tests.md --- + +--- START OF FILE: docs/resources/source-code/hb_http_client_sup.md --- +# [Module hb_http_client_sup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_client_sup.erl) + + + + +The supervisor for the gun HTTP client wrapper. + +__Behaviours:__ [`supervisor`](supervisor.md). + + + +## Function Index ## + + +
init/1
start_link/1
+ + + + +## Function Details ## + + + +### init/1 ### + +`init(Opts) -> any()` + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_http_client_sup.md --- + +--- START OF FILE: docs/resources/source-code/hb_http_client.md --- +# [Module hb_http_client.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_client.erl) + + + + +A wrapper library for gun. + +__Behaviours:__ [`gen_server`](gen_server.md). + + + +## Description ## +This module originates from the Arweave +project, and has been modified for use in HyperBEAM. + +## Function Index ## + + +
await_response/2*
dec_prometheus_gauge/1*Safe wrapper for prometheus_gauge:dec/2.
download_metric/2*
get_status_class/1*Return the HTTP status class label for cowboy_requests_total and +gun_requests_total metrics.
gun_req/3*
handle_call/3
handle_cast/2
handle_info/2
httpc_req/3*
inc_prometheus_counter/3*
inc_prometheus_gauge/1*Safe wrapper for prometheus_gauge:inc/2.
init/1
init_prometheus/1*
log/5*
maybe_invoke_monitor/2*Invoke the HTTP monitor message with AO-Core, if it is set in the +node message key.
method_to_bin/1*
open_connection/2*
parse_peer/2*
record_duration/2*Record the duration of the request in an async process.
record_response_status/3*
reply_error/2*
req/2
req/3*
request/3*
start_link/1
terminate/2
upload_metric/1*
+ + + + +## Function Details ## + + + +### await_response/2 * ### + +`await_response(Args, Opts) -> any()` + + + +### dec_prometheus_gauge/1 * ### + +`dec_prometheus_gauge(Name) -> any()` + +Safe wrapper for prometheus_gauge:dec/2. + + + +### download_metric/2 * ### + +`download_metric(Data, X2) -> any()` + + + +### get_status_class/1 * ### + +`get_status_class(Data) -> any()` + +Return the HTTP status class label for cowboy_requests_total and +gun_requests_total metrics. + + + +### gun_req/3 * ### + +`gun_req(Args, ReestablishedConnection, Opts) -> any()` + + + +### handle_call/3 ### + +`handle_call(Request, From, State) -> any()` + + + +### handle_cast/2 ### + +`handle_cast(Cast, State) -> any()` + + + +### handle_info/2 ### + +`handle_info(Message, State) -> any()` + + + +### httpc_req/3 * ### + +`httpc_req(Args, X2, Opts) -> any()` + + + +### inc_prometheus_counter/3 * ### + +`inc_prometheus_counter(Name, Labels, Value) -> any()` + + + +### inc_prometheus_gauge/1 * ### + +`inc_prometheus_gauge(Name) -> any()` + +Safe wrapper for prometheus_gauge:inc/2. + + + +### init/1 ### + +`init(Opts) -> any()` + + + +### init_prometheus/1 * ### + +`init_prometheus(Opts) -> any()` + + + +### log/5 * ### + +`log(Type, Event, X3, Reason, Opts) -> any()` + + + +### maybe_invoke_monitor/2 * ### + +`maybe_invoke_monitor(Details, Opts) -> any()` + +Invoke the HTTP monitor message with AO-Core, if it is set in the +node message key. We invoke the given message with the `body` set to a signed +version of the details. This allows node operators to configure their machine +to record duration statistics into customized data stores, computations, or +processes etc. Additionally, we include the `http_reference` value, if set in +the given `opts`. + +We use `hb_ao:get` rather than `hb_opts:get`, as settings configured +by the `~router@1.0` route `opts` key are unable to generate atoms. + + + +### method_to_bin/1 * ### + +`method_to_bin(X1) -> any()` + + + +### open_connection/2 * ### + +`open_connection(X1, Opts) -> any()` + + + +### parse_peer/2 * ### + +`parse_peer(Peer, Opts) -> any()` + + + +### record_duration/2 * ### + +`record_duration(Details, Opts) -> any()` + +Record the duration of the request in an async process. We write the +data to prometheus if the application is enabled, as well as invoking the +`http_monitor` if appropriate. + + + +### record_response_status/3 * ### + +`record_response_status(Method, Path, Response) -> any()` + + + +### reply_error/2 * ### + +`reply_error(PendingRequests, Reason) -> any()` + + + +### req/2 ### + +`req(Args, Opts) -> any()` + + + +### req/3 * ### + +`req(Args, ReestablishedConnection, Opts) -> any()` + + + +### request/3 * ### + +`request(PID, Args, Opts) -> any()` + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + + + +### terminate/2 ### + +`terminate(Reason, State) -> any()` + + + +### upload_metric/1 * ### + +`upload_metric(X1) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_http_client.md --- + +--- START OF FILE: docs/resources/source-code/hb_http_server.md --- +# [Module hb_http_server.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_server.erl) + + + + +A router that attaches a HTTP server to the AO-Core resolver. + + + +## Description ## + +Because AO-Core is built to speak in HTTP semantics, this module +only has to marshal the HTTP request into a message, and then +pass it to the AO-Core resolver. + +`hb_http:reply/4` is used to respond to the client, handling the +process of converting a message back into an HTTP response. + +The router uses an `Opts` message as its Cowboy initial state, +such that changing it on start of the router server allows for +the execution parameters of all downstream requests to be controlled. + +## Function Index ## + + +
allowed_methods/2Return the list of allowed methods for the HTTP server.
cors_reply/2*Reply to CORS preflight requests.
get_opts/1
handle_request/3*Handle all non-CORS preflight requests as AO-Core requests.
http3_conn_sup_loop/0*
init/2Entrypoint for all HTTP requests.
new_server/1*Trigger the creation of a new HTTP server node.
read_body/1*Helper to grab the full body of a HTTP request, even if it's chunked.
read_body/2*
set_default_opts/1
set_node_opts_test/0*Ensure that the start hook can be used to modify the node options.
set_opts/1Merges the provided Opts with uncommitted values from Request, +preserves the http_server value, and updates node_history by prepending +the Request.
set_opts/2
start/0Starts the HTTP server.
start/1
start_http2/3*
start_http3/3*
start_node/0Test that we can start the server, send a message, and get a response.
start_node/1
+ + + + +## Function Details ## + + + +### allowed_methods/2 ### + +`allowed_methods(Req, State) -> any()` + +Return the list of allowed methods for the HTTP server. + + + +### cors_reply/2 * ### + +`cors_reply(Req, ServerID) -> any()` + +Reply to CORS preflight requests. + + + +### get_opts/1 ### + +`get_opts(NodeMsg) -> any()` + + + +### handle_request/3 * ### + +`handle_request(RawReq, Body, ServerID) -> any()` + +Handle all non-CORS preflight requests as AO-Core requests. Execution +starts by parsing the HTTP request into HyerBEAM's message format, then +passing the message directly to `meta@1.0` which handles calling AO-Core in +the appropriate way. + + + +### http3_conn_sup_loop/0 * ### + +`http3_conn_sup_loop() -> any()` + + + +### init/2 ### + +`init(Req, ServerID) -> any()` + +Entrypoint for all HTTP requests. Receives the Cowboy request option and +the server ID, which can be used to lookup the node message. + + + +### new_server/1 * ### + +`new_server(RawNodeMsg) -> any()` + +Trigger the creation of a new HTTP server node. Accepts a `NodeMsg` +message, which is used to configure the server. This function executed the +`start` hook on the node, giving it the opportunity to modify the `NodeMsg` +before it is used to configure the server. The `start` hook expects gives and +expects the node message to be in the `body` key. + + + +### read_body/1 * ### + +`read_body(Req) -> any()` + +Helper to grab the full body of a HTTP request, even if it's chunked. + + + +### read_body/2 * ### + +`read_body(Req0, Acc) -> any()` + + + +### set_default_opts/1 ### + +`set_default_opts(Opts) -> any()` + + + +### set_node_opts_test/0 * ### + +`set_node_opts_test() -> any()` + +Ensure that the `start` hook can be used to modify the node options. We +do this by creating a message with a device that has a `start` key. This +key takes the message's body (the anticipated node options) and returns a +modified version of that body, which will be used to configure the node. We +then check that the node options were modified as we expected. + + + +### set_opts/1 ### + +`set_opts(Opts) -> any()` + +Merges the provided `Opts` with uncommitted values from `Request`, +preserves the http_server value, and updates node_history by prepending +the `Request`. If a server reference exists, updates the Cowboy environment +variable 'node_msg' with the resulting options map. + + + +### set_opts/2 ### + +`set_opts(Request, Opts) -> any()` + + + +### start/0 ### + +`start() -> any()` + +Starts the HTTP server. Optionally accepts an `Opts` message, which +is used as the source for server configuration settings, as well as the +`Opts` argument to use for all AO-Core resolution requests downstream. + + + +### start/1 ### + +`start(Opts) -> any()` + + + +### start_http2/3 * ### + +`start_http2(ServerID, ProtoOpts, NodeMsg) -> any()` + + + +### start_http3/3 * ### + +`start_http3(ServerID, ProtoOpts, NodeMsg) -> any()` + + + +### start_node/0 ### + +`start_node() -> any()` + +Test that we can start the server, send a message, and get a response. + + + +### start_node/1 ### + +`start_node(Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_http_server.md --- + +--- START OF FILE: docs/resources/source-code/hb_http.md --- +# [Module hb_http.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http.erl) + + + + + + +## Function Index ## + + +
accept_to_codec/2Calculate the codec name to use for a reply given its initiating Cowboy +request, the parsed TABM request, and the response message.
add_cors_headers/2*Add permissive CORS headers to a message, if the message has not already +specified CORS headers.
allowed_status/2*Check if a status is allowed, according to the configuration.
ans104_wasm_test/0*
codec_to_content_type/2*Call the content-type key on a message with the given codec, using +a fast-path for options that are not needed for this one-time lookup.
cors_get_test/0*
default_codec/1*Return the default codec for the given options.
empty_inbox/1*Empty the inbox of the current process for all messages with the given +reference.
encode_reply/3*Generate the headers and body for a HTTP response message.
get/2Gets a URL via HTTP and returns the resulting message in deserialized +form.
get/3
get_deep_signed_wasm_state_test/0*
get_deep_unsigned_wasm_state_test/0*
http_response_to_httpsig/4*Convert a HTTP response to a httpsig message.
httpsig_to_tabm_singleton/3*HTTPSig messages are inherently mixed into the transport layer, so they +require special handling in order to be converted to a normalized message.
maybe_add_unsigned/3*Add the method and path to a message, if they are not already present.
message_to_request/2*Given a message, return the information needed to make the request.
mime_to_codec/2*Find a codec name from a mime-type.
multirequest/5*Dispatch the same HTTP request to many nodes.
multirequest_opt/5*Get a value for a multirequest option from the config or message.
multirequest_opts/3*Get the multirequest options from the config or message.
nested_ao_resolve_test/0*
parallel_multirequest/8*Dispatch the same HTTP request to many nodes in parallel.
parallel_responses/7*Collect the necessary number of responses, and stop workers if +configured to do so.
post/3Posts a message to a URL on a remote peer via HTTP.
post/4
prepare_request/6*Turn a set of request arguments into a request message, formatted in the +preferred format.
remove_unsigned_fields/2*
reply/4Reply to the client's HTTP request with a message.
reply/5*
req_to_tabm_singleton/3Convert a cowboy request to a normalized message.
request/2Posts a binary to a URL on a remote peer via HTTP, returning the raw +binary body.
request/4
request/5
route_to_request/3*Parse a dev_router:route response and return a tuple of request +parameters.
run_wasm_signed_test/0*
run_wasm_unsigned_test/0*
send_encoded_node_message_test/2*
send_flat_encoded_node_message_test/0*
send_json_encoded_node_message_test/0*
send_large_signed_request_test/0*
serial_multirequest/7*Serially request a message, collecting responses until the required +number of responses have been gathered.
simple_ao_resolve_signed_test/0*
simple_ao_resolve_unsigned_test/0*
start/0
wasm_compute_request/3*
wasm_compute_request/4*
+ + + + +## Function Details ## + + + +### accept_to_codec/2 ### + +`accept_to_codec(TABMReq, Opts) -> any()` + +Calculate the codec name to use for a reply given its initiating Cowboy +request, the parsed TABM request, and the response message. The precidence +order for finding the codec is: +1. The `accept-codec` field in the message +2. The `accept` field in the request headers +3. The default codec +Options can be specified in mime-type format (`application/*`) or in +AO device format (`device@1.0`). + + + +### add_cors_headers/2 * ### + +`add_cors_headers(Msg, ReqHdr) -> any()` + +Add permissive CORS headers to a message, if the message has not already +specified CORS headers. + + + +### allowed_status/2 * ### + +`allowed_status(ResponseMsg, Statuses) -> any()` + +Check if a status is allowed, according to the configuration. + + + +### ans104_wasm_test/0 * ### + +`ans104_wasm_test() -> any()` + + + +### codec_to_content_type/2 * ### + +`codec_to_content_type(Codec, Opts) -> any()` + +Call the `content-type` key on a message with the given codec, using +a fast-path for options that are not needed for this one-time lookup. + + + +### cors_get_test/0 * ### + +`cors_get_test() -> any()` + + + +### default_codec/1 * ### + +`default_codec(Opts) -> any()` + +Return the default codec for the given options. + + + +### empty_inbox/1 * ### + +`empty_inbox(Ref) -> any()` + +Empty the inbox of the current process for all messages with the given +reference. + + + +### encode_reply/3 * ### + +`encode_reply(TABMReq, Message, Opts) -> any()` + +Generate the headers and body for a HTTP response message. + + + +### get/2 ### + +`get(Node, Opts) -> any()` + +Gets a URL via HTTP and returns the resulting message in deserialized +form. + + + +### get/3 ### + +`get(Node, PathBin, Opts) -> any()` + + + +### get_deep_signed_wasm_state_test/0 * ### + +`get_deep_signed_wasm_state_test() -> any()` + + + +### get_deep_unsigned_wasm_state_test/0 * ### + +`get_deep_unsigned_wasm_state_test() -> any()` + + + +### http_response_to_httpsig/4 * ### + +`http_response_to_httpsig(Status, HeaderMap, Body, Opts) -> any()` + +Convert a HTTP response to a httpsig message. + + + +### httpsig_to_tabm_singleton/3 * ### + +`httpsig_to_tabm_singleton(Req, Body, Opts) -> any()` + +HTTPSig messages are inherently mixed into the transport layer, so they +require special handling in order to be converted to a normalized message. +In particular, the signatures are verified if present and required by the +node configuration. Additionally, non-committed fields are removed from the +message if it is signed, with the exception of the `path` and `method` fields. + + + +### maybe_add_unsigned/3 * ### + +`maybe_add_unsigned(Req, Msg, Opts) -> any()` + +Add the method and path to a message, if they are not already present. +The precidence order for finding the path is: +1. The path in the message +2. The path in the request URI + + + +### message_to_request/2 * ### + +`message_to_request(M, Opts) -> any()` + +Given a message, return the information needed to make the request. + + + +### mime_to_codec/2 * ### + +`mime_to_codec(X1, Opts) -> any()` + +Find a codec name from a mime-type. + + + +### multirequest/5 * ### + +`multirequest(Config, Method, Path, Message, Opts) -> any()` + +Dispatch the same HTTP request to many nodes. Can be configured to +await responses from all nodes or just one, and to halt all requests after +after it has received the required number of responses, or to leave all +requests running until they have all completed. Default: Race for first +response. + +Expects a config message of the following form: +/Nodes/1..n: Hostname | #{ hostname => Hostname, address => Address } +/Responses: Number of responses to gather +/Stop-After: Should we stop after the required number of responses? +/Parallel: Should we run the requests in parallel? + + + +### multirequest_opt/5 * ### + +`multirequest_opt(Key, Config, Message, Default, Opts) -> any()` + +Get a value for a multirequest option from the config or message. + + + +### multirequest_opts/3 * ### + +`multirequest_opts(Config, Message, Opts) -> any()` + +Get the multirequest options from the config or message. The options in +the message take precidence over the options in the config. + + + +### nested_ao_resolve_test/0 * ### + +`nested_ao_resolve_test() -> any()` + + + +### parallel_multirequest/8 * ### + +`parallel_multirequest(Nodes, Responses, StopAfter, Method, Path, Message, Statuses, Opts) -> any()` + +Dispatch the same HTTP request to many nodes in parallel. + + + +### parallel_responses/7 * ### + +`parallel_responses(Res, Procs, Ref, Awaiting, StopAfter, Statuses, Opts) -> any()` + +Collect the necessary number of responses, and stop workers if +configured to do so. + + + +### post/3 ### + +`post(Node, Message, Opts) -> any()` + +Posts a message to a URL on a remote peer via HTTP. Returns the +resulting message in deserialized form. + + + +### post/4 ### + +`post(Node, Path, Message, Opts) -> any()` + + + +### prepare_request/6 * ### + +`prepare_request(Format, Method, Peer, Path, RawMessage, Opts) -> any()` + +Turn a set of request arguments into a request message, formatted in the +preferred format. + + + +### remove_unsigned_fields/2 * ### + +`remove_unsigned_fields(Msg, Opts) -> any()` + + + +### reply/4 ### + +`reply(Req, TABMReq, Message, Opts) -> any()` + +Reply to the client's HTTP request with a message. + + + +### reply/5 * ### + +`reply(Req, TABMReq, BinStatus, RawMessage, Opts) -> any()` + + + +### req_to_tabm_singleton/3 ### + +`req_to_tabm_singleton(Req, Body, Opts) -> any()` + +Convert a cowboy request to a normalized message. + + + +### request/2 ### + +`request(Message, Opts) -> any()` + +Posts a binary to a URL on a remote peer via HTTP, returning the raw +binary body. + + + +### request/4 ### + +`request(Method, Peer, Path, Opts) -> any()` + + + +### request/5 ### + +`request(Method, Config, Path, Message, Opts) -> any()` + + + +### route_to_request/3 * ### + +`route_to_request(M, X2, Opts) -> any()` + +Parse a `dev_router:route` response and return a tuple of request +parameters. + + + +### run_wasm_signed_test/0 * ### + +`run_wasm_signed_test() -> any()` + + + +### run_wasm_unsigned_test/0 * ### + +`run_wasm_unsigned_test() -> any()` + + + +### send_encoded_node_message_test/2 * ### + +`send_encoded_node_message_test(Config, Codec) -> any()` + + + +### send_flat_encoded_node_message_test/0 * ### + +`send_flat_encoded_node_message_test() -> any()` + + + +### send_json_encoded_node_message_test/0 * ### + +`send_json_encoded_node_message_test() -> any()` + + + +### send_large_signed_request_test/0 * ### + +`send_large_signed_request_test() -> any()` + + + +### serial_multirequest/7 * ### + +`serial_multirequest(Nodes, Remaining, Method, Path, Message, Statuses, Opts) -> any()` + +Serially request a message, collecting responses until the required +number of responses have been gathered. Ensure that the statuses are +allowed, according to the configuration. + + + +### simple_ao_resolve_signed_test/0 * ### + +`simple_ao_resolve_signed_test() -> any()` + + + +### simple_ao_resolve_unsigned_test/0 * ### + +`simple_ao_resolve_unsigned_test() -> any()` + + + +### start/0 ### + +`start() -> any()` + + + +### wasm_compute_request/3 * ### + +`wasm_compute_request(ImageFile, Func, Params) -> any()` + + + +### wasm_compute_request/4 * ### + +`wasm_compute_request(ImageFile, Func, Params, ResultPath) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_http.md --- + +--- START OF FILE: docs/resources/source-code/hb_json.md --- +# [Module hb_json.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_json.erl) + + + + +Wrapper for encoding and decoding JSON. + + + +## Description ## +Supports maps and Jiffy's old +`ejson` format. This module abstracts the underlying JSON library, allowing +us to switch between libraries as needed in the future. + +## Function Index ## + + +
decode/1Takes a JSON string and decodes it into an Erlang term.
decode/2
encode/1Takes a term in Erlang's native form and encodes it as a JSON string.
+ + + + +## Function Details ## + + + +### decode/1 ### + +`decode(Bin) -> any()` + +Takes a JSON string and decodes it into an Erlang term. + + + +### decode/2 ### + +`decode(Bin, Opts) -> any()` + + + +### encode/1 ### + +`encode(Term) -> any()` + +Takes a term in Erlang's native form and encodes it as a JSON string. + + +--- END OF FILE: docs/resources/source-code/hb_json.md --- + +--- START OF FILE: docs/resources/source-code/hb_keccak.md --- +# [Module hb_keccak.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_keccak.erl) + + + + + + +## Function Index ## + + +
hash_to_checksum_address/2*
init/0*
keccak_256/1
keccak_256_key_test/0*
keccak_256_key_to_address_test/0*
keccak_256_test/0*
key_to_ethereum_address/1
sha3_256/1
sha3_256_test/0*
to_hex/1*
+ + + + +## Function Details ## + + + +### hash_to_checksum_address/2 * ### + +`hash_to_checksum_address(Last40, Hash) -> any()` + + + +### init/0 * ### + +`init() -> any()` + + + +### keccak_256/1 ### + +`keccak_256(Bin) -> any()` + + + +### keccak_256_key_test/0 * ### + +`keccak_256_key_test() -> any()` + + + +### keccak_256_key_to_address_test/0 * ### + +`keccak_256_key_to_address_test() -> any()` + + + +### keccak_256_test/0 * ### + +`keccak_256_test() -> any()` + + + +### key_to_ethereum_address/1 ### + +`key_to_ethereum_address(Key) -> any()` + + + +### sha3_256/1 ### + +`sha3_256(Bin) -> any()` + + + +### sha3_256_test/0 * ### + +`sha3_256_test() -> any()` + + + +### to_hex/1 * ### + +`to_hex(Bin) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_keccak.md --- + +--- START OF FILE: docs/resources/source-code/hb_logger.md --- +# [Module hb_logger.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_logger.erl) + + + + + + +## Function Index ## + + +
console/2*
log/2
loop/1*
register/1
report/1
start/0
start/1
+ + + + +## Function Details ## + + + +### console/2 * ### + +`console(State, Act) -> any()` + + + +### log/2 ### + +`log(Monitor, Data) -> any()` + + + +### loop/1 * ### + +`loop(State) -> any()` + + + +### register/1 ### + +`register(Monitor) -> any()` + + + +### report/1 ### + +`report(Monitor) -> any()` + + + +### start/0 ### + +`start() -> any()` + + + +### start/1 ### + +`start(Client) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_logger.md --- + +--- START OF FILE: docs/resources/source-code/hb_message.md --- +# [Module hb_message.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_message.erl) + + + + +This module acts an adapter between messages, as modeled in the +AO-Core protocol, and their uderlying binary representations and formats. + + + +## Description ## + +Unless you are implementing a new message serialization codec, you should +not need to interact with this module directly. Instead, use the +`hb_ao` interfaces to interact with all messages. The `dev_message` +module implements a device interface for abstracting over the different +message formats. + +`hb_message` and the HyperBEAM caches can interact with multiple different +types of message formats: + +- Richly typed AO-Core structured messages. +- Arweave transations. +- ANS-104 data items. +- HTTP Signed Messages. +- Flat Maps. + +This module is responsible for converting between these formats. It does so +by normalizing messages to a common format: `Type Annotated Binary Messages` +(TABM). TABMs are deep Erlang maps with keys than only contain either other +TABMs or binary values. By marshalling all messages into this format, they +can easily be coerced into other output formats. For example, generating a +`HTTP Signed Message` format output from an Arweave transaction. TABM is +also a simple format from a computational perspective (only binary literals +and O(1) access maps), such that operations upon them are efficient. + +The structure of the conversions is as follows: + +
+Arweave TX/ANS-104 ==> dev_codec_ans104:from/1 ==> TABM
+HTTP Signed Message ==> dev_codec_httpsig_conv:from/1 ==> TABM
+Flat Maps ==> dev_codec_flat:from/1 ==> TABM
+
+TABM ==> dev_codec_structured:to/1 ==> AO-Core Message
+AO-Core Message ==> dev_codec_structured:from/1 ==> TABM
+
+TABM ==> dev_codec_ans104:to/1 ==> Arweave TX/ANS-104
+TABM ==> dev_codec_httpsig_conv:to/1 ==> HTTP Signed Message
+TABM ==> dev_codec_flat:to/1 ==> Flat Maps
+...
+
+ +Additionally, this module provides a number of utility functions for +manipulating messages. For example, `hb_message:sign/2` to sign a message of +arbitrary type, or `hb_message:format/1` to print an AO-Core/TABM message in +a human-readable format. + +The `hb_cache` module is responsible for storing and retrieving messages in +the HyperBEAM stores configured on the node. Each store has its own storage +backend, but each works with simple key-value pairs. Subsequently, the +`hb_cache` module uses TABMs as the internal format for storing and +retrieving messages. + +## Function Index ## + + +
basic_map_codec_test/1*
binary_to_binary_test/1*
commit/2Sign a message with the given wallet.
commit/3
commitment/2Extract a commitment from a message given a committer ID, or a spec +message to match against.
commitment/3
committed/1Return the list of committed keys from a message.
committed/2
committed/3
committed_empty_keys_test/1*
committed_keys_test/1*
complex_signed_message_test/1*
convert/3Convert a message from one format to another.
convert/4
deep_multisignature_test/0*
deeply_nested_committed_keys_test/0*
deeply_nested_message_with_content_test/1*Test that we can convert a 3 layer nested message into a tx record and back.
deeply_nested_message_with_only_content/1*
default_keys_removed_test/0*Test that the filter_default_keys/1 function removes TX fields +that have the default values found in the tx record, but not those that +have been set by the user.
default_tx_list/0Get the ordered list of fields as AO-Core keys and default values of +the tx record.
default_tx_message/0*Get the normalized fields and default values of the tx record.
empty_string_in_tag_test/1*
encode_balance_table/2*
encode_large_balance_table_test/1*
encode_small_balance_table_test/1*
filter_default_keys/1Remove keys from a map that have the default values found in the tx +record.
find_target/3Implements a standard pattern in which the target for an operation is +found by looking for a target key in the request.
format/1Format a message for printing, optionally taking an indentation level +to start from.
format/2
from_tabm/4*
generate_test_suite/1*
get_codec/2*Get a codec from the options.
hashpath_sign_verify_test/1*
id/1Return the ID of a message.
id/2
id/3
large_body_committed_keys_test/1*
match/2Check if two maps match, including recursively checking nested maps.
match/3
match_modes_test/0*
match_test/1*Test that the message matching function works.
matchable_keys/1*
message_suite_test_/0*
message_with_large_keys_test/1*Test that the data field is correctly managed when we have multiple +uses for it (the 'data' key itself, as well as keys that cannot fit in +tags).
message_with_simple_embedded_list_test/1*
minimization_test/0*
minimize/1Remove keys from the map that can be regenerated.
minimize/2*
nested_body_list_test/1*
nested_empty_map_test/1*
nested_message_with_large_content_test/1*Test that the data field is correctly managed when we have multiple +uses for it (the 'data' key itself, as well as keys that cannot fit in +tags).
nested_message_with_large_keys_and_content_test/1*Check that large keys and data fields are correctly handled together.
nested_message_with_large_keys_test/1*
nested_structured_fields_test/1*
normalize/1*Return a map with only the keys that necessary, without those that can +be regenerated.
print/1Pretty-print a message.
print/2*
priv_survives_conversion_test/1*
recursive_nested_list_test/1*
restore_priv/2*Add the existing priv sub-map back to a converted message, honoring +any existing priv sub-map that may already be present.
run_test/0*
set_body_codec_test/1*
sign_node_message_test/1*
signed_deep_message_test/1*
signed_list_test/1*
signed_message_encode_decode_verify_test/1*
signed_message_with_derived_components_test/1*
signed_nested_data_key_test/1*
signed_only_committed_data_field_test/1*
signed_with_inner_signed_message_test/1*
signers/1Return all of the committers on a message that have 'normal', 256 bit, +addresses.
simple_nested_message_test/1*
single_layer_message_to_encoding_test/1*Test that we can convert a message into a tx record and back.
structured_field_atom_parsing_test/1*Structured field parsing tests.
structured_field_decimal_parsing_test/1*
tabm_ao_ids_equal_test/1*
test_codecs/0*
to_tabm/3*
type/1Return the type of an encoded message.
uncommitted/1Return the unsigned version of a message in AO-Core format.
unsigned_id_test/1*
verify/1wrapper function to verify a message.
verify/2
with_commitments/2Filter messages that do not match the 'spec' given.
with_commitments/3*
with_only_committed/1Return a message with only the committed keys.
with_only_committed/2
with_only_committers/2Return the message with only the specified committers attached.
without_commitments/2Filter messages that match the 'spec' given.
without_commitments/3*
+ + + + +## Function Details ## + + + +### basic_map_codec_test/1 * ### + +`basic_map_codec_test(Codec) -> any()` + + + +### binary_to_binary_test/1 * ### + +`binary_to_binary_test(Codec) -> any()` + + + +### commit/2 ### + +`commit(Msg, WalletOrOpts) -> any()` + +Sign a message with the given wallet. + + + +### commit/3 ### + +`commit(Msg, Wallet, Format) -> any()` + + + +### commitment/2 ### + +`commitment(Committer, Msg) -> any()` + +Extract a commitment from a message given a `committer` ID, or a spec +message to match against. Returns only the first matching commitment, or +`not_found`. + + + +### commitment/3 ### + +`commitment(CommitterID, Msg, Opts) -> any()` + + + +### committed/1 ### + +`committed(Msg) -> any()` + +Return the list of committed keys from a message. + + + +### committed/2 ### + +`committed(Msg, Committers) -> any()` + + + +### committed/3 ### + +`committed(Msg, List, Opts) -> any()` + + + +### committed_empty_keys_test/1 * ### + +`committed_empty_keys_test(Codec) -> any()` + + + +### committed_keys_test/1 * ### + +`committed_keys_test(Codec) -> any()` + + + +### complex_signed_message_test/1 * ### + +`complex_signed_message_test(Codec) -> any()` + + + +### convert/3 ### + +`convert(Msg, TargetFormat, Opts) -> any()` + +Convert a message from one format to another. Taking a message in the +source format, a target format, and a set of opts. If not given, the source +is assumed to be `structured@1.0`. Additional codecs can be added by ensuring they +are part of the `Opts` map -- either globally, or locally for a computation. + +The encoding happens in two phases: +1. Convert the message to a TABM. +2. Convert the TABM to the target format. + +The conversion to a TABM is done by the `structured@1.0` codec, which is always +available. The conversion from a TABM is done by the target codec. + + + +### convert/4 ### + +`convert(Msg, TargetFormat, SourceFormat, Opts) -> any()` + + + +### deep_multisignature_test/0 * ### + +`deep_multisignature_test() -> any()` + + + +### deeply_nested_committed_keys_test/0 * ### + +`deeply_nested_committed_keys_test() -> any()` + + + +### deeply_nested_message_with_content_test/1 * ### + +`deeply_nested_message_with_content_test(Codec) -> any()` + +Test that we can convert a 3 layer nested message into a tx record and back. + + + +### deeply_nested_message_with_only_content/1 * ### + +`deeply_nested_message_with_only_content(Codec) -> any()` + + + +### default_keys_removed_test/0 * ### + +`default_keys_removed_test() -> any()` + +Test that the filter_default_keys/1 function removes TX fields +that have the default values found in the tx record, but not those that +have been set by the user. + + + +### default_tx_list/0 ### + +`default_tx_list() -> any()` + +Get the ordered list of fields as AO-Core keys and default values of +the tx record. + + + +### default_tx_message/0 * ### + +`default_tx_message() -> any()` + +Get the normalized fields and default values of the tx record. + + + +### empty_string_in_tag_test/1 * ### + +`empty_string_in_tag_test(Codec) -> any()` + + + +### encode_balance_table/2 * ### + +`encode_balance_table(Size, Codec) -> any()` + + + +### encode_large_balance_table_test/1 * ### + +`encode_large_balance_table_test(Codec) -> any()` + + + +### encode_small_balance_table_test/1 * ### + +`encode_small_balance_table_test(Codec) -> any()` + + + +### filter_default_keys/1 ### + +`filter_default_keys(Map) -> any()` + +Remove keys from a map that have the default values found in the tx +record. + + + +### find_target/3 ### + +`find_target(Self, Req, Opts) -> any()` + +Implements a standard pattern in which the target for an operation is +found by looking for a `target` key in the request. If the target is `self`, +or not present, the operation is performed on the original message. Otherwise, +the target is expected to be a key in the message, and the operation is +performed on the value of that key. + + + +### format/1 ### + +`format(Item) -> any()` + +Format a message for printing, optionally taking an indentation level +to start from. + + + +### format/2 ### + +`format(Bin, Indent) -> any()` + + + +### from_tabm/4 * ### + +`from_tabm(Msg, TargetFormat, OldPriv, Opts) -> any()` + + + +### generate_test_suite/1 * ### + +`generate_test_suite(Suite) -> any()` + + + +### get_codec/2 * ### + +`get_codec(TargetFormat, Opts) -> any()` + +Get a codec from the options. + + + +### hashpath_sign_verify_test/1 * ### + +`hashpath_sign_verify_test(Codec) -> any()` + + + +### id/1 ### + +`id(Msg) -> any()` + +Return the ID of a message. + + + +### id/2 ### + +`id(Msg, Committers) -> any()` + + + +### id/3 ### + +`id(Msg, RawCommitters, Opts) -> any()` + + + +### large_body_committed_keys_test/1 * ### + +`large_body_committed_keys_test(Codec) -> any()` + + + +### match/2 ### + +`match(Map1, Map2) -> any()` + +Check if two maps match, including recursively checking nested maps. +Takes an optional mode argument to control the matching behavior: +`strict`: All keys in both maps be present and match. +`only_present`: Only present keys in both maps must match. +`primary`: Only the primary map's keys must be present. + + + +### match/3 ### + +`match(Map1, Map2, Mode) -> any()` + + + +### match_modes_test/0 * ### + +`match_modes_test() -> any()` + + + +### match_test/1 * ### + +`match_test(Codec) -> any()` + +Test that the message matching function works. + + + +### matchable_keys/1 * ### + +`matchable_keys(Map) -> any()` + + + +### message_suite_test_/0 * ### + +`message_suite_test_() -> any()` + + + +### message_with_large_keys_test/1 * ### + +`message_with_large_keys_test(Codec) -> any()` + +Test that the data field is correctly managed when we have multiple +uses for it (the 'data' key itself, as well as keys that cannot fit in +tags). + + + +### message_with_simple_embedded_list_test/1 * ### + +`message_with_simple_embedded_list_test(Codec) -> any()` + + + +### minimization_test/0 * ### + +`minimization_test() -> any()` + + + +### minimize/1 ### + +`minimize(Msg) -> any()` + +Remove keys from the map that can be regenerated. Optionally takes an +additional list of keys to include in the minimization. + + + +### minimize/2 * ### + +`minimize(RawVal, ExtraKeys) -> any()` + + + +### nested_body_list_test/1 * ### + +`nested_body_list_test(Codec) -> any()` + + + +### nested_empty_map_test/1 * ### + +`nested_empty_map_test(Codec) -> any()` + + + +### nested_message_with_large_content_test/1 * ### + +`nested_message_with_large_content_test(Codec) -> any()` + +Test that the data field is correctly managed when we have multiple +uses for it (the 'data' key itself, as well as keys that cannot fit in +tags). + + + +### nested_message_with_large_keys_and_content_test/1 * ### + +`nested_message_with_large_keys_and_content_test(Codec) -> any()` + +Check that large keys and data fields are correctly handled together. + + + +### nested_message_with_large_keys_test/1 * ### + +`nested_message_with_large_keys_test(Codec) -> any()` + + + +### nested_structured_fields_test/1 * ### + +`nested_structured_fields_test(Codec) -> any()` + + + +### normalize/1 * ### + +`normalize(Map) -> any()` + +Return a map with only the keys that necessary, without those that can +be regenerated. + + + +### print/1 ### + +`print(Msg) -> any()` + +Pretty-print a message. + + + +### print/2 * ### + +`print(Msg, Indent) -> any()` + + + +### priv_survives_conversion_test/1 * ### + +`priv_survives_conversion_test(Codec) -> any()` + + + +### recursive_nested_list_test/1 * ### + +`recursive_nested_list_test(Codec) -> any()` + + + +### restore_priv/2 * ### + +`restore_priv(Msg, EmptyPriv) -> any()` + +Add the existing `priv` sub-map back to a converted message, honoring +any existing `priv` sub-map that may already be present. + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### set_body_codec_test/1 * ### + +`set_body_codec_test(Codec) -> any()` + + + +### sign_node_message_test/1 * ### + +`sign_node_message_test(Codec) -> any()` + + + +### signed_deep_message_test/1 * ### + +`signed_deep_message_test(Codec) -> any()` + + + +### signed_list_test/1 * ### + +`signed_list_test(Codec) -> any()` + + + +### signed_message_encode_decode_verify_test/1 * ### + +`signed_message_encode_decode_verify_test(Codec) -> any()` + + + +### signed_message_with_derived_components_test/1 * ### + +`signed_message_with_derived_components_test(Codec) -> any()` + + + +### signed_nested_data_key_test/1 * ### + +`signed_nested_data_key_test(Codec) -> any()` + + + +### signed_only_committed_data_field_test/1 * ### + +`signed_only_committed_data_field_test(Codec) -> any()` + + + +### signed_with_inner_signed_message_test/1 * ### + +`signed_with_inner_signed_message_test(Codec) -> any()` + + + +### signers/1 ### + +`signers(Msg) -> any()` + +Return all of the committers on a message that have 'normal', 256 bit, +addresses. + + + +### simple_nested_message_test/1 * ### + +`simple_nested_message_test(Codec) -> any()` + + + +### single_layer_message_to_encoding_test/1 * ### + +`single_layer_message_to_encoding_test(Codec) -> any()` + +Test that we can convert a message into a tx record and back. + + + +### structured_field_atom_parsing_test/1 * ### + +`structured_field_atom_parsing_test(Codec) -> any()` + +Structured field parsing tests. + + + +### structured_field_decimal_parsing_test/1 * ### + +`structured_field_decimal_parsing_test(Codec) -> any()` + + + +### tabm_ao_ids_equal_test/1 * ### + +`tabm_ao_ids_equal_test(Codec) -> any()` + + + +### test_codecs/0 * ### + +`test_codecs() -> any()` + + + +### to_tabm/3 * ### + +`to_tabm(Msg, SourceFormat, Opts) -> any()` + + + +### type/1 ### + +`type(TX) -> any()` + +Return the type of an encoded message. + + + +### uncommitted/1 ### + +`uncommitted(Bin) -> any()` + +Return the unsigned version of a message in AO-Core format. + + + +### unsigned_id_test/1 * ### + +`unsigned_id_test(Codec) -> any()` + + + +### verify/1 ### + +`verify(Msg) -> any()` + +wrapper function to verify a message. + + + +### verify/2 ### + +`verify(Msg, Committers) -> any()` + + + +### with_commitments/2 ### + +`with_commitments(Spec, Msg) -> any()` + +Filter messages that do not match the 'spec' given. The underlying match +is performed in the `only_present` mode, such that match specifications only +need to specify the keys that must be present. + + + +### with_commitments/3 * ### + +`with_commitments(Spec, Msg, Opts) -> any()` + + + +### with_only_committed/1 ### + +`with_only_committed(Msg) -> any()` + +Return a message with only the committed keys. If no commitments are +present, the message is returned unchanged. This means that you need to +check if the message is: +- Committed +- Verifies +...before using the output of this function as the 'canonical' message. This +is such that expensive operations like signature verification are not +performed unless necessary. + + + +### with_only_committed/2 ### + +`with_only_committed(Msg, Opts) -> any()` + + + +### with_only_committers/2 ### + +`with_only_committers(Msg, Committers) -> any()` + +Return the message with only the specified committers attached. + + + +### without_commitments/2 ### + +`without_commitments(Spec, Msg) -> any()` + +Filter messages that match the 'spec' given. Inverts the `with_commitments/2` +function, such that only messages that do _not_ match the spec are returned. + + + +### without_commitments/3 * ### + +`without_commitments(Spec, Msg, Opts) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_message.md --- + +--- START OF FILE: docs/resources/source-code/hb_metrics_collector.md --- +# [Module hb_metrics_collector.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_metrics_collector.erl) + + + + +__Behaviours:__ [`prometheus_collector`](prometheus_collector.md). + + + +## Function Index ## + + +
collect_metrics/2
collect_mf/2
create_gauge/3*
deregister_cleanup/1
+ + + + +## Function Details ## + + + +### collect_metrics/2 ### + +`collect_metrics(X1, SystemLoad) -> any()` + + + +### collect_mf/2 ### + +`collect_mf(Registry, Callback) -> any()` + + + +### create_gauge/3 * ### + +`create_gauge(Name, Help, Data) -> any()` + + + +### deregister_cleanup/1 ### + +`deregister_cleanup(X1) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_metrics_collector.md --- + +--- START OF FILE: docs/resources/source-code/hb_name.md --- +# [Module hb_name.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_name.erl) + + + + +An abstraction for name registration/deregistration in Hyperbeam. + + + +## Description ## +Its motivation is to provide a way to register names that are not necessarily +atoms, but can be any term (for example: hashpaths or `process@1.0` IDs). +An important characteristic of these functions is that they are atomic: +There can only ever be one registrant for a given name at a time. + +## Function Index ## + + +
all/0List the names in the registry.
all_test/0*
atom_test/0*
basic_test/1*
cleanup_test/0*
concurrency_test/0*
dead_process_test/0*
ets_lookup/1*
lookup/1Lookup a name -> PID.
register/1Register a name.
register/2
spawn_test_workers/1*
start/0
start_ets/0*
term_test/0*
unregister/1Unregister a name.
wait_for_cleanup/2*
+ + + + +## Function Details ## + + + +### all/0 ### + +`all() -> any()` + +List the names in the registry. + + + +### all_test/0 * ### + +`all_test() -> any()` + + + +### atom_test/0 * ### + +`atom_test() -> any()` + + + +### basic_test/1 * ### + +`basic_test(Term) -> any()` + + + +### cleanup_test/0 * ### + +`cleanup_test() -> any()` + + + +### concurrency_test/0 * ### + +`concurrency_test() -> any()` + + + +### dead_process_test/0 * ### + +`dead_process_test() -> any()` + + + +### ets_lookup/1 * ### + +`ets_lookup(Name) -> any()` + + + +### lookup/1 ### + +`lookup(Name) -> any()` + +Lookup a name -> PID. + + + +### register/1 ### + +`register(Name) -> any()` + +Register a name. If the name is already registered, the registration +will fail. The name can be any Erlang term. + + + +### register/2 ### + +`register(Name, Pid) -> any()` + + + +### spawn_test_workers/1 * ### + +`spawn_test_workers(Name) -> any()` + + + +### start/0 ### + +`start() -> any()` + + + +### start_ets/0 * ### + +`start_ets() -> any()` + + + +### term_test/0 * ### + +`term_test() -> any()` + + + +### unregister/1 ### + +`unregister(Name) -> any()` + +Unregister a name. + + + +### wait_for_cleanup/2 * ### + +`wait_for_cleanup(Name, Retries) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_name.md --- + +--- START OF FILE: docs/resources/source-code/hb_opts.md --- +# [Module hb_opts.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_opts.erl) + + + + +A module for interacting with local and global options inside +HyperBEAM. + + + +## Description ## + +Options are set globally, but can also be overridden using an +an optional local `Opts` map argument. Many functions across the HyperBEAM +environment accept an `Opts` argument, which can be used to customize +behavior. + +Options set in an `Opts` map must _never_ change the behavior of a function +that should otherwise be deterministic. Doing so may lead to loss of funds +by the HyperBEAM node operator, as the results of their executions will be +different than those of other node operators. If they are economically +staked on the correctness of these results, they may experience punishments +for non-verifiable behavior. Instead, if a local node setting makes +deterministic behavior impossible, the caller should fail the execution +with a refusal to execute. + +## Function Index ## + + +
cached_os_env/2*Cache the result of os:getenv/1 in the process dictionary, as it never +changes during the lifetime of a node.
check_required_opts/2Utility function to check for required options in a list.
config_lookup/2*An abstraction for looking up configuration variables.
default_message/0The default configuration options of the hyperbeam node.
get/1Get an option from the global options, optionally overriding with a +local Opts map if prefer or only is set to local.
get/2
get/3
global_get/2*Get an environment variable or configuration key.
load/1Parse a flat@1.0 encoded file into a map, matching the types of the +keys to those in the default message.
load_bin/1
mimic_default_types/2Mimic the types of the default message for a given map.
normalize_default/1*Get an option from environment variables, optionally consulting the +hb_features of the node if a conditional default tuple is provided.
validate_node_history/1Validate that the node_history length is within an acceptable range.
validate_node_history/3
+ + + + +## Function Details ## + + + +### cached_os_env/2 * ### + +`cached_os_env(Key, DefaultValue) -> any()` + +Cache the result of os:getenv/1 in the process dictionary, as it never +changes during the lifetime of a node. + + + +### check_required_opts/2 ### + +

+check_required_opts(KeyValuePairs::[{binary(), term()}], Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`KeyValuePairs`: A list of {Name, Value} pairs to check.
`Opts`: The original options map to return if validation succeeds.
+ +returns: `{ok, Opts}` if all required options are present, or +`{error, <<"Missing required parameters: ", MissingOptsStr/binary>>}` +where `MissingOptsStr` is a comma-separated list of missing option names. + +Utility function to check for required options in a list. +Takes a list of {Name, Value} pairs and returns: +- {ok, Opts} when all required options are present (Value =/= not_found) +- {error, ErrorMsg} with a message listing all missing options when any are not_found + + + +### config_lookup/2 * ### + +`config_lookup(Key, Default) -> any()` + +An abstraction for looking up configuration variables. In the future, +this is the function that we will want to change to support a more dynamic +configuration system. + + + +### default_message/0 ### + +`default_message() -> any()` + +The default configuration options of the hyperbeam node. + + + +### get/1 ### + +`get(Key) -> any()` + +Get an option from the global options, optionally overriding with a +local `Opts` map if `prefer` or `only` is set to `local`. If the `only` +option is provided in the `local` map, only keys found in the corresponding +(`local` or `global`) map will be returned. This function also offers users +a way to specify a default value to return if the option is not set. + +`prefer` defaults to `local`. + + + +### get/2 ### + +`get(Key, Default) -> any()` + + + +### get/3 ### + +`get(Key, Default, Opts) -> any()` + + + +### global_get/2 * ### + +`global_get(Key, Default) -> any()` + +Get an environment variable or configuration key. + + + +### load/1 ### + +`load(Path) -> any()` + +Parse a `flat@1.0` encoded file into a map, matching the types of the +keys to those in the default message. + + + +### load_bin/1 ### + +`load_bin(Bin) -> any()` + + + +### mimic_default_types/2 ### + +`mimic_default_types(Map, Mode) -> any()` + +Mimic the types of the default message for a given map. + + + +### normalize_default/1 * ### + +`normalize_default(Default) -> any()` + +Get an option from environment variables, optionally consulting the +`hb_features` of the node if a conditional default tuple is provided. + + + +### validate_node_history/1 ### + +`validate_node_history(Opts) -> any()` + +Validate that the node_history length is within an acceptable range. + + + +### validate_node_history/3 ### + +`validate_node_history(Opts, MinLength, MaxLength) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_opts.md --- + +--- START OF FILE: docs/resources/source-code/hb_path.md --- +# [Module hb_path.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_path.erl) + + + + +This module provides utilities for manipulating the paths of a +message: Its request path (referred to in messages as just the `Path`), and +its HashPath. + + + +## Description ## + +A HashPath is a rolling Merkle list of the messages that have been applied +in order to generate a given message. Because applied messages can +themselves be the result of message applications with the AO-Core protocol, +the HashPath can be thought of as the tree of messages that represent the +history of a given message. The initial message on a HashPath is referred to +by its ID and serves as its user-generated 'root'. + +Specifically, the HashPath can be generated by hashing the previous HashPath +and the current message. This means that each message in the HashPath is +dependent on all previous messages. + +``` + + Msg1.HashPath = Msg1.ID + Msg3.HashPath = Msg1.Hash(Msg1.HashPath, Msg2.ID) + Msg3.{...} = AO-Core.apply(Msg1, Msg2) + ... +``` + +A message's ID itself includes its HashPath, leading to the mixing of +a Msg2's merkle list into the resulting Msg3's HashPath. This allows a single +message to represent a history _tree_ of all of the messages that were +applied to generate it -- rather than just a linear history. + +A message may also specify its own algorithm for generating its HashPath, +which allows for custom logic to be used for representing the history of a +message. When Msg2's are applied to a Msg1, the resulting Msg3's HashPath +will be generated according to Msg1's algorithm choice. + +## Function Index ## + + +
do_to_binary/1*
from_message/2Extract the request path or hashpath from a message.
hashpath/2Add an ID of a Msg2 to the HashPath of another message.
hashpath/3
hashpath/4
hashpath_alg/1Get the hashpath function for a message from its HashPath-Alg.
hashpath_direct_msg2_test/0*
hashpath_test/0*
hd/2Extract the first key from a Message2's Path field.
hd_test/0*
matches/2Check if two keys match.
multiple_hashpaths_test/0*
normalize/1Normalize a path to a binary, removing the leading slash if present.
pop_from_message_test/0*
pop_from_path_list_test/0*
pop_request/2Pop the next element from a request path or path list.
priv_remaining/2Return the Remaining-Path of a message, from its hidden AO-Core +key.
priv_store_remaining/2Store the remaining path of a message in its hidden AO-Core key.
push_request/2Add a message to the head (next to execute) of a request path.
queue_request/2Queue a message at the back of a request path.
regex_matches/2Check if two keys match using regex.
regex_matches_test/0*
term_to_path_parts/1Convert a term into an executable path.
term_to_path_parts/2
term_to_path_parts_test/0*
tl/2Return the message without its first path element.
tl_test/0*
to_binary/1Convert a path of any form to a binary.
to_binary_test/0*
validate_path_transitions/2*
verify_hashpath/2Verify the HashPath of a message, given a list of messages that +represent its history.
verify_hashpath_test/0*
+ + + + +## Function Details ## + + + +### do_to_binary/1 * ### + +`do_to_binary(Path) -> any()` + + + +### from_message/2 ### + +`from_message(X1, Msg) -> any()` + +Extract the request path or hashpath from a message. We do not use +AO-Core for this resolution because this function is called from inside AO-Core +itself. This imparts a requirement: the message's device must store a +viable hashpath and path in its Erlang map at all times, unless the message +is directly from a user (in which case paths and hashpaths will not have +been assigned yet). + + + +### hashpath/2 ### + +`hashpath(Bin, Opts) -> any()` + +Add an ID of a Msg2 to the HashPath of another message. + + + +### hashpath/3 ### + +`hashpath(Msg1, Msg2, Opts) -> any()` + + + +### hashpath/4 ### + +`hashpath(Msg1, Msg2, HashpathAlg, Opts) -> any()` + + + +### hashpath_alg/1 ### + +`hashpath_alg(Msg) -> any()` + +Get the hashpath function for a message from its HashPath-Alg. +If no hashpath algorithm is specified, the protocol defaults to +`sha-256-chain`. + + + +### hashpath_direct_msg2_test/0 * ### + +`hashpath_direct_msg2_test() -> any()` + + + +### hashpath_test/0 * ### + +`hashpath_test() -> any()` + + + +### hd/2 ### + +`hd(Msg2, Opts) -> any()` + +Extract the first key from a `Message2`'s `Path` field. +Note: This function uses the `dev_message:get/2` function, rather than +a generic call as the path should always be an explicit key in the message. + + + +### hd_test/0 * ### + +`hd_test() -> any()` + + + +### matches/2 ### + +`matches(Key1, Key2) -> any()` + +Check if two keys match. + + + +### multiple_hashpaths_test/0 * ### + +`multiple_hashpaths_test() -> any()` + + + +### normalize/1 ### + +`normalize(Path) -> any()` + +Normalize a path to a binary, removing the leading slash if present. + + + +### pop_from_message_test/0 * ### + +`pop_from_message_test() -> any()` + + + +### pop_from_path_list_test/0 * ### + +`pop_from_path_list_test() -> any()` + + + +### pop_request/2 ### + +`pop_request(Msg, Opts) -> any()` + +Pop the next element from a request path or path list. + + + +### priv_remaining/2 ### + +`priv_remaining(Msg, Opts) -> any()` + +Return the `Remaining-Path` of a message, from its hidden `AO-Core` +key. Does not use the `get` or set `hb_private` functions, such that it +can be safely used inside the main AO-Core resolve function. + + + +### priv_store_remaining/2 ### + +`priv_store_remaining(Msg, RemainingPath) -> any()` + +Store the remaining path of a message in its hidden `AO-Core` key. + + + +### push_request/2 ### + +`push_request(Msg, Path) -> any()` + +Add a message to the head (next to execute) of a request path. + + + +### queue_request/2 ### + +`queue_request(Msg, Path) -> any()` + +Queue a message at the back of a request path. `path` is the only +key that we cannot use dev_message's `set/3` function for (as it expects +the compute path to be there), so we use `maps:put/3` instead. + + + +### regex_matches/2 ### + +`regex_matches(Path1, Path2) -> any()` + +Check if two keys match using regex. + + + +### regex_matches_test/0 * ### + +`regex_matches_test() -> any()` + + + +### term_to_path_parts/1 ### + +`term_to_path_parts(Path) -> any()` + +Convert a term into an executable path. Supports binaries, lists, and +atoms. Notably, it does not support strings as lists of characters. + + + +### term_to_path_parts/2 ### + +`term_to_path_parts(Binary, Opts) -> any()` + + + +### term_to_path_parts_test/0 * ### + +`term_to_path_parts_test() -> any()` + + + +### tl/2 ### + +`tl(Msg2, Opts) -> any()` + +Return the message without its first path element. Note that this +is the only transformation in AO-Core that does _not_ make a log of its +transformation. Subsequently, the message's IDs will not be verifiable +after executing this transformation. +This may or may not be the mainnet behavior we want. + + + +### tl_test/0 * ### + +`tl_test() -> any()` + + + +### to_binary/1 ### + +`to_binary(Path) -> any()` + +Convert a path of any form to a binary. + + + +### to_binary_test/0 * ### + +`to_binary_test() -> any()` + + + +### validate_path_transitions/2 * ### + +`validate_path_transitions(X, Opts) -> any()` + + + +### verify_hashpath/2 ### + +`verify_hashpath(Rest, Opts) -> any()` + +Verify the HashPath of a message, given a list of messages that +represent its history. + + + +### verify_hashpath_test/0 * ### + +`verify_hashpath_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_path.md --- + +--- START OF FILE: docs/resources/source-code/hb_persistent.md --- +# [Module hb_persistent.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_persistent.erl) + + + + +Creates and manages long-lived AO-Core resolution processes. + + + +## Description ## + +These can be useful for situations where a message is large and expensive +to serialize and deserialize, or when executions should be deliberately +serialized to avoid parallel executions of the same computation. This +module is called during the core `hb_ao` execution process, so care +must be taken to avoid recursive spawns/loops. + +Built using the `pg` module, which is a distributed Erlang process group +manager. + +## Function Index ## + + +
await/4If there was already an Erlang process handling this execution, +we should register with them and wait for them to notify us of +completion.
deduplicated_execution_test/0*Test merging and returning a value with a persistent worker.
default_await/5Default await function that waits for a resolution from a worker.
default_grouper/3Create a group name from a Msg1 and Msg2 pair as a tuple.
default_worker/3A server function for handling persistent executions.
do_monitor/1*
do_monitor/2*
find_execution/2*Find a group with the given name.
find_or_register/3Register the process to lead an execution if none is found, otherwise +signal that we should await resolution.
find_or_register/4*
forward_work/2Forward requests to a newly delegated execution process.
group/3Calculate the group name for a Msg1 and Msg2 pair.
notify/4Check our inbox for processes that are waiting for the resolution +of this execution.
persistent_worker_test/0*Test spawning a default persistent worker.
register_groupname/2*Register for performing an AO-Core resolution.
send_response/4*Helper function that wraps responding with a new Msg3.
spawn_after_execution_test/0*
spawn_test_client/2*
spawn_test_client/3*
start/0*Ensure that the pg module is started.
start_monitor/0Start a monitor that prints the current members of the group every +n seconds.
start_monitor/1
start_worker/2Start a worker process that will hold a message in memory for +future executions.
start_worker/3
stop_monitor/1
test_device/0*
test_device/1*
unregister/3*Unregister for being the leader on an AO-Core resolution.
unregister_groupname/2*
unregister_notify/4Unregister as the leader for an execution and notify waiting processes.
wait_for_test_result/1*
worker_event/5*Log an event with the worker process.
+ + + + +## Function Details ## + + + +### await/4 ### + +`await(Worker, Msg1, Msg2, Opts) -> any()` + +If there was already an Erlang process handling this execution, +we should register with them and wait for them to notify us of +completion. + + + +### deduplicated_execution_test/0 * ### + +`deduplicated_execution_test() -> any()` + +Test merging and returning a value with a persistent worker. + + + +### default_await/5 ### + +`default_await(Worker, GroupName, Msg1, Msg2, Opts) -> any()` + +Default await function that waits for a resolution from a worker. + + + +### default_grouper/3 ### + +`default_grouper(Msg1, Msg2, Opts) -> any()` + +Create a group name from a Msg1 and Msg2 pair as a tuple. + + + +### default_worker/3 ### + +`default_worker(GroupName, Msg1, Opts) -> any()` + +A server function for handling persistent executions. + + + +### do_monitor/1 * ### + +`do_monitor(Group) -> any()` + + + +### do_monitor/2 * ### + +`do_monitor(Group, Last) -> any()` + + + +### find_execution/2 * ### + +`find_execution(Groupname, Opts) -> any()` + +Find a group with the given name. + + + +### find_or_register/3 ### + +`find_or_register(Msg1, Msg2, Opts) -> any()` + +Register the process to lead an execution if none is found, otherwise +signal that we should await resolution. + + + +### find_or_register/4 * ### + +`find_or_register(GroupName, Msg1, Msg2, Opts) -> any()` + + + +### forward_work/2 ### + +`forward_work(NewPID, Opts) -> any()` + +Forward requests to a newly delegated execution process. + + + +### group/3 ### + +`group(Msg1, Msg2, Opts) -> any()` + +Calculate the group name for a Msg1 and Msg2 pair. Uses the Msg1's +`group` function if it is found in the `info`, otherwise uses the default. + + + +### notify/4 ### + +`notify(GroupName, Msg2, Msg3, Opts) -> any()` + +Check our inbox for processes that are waiting for the resolution +of this execution. Comes in two forms: +1. Notify on group name alone. +2. Notify on group name and Msg2. + + + +### persistent_worker_test/0 * ### + +`persistent_worker_test() -> any()` + +Test spawning a default persistent worker. + + + +### register_groupname/2 * ### + +`register_groupname(Groupname, Opts) -> any()` + +Register for performing an AO-Core resolution. + + + +### send_response/4 * ### + +`send_response(Listener, GroupName, Msg2, Msg3) -> any()` + +Helper function that wraps responding with a new Msg3. + + + +### spawn_after_execution_test/0 * ### + +`spawn_after_execution_test() -> any()` + + + +### spawn_test_client/2 * ### + +`spawn_test_client(Msg1, Msg2) -> any()` + + + +### spawn_test_client/3 * ### + +`spawn_test_client(Msg1, Msg2, Opts) -> any()` + + + +### start/0 * ### + +`start() -> any()` + +Ensure that the `pg` module is started. + + + +### start_monitor/0 ### + +`start_monitor() -> any()` + +Start a monitor that prints the current members of the group every +n seconds. + + + +### start_monitor/1 ### + +`start_monitor(Group) -> any()` + + + +### start_worker/2 ### + +`start_worker(Msg, Opts) -> any()` + +Start a worker process that will hold a message in memory for +future executions. + + + +### start_worker/3 ### + +`start_worker(GroupName, NotMsg, Opts) -> any()` + + + +### stop_monitor/1 ### + +`stop_monitor(PID) -> any()` + + + +### test_device/0 * ### + +`test_device() -> any()` + + + +### test_device/1 * ### + +`test_device(Base) -> any()` + + + +### unregister/3 * ### + +`unregister(Msg1, Msg2, Opts) -> any()` + +Unregister for being the leader on an AO-Core resolution. + + + +### unregister_groupname/2 * ### + +`unregister_groupname(Groupname, Opts) -> any()` + + + +### unregister_notify/4 ### + +`unregister_notify(GroupName, Msg2, Msg3, Opts) -> any()` + +Unregister as the leader for an execution and notify waiting processes. + + + +### wait_for_test_result/1 * ### + +`wait_for_test_result(Ref) -> any()` + + + +### worker_event/5 * ### + +`worker_event(Group, Data, Msg1, Msg2, Opts) -> any()` + +Log an event with the worker process. If we used the default grouper +function, we should also include the Msg1 and Msg2 in the event. If we did not, +we assume that the group name expresses enough information to identify the +request. + + +--- END OF FILE: docs/resources/source-code/hb_persistent.md --- + +--- START OF FILE: docs/resources/source-code/hb_private.md --- +# [Module hb_private.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_private.erl) + + + + +This module provides basic helper utilities for managing the +private element of a message, which can be used to store state that is +not included in serialized messages, or those granted to users via the +APIs. + + + +## Description ## + +Private elements of a message can be useful for storing state that +is only relevant temporarily. For example, a device might use the private +element to store a cache of values that are expensive to recompute. They +should _not_ be used for encoding state that makes the execution of a +device non-deterministic (unless you are sure you know what you are doing). + +The `set` and `get` functions of this module allow you to run those keys +as AO-Core paths if you would like to have private `devices` in the +messages non-public zone. + +See `hb_ao` for more information about the AO-Core protocol +and private elements of messages. + +## Function Index ## + + +
from_message/1Return the private key from a message.
get/3Helper for getting a value from the private element of a message.
get/4
get_private_key_test/0*
is_private/1Check if a key is private.
priv_ao_opts/1*The opts map that should be used when resolving paths against the +private element of a message.
remove_private_specifier/1*Remove the first key from the path if it is a private specifier.
reset/1Unset all of the private keys in a message.
set/3
set/4Helper function for setting a key in the private element of a message.
set_priv/2Helper function for setting the complete private element of a message.
set_private_test/0*
+ + + + +## Function Details ## + + + +### from_message/1 ### + +`from_message(Msg) -> any()` + +Return the `private` key from a message. If the key does not exist, an +empty map is returned. + + + +### get/3 ### + +`get(Key, Msg, Opts) -> any()` + +Helper for getting a value from the private element of a message. Uses +AO-Core resolve under-the-hood, removing the private specifier from the +path if it exists. + + + +### get/4 ### + +`get(InputPath, Msg, Default, Opts) -> any()` + + + +### get_private_key_test/0 * ### + +`get_private_key_test() -> any()` + + + +### is_private/1 ### + +`is_private(Key) -> any()` + +Check if a key is private. + + + +### priv_ao_opts/1 * ### + +`priv_ao_opts(Opts) -> any()` + +The opts map that should be used when resolving paths against the +private element of a message. + + + +### remove_private_specifier/1 * ### + +`remove_private_specifier(InputPath) -> any()` + +Remove the first key from the path if it is a private specifier. + + + +### reset/1 ### + +`reset(Msg) -> any()` + +Unset all of the private keys in a message. + + + +### set/3 ### + +`set(Msg, PrivMap, Opts) -> any()` + + + +### set/4 ### + +`set(Msg, InputPath, Value, Opts) -> any()` + +Helper function for setting a key in the private element of a message. + + + +### set_priv/2 ### + +`set_priv(Msg, PrivMap) -> any()` + +Helper function for setting the complete private element of a message. + + + +### set_private_test/0 * ### + +`set_private_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_private.md --- + +--- START OF FILE: docs/resources/source-code/hb_process_monitor.md --- +# [Module hb_process_monitor.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_process_monitor.erl) + + + + + + +## Function Index ## + + +
handle_crons/1*
server/1*
start/1
start/2
start/3
stop/1
ticker/2*
+ + + + +## Function Details ## + + + +### handle_crons/1 * ### + +`handle_crons(State) -> any()` + + + +### server/1 * ### + +`server(State) -> any()` + + + +### start/1 ### + +`start(ProcID) -> any()` + + + +### start/2 ### + +`start(ProcID, Rate) -> any()` + + + +### start/3 ### + +`start(ProcID, Rate, Cursor) -> any()` + + + +### stop/1 ### + +`stop(PID) -> any()` + + + +### ticker/2 * ### + +`ticker(Monitor, Rate) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_process_monitor.md --- + +--- START OF FILE: docs/resources/source-code/hb_router.md --- +# [Module hb_router.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_router.erl) + + + + + + +## Function Index ## + + +
find/2
find/3
+ + + + +## Function Details ## + + + +### find/2 ### + +`find(Type, ID) -> any()` + + + +### find/3 ### + +`find(Type, ID, Address) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_router.md --- + +--- START OF FILE: docs/resources/source-code/hb_singleton.md --- +# [Module hb_singleton.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_singleton.erl) + + + + +A parser that translates AO-Core HTTP API requests in TABM format +into an ordered list of messages to evaluate. + + + +## Description ## + +The details of this format +are described in `docs/ao-core-http-api.md`. + +Syntax overview: + +``` + + Singleton: Message containing keys and a path field, + which may also contain a query string of key-value pairs. + Path: + - /Part1/Part2/.../PartN/ => [Part1, Part2, ..., PartN] + - /ID/Part2/.../PartN => [ID, Part2, ..., PartN] + Part: (Key + Resolution), Device?, #{ K => V}? + - Part => #{ path => Part } + - Part&Key=Value => #{ path => Part, Key => Value } + - Part&Key => #{ path => Part, Key => true } + - Part&k1=v1&k2=v2 => #{ path => Part, k1 => `<<"v1">>, k2 => <<"v2">> }' + - Part~Device => {as, Device, #{ path => Part }} + - Part~D&K1=V1 => {as, D, #{ path => Part, K1 => `<<"v1">> }}' + - pt&k1+int=1 => #{ path => pt, k1 => 1 } + - pt~d&k1+int=1 => {as, d, #{ path => pt, k1 => 1 }} + - (/nested/path) => Resolution of the path /nested/path + - (/nested/path&k1=v1) => (resolve /nested/path)#{k1 => v1} + - (/nested/path~D&K1=V1) => (resolve /nested/path)#{K1 => V1} + - pt&k1+res=(/a/b/c) => #{ path => pt, k1 => (resolve /a/b/c) } + Key: + - key: <<"value">> => #{ key => <<"value">>, ... } for all messages + - n.key: <<"value">> => #{ key => <<"value">>, ... } for Nth message + - key+Int: 1 => #{ key => 1, ... } + - key+Res: /nested/path => #{ key => (resolve /nested/path), ... } + - N.Key+Res=(/a/b/c) => #{ Key => (resolve /a/b/c), ... } +``` + + + +## Data Types ## + + + + +### ao_message() ### + + +

+ao_message() = map() | binary()
+
+ + + + +### tabm_message() ### + + +

+tabm_message() = map()
+
+ + + +## Function Index ## + + +
all_path_parts/2*Extract all of the parts from the binary, given (a list of) separators.
append_path/2*
apply_types/1*Step 3: Apply types to values and remove specifiers.
basic_hashpath_test/0*
basic_hashpath_to_test/0*
build/3*
build_messages/2*Step 5: Merge the base message with the scoped messages.
decode_string/1*Attempt Cowboy URL decode, then sanitize the result.
from/1Normalize a singleton TABM message into a list of executable AO-Core +messages.
group_scoped/2*Step 4: Group headers/query by N-scope.
inlined_keys_test/0*
inlined_keys_to_test/0*
maybe_join/2*Join a list of items with a separator, or return the first item if there +is only one item.
maybe_subpath/1*Check if the string is a subpath, returning it in parsed form, +or the original string with a specifier.
maybe_typed/2*Parse a key's type (applying it to the value) and device name if present.
multiple_inlined_keys_test/0*
multiple_inlined_keys_to_test/0*
multiple_messages_test/0*
multiple_messages_to_test/0*
normalize_base/1*Normalize the base path.
parse_explicit_message_test/0*
parse_full_path/1*Parse the relative reference into path, query, and fragment.
parse_inlined_key_val/1*Extrapolate the inlined key-value pair from a path segment.
parse_inlined_keys/2*Parse inlined key-value pairs from a path segment.
parse_part/1*Parse a path part into a message or an ID.
parse_part_mods/2*Parse part modifiers: +1.
parse_scope/1*Get the scope of a key.
part/2*Extract the characters from the binary until a separator is found.
part/4*
path_messages/1*Step 2: Decode, split and sanitize the path.
path_parts/2*Split the path into segments, filtering out empty segments and +segments that are too long.
path_parts_test/0*
scoped_key_test/0*
scoped_key_to_test/0*
simple_to_test/0*
single_message_test/0*
subpath_in_inlined_test/0*
subpath_in_inlined_to_test/0*
subpath_in_key_test/0*
subpath_in_key_to_test/0*
subpath_in_path_test/0*
subpath_in_path_to_test/0*
to/1Convert a list of AO-Core message into TABM message.
to_suite_test_/0*
type/1*
typed_key_test/0*
typed_key_to_test/0*
+ + + + +## Function Details ## + + + +### all_path_parts/2 * ### + +`all_path_parts(Sep, Bin) -> any()` + +Extract all of the parts from the binary, given (a list of) separators. + + + +### append_path/2 * ### + +`append_path(PathPart, Message) -> any()` + + + +### apply_types/1 * ### + +`apply_types(Msg) -> any()` + +Step 3: Apply types to values and remove specifiers. + + + +### basic_hashpath_test/0 * ### + +`basic_hashpath_test() -> any()` + + + +### basic_hashpath_to_test/0 * ### + +`basic_hashpath_to_test() -> any()` + + + +### build/3 * ### + +`build(I, Rest, ScopedKeys) -> any()` + + + +### build_messages/2 * ### + +`build_messages(Msgs, ScopedModifications) -> any()` + +Step 5: Merge the base message with the scoped messages. + + + +### decode_string/1 * ### + +`decode_string(B) -> any()` + +Attempt Cowboy URL decode, then sanitize the result. + + + +### from/1 ### + +`from(Path) -> any()` + +Normalize a singleton TABM message into a list of executable AO-Core +messages. + + + +### group_scoped/2 * ### + +`group_scoped(Map, Msgs) -> any()` + +Step 4: Group headers/query by N-scope. +`N.Key` => applies to Nth step. Otherwise => `global` + + + +### inlined_keys_test/0 * ### + +`inlined_keys_test() -> any()` + + + +### inlined_keys_to_test/0 * ### + +`inlined_keys_to_test() -> any()` + + + +### maybe_join/2 * ### + +`maybe_join(Items, Sep) -> any()` + +Join a list of items with a separator, or return the first item if there +is only one item. If there are no items, return an empty binary. + + + +### maybe_subpath/1 * ### + +`maybe_subpath(Str) -> any()` + +Check if the string is a subpath, returning it in parsed form, +or the original string with a specifier. + + + +### maybe_typed/2 * ### + +`maybe_typed(Key, Value) -> any()` + +Parse a key's type (applying it to the value) and device name if present. + + + +### multiple_inlined_keys_test/0 * ### + +`multiple_inlined_keys_test() -> any()` + + + +### multiple_inlined_keys_to_test/0 * ### + +`multiple_inlined_keys_to_test() -> any()` + + + +### multiple_messages_test/0 * ### + +`multiple_messages_test() -> any()` + + + +### multiple_messages_to_test/0 * ### + +`multiple_messages_to_test() -> any()` + + + +### normalize_base/1 * ### + +`normalize_base(Rest) -> any()` + +Normalize the base path. + + + +### parse_explicit_message_test/0 * ### + +`parse_explicit_message_test() -> any()` + + + +### parse_full_path/1 * ### + +`parse_full_path(RelativeRef) -> any()` + +Parse the relative reference into path, query, and fragment. + + + +### parse_inlined_key_val/1 * ### + +`parse_inlined_key_val(Bin) -> any()` + +Extrapolate the inlined key-value pair from a path segment. If the +key has a value, it may provide a type (as with typical keys), but if a +value is not provided, it is assumed to be a boolean `true`. + + + +### parse_inlined_keys/2 * ### + +`parse_inlined_keys(InlinedMsgBin, Msg) -> any()` + +Parse inlined key-value pairs from a path segment. Each key-value pair +is separated by `&` and is of the form `K=V`. + + + +### parse_part/1 * ### + +`parse_part(ID) -> any()` + +Parse a path part into a message or an ID. +Applies the syntax rules outlined in the module doc, in the following order: +1. ID +2. Part subpath resolutions +3. Inlined key-value pairs +4. Device specifier + + + +### parse_part_mods/2 * ### + +`parse_part_mods(X1, Msg) -> any()` + +Parse part modifiers: +1. `~Device` => `{as, Device, Msg}` +2. `&K=V` => `Msg#{ K => V }` + + + +### parse_scope/1 * ### + +`parse_scope(KeyBin) -> any()` + +Get the scope of a key. Adds 1 to account for the base message. + + + +### part/2 * ### + +`part(Sep, Bin) -> any()` + +Extract the characters from the binary until a separator is found. +The first argument of the function is an explicit separator character, or +a list of separator characters. Returns a tuple with the separator, the +accumulated characters, and the rest of the binary. + + + +### part/4 * ### + +`part(Seps, X2, Depth, CurrAcc) -> any()` + + + +### path_messages/1 * ### + +`path_messages(RawBin) -> any()` + +Step 2: Decode, split and sanitize the path. Split by `/` but avoid +subpath components, such that their own path parts are not dissociated from +their parent path. + + + +### path_parts/2 * ### + +`path_parts(Sep, PathBin) -> any()` + +Split the path into segments, filtering out empty segments and +segments that are too long. + + + +### path_parts_test/0 * ### + +`path_parts_test() -> any()` + + + +### scoped_key_test/0 * ### + +`scoped_key_test() -> any()` + + + +### scoped_key_to_test/0 * ### + +`scoped_key_to_test() -> any()` + + + +### simple_to_test/0 * ### + +`simple_to_test() -> any()` + + + +### single_message_test/0 * ### + +`single_message_test() -> any()` + + + +### subpath_in_inlined_test/0 * ### + +`subpath_in_inlined_test() -> any()` + + + +### subpath_in_inlined_to_test/0 * ### + +`subpath_in_inlined_to_test() -> any()` + + + +### subpath_in_key_test/0 * ### + +`subpath_in_key_test() -> any()` + + + +### subpath_in_key_to_test/0 * ### + +`subpath_in_key_to_test() -> any()` + + + +### subpath_in_path_test/0 * ### + +`subpath_in_path_test() -> any()` + + + +### subpath_in_path_to_test/0 * ### + +`subpath_in_path_to_test() -> any()` + + + +### to/1 ### + +

+to(Messages::[ao_message()]) -> tabm_message()
+
+
+ +Convert a list of AO-Core message into TABM message. + + + +### to_suite_test_/0 * ### + +`to_suite_test_() -> any()` + + + +### type/1 * ### + +`type(Value) -> any()` + + + +### typed_key_test/0 * ### + +`typed_key_test() -> any()` + + + +### typed_key_to_test/0 * ### + +`typed_key_to_test() -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_singleton.md --- + +--- START OF FILE: docs/resources/source-code/hb_store_fs.md --- +# [Module hb_store_fs.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_fs.erl) + + + + + + +## Function Index ## + + +
add_prefix/2*Add the directory prefix to a path.
list/2List contents of a directory in the store.
make_group/2Create a directory (group) in the store.
make_link/3Create a symlink, handling the case where the link would point to itself.
read/1*
read/2Read a key from the store, following symlinks as needed.
remove_prefix/2*Remove the directory prefix from a path.
reset/1Reset the store by completely removing its directory and recreating it.
resolve/2Replace links in a path successively, returning the final path.
resolve/3*
scope/1The file-based store is always local, for now.
start/1Initialize the file system store with the given data directory.
stop/1Stop the file system store.
type/1*
type/2Determine the type of a key in the store.
write/3Write a value to the specified path in the store.
+ + + + +## Function Details ## + + + +### add_prefix/2 * ### + +`add_prefix(X1, Path) -> any()` + +Add the directory prefix to a path. + + + +### list/2 ### + +`list(Opts, Path) -> any()` + +List contents of a directory in the store. + + + +### make_group/2 ### + +`make_group(Opts, Path) -> any()` + +Create a directory (group) in the store. + + + +### make_link/3 ### + +`make_link(Opts, Link, New) -> any()` + +Create a symlink, handling the case where the link would point to itself. + + + +### read/1 * ### + +`read(Path) -> any()` + + + +### read/2 ### + +`read(Opts, Key) -> any()` + +Read a key from the store, following symlinks as needed. + + + +### remove_prefix/2 * ### + +`remove_prefix(X1, Path) -> any()` + +Remove the directory prefix from a path. + + + +### reset/1 ### + +`reset(X1) -> any()` + +Reset the store by completely removing its directory and recreating it. + + + +### resolve/2 ### + +`resolve(Opts, RawPath) -> any()` + +Replace links in a path successively, returning the final path. +Each element of the path is resolved in turn, with the result of each +resolution becoming the prefix for the next resolution. This allows +paths to resolve across many links. For example, a structure as follows: + +/a/b/c: "Not the right data" +/a/b -> /a/alt-b +/a/alt-b/c: "Correct data" + +will resolve "a/b/c" to "Correct data". + + + +### resolve/3 * ### + +`resolve(Opts, CurrPath, Rest) -> any()` + + + +### scope/1 ### + +`scope(X1) -> any()` + +The file-based store is always local, for now. In the future, we may +want to allow that an FS store is shared across a cluster and thus remote. + + + +### start/1 ### + +`start(X1) -> any()` + +Initialize the file system store with the given data directory. + + + +### stop/1 ### + +`stop(X1) -> any()` + +Stop the file system store. Currently a no-op. + + + +### type/1 * ### + +`type(Path) -> any()` + + + +### type/2 ### + +`type(Opts, Key) -> any()` + +Determine the type of a key in the store. + + + +### write/3 ### + +`write(Opts, PathComponents, Value) -> any()` + +Write a value to the specified path in the store. + + +--- END OF FILE: docs/resources/source-code/hb_store_fs.md --- + +--- START OF FILE: docs/resources/source-code/hb_store_gateway.md --- +# [Module hb_store_gateway.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_gateway.erl) + + + + +A store module that reads data from the nodes Arweave gateway and +GraphQL routes, additionally including additional store-specific routes. + + + +## Function Index ## + + +
cache_read_message_test/0*Ensure that saving to the gateway store works.
external_http_access_test/0*Test that the default node config allows for data to be accessed.
graphql_as_store_test_/0*Store is accessible via the default options.
graphql_from_cache_test/0*Stored messages are accessible via hb_cache accesses.
list/2
manual_local_cache_test/0*
maybe_cache/2*Cache the data if the cache is enabled.
read/2Read the data at the given key from the GraphQL route.
resolve/2
resolve_on_gateway_test_/0*
scope/1The scope of a GraphQL store is always remote, due to performance.
specific_route_test/0*Routes can be specified in the options, overriding the default routes.
store_opts_test/0*Test to verify store opts is being set for Data-Protocol ao.
type/2Get the type of the data at the given key.
+ + + + +## Function Details ## + + + +### cache_read_message_test/0 * ### + +`cache_read_message_test() -> any()` + +Ensure that saving to the gateway store works. + + + +### external_http_access_test/0 * ### + +`external_http_access_test() -> any()` + +Test that the default node config allows for data to be accessed. + + + +### graphql_as_store_test_/0 * ### + +`graphql_as_store_test_() -> any()` + +Store is accessible via the default options. + + + +### graphql_from_cache_test/0 * ### + +`graphql_from_cache_test() -> any()` + +Stored messages are accessible via `hb_cache` accesses. + + + +### list/2 ### + +`list(StoreOpts, Key) -> any()` + + + +### manual_local_cache_test/0 * ### + +`manual_local_cache_test() -> any()` + + + +### maybe_cache/2 * ### + +`maybe_cache(StoreOpts, Data) -> any()` + +Cache the data if the cache is enabled. The `store` option may either +be `false` to disable local caching, or a store definition to use as the +cache. + + + +### read/2 ### + +`read(StoreOpts, Key) -> any()` + +Read the data at the given key from the GraphQL route. Will only attempt +to read the data if the key is an ID. + + + +### resolve/2 ### + +`resolve(X1, Key) -> any()` + + + +### resolve_on_gateway_test_/0 * ### + +`resolve_on_gateway_test_() -> any()` + + + +### scope/1 ### + +`scope(X1) -> any()` + +The scope of a GraphQL store is always remote, due to performance. + + + +### specific_route_test/0 * ### + +`specific_route_test() -> any()` + +Routes can be specified in the options, overriding the default routes. +We test this by inversion: If the above cache read test works, then we know +that the default routes allow access to the item. If the test below were to +produce the same result, despite an empty 'only' route list, then we would +know that the module is not respecting the route list. + + + +### store_opts_test/0 * ### + +`store_opts_test() -> any()` + +Test to verify store opts is being set for Data-Protocol ao + + + +### type/2 ### + +`type(StoreOpts, Key) -> any()` + +Get the type of the data at the given key. We potentially cache the +result, so that we don't have to read the data from the GraphQL route +multiple times. + + +--- END OF FILE: docs/resources/source-code/hb_store_gateway.md --- + +--- START OF FILE: docs/resources/source-code/hb_store_remote_node.md --- +# [Module hb_store_remote_node.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_remote_node.erl) + + + + +A store module that reads data from another AO node. + + + +## Description ## +Notably, this store only provides the _read_ side of the store interface. +The write side could be added, returning an commitment that the data has +been written to the remote node. In that case, the node would probably want +to upload it to an Arweave bundler to ensure persistence, too. + +## Function Index ## + + +
make_link/3Link a source to a destination in the remote node.
read/2Read a key from the remote node.
read_test/0*Test that we can create a store, write a random message to it, then +start a remote node with that store, and read the message from it.
resolve/2Resolve a key path in the remote store.
scope/1Return the scope of this store.
type/2Determine the type of value at a given key.
write/3Write a key to the remote node.
+ + + + +## Function Details ## + + + +### make_link/3 ### + +`make_link(Opts, Source, Destination) -> any()` + +Link a source to a destination in the remote node. + +Constructs an HTTP POST link request. If a wallet is provided, +the message is signed. Returns {ok, Path} on HTTP 200, or +{error, Reason} on failure. + + + +### read/2 ### + +`read(Opts, Key) -> any()` + +Read a key from the remote node. + +Makes an HTTP GET request to the remote node and returns the +committed message. + + + +### read_test/0 * ### + +`read_test() -> any()` + +Test that we can create a store, write a random message to it, then +start a remote node with that store, and read the message from it. + + + +### resolve/2 ### + +`resolve(X1, Key) -> any()` + +Resolve a key path in the remote store. + +For the remote node store, the key is returned as-is. + + + +### scope/1 ### + +`scope(Arg) -> any()` + +Return the scope of this store. + +For the remote store, the scope is always `remote`. + + + +### type/2 ### + +`type(Opts, Key) -> any()` + +Determine the type of value at a given key. + +Remote nodes support only the `simple` type or `not_found`. + + + +### write/3 ### + +`write(Opts, Key, Value) -> any()` + +Write a key to the remote node. + +Constructs an HTTP POST write request. If a wallet is provided, +the message is signed. Returns {ok, Path} on HTTP 200, or +{error, Reason} on failure. + + +--- END OF FILE: docs/resources/source-code/hb_store_remote_node.md --- + +--- START OF FILE: docs/resources/source-code/hb_store_rocksdb.md --- +# [Module hb_store_rocksdb.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_rocksdb.erl) + + + + +A process wrapper over rocksdb storage. + +__Behaviours:__ [`gen_server`](gen_server.md), [`hb_store`](hb_store.md). + + + +## Description ## + +Replicates functionality of the +hb_fs_store module. + +Encodes the item types with the help of prefixes, see `encode_value/2` +and `decode_value/1` + + +## Data Types ## + + + + +### key() ### + + +

+key() = binary() | list()
+
+ + + + +### value() ### + + +

+value() = binary() | list()
+
+ + + + +### value_type() ### + + +

+value_type() = link | raw | group
+
+ + + +## Function Index ## + + +
add_path/3Add two path components together.
code_change/3
collect/1*
collect/2*
convert_if_list/1*
decode_value/1*
do_read/2*
do_resolve/3*
do_write/3*Write given Key and Value to the database.
enabled/0Returns whether the RocksDB store is enabled.
encode_value/2*
ensure_dir/2*
ensure_dir/3*
ensure_list/1*Ensure that the given filename is a list, not a binary.
handle_call/3
handle_cast/2
handle_info/2
init/1
join/1*
list/0List all items registered in rocksdb store.
list/2Returns the full list of items stored under the given path.
make_group/2Creates group under the given path.
make_link/3
maybe_append_key_to_group/2*
maybe_convert_to_binary/1*
maybe_create_dir/3*
open_rockdb/1*
path/2Return path.
read/2Read data by the key.
reset/1
resolve/2Replace links in a path with the target of the link.
scope/1Return scope (local).
start/1
start_link/1Start the RocksDB store.
stop/1
terminate/2
type/2Get type of the current item.
write/3Write given Key and Value to the database.
+ + + + +## Function Details ## + + + +### add_path/3 ### + +`add_path(Opts, Path1, Path2) -> any()` + +Add two path components together. // is not used + + + +### code_change/3 ### + +`code_change(OldVsn, State, Extra) -> any()` + + + +### collect/1 * ### + +`collect(Iterator) -> any()` + + + +### collect/2 * ### + +`collect(Iterator, Acc) -> any()` + + + +### convert_if_list/1 * ### + +`convert_if_list(Value) -> any()` + + + +### decode_value/1 * ### + +

+decode_value(X1::binary()) -> {value_type(), binary()}
+
+
+ + + +### do_read/2 * ### + +`do_read(Opts, Key) -> any()` + + + +### do_resolve/3 * ### + +`do_resolve(Opts, FinalPath, Rest) -> any()` + + + +### do_write/3 * ### + +

+do_write(Opts, Key, Value) -> Result
+
+ +
  • Opts = map()
  • Key = key()
  • Value = value()
  • Result = ok | {error, any()}
+ +Write given Key and Value to the database + + + +### enabled/0 ### + +`enabled() -> any()` + +Returns whether the RocksDB store is enabled. + + + +### encode_value/2 * ### + +

+encode_value(X1::value_type(), Value::binary()) -> binary()
+
+
+ + + +### ensure_dir/2 * ### + +`ensure_dir(DBHandle, BaseDir) -> any()` + + + +### ensure_dir/3 * ### + +`ensure_dir(DBHandle, CurrentPath, Rest) -> any()` + + + +### ensure_list/1 * ### + +`ensure_list(Value) -> any()` + +Ensure that the given filename is a list, not a binary. + + + +### handle_call/3 ### + +`handle_call(Request, From, State) -> any()` + + + +### handle_cast/2 ### + +`handle_cast(Request, State) -> any()` + + + +### handle_info/2 ### + +`handle_info(Info, State) -> any()` + + + +### init/1 ### + +`init(Dir) -> any()` + + + +### join/1 * ### + +`join(Key) -> any()` + + + +### list/0 ### + +`list() -> any()` + +List all items registered in rocksdb store. Should be used only +for testing/debugging, as the underlying operation is doing full traversal +on the KV storage, and is slow. + + + +### list/2 ### + +

+list(Opts, Path) -> Result
+
+ +
  • Opts = any()
  • Path = any()
  • Result = {ok, [string()]} | {error, term()}
+ +Returns the full list of items stored under the given path. Where the path +child items is relevant to the path of parentItem. (Same as in `hb_store_fs`). + + + +### make_group/2 ### + +

+make_group(Opts, Key) -> Result
+
+ +
  • Opts = any()
  • Key = binary()
  • Result = ok | {error, already_added}
+ +Creates group under the given path. + + + +### make_link/3 ### + +

+make_link(Opts::any(), Key1::key(), New::key()) -> ok
+
+
+ + + +### maybe_append_key_to_group/2 * ### + +`maybe_append_key_to_group(Key, CurrentDirContents) -> any()` + + + +### maybe_convert_to_binary/1 * ### + +`maybe_convert_to_binary(Value) -> any()` + + + +### maybe_create_dir/3 * ### + +`maybe_create_dir(DBHandle, DirPath, Value) -> any()` + + + +### open_rockdb/1 * ### + +`open_rockdb(RawDir) -> any()` + + + +### path/2 ### + +`path(Opts, Path) -> any()` + +Return path + + + +### read/2 ### + +

+read(Opts, Key) -> Result
+
+ +
  • Opts = map()
  • Key = key() | list()
  • Result = {ok, value()} | not_found | {error, {corruption, string()}} | {error, any()}
+ +Read data by the key. +Recursively follows link messages + + + +### reset/1 ### + +

+reset(Opts::[]) -> ok | no_return()
+
+
+ + + +### resolve/2 ### + +

+resolve(Opts, Path) -> Result
+
+ +
  • Opts = any()
  • Path = binary() | list()
  • Result = not_found | string()
+ +Replace links in a path with the target of the link. + + + +### scope/1 ### + +`scope(X1) -> any()` + +Return scope (local) + + + +### start/1 ### + +`start(Opts) -> any()` + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + +Start the RocksDB store. + + + +### stop/1 ### + +

+stop(Opts::any()) -> ok
+
+
+ + + +### terminate/2 ### + +`terminate(Reason, State) -> any()` + + + +### type/2 ### + +

+type(Opts, Key) -> Result
+
+ +
  • Opts = map()
  • Key = binary()
  • Result = composite | simple | not_found
+ +Get type of the current item + + + +### write/3 ### + +

+write(Opts, Key, Value) -> Result
+
+ +
  • Opts = map()
  • Key = key()
  • Value = value()
  • Result = ok | {error, any()}
+ +Write given Key and Value to the database + + +--- END OF FILE: docs/resources/source-code/hb_store_rocksdb.md --- + +--- START OF FILE: docs/resources/source-code/hb_store.md --- +# [Module hb_store.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store.erl) + + + + + + +## Function Index ## + + +
add_path/2Add two path components together.
add_path/3
behavior_info/1
call_all/3*Call a function on all modules in the store.
call_function/3*Call a function on the first store module that succeeds.
filter/2Takes a store object and a filter function or match spec, returning a +new store object with only the modules that match the filter.
generate_test_suite/1
generate_test_suite/2
get_store_scope/1*Ask a store for its own scope.
hierarchical_path_resolution_test/1*Ensure that we can resolve links through a directory.
join/1Join a list of path components together.
list/2List the keys in a group in the store.
make_group/2Make a group in the store.
make_link/3Make a link from one path to another in the store.
path/1Create a path from a list of path components.
path/2
read/2Read a key from the store.
reset/1Delete all of the keys in a store.
resolve/2Follow links through the store to resolve a path to its ultimate target.
resursive_path_resolution_test/1*Ensure that we can resolve links recursively.
scope/2Limit the store scope to only a specific (set of) option(s).
simple_path_resolution_test/1*Test path resolution dynamics.
sort/2Order a store by a preference of its scopes.
start/1
stop/1
store_suite_test_/0*
test_stores/0
type/2Get the type of element of a given path in the store.
write/3Write a key with a value to the store.
+ + + + +## Function Details ## + + + +### add_path/2 ### + +`add_path(Path1, Path2) -> any()` + +Add two path components together. If no store implements the add_path +function, we concatenate the paths. + + + +### add_path/3 ### + +`add_path(Store, Path1, Path2) -> any()` + + + +### behavior_info/1 ### + +`behavior_info(X1) -> any()` + + + +### call_all/3 * ### + +`call_all(X, Function, Args) -> any()` + +Call a function on all modules in the store. + + + +### call_function/3 * ### + +`call_function(X, Function, Args) -> any()` + +Call a function on the first store module that succeeds. Returns its +result, or no_viable_store if none of the stores succeed. + + + +### filter/2 ### + +`filter(Module, Filter) -> any()` + +Takes a store object and a filter function or match spec, returning a +new store object with only the modules that match the filter. The filter +function takes 2 arguments: the scope and the options. It calls the store's +scope function to get the scope of the module. + + + +### generate_test_suite/1 ### + +`generate_test_suite(Suite) -> any()` + + + +### generate_test_suite/2 ### + +`generate_test_suite(Suite, Stores) -> any()` + + + +### get_store_scope/1 * ### + +`get_store_scope(Store) -> any()` + +Ask a store for its own scope. If it doesn't have one, return the +default scope (local). + + + +### hierarchical_path_resolution_test/1 * ### + +`hierarchical_path_resolution_test(Opts) -> any()` + +Ensure that we can resolve links through a directory. + + + +### join/1 ### + +`join(Path) -> any()` + +Join a list of path components together. + + + +### list/2 ### + +`list(Modules, Path) -> any()` + +List the keys in a group in the store. Use only in debugging. +The hyperbeam model assumes that stores are built as efficient hash-based +structures, so this is likely to be very slow for most stores. + + + +### make_group/2 ### + +`make_group(Modules, Path) -> any()` + +Make a group in the store. A group can be seen as a namespace or +'directory' in a filesystem. + + + +### make_link/3 ### + +`make_link(Modules, Existing, New) -> any()` + +Make a link from one path to another in the store. + + + +### path/1 ### + +`path(Path) -> any()` + +Create a path from a list of path components. If no store implements +the path function, we return the path with the 'default' transformation (id). + + + +### path/2 ### + +`path(X1, Path) -> any()` + + + +### read/2 ### + +`read(Modules, Key) -> any()` + +Read a key from the store. + + + +### reset/1 ### + +`reset(Modules) -> any()` + +Delete all of the keys in a store. Should be used with extreme +caution. Lost data can lose money in many/most of hyperbeam's use cases. + + + +### resolve/2 ### + +`resolve(Modules, Path) -> any()` + +Follow links through the store to resolve a path to its ultimate target. + + + +### resursive_path_resolution_test/1 * ### + +`resursive_path_resolution_test(Opts) -> any()` + +Ensure that we can resolve links recursively. + + + +### scope/2 ### + +`scope(Scope, Opts) -> any()` + +Limit the store scope to only a specific (set of) option(s). +Takes either an Opts message or store, and either a single scope or a list +of scopes. + + + +### simple_path_resolution_test/1 * ### + +`simple_path_resolution_test(Opts) -> any()` + +Test path resolution dynamics. + + + +### sort/2 ### + +`sort(Stores, PreferenceOrder) -> any()` + +Order a store by a preference of its scopes. This is useful for making +sure that faster (or perhaps cheaper) stores are used first. If a list is +provided, it will be used as a preference order. If a map is provided, +scopes will be ordered by the scores in the map. Any unknown scopes will +default to a score of 0. + + + +### start/1 ### + +`start(Modules) -> any()` + + + +### stop/1 ### + +`stop(Modules) -> any()` + + + +### store_suite_test_/0 * ### + +`store_suite_test_() -> any()` + + + +### test_stores/0 ### + +`test_stores() -> any()` + + + +### type/2 ### + +`type(Modules, Path) -> any()` + +Get the type of element of a given path in the store. This can be +a performance killer if the store is remote etc. Use only when necessary. + + + +### write/3 ### + +`write(Modules, Key, Value) -> any()` + +Write a key with a value to the store. + + +--- END OF FILE: docs/resources/source-code/hb_store.md --- + +--- START OF FILE: docs/resources/source-code/hb_structured_fields.md --- +# [Module hb_structured_fields.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_structured_fields.erl) + + + + +A module for parsing and converting between Erlang and HTTP Structured +Fields, as described in RFC-9651. + + + +## Description ## + +The mapping between Erlang and structured headers types is as follow: + +List: list() +Inner list: {list, [item()], params()} +Dictionary: [{binary(), item()}] +There is no distinction between empty list and empty dictionary. +Item with parameters: {item, bare_item(), params()} +Parameters: [{binary(), bare_item()}] +Bare item: one bare_item() that can be of type: +Integer: integer() +Decimal: {decimal, {integer(), integer()}} +String: {string, binary()} +Token: {token, binary()} +Byte sequence: {binary, binary()} +Boolean: boolean() + + +## Data Types ## + + + + +### sh_bare_item() ### + + +

+sh_bare_item() = integer() | sh_decimal() | boolean() | {string | token | binary, binary()}
+
+ + + + +### sh_decimal() ### + + +

+sh_decimal() = {decimal, {integer(), integer()}}
+
+ + + + +### sh_dictionary() ### + + +

+sh_dictionary() = [{binary(), sh_item() | sh_inner_list()}]
+
+ + + + +### sh_inner_list() ### + + +

+sh_inner_list() = {list, [sh_item()], sh_params()}
+
+ + + + +### sh_item() ### + + +

+sh_item() = {item, sh_bare_item(), sh_params()}
+
+ + + + +### sh_list() ### + + +

+sh_list() = [sh_item() | sh_inner_list()]
+
+ + + + +### sh_params() ### + + +

+sh_params() = [{binary(), sh_bare_item()}]
+
+ + + +## Function Index ## + + +
bare_item/1
dictionary/1
e2t/1*
e2tb/1*
e2tp/1*
escape_string/2*
exp_div/1*
expected_to_term/1*
from_bare_item/1Convert an SF bare_item to an Erlang term.
inner_list/1*
item/1
item_or_inner_list/1*
key_to_binary/1*Convert an Erlang term to a binary key.
list/1
params/1*
parse_bare_item/1Parse an integer or decimal.
parse_before_param/2*
parse_binary/2*Parse a byte sequence binary.
parse_decimal/5*Parse a decimal binary.
parse_dict_before_member/2*Parse a binary SF dictionary before a member.
parse_dict_before_sep/2*Parse a binary SF dictionary before a separator.
parse_dict_key/3*
parse_dictionary/1Parse a binary SF dictionary.
parse_inner_list/2*
parse_item/1Parse a binary SF item to an SF item.
parse_item1/1*
parse_list/1Parse a binary SF list.
parse_list_before_member/2*Parse a binary SF list before a member.
parse_list_before_sep/2*Parse a binary SF list before a separator.
parse_list_member/2*Parse a binary SF list before a member.
parse_number/3*Parse an integer or decimal binary.
parse_param/3*
parse_string/2*Parse a string binary.
parse_struct_hd_test_/0*
parse_token/2*Parse a token binary.
raw_to_binary/1*
struct_hd_identity_test_/0*
to_bare_item/1*Convert an Erlang term to an SF bare_item.
to_dictionary/1Convert a map to a dictionary.
to_dictionary/2*
to_dictionary_depth_test/0*
to_dictionary_test/0*
to_inner_item/1*Convert an Erlang term to an SF item.
to_inner_list/1*Convert an inner list to an SF term.
to_inner_list/2*
to_inner_list/3*
to_item/1Convert an item to a dictionary.
to_item/2
to_item_or_inner_list/1*Convert an Erlang term to an SF item or inner_list.
to_item_test/0*
to_list/1Convert a list to an SF term.
to_list/2*
to_list_depth_test/0*
to_list_test/0*
to_param/1*Convert an Erlang term to an SF parameter.
trim_ws/1*
trim_ws_end/2*
+ + + + +## Function Details ## + + + +### bare_item/1 ### + +`bare_item(Integer) -> any()` + + + +### dictionary/1 ### + +

+dictionary(Map::#{binary() => sh_item() | sh_inner_list()} | sh_dictionary()) -> iolist()
+
+
+ + + +### e2t/1 * ### + +`e2t(Dict) -> any()` + + + +### e2tb/1 * ### + +`e2tb(V) -> any()` + + + +### e2tp/1 * ### + +`e2tp(Params) -> any()` + + + +### escape_string/2 * ### + +`escape_string(X1, Acc) -> any()` + + + +### exp_div/1 * ### + +`exp_div(N) -> any()` + + + +### expected_to_term/1 * ### + +`expected_to_term(Dict) -> any()` + + + +### from_bare_item/1 ### + +`from_bare_item(BareItem) -> any()` + +Convert an SF `bare_item` to an Erlang term. + + + +### inner_list/1 * ### + +`inner_list(X1) -> any()` + + + +### item/1 ### + +

+item(X1::sh_item()) -> iolist()
+
+
+ + + +### item_or_inner_list/1 * ### + +`item_or_inner_list(Value) -> any()` + + + +### key_to_binary/1 * ### + +`key_to_binary(Key) -> any()` + +Convert an Erlang term to a binary key. + + + +### list/1 ### + +

+list(List::sh_list()) -> iolist()
+
+
+ + + +### params/1 * ### + +`params(Params) -> any()` + + + +### parse_bare_item/1 ### + +`parse_bare_item(X1) -> any()` + +Parse an integer or decimal. + + + +### parse_before_param/2 * ### + +`parse_before_param(X1, Acc) -> any()` + + + +### parse_binary/2 * ### + +`parse_binary(X1, Acc) -> any()` + +Parse a byte sequence binary. + + + +### parse_decimal/5 * ### + +`parse_decimal(R, L1, L2, IntAcc, FracAcc) -> any()` + +Parse a decimal binary. + + + +### parse_dict_before_member/2 * ### + +`parse_dict_before_member(X1, Acc) -> any()` + +Parse a binary SF dictionary before a member. + + + +### parse_dict_before_sep/2 * ### + +`parse_dict_before_sep(X1, Acc) -> any()` + +Parse a binary SF dictionary before a separator. + + + +### parse_dict_key/3 * ### + +`parse_dict_key(R, Acc, K) -> any()` + + + +### parse_dictionary/1 ### + +

+parse_dictionary(X1::binary()) -> sh_dictionary()
+
+
+ +Parse a binary SF dictionary. + + + +### parse_inner_list/2 * ### + +`parse_inner_list(R0, Acc) -> any()` + + + +### parse_item/1 ### + +

+parse_item(Bin::binary()) -> sh_item()
+
+
+ +Parse a binary SF item to an SF `item`. + + + +### parse_item1/1 * ### + +`parse_item1(Bin) -> any()` + + + +### parse_list/1 ### + +

+parse_list(Bin::binary()) -> sh_list()
+
+
+ +Parse a binary SF list. + + + +### parse_list_before_member/2 * ### + +`parse_list_before_member(R, Acc) -> any()` + +Parse a binary SF list before a member. + + + +### parse_list_before_sep/2 * ### + +`parse_list_before_sep(X1, Acc) -> any()` + +Parse a binary SF list before a separator. + + + +### parse_list_member/2 * ### + +`parse_list_member(R0, Acc) -> any()` + +Parse a binary SF list before a member. + + + +### parse_number/3 * ### + +`parse_number(R, L, Acc) -> any()` + +Parse an integer or decimal binary. + + + +### parse_param/3 * ### + +`parse_param(R, Acc, K) -> any()` + + + +### parse_string/2 * ### + +`parse_string(X1, Acc) -> any()` + +Parse a string binary. + + + +### parse_struct_hd_test_/0 * ### + +`parse_struct_hd_test_() -> any()` + + + +### parse_token/2 * ### + +`parse_token(R, Acc) -> any()` + +Parse a token binary. + + + +### raw_to_binary/1 * ### + +`raw_to_binary(RawList) -> any()` + + + +### struct_hd_identity_test_/0 * ### + +`struct_hd_identity_test_() -> any()` + + + +### to_bare_item/1 * ### + +`to_bare_item(BareItem) -> any()` + +Convert an Erlang term to an SF `bare_item`. + + + +### to_dictionary/1 ### + +`to_dictionary(Map) -> any()` + +Convert a map to a dictionary. + + + +### to_dictionary/2 * ### + +`to_dictionary(Dict, Rest) -> any()` + + + +### to_dictionary_depth_test/0 * ### + +`to_dictionary_depth_test() -> any()` + + + +### to_dictionary_test/0 * ### + +`to_dictionary_test() -> any()` + + + +### to_inner_item/1 * ### + +`to_inner_item(Item) -> any()` + +Convert an Erlang term to an SF `item`. + + + +### to_inner_list/1 * ### + +`to_inner_list(Inner) -> any()` + +Convert an inner list to an SF term. + + + +### to_inner_list/2 * ### + +`to_inner_list(Inner, Params) -> any()` + + + +### to_inner_list/3 * ### + +`to_inner_list(Inner, Rest, Params) -> any()` + + + +### to_item/1 ### + +`to_item(Item) -> any()` + +Convert an item to a dictionary. + + + +### to_item/2 ### + +`to_item(Item, Params) -> any()` + + + +### to_item_or_inner_list/1 * ### + +`to_item_or_inner_list(ItemOrInner) -> any()` + +Convert an Erlang term to an SF `item` or `inner_list`. + + + +### to_item_test/0 * ### + +`to_item_test() -> any()` + + + +### to_list/1 ### + +`to_list(List) -> any()` + +Convert a list to an SF term. + + + +### to_list/2 * ### + +`to_list(Acc, Rest) -> any()` + + + +### to_list_depth_test/0 * ### + +`to_list_depth_test() -> any()` + + + +### to_list_test/0 * ### + +`to_list_test() -> any()` + + + +### to_param/1 * ### + +`to_param(X1) -> any()` + +Convert an Erlang term to an SF `parameter`. + + + +### trim_ws/1 * ### + +`trim_ws(R) -> any()` + + + +### trim_ws_end/2 * ### + +`trim_ws_end(Value, N) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_structured_fields.md --- + +--- START OF FILE: docs/resources/source-code/hb_sup.md --- +# [Module hb_sup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_sup.erl) + + + + +__Behaviours:__ [`supervisor`](supervisor.md). + + + +## Function Index ## + + +
init/1
start_link/0
start_link/1
store_children/1*Generate a child spec for stores in the given Opts.
+ + + + +## Function Details ## + + + +### init/1 ### + +`init(Opts) -> any()` + + + +### start_link/0 ### + +`start_link() -> any()` + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + + + +### store_children/1 * ### + +`store_children(Store) -> any()` + +Generate a child spec for stores in the given Opts. + + +--- END OF FILE: docs/resources/source-code/hb_sup.md --- + +--- START OF FILE: docs/resources/source-code/hb_test_utils.md --- +# [Module hb_test_utils.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_test_utils.erl) + + + + +Simple utilities for testing HyperBEAM. + + + +## Function Index ## + + +
run/4
satisfies_requirements/1*Determine if the environment satisfies the given test requirements.
suite_with_opts/2Run each test in a suite with each set of options.
+ + + + +## Function Details ## + + + +### run/4 ### + +`run(Name, OptsName, Suite, OptsList) -> any()` + + + +### satisfies_requirements/1 * ### + +`satisfies_requirements(Requirements) -> any()` + +Determine if the environment satisfies the given test requirements. +Requirements is a list of atoms, each corresponding to a module that must +return true if it exposes an `enabled/0` function. + + + +### suite_with_opts/2 ### + +`suite_with_opts(Suite, OptsList) -> any()` + +Run each test in a suite with each set of options. Start and reset +the store(s) for each test. Expects suites to be a list of tuples with +the test name, description, and test function. +The list of `Opts` should contain maps with the `name` and `opts` keys. +Each element may also contain a `skip` key with a list of test names to skip. +They can also contain a `desc` key with a description of the options. + + +--- END OF FILE: docs/resources/source-code/hb_test_utils.md --- + +--- START OF FILE: docs/resources/source-code/hb_tracer.md --- +# [Module hb_tracer.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_tracer.erl) + + + + +A module for tracing the flow of requests through the system. + + + +## Description ## +This allows for tracking the lifecycle of a request from HTTP receipt through processing and response. + +## Function Index ## + + +
checkmark_emoji/0*
failure_emoji/0*
format_error_trace/1Format a trace for error in a user-friendly emoji oriented output.
get_trace/1Exports the complete queue of events.
record_step/2Register a new step into a tracer.
stage_to_emoji/1*
start_trace/0Start a new tracer acting as queue of events registered.
trace_loop/1*
+ + + + +## Function Details ## + + + +### checkmark_emoji/0 * ### + +`checkmark_emoji() -> any()` + + + +### failure_emoji/0 * ### + +`failure_emoji() -> any()` + + + +### format_error_trace/1 ### + +`format_error_trace(Trace) -> any()` + +Format a trace for error in a user-friendly emoji oriented output + + + +### get_trace/1 ### + +`get_trace(TracePID) -> any()` + +Exports the complete queue of events + + + +### record_step/2 ### + +`record_step(TracePID, Step) -> any()` + +Register a new step into a tracer + + + +### stage_to_emoji/1 * ### + +`stage_to_emoji(Stage) -> any()` + + + +### start_trace/0 ### + +`start_trace() -> any()` + +Start a new tracer acting as queue of events registered. + + + +### trace_loop/1 * ### + +`trace_loop(Trace) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb_tracer.md --- + +--- START OF FILE: docs/resources/source-code/hb_util.md --- +# [Module hb_util.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_util.erl) + + + + +A collection of utility functions for building with HyperBEAM. + + + +## Function Index ## + + +
add_commas/1*
all_hb_modules/0Get all loaded modules that are loaded and are part of HyperBEAM.
atom/1Coerce a string to an atom.
bin/1Coerce a value to a binary.
count/2
debug_fmt/1Convert a term to a string for debugging print purposes.
debug_fmt/2
debug_print/4Print a message to the standard error stream, prefixed by the amount +of time that has elapsed since the last call to this function.
decode/1Try to decode a URL safe base64 into a binary or throw an error when +invalid.
deep_merge/2Deep merge two maps, recursively merging nested maps.
do_debug_fmt/2*
do_to_lines/1*
encode/1Encode a binary to URL safe base64 binary string.
eunit_print/2Format and print an indented string to standard error.
find_value/2Find the value associated with a key in parsed a JSON structure list.
find_value/3
float/1Coerce a string to a float.
format_address/2*If the user attempts to print a wallet, format it as an address.
format_binary/1Format a binary as a short string suitable for printing.
format_debug_trace/3*Generate the appropriate level of trace for a given call.
format_indented/2Format a string with an indentation level.
format_indented/3
format_maybe_multiline/2Format a map as either a single line or a multi-line string depending +on the value of the debug_print_map_line_threshold runtime option.
format_trace/1Format a stack trace as a list of strings, one for each stack frame.
format_trace/2*
format_trace_short/1Format a trace to a short string.
format_trace_short/4*
format_tuple/2*Helper function to format tuples with arity greater than 2.
get_trace/0*Get the trace of the current process.
hd/1Get the first element (the lowest integer key >= 1) of a numbered map.
hd/2
hd/3
hd/5*
human_id/1Convert a native binary ID to a human readable ID.
human_int/1Add , characters to a number every 3 digits to make it human readable.
id/1Return the human-readable form of an ID of a message when given either +a message explicitly, raw encoded ID, or an Erlang Arweave tx record.
id/2
int/1Coerce a string to an integer.
is_hb_module/1Is the given module part of HyperBEAM?.
is_hb_module/2
is_human_binary/1*Determine whether a binary is human-readable.
is_ordered_list/1Determine if the message given is an ordered list, starting from 1.
is_ordered_list/2*
is_string_list/1Is the given term a string list?.
key_to_atom/2Convert keys in a map to atoms, lowering - to _.
list/1Coerce a value to a list.
list_to_numbered_map/1Convert a list of elements to a map with numbered keys.
maybe_throw/2Throw an exception if the Opts map has an error_strategy key with the +value throw.
mean/1
message_to_ordered_list/1Take a message with numbered keys and convert it to a list of tuples +with the associated key as an integer and a value.
message_to_ordered_list/2
message_to_ordered_list/4*
native_id/1Convert a human readable ID to a native binary ID.
normalize_trace/1*Remove all calls from this module from the top of a trace.
number/1Label a list of elements with a number.
ok/1Unwrap a tuple of the form {ok, Value}, or throw/return, depending on +the value of the error_strategy option.
ok/2
pick_weighted/2*
print_trace/3*
print_trace/4Print the trace of the current stack, up to the first non-hyperbeam +module.
print_trace_short/4Print a trace to the standard error stream.
remove_common/2Remove the common prefix from two strings, returning the remainder of the +first string.
remove_trailing_noise/1*
remove_trailing_noise/2
safe_decode/1Safely decode a URL safe base64 into a binary returning an ok or error +tuple.
safe_encode/1Safely encode a binary to URL safe base64.
short_id/1Return a short ID for the different types of IDs used in AO-Core.
shuffle/1*Shuffle a list.
stddev/1
to_hex/1Convert a binary to a hex string.
to_lines/1*
to_lower/1Convert a binary to a lowercase.
to_sorted_keys/1Given a map or KVList, return a deterministically ordered list of its keys.
to_sorted_list/1Given a map or KVList, return a deterministically sorted list of its +key-value pairs.
trace_macro_helper/5Utility function to help macro ?trace/0 remove the first frame of the +stack trace.
until/1Utility function to wait for a condition to be true.
until/2
until/3
variance/1
weighted_random/1Return a random element from a list, weighted by the values in the list.
+ + + + +## Function Details ## + + + +### add_commas/1 * ### + +`add_commas(Rest) -> any()` + + + +### all_hb_modules/0 ### + +`all_hb_modules() -> any()` + +Get all loaded modules that are loaded and are part of HyperBEAM. + + + +### atom/1 ### + +`atom(Str) -> any()` + +Coerce a string to an atom. + + + +### bin/1 ### + +`bin(Value) -> any()` + +Coerce a value to a binary. + + + +### count/2 ### + +`count(Item, List) -> any()` + + + +### debug_fmt/1 ### + +`debug_fmt(X) -> any()` + +Convert a term to a string for debugging print purposes. + + + +### debug_fmt/2 ### + +`debug_fmt(X, Indent) -> any()` + + + +### debug_print/4 ### + +`debug_print(X, Mod, Func, LineNum) -> any()` + +Print a message to the standard error stream, prefixed by the amount +of time that has elapsed since the last call to this function. + + + +### decode/1 ### + +`decode(Input) -> any()` + +Try to decode a URL safe base64 into a binary or throw an error when +invalid. + + + +### deep_merge/2 ### + +`deep_merge(Map1, Map2) -> any()` + +Deep merge two maps, recursively merging nested maps. + + + +### do_debug_fmt/2 * ### + +`do_debug_fmt(Wallet, Indent) -> any()` + + + +### do_to_lines/1 * ### + +`do_to_lines(In) -> any()` + + + +### encode/1 ### + +`encode(Bin) -> any()` + +Encode a binary to URL safe base64 binary string. + + + +### eunit_print/2 ### + +`eunit_print(FmtStr, FmtArgs) -> any()` + +Format and print an indented string to standard error. + + + +### find_value/2 ### + +`find_value(Key, List) -> any()` + +Find the value associated with a key in parsed a JSON structure list. + + + +### find_value/3 ### + +`find_value(Key, Map, Default) -> any()` + + + +### float/1 ### + +`float(Str) -> any()` + +Coerce a string to a float. + + + +### format_address/2 * ### + +`format_address(Wallet, Indent) -> any()` + +If the user attempts to print a wallet, format it as an address. + + + +### format_binary/1 ### + +`format_binary(Bin) -> any()` + +Format a binary as a short string suitable for printing. + + + +### format_debug_trace/3 * ### + +`format_debug_trace(Mod, Func, Line) -> any()` + +Generate the appropriate level of trace for a given call. + + + +### format_indented/2 ### + +`format_indented(Str, Indent) -> any()` + +Format a string with an indentation level. + + + +### format_indented/3 ### + +`format_indented(RawStr, Fmt, Ind) -> any()` + + + +### format_maybe_multiline/2 ### + +`format_maybe_multiline(X, Indent) -> any()` + +Format a map as either a single line or a multi-line string depending +on the value of the `debug_print_map_line_threshold` runtime option. + + + +### format_trace/1 ### + +`format_trace(Stack) -> any()` + +Format a stack trace as a list of strings, one for each stack frame. +Each stack frame is formatted if it matches the `stack_print_prefixes` +option. At the first frame that does not match a prefix in the +`stack_print_prefixes` option, the rest of the stack is not formatted. + + + +### format_trace/2 * ### + +`format_trace(Rest, Prefixes) -> any()` + + + +### format_trace_short/1 ### + +`format_trace_short(Trace) -> any()` + +Format a trace to a short string. + + + +### format_trace_short/4 * ### + +`format_trace_short(Max, Latch, Trace, Prefixes) -> any()` + + + +### format_tuple/2 * ### + +`format_tuple(Tuple, Indent) -> any()` + +Helper function to format tuples with arity greater than 2. + + + +### get_trace/0 * ### + +`get_trace() -> any()` + +Get the trace of the current process. + + + +### hd/1 ### + +`hd(Message) -> any()` + +Get the first element (the lowest integer key >= 1) of a numbered map. +Optionally, it takes a specifier of whether to return the key or the value, +as well as a standard map of HyperBEAM runtime options. + + + +### hd/2 ### + +`hd(Message, ReturnType) -> any()` + + + +### hd/3 ### + +`hd(Message, ReturnType, Opts) -> any()` + + + +### hd/5 * ### + +`hd(Map, Rest, Index, ReturnType, Opts) -> any()` + + + +### human_id/1 ### + +`human_id(Bin) -> any()` + +Convert a native binary ID to a human readable ID. If the ID is already +a human readable ID, it is returned as is. If it is an ethereum address, it +is returned as is. + + + +### human_int/1 ### + +`human_int(Int) -> any()` + +Add `,` characters to a number every 3 digits to make it human readable. + + + +### id/1 ### + +`id(Item) -> any()` + +Return the human-readable form of an ID of a message when given either +a message explicitly, raw encoded ID, or an Erlang Arweave `tx` record. + + + +### id/2 ### + +`id(TX, Type) -> any()` + + + +### int/1 ### + +`int(Str) -> any()` + +Coerce a string to an integer. + + + +### is_hb_module/1 ### + +`is_hb_module(Atom) -> any()` + +Is the given module part of HyperBEAM? + + + +### is_hb_module/2 ### + +`is_hb_module(Atom, Prefixes) -> any()` + + + +### is_human_binary/1 * ### + +`is_human_binary(Bin) -> any()` + +Determine whether a binary is human-readable. + + + +### is_ordered_list/1 ### + +`is_ordered_list(Msg) -> any()` + +Determine if the message given is an ordered list, starting from 1. + + + +### is_ordered_list/2 * ### + +`is_ordered_list(N, Msg) -> any()` + + + +### is_string_list/1 ### + +`is_string_list(MaybeString) -> any()` + +Is the given term a string list? + + + +### key_to_atom/2 ### + +`key_to_atom(Key, Mode) -> any()` + +Convert keys in a map to atoms, lowering `-` to `_`. + + + +### list/1 ### + +`list(Value) -> any()` + +Coerce a value to a list. + + + +### list_to_numbered_map/1 ### + +`list_to_numbered_map(List) -> any()` + +Convert a list of elements to a map with numbered keys. + + + +### maybe_throw/2 ### + +`maybe_throw(Val, Opts) -> any()` + +Throw an exception if the Opts map has an `error_strategy` key with the +value `throw`. Otherwise, return the value. + + + +### mean/1 ### + +`mean(List) -> any()` + + + +### message_to_ordered_list/1 ### + +`message_to_ordered_list(Message) -> any()` + +Take a message with numbered keys and convert it to a list of tuples +with the associated key as an integer and a value. Optionally, it takes a +standard map of HyperBEAM runtime options. + + + +### message_to_ordered_list/2 ### + +`message_to_ordered_list(Message, Opts) -> any()` + + + +### message_to_ordered_list/4 * ### + +`message_to_ordered_list(Message, Keys, Key, Opts) -> any()` + + + +### native_id/1 ### + +`native_id(Bin) -> any()` + +Convert a human readable ID to a native binary ID. If the ID is already +a native binary ID, it is returned as is. + + + +### normalize_trace/1 * ### + +`normalize_trace(Rest) -> any()` + +Remove all calls from this module from the top of a trace. + + + +### number/1 ### + +`number(List) -> any()` + +Label a list of elements with a number. + + + +### ok/1 ### + +`ok(Value) -> any()` + +Unwrap a tuple of the form `{ok, Value}`, or throw/return, depending on +the value of the `error_strategy` option. + + + +### ok/2 ### + +`ok(Other, Opts) -> any()` + + + +### pick_weighted/2 * ### + +`pick_weighted(Rest, Remaining) -> any()` + + + +### print_trace/3 * ### + +`print_trace(Stack, Label, CallerInfo) -> any()` + + + +### print_trace/4 ### + +`print_trace(Stack, CallMod, CallFunc, CallLine) -> any()` + +Print the trace of the current stack, up to the first non-hyperbeam +module. Prints each stack frame on a new line, until it finds a frame that +does not start with a prefix in the `stack_print_prefixes` hb_opts. +Optionally, you may call this function with a custom label and caller info, +which will be used instead of the default. + + + +### print_trace_short/4 ### + +`print_trace_short(Trace, Mod, Func, Line) -> any()` + +Print a trace to the standard error stream. + + + +### remove_common/2 ### + +`remove_common(MainStr, SubStr) -> any()` + +Remove the common prefix from two strings, returning the remainder of the +first string. This function also coerces lists to binaries where appropriate, +returning the type of the first argument. + + + +### remove_trailing_noise/1 * ### + +`remove_trailing_noise(Str) -> any()` + + + +### remove_trailing_noise/2 ### + +`remove_trailing_noise(Str, Noise) -> any()` + + + +### safe_decode/1 ### + +`safe_decode(E) -> any()` + +Safely decode a URL safe base64 into a binary returning an ok or error +tuple. + + + +### safe_encode/1 ### + +`safe_encode(Bin) -> any()` + +Safely encode a binary to URL safe base64. + + + +### short_id/1 ### + +`short_id(Bin) -> any()` + +Return a short ID for the different types of IDs used in AO-Core. + + + +### shuffle/1 * ### + +`shuffle(List) -> any()` + +Shuffle a list. + + + +### stddev/1 ### + +`stddev(List) -> any()` + + + +### to_hex/1 ### + +`to_hex(Bin) -> any()` + +Convert a binary to a hex string. Do not use this for anything other than +generating a lower-case, non-special character id. It should not become part of +the core protocol. We use b64u for efficient encoding. + + + +### to_lines/1 * ### + +`to_lines(Elems) -> any()` + + + +### to_lower/1 ### + +`to_lower(Str) -> any()` + +Convert a binary to a lowercase. + + + +### to_sorted_keys/1 ### + +`to_sorted_keys(Msg) -> any()` + +Given a map or KVList, return a deterministically ordered list of its keys. + + + +### to_sorted_list/1 ### + +`to_sorted_list(Msg) -> any()` + +Given a map or KVList, return a deterministically sorted list of its +key-value pairs. + + + +### trace_macro_helper/5 ### + +`trace_macro_helper(Fun, X2, Mod, Func, Line) -> any()` + +Utility function to help macro `?trace/0` remove the first frame of the +stack trace. + + + +### until/1 ### + +`until(Condition) -> any()` + +Utility function to wait for a condition to be true. Optionally, +you can pass a function that will be called with the current count of +iterations, returning an integer that will be added to the count. Once the +condition is true, the function will return the count. + + + +### until/2 ### + +`until(Condition, Count) -> any()` + + + +### until/3 ### + +`until(Condition, Fun, Count) -> any()` + + + +### variance/1 ### + +`variance(List) -> any()` + + + +### weighted_random/1 ### + +`weighted_random(List) -> any()` + +Return a random element from a list, weighted by the values in the list. + + +--- END OF FILE: docs/resources/source-code/hb_util.md --- + +--- START OF FILE: docs/resources/source-code/hb_volume.md --- +# [Module hb_volume.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_volume.erl) + + + + + + +## Function Index ## + + +
change_node_store/2
check_for_device/1
create_actual_partition/2*
create_mount_info/3*
create_partition/2
format_disk/2
get_partition_info/1*
list_partitions/0
mount_disk/4
mount_opened_volume/3*
parse_disk_info/2*
parse_disk_line/2*
parse_disk_model_line/2*
parse_disk_units_line/2*
parse_io_size_line/2*
parse_sector_size_line/2*
process_disk_line/2*
update_store_config/2*
+ + + + +## Function Details ## + + + +### change_node_store/2 ### + +

+change_node_store(StorePath::binary(), CurrentStore::list()) -> {ok, map()} | {error, binary()}
+
+
+ + + +### check_for_device/1 ### + +

+check_for_device(Device::binary()) -> boolean()
+
+
+ + + +### create_actual_partition/2 * ### + +`create_actual_partition(Device, PartType) -> any()` + + + +### create_mount_info/3 * ### + +`create_mount_info(Partition, MountPoint, VolumeName) -> any()` + + + +### create_partition/2 ### + +

+create_partition(Device::binary(), PartType::binary()) -> {ok, map()} | {error, binary()}
+
+
+ + + +### format_disk/2 ### + +

+format_disk(Partition::binary(), EncKey::binary()) -> {ok, map()} | {error, binary()}
+
+
+ + + +### get_partition_info/1 * ### + +`get_partition_info(Device) -> any()` + + + +### list_partitions/0 ### + +

+list_partitions() -> {ok, map()} | {error, binary()}
+
+
+ + + +### mount_disk/4 ### + +

+mount_disk(Partition::binary(), EncKey::binary(), MountPoint::binary(), VolumeName::binary()) -> {ok, map()} | {error, binary()}
+
+
+ + + +### mount_opened_volume/3 * ### + +`mount_opened_volume(Partition, MountPoint, VolumeName) -> any()` + + + +### parse_disk_info/2 * ### + +`parse_disk_info(Device, Lines) -> any()` + + + +### parse_disk_line/2 * ### + +`parse_disk_line(Line, Info) -> any()` + + + +### parse_disk_model_line/2 * ### + +`parse_disk_model_line(Line, Info) -> any()` + + + +### parse_disk_units_line/2 * ### + +`parse_disk_units_line(Line, Info) -> any()` + + + +### parse_io_size_line/2 * ### + +`parse_io_size_line(Line, Info) -> any()` + + + +### parse_sector_size_line/2 * ### + +`parse_sector_size_line(Line, Info) -> any()` + + + +### process_disk_line/2 * ### + +`process_disk_line(Line, X2) -> any()` + + + +### update_store_config/2 * ### + +

+update_store_config(StoreConfig::term(), NewPath::binary()) -> term()
+
+
+ + +--- END OF FILE: docs/resources/source-code/hb_volume.md --- + +--- START OF FILE: docs/resources/source-code/hb.md --- +# [Module hb.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb.erl) + + + + +Hyperbeam is a decentralized node implementing the AO-Core protocol +on top of Arweave. + + + +## Description ## + +This protocol offers a computation layer for executing arbitrary logic on +top of the network's data. + +Arweave is built to offer a robust, permanent storage layer for static data +over time. It can be seen as a globally distributed key-value store that +allows users to lookup IDs to retrieve data at any point in time: + +`Arweave(ID) => Message` + +Hyperbeam adds another layer of functionality on top of Arweave's protocol: +Allowing users to store and retrieve not only arbitrary bytes, but also to +perform execution of computation upon that data: + +`Hyperbeam(Message1, Message2) => Message3` + +When Hyperbeam executes a message, it will return a new message containing +the result of that execution, as well as signed commitments of its +correctness. If the computation that is executed is deterministic, recipients +of the new message are able to verify that the computation was performed +correctly. The new message may be stored back to Arweave if desired, +forming a permanent, verifiable, and decentralized log of computation. + +The mechanisms described above form the basis of a decentralized and +verifiable compute engine without any relevant protocol-enforced +scalability limits. It is an implementation of a global, shared +supercomputer. + +Hyperbeam can be used for an extremely large variety of applications, from +serving static Arweave data with signed commitments of correctness, to +executing smart contracts that have _built-in_ HTTP APIs. The Hyperbeam +node implementation implements AO, an Actor-Oriented process-based +environment for orchestrating computation over Arweave messages in order to +facilitate the execution of more traditional, consensus-based smart +contracts. + +The core abstractions of the Hyperbeam node are broadly as follows: + +1. The `hb` and `hb_opts` modules manage the node's configuration, +environment variables, and debugging tools. + +2. The `hb_http` and `hb_http_server` modules manage all HTTP-related +functionality. `hb_http_server` handles turning received HTTP requests +into messages and applying those messages with the appropriate devices. +`hb_http` handles making requests and responding with messages. `cowboy` +is used to implement the underlying HTTP server. + +3. `hb_ao` implements the computation logic of the node: A mechanism +for resolving messages to other messages, via the application of logic +implemented in `devices`. `hb_ao` also manages the loading of Erlang +modules for each device into the node's environment. There are many +different default devices implemented in the hyperbeam node, using the +namespace `dev_*`. Some of the critical components are: + +- `dev_message`: The default handler for all messages that do not +specify their own device. The message device is also used to resolve +keys that are not implemented by the device specified in a message, +unless otherwise signalled. + +- `dev_stack`: The device responsible for creating and executing stacks +of other devices on messages that request it. There are many uses for +this device, one of which is the resolution of AO processes. + +- `dev_p4`: The device responsible for managing payments for the services +provided by the node. + +4. `hb_store`, `hb_cache` and the store implementations forms a layered +system for managing the node's access to persistent storage. `hb_cache` +is used as a resolution mechanism for reading and writing messages, while +`hb_store` provides an abstraction over the underlying persistent key-value +byte storage mechanisms. Example `hb_store` mechanisms can be found in +`hb_store_fs` and `hb_store_remote_node`. + +5. `ar_*` modules implement functionality related to the base-layer Arweave +protocol and are largely unchanged from their counterparts in the Arweave +node codebase presently maintained by the Digital History Association +(@dha-team/Arweave). + +You can find documentation of a similar form to this note in each of the core +modules of the hyperbeam node. + +## Function Index ## + + +
address/0Get the address of a wallet.
address/1*
benchmark/2Run a function as many times as possible in a given amount of time.
benchmark/3Run multiple instances of a function in parallel for a given amount of time.
build/0Utility function to hot-recompile and load the hyperbeam environment.
debug_wait/4Utility function to wait for a given amount of time, printing a debug +message to the console first.
do_start_simple_pay/1*
init/0Initialize system-wide settings for the hyperbeam node.
no_prod/3Utility function to throw an error if the current mode is prod and +non-prod ready code is being executed.
now/0Utility function to get the current time in milliseconds.
profile/1Utility function to start a profiling session and run a function, +then analyze the results.
read/1Debugging function to read a message from the cache.
read/2
start_mainnet/0Start a mainnet server without payments.
start_mainnet/1
start_simple_pay/0Start a server with a simple-pay@1.0 pre-processor.
start_simple_pay/1
start_simple_pay/2
topup/3Helper for topping up a user's balance on a simple-pay node.
topup/4
wallet/0
wallet/1
+ + + + +## Function Details ## + + + +### address/0 ### + +`address() -> any()` + +Get the address of a wallet. Defaults to the address of the wallet +specified by the `priv_key_location` configuration key. It can also take a +wallet tuple as an argument. + + + +### address/1 * ### + +`address(Wallet) -> any()` + + + +### benchmark/2 ### + +`benchmark(Fun, TLen) -> any()` + +Run a function as many times as possible in a given amount of time. + + + +### benchmark/3 ### + +`benchmark(Fun, TLen, Procs) -> any()` + +Run multiple instances of a function in parallel for a given amount of time. + + + +### build/0 ### + +`build() -> any()` + +Utility function to hot-recompile and load the hyperbeam environment. + + + +### debug_wait/4 ### + +`debug_wait(T, Mod, Func, Line) -> any()` + +Utility function to wait for a given amount of time, printing a debug +message to the console first. + + + +### do_start_simple_pay/1 * ### + +`do_start_simple_pay(Opts) -> any()` + + + +### init/0 ### + +`init() -> any()` + +Initialize system-wide settings for the hyperbeam node. + + + +### no_prod/3 ### + +`no_prod(X, Mod, Line) -> any()` + +Utility function to throw an error if the current mode is prod and +non-prod ready code is being executed. You can find these in the codebase +by looking for ?NO_PROD calls. + + + +### now/0 ### + +`now() -> any()` + +Utility function to get the current time in milliseconds. + + + +### profile/1 ### + +`profile(Fun) -> any()` + +Utility function to start a profiling session and run a function, +then analyze the results. Obviously -- do not use in production. + + + +### read/1 ### + +`read(ID) -> any()` + +Debugging function to read a message from the cache. +Specify either a scope atom (local or remote) or a store tuple +as the second argument. + + + +### read/2 ### + +`read(ID, ScopeAtom) -> any()` + + + +### start_mainnet/0 ### + +`start_mainnet() -> any()` + +Start a mainnet server without payments. + + + +### start_mainnet/1 ### + +`start_mainnet(Port) -> any()` + + + +### start_simple_pay/0 ### + +`start_simple_pay() -> any()` + +Start a server with a `simple-pay@1.0` pre-processor. + + + +### start_simple_pay/1 ### + +`start_simple_pay(Addr) -> any()` + + + +### start_simple_pay/2 ### + +`start_simple_pay(Addr, Port) -> any()` + + + +### topup/3 ### + +`topup(Node, Amount, Recipient) -> any()` + +Helper for topping up a user's balance on a simple-pay node. + + + +### topup/4 ### + +`topup(Node, Amount, Recipient, Wallet) -> any()` + + + +### wallet/0 ### + +`wallet() -> any()` + + + +### wallet/1 ### + +`wallet(Location) -> any()` + + +--- END OF FILE: docs/resources/source-code/hb.md --- + +--- START OF FILE: docs/resources/source-code/index.md --- +# Source Code Documentation + +Welcome to the source code documentation for HyperBEAM. This section provides detailed insights into the codebase, helping developers understand the structure, functionality, and implementation details of HyperBEAM and its components. + +## Overview + +HyperBEAM is built with a modular architecture to ensure scalability, maintainability, and extensibility. The source code is organized into distinct components, each serving a specific purpose within the ecosystem. + +## Sections + +- **HyperBEAM Core**: The main framework that orchestrates data processing, storage, and routing. +- **Compute Unit**: Handles computational tasks and integrates with the HyperBEAM core for distributed processing. +- **Trusted Execution Environment (TEE)**: Ensures secure execution of sensitive operations. +- **Client Libraries**: Tools and SDKs for interacting with HyperBEAM, including the JavaScript client. + +## Getting Started + +To explore the source code, you can clone the repository from [GitHub](https://github.com/permaweb/HyperBEAM). + +## Navigation + +Use the navigation menu to dive into specific parts of the codebase. Each module includes detailed documentation, code comments, and examples to assist in understanding and contributing to the project. + + +--- END OF FILE: docs/resources/source-code/index.md --- + +--- START OF FILE: docs/resources/source-code/README.md --- + + +# The hb application # + + +## Modules ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ar_bundles
ar_deep_hash
ar_rate_limiter
ar_timestamp
ar_tx
ar_wallet
dev_cache
dev_cacheviz
dev_codec_ans104
dev_codec_flat
dev_codec_httpsig
dev_codec_httpsig_conv
dev_codec_json
dev_codec_structured
dev_cron
dev_cu
dev_dedup
dev_delegated_compute
dev_faff
dev_genesis_wasm
dev_green_zone
dev_hook
dev_hyperbuddy
dev_json_iface
dev_local_name
dev_lookup
dev_lua
dev_lua_lib
dev_lua_test
dev_manifest
dev_message
dev_meta
dev_monitor
dev_multipass
dev_name
dev_node_process
dev_p4
dev_patch
dev_poda
dev_process
dev_process_cache
dev_process_worker
dev_push
dev_relay
dev_router
dev_scheduler
dev_scheduler_cache
dev_scheduler_formats
dev_scheduler_registry
dev_scheduler_server
dev_simple_pay
dev_snp
dev_snp_nif
dev_stack
dev_test
dev_volume
dev_wasi
dev_wasm
hb
hb_ao
hb_ao_test_vectors
hb_app
hb_beamr
hb_beamr_io
hb_cache
hb_cache_control
hb_cache_render
hb_client
hb_crypto
hb_debugger
hb_escape
hb_event
hb_examples
hb_features
hb_gateway_client
hb_http
hb_http_benchmark_tests
hb_http_client
hb_http_client_sup
hb_http_server
hb_json
hb_keccak
hb_logger
hb_message
hb_metrics_collector
hb_name
hb_opts
hb_path
hb_persistent
hb_private
hb_process_monitor
hb_router
hb_singleton
hb_store
hb_store_fs
hb_store_gateway
hb_store_remote_node
hb_store_rocksdb
hb_structured_fields
hb_sup
hb_test_utils
hb_tracer
hb_util
hb_volume
rsa_pss
+ + +--- END OF FILE: docs/resources/source-code/README.md --- + +--- START OF FILE: docs/resources/source-code/rsa_pss.md --- +# [Module rsa_pss.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/rsa_pss.erl) + + + + +Distributed under the Mozilla Public License v2.0. + +Copyright (c) 2014-2015, Andrew Bennett + +__Authors:__ Andrew Bennett ([`andrew@pixid.com`](mailto:andrew@pixid.com)). + + + +## Description ## +Original available at: +https://github.com/potatosalad/erlang-crypto_rsassa_pss + + +## Data Types ## + + + + +### rsa_digest_type() ### + + +

+rsa_digest_type() = md5 | sha | sha224 | sha256 | sha384 | sha512
+
+ + + + +### rsa_private_key() ### + + +

+rsa_private_key() = #RSAPrivateKey{}
+
+ + + + +### rsa_public_key() ### + + +

+rsa_public_key() = #RSAPublicKey{}
+
+ + + +## Function Index ## + + +
dp/2*
ep/2*
int_to_bit_size/1*
int_to_bit_size/2*
int_to_byte_size/1*
int_to_byte_size/2*
mgf1/3*
mgf1/5*
normalize_to_key_size/2*
pad_to_key_size/2*
sign/3
sign/4
verify/4
verify_legacy/4
+ + + + +## Function Details ## + + + +### dp/2 * ### + +`dp(B, X2) -> any()` + + + +### ep/2 * ### + +`ep(B, X2) -> any()` + + + +### int_to_bit_size/1 * ### + +`int_to_bit_size(I) -> any()` + + + +### int_to_bit_size/2 * ### + +`int_to_bit_size(I, B) -> any()` + + + +### int_to_byte_size/1 * ### + +`int_to_byte_size(I) -> any()` + + + +### int_to_byte_size/2 * ### + +`int_to_byte_size(I, B) -> any()` + + + +### mgf1/3 * ### + +`mgf1(DigestType, Seed, Len) -> any()` + + + +### mgf1/5 * ### + +`mgf1(DigestType, Seed, Len, T, Counter) -> any()` + + + +### normalize_to_key_size/2 * ### + +`normalize_to_key_size(Bits, A) -> any()` + + + +### pad_to_key_size/2 * ### + +`pad_to_key_size(Bytes, Data) -> any()` + + + +### sign/3 ### + +

+sign(Message, DigestType, PrivateKey) -> Signature
+
+ + + + + +### sign/4 ### + +

+sign(Message, DigestType, Salt, PrivateKey) -> Signature
+
+ + + + + +### verify/4 ### + +

+verify(Message, DigestType, Signature, PublicKey) -> boolean()
+
+ + + + + +### verify_legacy/4 ### + +`verify_legacy(Message, DigestType, Signature, PublicKey) -> any()` + + +--- END OF FILE: docs/resources/source-code/rsa_pss.md --- + +--- START OF FILE: docs/run/configuring-your-machine.md --- +# Configuring Your HyperBEAM Node + +This guide details the various ways to configure your HyperBEAM node's behavior, including ports, storage, keys, and logging. + +## Configuration (`config.flat`) + +The primary way to configure your HyperBEAM node is through a `config.flat` file located in the node's working directory or specified by the `HB_CONFIG_LOCATION` environment variable. + +This file uses a simple `Key = Value.` format (note the period at the end of each line). + +**Example `config.flat`:** + +```erlang +% Set the HTTP port +port = 8080. + +% Specify the Arweave key file +priv_key_location = "/path/to/your/wallet.json". + +% Set the data store directory +% Note: Storage configuration can be complex. See below. +% store = [{local, [{root, <<"./node_data_mainnet">>}]}]. % Example of complex config, not for config.flat + +% Enable verbose logging for specific modules +% debug_print = [hb_http, dev_router]. % Example of complex config, not for config.flat +``` + +Below is a reference of commonly used configuration keys. Remember that `config.flat` only supports simple key-value pairs (Atoms, Strings, Integers, Booleans). For complex configurations (Lists, Maps), you must use environment variables or `hb:start_mainnet/1`. + +### Core Configuration + +These options control fundamental HyperBEAM behavior. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `port` | Integer | 8734 | HTTP API port | +| `hb_config_location` | String | "config.flat" | Path to configuration file | +| `priv_key_location` | String | "hyperbeam-key.json" | Path to operator wallet key file | +| `mode` | Atom | debug | Execution mode (debug, prod) | + +### Server & Network Configuration + +These options control networking behavior and HTTP settings. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `host` | String | "localhost" | Choice of remote node for non-local tasks | +| `gateway` | String | "https://arweave.net" | Default gateway | +| `bundler_ans104` | String | "https://up.arweave.net:443" | Location of ANS-104 bundler | +| `protocol` | Atom | http2 | Protocol for HTTP requests (http1, http2, http3) | +| `http_client` | Atom | gun | HTTP client to use (gun, httpc) | +| `http_connect_timeout` | Integer | 5000 | HTTP connection timeout in milliseconds | +| `http_keepalive` | Integer | 120000 | HTTP keepalive time in milliseconds | +| `http_request_send_timeout` | Integer | 60000 | HTTP request send timeout in milliseconds | +| `relay_http_client` | Atom | httpc | HTTP client for the relay device | + + +### Security & Identity + +These options control identity and security settings. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `scheduler_location_ttl` | Integer | 604800000 | TTL for scheduler registration (7 days in ms) | + + +### Caching & Storage + +These options control caching behavior. **Note:** Detailed storage configuration (`store` option) involves complex data structures and cannot be set via `config.flat`. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `cache_lookup_hueristics` | Boolean | false | Whether to use caching heuristics or always consult the local data store | +| `access_remote_cache_for_client` | Boolean | false | Whether to access data from remote caches for client requests | +| `store_all_signed` | Boolean | true | Whether the node should store all signed messages | +| `await_inprogress` | Atom/Boolean | named | Whether to await in-progress executions (false, named, true) | + + +### Execution & Processing + +These options control how HyperBEAM executes messages and processes. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `scheduling_mode` | Atom | local_confirmation | When to inform recipients about scheduled assignments (aggressive, local_confirmation, remote_confirmation) | +| `compute_mode` | Atom | lazy | Whether to execute more messages after returning a result (aggressive, lazy) | +| `process_workers` | Boolean | true | Whether the node should use persistent processes | +| `client_error_strategy` | Atom | throw | What to do if a client error occurs | +| `wasm_allow_aot` | Boolean | false | Allow ahead-of-time compilation for WASM | + +### Device Management + +These options control how HyperBEAM manages devices. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `load_remote_devices` | Boolean | false | Whether to load devices from remote signers | + + +### Debug & Development + +These options control debugging and development features. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `debug_stack_depth` | Integer | 40 | Maximum stack depth for debug printing | +| `debug_print_map_line_threshold` | Integer | 30 | Maximum lines for map printing | +| `debug_print_binary_max` | Integer | 60 | Maximum binary size for debug printing | +| `debug_print_indent` | Integer | 2 | Indentation for debug printing | +| `debug_print_trace` | Atom | short | Trace mode (short, false) | +| `short_trace_len` | Integer | 5 | Length of short traces | +| `debug_hide_metadata` | Boolean | true | Whether to hide metadata in debug output | +| `debug_ids` | Boolean | false | Whether to print IDs in debug output | +| `debug_hide_priv` | Boolean | true | Whether to hide private data in debug output | + + +**Note:** For the *absolute complete* and most up-to-date list, including complex options not suitable for `config.flat`, refer to the `default_message/0` function in the `hb_opts` module source code. + +## Overrides (Environment Variables & Args) + +You can override settings from `config.flat` or provide values if the file is missing using environment variables or command-line arguments. + +**Using Environment Variables:** + +Environment variables typically use an `HB_` prefix followed by the configuration key in uppercase. + +* **`HB_PORT=`:** Overrides `hb_port`. + * Example: `HB_PORT=8080 rebar3 shell` +* **`HB_KEY=`:** Overrides `hb_key`. + * Example: `HB_KEY=~/.keys/arweave_key.json rebar3 shell` +* **`HB_STORE=`:** Overrides `hb_store`. + * Example: `HB_STORE=./node_data_1 rebar3 shell` +* **`HB_PRINT=`:** Overrides `hb_print`. `` can be `true` (or `1`), or a comma-separated list of modules/topics (e.g., `hb_path,hb_ao,ao_result`). + * Example: `HB_PRINT=hb_http,dev_router rebar3 shell` +* **`HB_CONFIG_LOCATION=`:** Specifies a custom location for the configuration file. + +**Using `erl_opts` (Direct Erlang VM Arguments):** + +You can also pass arguments directly to the Erlang VM using the `- ` format within `erl_opts`. This is generally less common for application configuration than `config.flat` or environment variables. + +```bash +rebar3 shell --erl_opts "-hb_port 8080 -hb_key path/to/key.json" +``` + +**Order of Precedence:** + +1. Command-line arguments (`erl_opts`). +2. Settings in `config.flat`. +3. Environment variables (`HB_*`). +4. Default values from `hb_opts.erl`. + +## Configuration in Releases + +When running a release build (see [Running a HyperBEAM Node](./running-a-hyperbeam-node.md)), configuration works similarly: + +1. A `config.flat` file will be present in the release directory (e.g., `_build/default/rel/hb/config.flat`). Edit this file to set your desired parameters for the release environment. +2. Environment variables (`HB_*`) can still be used to override the settings in the release's `config.flat` when starting the node using the `bin/hb` script. + +--- END OF FILE: docs/run/configuring-your-machine.md --- + +--- START OF FILE: docs/run/joining-running-a-router.md --- +# Joining or Running a Router Node + +Router nodes play a crucial role in the HyperBEAM network by directing incoming HTTP requests to appropriate worker nodes capable of handling the requested computation or data retrieval. They act as intelligent load balancers and entry points into the AO ecosystem. + +!!! info "Advanced Topic" + Configuring and running a production-grade router involves considerations beyond the scope of this introductory guide, including network topology, security, high availability, and performance tuning. + +## What is a Router? + +In HyperBEAM, the `dev_router` module (and associated logic) implements routing functionality. A node configured as a router typically: + +1. Receives external HTTP requests (HyperPATH calls). +2. Parses the request path to determine the target process, device, and desired operation. +3. Consults its routing table or logic to select an appropriate downstream worker node (which might be itself or another node). +4. Forwards the request to the selected worker. +5. Receives the response from the worker. +6. Returns the response to the original client. + +Routers often maintain information about the capabilities and load of worker nodes they know about. + +## Configuring Routing Behavior + +Routing logic is primarily configured through node options, often managed via `hb_opts` or environment variables when starting the node. Key aspects include: + +* **Route Definitions:** Defining patterns (templates) and corresponding downstream targets (worker node URLs or internal handlers). Routes are typically ordered by precedence. +* **Load Balancing Strategy:** How the router chooses among multiple potential workers for a given route (e.g., round-robin, least connections, latency-based). +* **Worker Discovery/Management:** How the router learns about available worker nodes and their status. + +**Example Configuration Snippet (Conceptual - from `hb_opts` or config file):** + +```erlang +{ + routes, + [ + #{ template => "/~meta@1.0/.*", target => self }, % Handle meta locally + #{ template => "/PROCESS_ID1~process@1.0/.*", target => "http://worker1.example.com" }, + #{ template => "/PROCESS_ID2~process@1.0/.*", target => "http://worker2.example.com" }, + #{ template => "/.*~wasm64@1.0/.*", target => ["http://wasm_worker1", "http://wasm_worker2"], strategy => round_robin }, % Route WASM requests + #{ template => "/.*", target => "http://default_worker.example.com" } % Default fallback + ] +}, +{ router_load_balancing_strategy, latency_aware } +``` + +*(Note: The actual configuration format and options should be verified in the `hb_opts.erl` and `dev_router.erl` source code.)* + +## Running a Simple Router + +While a dedicated router setup is complex, any HyperBEAM node implicitly performs some level of routing, especially if it needs to interact with other nodes (e.g., via the `~relay@1.0` device). The default configuration might route certain requests internally or have basic forwarding capabilities. + +To run a node that explicitly acts *more* like a router, you would typically configure it with specific `routes` pointing to other worker nodes, potentially disabling local execution for certain devices it intends to forward. + +## Joining an Existing Router Network + +As a user or developer, you typically don't *run* the main public routers (like `router-1.forward.computer`). Instead, you configure your client applications (or your own local node if it needs to relay requests) to *use* these public routers as entry points. + +When making HyperPATH calls, you simply target the public router's URL: + +``` +https:///~/... +``` +The router handles directing your request to an appropriate compute node. + +## Further Exploration + +* Examine the `dev_router.erl` source code for detailed implementation. +* Review the available configuration options in `hb_opts.erl` related to routing (`routes`, strategies, etc.). +* Consult community channels or advanced documentation for best practices on deploying production routers. + +--- END OF FILE: docs/run/joining-running-a-router.md --- + +--- START OF FILE: docs/run/running-a-hyperbeam-node.md --- +# Running a HyperBEAM Node + +This guide provides the basics for running your own HyperBEAM node, installing dependencies, and connecting to the AO network. + +## System Dependencies + +To successfully build and run a HyperBEAM node, your system needs several software dependencies installed. + +=== "macOS" + Install core dependencies using [Homebrew](https://brew.sh/): + + ```bash + brew install cmake git pkg-config openssl ncurses + ``` + +=== "Linux (Debian/Ubuntu)" + Install core dependencies using `apt`: + ```bash + sudo apt-get update && sudo apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + git \ + pkg-config \ + ncurses-dev \ + libssl-dev \ + sudo \ + curl + ca-certificates + ``` + +=== "Windows (WSL)" + Using the Windows Subsystem for Linux (WSL) with a distribution like Ubuntu is recommended. Follow the Linux (Debian/Ubuntu) instructions within your WSL environment. + + + +### Erlang/OTP + +HyperBEAM is built on Erlang/OTP. You need a compatible version installed (check the `rebar.config` or project documentation for specific version requirements, **typically OTP 27**). + +Installation methods: + +=== "macOS (brew)" + ```bash + brew install erlang + ``` + +=== "Linux (apt)" + ```bash + sudo apt install erlang + ``` + + +=== "Source Build" + Download from [erlang.org](https://www.erlang.org/downloads) and follow the build instructions for your platform. + +### Rebar3 + +Rebar3 is the build tool for Erlang projects. + +Installation methods: + +=== "macOS (brew)" + ```bash + brew install rebar3 + ``` + +=== "Linux / macOS (Direct Download)" + Get the `rebar3` binary from the [official website](https://rebar3.org/). Place the downloaded `rebar3` file in your system's `PATH` (e.g., `/usr/local/bin`) and make it executable (`chmod +x rebar3`). + + + +### Node.js + +Node.js might be required for certain JavaScript-related tools or dependencies. + +Installation methods: + +=== "macOS (brew)" + ```bash + brew install node + ``` + +=== "Linux (apt)" + ```bash + # Check your distribution's recommended method, might need nodesource repo + sudo apt install nodejs npm + ``` + +=== "asdf (Recommended)" + `asdf-vm` with the `asdf-nodejs` plugin is recommended. + ```bash + asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git + asdf install nodejs # e.g., lts + asdf global nodejs + ``` + +### Rust + +Rust is needed if you intend to work with or build components involving WebAssembly (WASM) or certain Native Implemented Functions (NIFs) used by some devices (like `~snp@1.0`). + +The recommended way to install Rust on **all platforms** is via `rustup`: + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +source "$HOME/.cargo/env" # Or follow the instructions provided by rustup +``` + +## Prerequisites for Running + +Before starting a node, ensure you have: + +* Installed the [system dependencies](#system-dependencies) mentioned above. +* Cloned the [HyperBEAM repository](https://github.com/permaweb/HyperBEAM) (`git clone ...`). +* Compiled the source code (`rebar3 compile` in the repo directory). +* An Arweave **wallet keyfile** (e.g., generated via [Wander](https://www.wander.app)). The path to this file is typically set via the `hb_key` configuration option (see [Configuring Your HyperBEAM Node](./configuring-your-machine.md)). + +## Starting a Basic Node + +The simplest way to start a HyperBEAM node for development or testing is using `rebar3` from the repository's root directory: + +```bash +rebar3 shell +``` + +This command: + +1. Starts the Erlang Virtual Machine (BEAM) with all HyperBEAM modules loaded. +2. Initializes the node with default settings (from `hb_opts.erl`). +3. Starts the default HTTP server (typically on **port 10000**), making the node accessible via HyperPATHs. +4. Drops you into an interactive Erlang shell where you can interact with the running node. + +This basic setup is suitable for local development and exploring HyperBEAM's functionalities. + +## Optional Build Profiles + +HyperBEAM uses build profiles to enable optional features, often requiring extra dependencies. To run a node with specific profiles enabled, use `rebar3 as ... shell`: + +**Available Profiles (Examples):** + +* `genesis_wasm`: Enables Genesis WebAssembly support. +* `rocksdb`: Enables the RocksDB storage backend. +* `http3`: Enables HTTP/3 support. + +**Example Usage:** + +```bash +# Start with RocksDB profile +rebar3 as rocksdb shell + +# Start with RocksDB and Genesis WASM profiles +rebar3 as rocksdb, genesis_wasm shell +``` + +*Note: Choose profiles **before** starting the shell, as they affect compile-time options.* + +## Node Configuration + +HyperBEAM offers various configuration options (port, key file, data storage, logging, etc.). These are primarily set using a `config.flat` file and can be overridden by environment variables or command-line arguments. + +See the dedicated **[Configuring Your HyperBEAM Node](./configuring-your-machine.md)** guide for detailed information on all configuration methods and options. + +## Verify Installation + +To quickly check if your node is running and accessible, you can send a request to its `~meta@1.0` device (assuming default port 10000): + +```bash +curl http://localhost:10000/~meta@1.0/info +``` + +A JSON response containing node information indicates success. + +## Running for Production (Mainnet) + +While you can connect to the main AO network using the `rebar3 shell` for testing purposes (potentially using specific configurations or helper functions like `hb:start_mainnet/1` if available and applicable), the standard and recommended method for a stable production deployment (like running on the mainnet) is to build and run a **release**. + +**1. Build the Release:** + +From the root of the HyperBEAM repository, build the release package. You might include specific profiles needed for your mainnet setup (e.g., `rocksdb` if you intend to use it): + +```bash +# Build release with default profile +rebar3 release + +# Or, build with specific profiles (example) +# rebar3 as rocksdb release +``` + +This command compiles the project and packages it along with the Erlang Runtime System (ERTS) and all dependencies into a directory, typically `_build/default/rel/hb`. + +**2. Configure the Release:** + +Navigate into the release directory (e.g., `cd _build/default/rel/hb`). Ensure you have a correctly configured `config.flat` file here. See the [configuration guide](./configuring-your-machine.md) for details on setting mainnet parameters (port, key file location, store path, specific peers, etc.). Environment variables can also be used to override settings in the release's `config.flat` when starting the node. + +**3. Start the Node:** + +Use the generated start script (`bin/hb`) to run the node: + +```bash +# Start the node in the foreground (logs to console) +./bin/hb console + +# Start the node as a background daemon +./bin/hb start + +# Check the status +./bin/hb ping +./bin/hb status + +# Stop the node +./bin/hb stop +``` + +Consult the generated `bin/hb` script or Erlang/OTP documentation for more advanced start-up options (e.g., attaching a remote shell). + +Running as a release provides a more robust, isolated, and manageable way to operate a node compared to running directly from the `rebar3 shell`. + +## Stopping the Node (rebar3 shell) + +To stop the node running *within the `rebar3 shell`*, press `Ctrl+C` twice or use the Erlang command `q().`. + +## Next Steps + +* **Configure Your Node:** Deep dive into [configuration options](./configuring-your-machine.md). +* **TEE Nodes:** Learn about running nodes in [Trusted Execution Environments](./tee-nodes.md) for enhanced security. +* **Routers:** Understand how to configure and run a [router node](./joining-running-a-router.md). + +--- END OF FILE: docs/run/running-a-hyperbeam-node.md --- + +--- START OF FILE: docs/run/tee-nodes.md --- +# Trusted Execution Environment (TEE) + +!!! info "Documentation Coming Soon" + Detailed documentation about Trusted Execution Environment support in HyperBEAM is currently being developed and will be available soon. + +## Overview + +HyperBEAM supports Trusted Execution Environments (TEEs) through the `~snp@1.0` device, which enables secure, trust-minimized computation on remote machines. TEEs provide hardware-level isolation and attestation capabilities that allow users to verify that their code is running in a protected environment, exactly as intended, even on untrusted hardware. + +The `~snp@1.0` device in HyperBEAM is used to generate and validate proofs that a node is executing inside a Trusted Execution Environment. Nodes executing inside these environments use an ephemeral key pair that provably only exists inside the TEE, and can sign attestations of AO-Core executions in a trust-minimized way. + +## Key Features + +- Hardware-level isolation for secure computation +- Remote attestation capabilities +- Protected execution state +- Confidential computing support +- Compatibility with AMD SEV-SNP technology + +## Coming Soon + +Detailed documentation on the following topics will be added: + +- TEE setup and configuration +- Using the `~snp@1.0` device +- Verifying TEE attestations +- Developing for TEEs +- Security considerations +- Performance characteristics + +If you intend to offer TEE-based computation of AO-Core devices, please see the [HyperBEAM OS repository](https://github.com/permaweb/hb-os) for preliminary details on configuration and deployment. +--- END OF FILE: docs/run/tee-nodes.md --- + diff --git a/docs/llms.txt b/docs/llms.txt new file mode 100644 index 000000000..31023fbde --- /dev/null +++ b/docs/llms.txt @@ -0,0 +1,155 @@ +Generated: 2025-05-15T13:32:25Z + +## HyperBEAM Documentation Summary + +This document provides an overview and routes for the HyperBEAM documentation, intended for LLM consumption. +Key sections include: Getting Started (begin), Running HyperBEAM (run), Developer Guides (guides), Device Integration (devices), and Resources (resources). + +## Documentation Pages by Section + +### introduction + +* [AO Devices](./introduction/ao-devices.html) +* [Pathing in AO-Core](./introduction/pathing-in-ao-core.html) +* [What is AO-Core?](./introduction/what-is-ao-core.html) +* [What is HyperBEAM?](./introduction/what-is-hyperbeam.html) + +### run + +* [Configuring Your HyperBEAM Node](./run/configuring-your-machine.html) +* [Joining or Running a Router Node](./run/joining-running-a-router.html) +* [Running a HyperBEAM Node](./run/running-a-hyperbeam-node.html) +* [Trusted Execution Environment (TEE)](./run/tee-nodes.html) + +### uuild + +* [Exposing Process State with the Patch Device](./build/exposing-process-state.html) +* [Extending HyperBEAM](./build/extending-hyperbeam.html) +* [Getting Started Building on AO-Core](./build/get-started-building-on-ao-core.html) +* [Serverless Decentralized Compute on AO](./build/serverless-decentralized-compute.html) + +### devices + +* [Device: ~json@1.0](./devices/json-at-1-0.html) +* [Device: ~lua@5.3a](./devices/lua-at-5-3a.html) +* [Device: ~message@1.0](./devices/message-at-1-0.html) +* [Device: ~meta@1.0](./devices/meta-at-1-0.html) +* [Devices](./devices/overview.html) +* [Device: ~process@1.0](./devices/process-at-1-0.html) +* [Device: ~relay@1.0](./devices/relay-at-1-0.html) +* [Device: ~scheduler@1.0](./devices/scheduler-at-1-0.html) +* [Device: ~wasm64@1.0](./devices/wasm64-at-1-0.html) + +### resources + +* [LLM Context Files](./resources/llms.html) +* [Frequently Asked Questions](./resources/reference/faq.html) +* [Glossary](./resources/reference/glossary.html) +* [Troubleshooting Guide](./resources/reference/troubleshooting.html) +* [[Module ar_bundles.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_bundles.erl)](./resources/source-code/ar_bundles.html) +* [[Module ar_deep_hash.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_deep_hash.erl)](./resources/source-code/ar_deep_hash.html) +* [[Module ar_rate_limiter.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_rate_limiter.erl)](./resources/source-code/ar_rate_limiter.html) +* [[Module ar_timestamp.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_timestamp.erl)](./resources/source-code/ar_timestamp.html) +* [[Module ar_tx.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_tx.erl)](./resources/source-code/ar_tx.html) +* [[Module ar_wallet.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_wallet.erl)](./resources/source-code/ar_wallet.html) +* [[Module dev_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cache.erl)](./resources/source-code/dev_cache.html) +* [[Module dev_cacheviz.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cacheviz.erl)](./resources/source-code/dev_cacheviz.html) +* [[Module dev_codec_ans104.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_ans104.erl)](./resources/source-code/dev_codec_ans104.html) +* [[Module dev_codec_flat.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_flat.erl)](./resources/source-code/dev_codec_flat.html) +* [[Module dev_codec_httpsig.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_httpsig.erl)](./resources/source-code/dev_codec_httpsig.html) +* [[Module dev_codec_httpsig_conv.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_httpsig_conv.erl)](./resources/source-code/dev_codec_httpsig_conv.html) +* [[Module dev_codec_json.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_json.erl)](./resources/source-code/dev_codec_json.html) +* [[Module dev_codec_structured.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_structured.erl)](./resources/source-code/dev_codec_structured.html) +* [[Module dev_cron.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cron.erl)](./resources/source-code/dev_cron.html) +* [[Module dev_cu.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cu.erl)](./resources/source-code/dev_cu.html) +* [[Module dev_dedup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_dedup.erl)](./resources/source-code/dev_dedup.html) +* [[Module dev_delegated_compute.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_delegated_compute.erl)](./resources/source-code/dev_delegated_compute.html) +* [[Module dev_faff.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_faff.erl)](./resources/source-code/dev_faff.html) +* [[Module dev_genesis_wasm.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_genesis_wasm.erl)](./resources/source-code/dev_genesis_wasm.html) +* [[Module dev_green_zone.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_green_zone.erl)](./resources/source-code/dev_green_zone.html) +* [[Module dev_hook.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_hook.erl)](./resources/source-code/dev_hook.html) +* [[Module dev_hyperbuddy.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_hyperbuddy.erl)](./resources/source-code/dev_hyperbuddy.html) +* [[Module dev_json_iface.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_json_iface.erl)](./resources/source-code/dev_json_iface.html) +* [[Module dev_local_name.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_local_name.erl)](./resources/source-code/dev_local_name.html) +* [[Module dev_lookup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lookup.erl)](./resources/source-code/dev_lookup.html) +* [[Module dev_lua.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lua.erl)](./resources/source-code/dev_lua.html) +* [[Module dev_lua_lib.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lua_lib.erl)](./resources/source-code/dev_lua_lib.html) +* [[Module dev_lua_test.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lua_test.erl)](./resources/source-code/dev_lua_test.html) +* [[Module dev_manifest.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_manifest.erl)](./resources/source-code/dev_manifest.html) +* [[Module dev_message.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_message.erl)](./resources/source-code/dev_message.html) +* [[Module dev_meta.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_meta.erl)](./resources/source-code/dev_meta.html) +* [[Module dev_monitor.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_monitor.erl)](./resources/source-code/dev_monitor.html) +* [[Module dev_multipass.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_multipass.erl)](./resources/source-code/dev_multipass.html) +* [[Module dev_name.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_name.erl)](./resources/source-code/dev_name.html) +* [[Module dev_node_process.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_node_process.erl)](./resources/source-code/dev_node_process.html) +* [[Module dev_p4.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_p4.erl)](./resources/source-code/dev_p4.html) +* [[Module dev_patch.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_patch.erl)](./resources/source-code/dev_patch.html) +* [[Module dev_poda.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_poda.erl)](./resources/source-code/dev_poda.html) +* [[Module dev_process.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_process.erl)](./resources/source-code/dev_process.html) +* [[Module dev_process_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_process_cache.erl)](./resources/source-code/dev_process_cache.html) +* [[Module dev_process_worker.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_process_worker.erl)](./resources/source-code/dev_process_worker.html) +* [[Module dev_push.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_push.erl)](./resources/source-code/dev_push.html) +* [[Module dev_relay.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_relay.erl)](./resources/source-code/dev_relay.html) +* [[Module dev_router.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_router.erl)](./resources/source-code/dev_router.html) +* [[Module dev_scheduler.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler.erl)](./resources/source-code/dev_scheduler.html) +* [[Module dev_scheduler_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_cache.erl)](./resources/source-code/dev_scheduler_cache.html) +* [[Module dev_scheduler_formats.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_formats.erl)](./resources/source-code/dev_scheduler_formats.html) +* [[Module dev_scheduler_registry.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_registry.erl)](./resources/source-code/dev_scheduler_registry.html) +* [[Module dev_scheduler_server.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_server.erl)](./resources/source-code/dev_scheduler_server.html) +* [[Module dev_simple_pay.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_simple_pay.erl)](./resources/source-code/dev_simple_pay.html) +* [[Module dev_snp.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_snp.erl)](./resources/source-code/dev_snp.html) +* [[Module dev_snp_nif.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_snp_nif.erl)](./resources/source-code/dev_snp_nif.html) +* [[Module dev_stack.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_stack.erl)](./resources/source-code/dev_stack.html) +* [[Module dev_test.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_test.erl)](./resources/source-code/dev_test.html) +* [[Module dev_volume.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_volume.erl)](./resources/source-code/dev_volume.html) +* [[Module dev_wasi.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_wasi.erl)](./resources/source-code/dev_wasi.html) +* [[Module dev_wasm.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_wasm.erl)](./resources/source-code/dev_wasm.html) +* [[Module hb.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb.erl)](./resources/source-code/hb.html) +* [[Module hb_ao.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_ao.erl)](./resources/source-code/hb_ao.html) +* [[Module hb_ao_test_vectors.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_ao_test_vectors.erl)](./resources/source-code/hb_ao_test_vectors.html) +* [[Module hb_app.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_app.erl)](./resources/source-code/hb_app.html) +* [[Module hb_beamr.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_beamr.erl)](./resources/source-code/hb_beamr.html) +* [[Module hb_beamr_io.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_beamr_io.erl)](./resources/source-code/hb_beamr_io.html) +* [[Module hb_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_cache.erl)](./resources/source-code/hb_cache.html) +* [[Module hb_cache_control.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_cache_control.erl)](./resources/source-code/hb_cache_control.html) +* [[Module hb_cache_render.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_cache_render.erl)](./resources/source-code/hb_cache_render.html) +* [[Module hb_client.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_client.erl)](./resources/source-code/hb_client.html) +* [[Module hb_crypto.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_crypto.erl)](./resources/source-code/hb_crypto.html) +* [[Module hb_debugger.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_debugger.erl)](./resources/source-code/hb_debugger.html) +* [[Module hb_escape.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_escape.erl)](./resources/source-code/hb_escape.html) +* [[Module hb_event.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_event.erl)](./resources/source-code/hb_event.html) +* [[Module hb_examples.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_examples.erl)](./resources/source-code/hb_examples.html) +* [[Module hb_features.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_features.erl)](./resources/source-code/hb_features.html) +* [[Module hb_gateway_client.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_gateway_client.erl)](./resources/source-code/hb_gateway_client.html) +* [[Module hb_http.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http.erl)](./resources/source-code/hb_http.html) +* [[Module hb_http_benchmark_tests.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_benchmark_tests.erl)](./resources/source-code/hb_http_benchmark_tests.html) +* [[Module hb_http_client.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_client.erl)](./resources/source-code/hb_http_client.html) +* [[Module hb_http_client_sup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_client_sup.erl)](./resources/source-code/hb_http_client_sup.html) +* [[Module hb_http_server.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_server.erl)](./resources/source-code/hb_http_server.html) +* [[Module hb_json.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_json.erl)](./resources/source-code/hb_json.html) +* [[Module hb_keccak.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_keccak.erl)](./resources/source-code/hb_keccak.html) +* [[Module hb_logger.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_logger.erl)](./resources/source-code/hb_logger.html) +* [[Module hb_message.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_message.erl)](./resources/source-code/hb_message.html) +* [[Module hb_metrics_collector.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_metrics_collector.erl)](./resources/source-code/hb_metrics_collector.html) +* [[Module hb_name.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_name.erl)](./resources/source-code/hb_name.html) +* [[Module hb_opts.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_opts.erl)](./resources/source-code/hb_opts.html) +* [[Module hb_path.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_path.erl)](./resources/source-code/hb_path.html) +* [[Module hb_persistent.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_persistent.erl)](./resources/source-code/hb_persistent.html) +* [[Module hb_private.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_private.erl)](./resources/source-code/hb_private.html) +* [[Module hb_process_monitor.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_process_monitor.erl)](./resources/source-code/hb_process_monitor.html) +* [[Module hb_router.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_router.erl)](./resources/source-code/hb_router.html) +* [[Module hb_singleton.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_singleton.erl)](./resources/source-code/hb_singleton.html) +* [[Module hb_store.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store.erl)](./resources/source-code/hb_store.html) +* [[Module hb_store_fs.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_fs.erl)](./resources/source-code/hb_store_fs.html) +* [[Module hb_store_gateway.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_gateway.erl)](./resources/source-code/hb_store_gateway.html) +* [[Module hb_store_remote_node.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_remote_node.erl)](./resources/source-code/hb_store_remote_node.html) +* [[Module hb_store_rocksdb.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_rocksdb.erl)](./resources/source-code/hb_store_rocksdb.html) +* [[Module hb_structured_fields.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_structured_fields.erl)](./resources/source-code/hb_structured_fields.html) +* [[Module hb_sup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_sup.erl)](./resources/source-code/hb_sup.html) +* [[Module hb_test_utils.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_test_utils.erl)](./resources/source-code/hb_test_utils.html) +* [[Module hb_tracer.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_tracer.erl)](./resources/source-code/hb_tracer.html) +* [[Module hb_util.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_util.erl)](./resources/source-code/hb_util.html) +* [[Module hb_volume.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_volume.erl)](./resources/source-code/hb_volume.html) +* [Source Code Documentation](./resources/source-code/index.html) +* [The hb application #](./resources/source-code/README.html) +* [[Module rsa_pss.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/rsa_pss.erl)](./resources/source-code/rsa_pss.html) diff --git a/docs/converge-protocol.md b/docs/misc/ao-core-protocol.md similarity index 50% rename from docs/converge-protocol.md rename to docs/misc/ao-core-protocol.md index 3b723e883..dfc026d9b 100644 --- a/docs/converge-protocol.md +++ b/docs/misc/ao-core-protocol.md @@ -1,10 +1,10 @@ -# The Converge Protocol. +# The AO-Core protocol. ## Status: DRAFT-1 ## Authors: Sam Williams, Tom Wilson, Tyler Hall, James Pichota, Vince Juliano < {first}@arweave.org> -This document contains a rough specification, in `engineering note` form, for the Converge Protocol. Converge is a method through which the permaweb can be interpreted as not only a flat, permanent ledger of data, but also as a mutable and reactive computation environment. Through this frame, the permaweb can be interpreted as a single, shared ['system image'](https://en.wikipedia.org/wiki/Single_system_image) that can be accessed and added to permissionlessly. Notably, the Converge itself intends to be a truly minimal computation model: It does not enforce any forms of consensus upon its execution directly, nor the use of any particular virtual machine. Even the requirements it imparts upon the host runtime environment are minimal. Instead, the Converge focuses on offering the simplest possible representation of _data_ and _computation_ upon that data, tailored for Arweave's distributed environment and [HTTP](https://datatracker.ietf.org/doc/html/rfc9114) access methods. +This document contains a rough specification, in `engineering note` form, for the AO-Core protocol. AO-Core is a method through which the permaweb can be interpreted as not only a flat, permanent ledger of data, but also as a mutable and reactive computation environment. Through this frame, the permaweb can be interpreted as a single, shared ['system image'](https://en.wikipedia.org/wiki/Single_system_image) that can be accessed and added to permissionlessly. Notably, the AO-Core itself intends to be a truly minimal computation model: It does not enforce any forms of consensus upon its execution directly, nor the use of any particular virtual machine. Even the requirements it imparts upon the host runtime environment are minimal. Instead, the AO-Core focuses on offering the simplest possible representation of _data_ and _computation_ upon that data, tailored for Arweave's distributed environment and [HTTP](https://datatracker.ietf.org/doc/html/rfc9114) access methods. -[HyperBEAM](https://github.com/permaweb/HyperBEAM) is an implementation of the converge protocol, as well as [AO](https://ao.arweave.net), a framework ontop of the Converge that constructs an environment for _trustless_ -- not just _permissionless_ -- computation. +[HyperBEAM](https://github.com/permaweb/HyperBEAM) is an implementation of the AO-Core protocol, as well as [AO](https://ao.arweave.net), a framework ontop of the AO-Core that constructs an environment for _trustless_ -- not just _permissionless_ -- computation. ## Context @@ -16,13 +16,13 @@ In this specification we refer to a number of abstract properties of computation `Trustlessness`: Users can participate in the network without needing to trust other parties are not acting maliciously. -Both of these properties are extremely powerful and can only be offered by decentralized computation machines. These properties can also, unfortunately, only presently be offered in degrees. For example, Bitcoin may offer a high level of `permissionlessness`, but it is not absolute: A user still requires that a majority of the mining power does not censor their transactions or blocks containing them. Similarly, while your recipient of a Bitcoin transfer has a high degree of `trustlessness`, at minimum the user must still trust the implementors of the cryptographic verification algorithms of their client in order to use the system. Rather than offering a single, standardized approach to the problem of offering permissionless and trustlessness computation, which would necessitate enforcing the same trade-offs upon all parties, Converge and AO instead focus on allowing users to make their own appropriate choices amongst a variety of options while still being able to interoperate together. +Both of these properties are extremely powerful and can only be offered by decentralized computation machines. These properties can also, unfortunately, only presently be offered in degrees. For example, Bitcoin may offer a high level of `permissionlessness`, but it is not absolute: A user still requires that a majority of the mining power does not censor their transactions or blocks containing them. Similarly, while your recipient of a Bitcoin transfer has a high degree of `trustlessness`, at minimum the user must still trust the implementors of the cryptographic verification algorithms of their client in order to use the system. Rather than offering a single, standardized approach to the problem of offering permissionless and trustlessness computation, which would necessitate enforcing the same trade-offs upon all parties, AO-Core and AO instead focus on allowing users to make their own appropriate choices amongst a variety of options while still being able to interoperate together. ## Machine Definition -Every item on the permaweb is described as a `Message`. Each `Message` is interpretable by Converge as a `map` of named functions, or as a concrete binary term. Each function in a message may be called through the creation of another message, providing a map of arguments to the execution. +Every item on the permaweb is described as a `Message`. Each `Message` is interpretable by AO-Core as a `map` of named functions, or as a concrete binary term. Each function in a message may be called through the creation of another message, providing a map of arguments to the execution. -Each `message` on the permaweb may optionally state a `Device` which should be used by Converge-compatible systems to interpret its contents. If no `Device` key is explicitly stated by the message, it must be infered as `Message`. The `Message` device should be implemented to simply return the binary or message at a given function name (a `key` in the map). Every `Device` must implement functions with the names `ID` and `Keys`. An `ID` is a function that can be used in order to refer to a message at a later time, while `Keys` should return a binary representation (with `Encoding` optionally specified as a parameter in the argument message) of each key in the message. +Each `message` on the permaweb may optionally state a `Device` which should be used by AO-Core-compatible systems to interpret its contents. If no `Device` key is explicitly stated by the message, it must be infered as `Message`. The `Message` device should be implemented to simply return the binary or message at a given function name (a `key` in the map). Every `Device` must implement functions with the names `ID` and `Keys`. An `ID` is a function that can be used in order to refer to a message at a later time, while `Keys` should return a binary representation (with `Encoding` optionally specified as a parameter in the argument message) of each key in the message. Concretely, these relations can be expressed as follows: ``` @@ -31,21 +31,21 @@ Types: Message :: Map< Name :: Binary, (Message?) => Message > | Binary Functions: - Message1(Message2) => Message3 :: Converge.apply(Message1, Message2) => Message3 - Converge.apply(Message1[Device], Message2, RuntimeEnv? :: Message) => Message3 + Message1(Message2) => Message3 :: AO-Core.apply(Message1, Message2) => Message3 + AO-Core.apply(Message1[Device], Message2, RuntimeEnv? :: Message) => Message3 Invariants: ∀ message ∈ Permaweb, ∃ <<"ID">> ∈ Message ∀ message ∈ Permaweb, ∃ <<"Keys">> ∈ Message - ∀ message ∈ Permaweb, ∃ (<<"Device">> ∈ Message ∨ Converge.apply(Message, <<"Device">>) => <<"Message">>) + ∀ message ∈ Permaweb, ∃ (<<"Device">> ∈ Message ∨ AO-Core.apply(Message, <<"Device">>) => <<"Message">>) ``` -The Converge protocol intends to be a computer native to the technologies of the internet. More specifically, we have focused its representation on compatibility with the HTTP family of protocols (`HTTP/{1.1,2,3}`). As such, every message in this system can be refered to via `Path`s. A path starts from a given message in the system (whether written to the permaweb yet or not), and applies a series of additional messages on top of it. Each resulting message itself must have an `ID` resolvable via its device -- subsequently enabling additional paths to be described atop the intermediate message. +The AO-Core protocol intends to be a computer native to the technologies of the internet. More specifically, we have focused its representation on compatibility with the HTTP family of protocols (`HTTP/{1.1,2,3}`). As such, every message in this system can be refered to via `Path`s. A path starts from a given message in the system (whether written to the permaweb yet or not), and applies a series of additional messages on top of it. Each resulting message itself must have an `ID` resolvable via its device -- subsequently enabling additional paths to be described atop the intermediate message. For example: ``` /StartingID/Input1ID/Input2ID/Input3ID => - /{Converge.apply(StartingMsg, Input1)}/Input2ID/Input3ID => + /{AO-Core.apply(StartingMsg, Input1)}/Input2ID/Input3ID => /OutputID1/Input2ID/Input3ID => ... /Output3ID @@ -54,7 +54,7 @@ For example: ## Hyperbeam Devices -The hyperbeam environment implements Converge through the use of device modules. Each module in the base deployment is namespaced `dev_*`, for example `dev_scheduler`. Devices in hyperbeam can communicate the basic set of keys that they offer using the function exports. Each of these functions is interpreted as yielding the value for a key on a message containing that device, with the key's name being defined by its function name. Each of these functions may take one to three arguments of the following: `function(StateMessage, InputMessage, Environment)`. It must yield a result of the form `{Status, NewMessage}`, where `Status` is typically (but not necessarily) either `ok` or `error`. If a device does not offer one of the required functions (`ID` or `device`), the environment falls back to the underlying `message` device implementations. `tolowercase` is applied to all function names before execution, and `-` characters are replaced with `_` to allow for more readable device implementations. +The hyperbeam environment implements AO-Core through the use of device modules. Each module in the base deployment is namespaced `dev_*`, for example `dev_scheduler`. Devices in hyperbeam can communicate the basic set of keys that they offer using the function exports. Each of these functions is interpreted as yielding the value for a key on a message containing that device, with the key's name being defined by its function name. Each of these functions may take one to three arguments of the following: `function(StateMessage, InputMessage, Environment)`. It must yield a result of the form `{Status, NewMessage}`, where `Status` is typically (but not necessarily) either `ok` or `error`. If a device does not offer one of the required functions (`ID` or `device`), the environment falls back to the underlying `message` device implementations. `tolowercase` is applied to all function names before execution, and `-` characters are replaced with `_` to allow for more readable device implementations. The special case `info/{0,1,2}` function may be implemented by the device, signalling environment requirements to hyperbeam. The `info` function can optionally take the `message` in question and the environment variables as arguments. It should return a map of environmental information of the following form: @@ -68,7 +68,7 @@ The special case `info/{0,1,2}` function may be implemented by the device, signa } ``` -See `hb_converge.erl` for a full overview of supported keys. +See `hb_ao.erl` for a full overview of supported keys. If the `default` parameter is provided, the function will be used as the entrypoint all key resolution when a matching function is not found. The key's name is provided as an additional first argument in this case (`defaultFun/{2,3,4}`). @@ -83,11 +83,11 @@ The `uses` info key may be optionally utilized to signal to the environment whic ## The Stack Device -In order to allow messages to have more flexibility in their execution, hyperbeam offers an implementation of a Converge `stack`-style device, which combines a series of devices on a message into a single 'stack' of executable transformations. This device allows many complex forms of processors to be built inside the Converge environment -- for example, AO processes -- whiile transferring the architecture's modularity and flexibility to them. +In order to allow messages to have more flexibility in their execution, hyperbeam offers an implementation of an AO-Core `stack`-style device, which combines a series of devices on a message into a single 'stack' of executable transformations. This device allows many complex forms of processors to be built inside the AO-Core environment -- for example, AO processes -- whiile transferring the architecture's modularity and flexibility to them. When added as the highest `Device` tag on a message, the stack device scans the remainder of the message's tags looking for (and subsequently loading) any other messages it finds. When a user then calls an execution on top of a message containing a device, the device passes through each of the elements of the stack in turn, 'folding' over it. -## Paths, Hashpaths, and Attestations +## Paths, Hashpaths, and Commitments As described, all data in HyperBEAM is the result of the application of two messages together. Each piece of data has its own ID, which are 'mixed' cryptographically during execution resulting in a new value. This value is called the `hashpath`, and can be seen as a memoization of the tree of executions that were the source of a given piece of data. @@ -99,10 +99,10 @@ Hashpaths are derived as follows: Due to its Merklized form (each `Hashpath` is the result of cryptographically mixing two prior commitments), a hashpath is extremely short (32 bytes with the standard SHA2-256 `Hashpath-Alg`), yet can be used to reconstruct the entire tree of inputs necessary to generate a given state. -When given together with the message that it resulted from and a cryptographic assurance, a hashpath represents an attestation of correctness of an output: That `Message1(Message)` does in fact give rise to the given collection of keys. This output itself is addressable via a traditional cryptographic hash, or via a signature upon that hash. While Converge provides the framework for these security mechanisms and their mutual compatibility, it does not enforce any single set of choices, such that users and implementors can make design decisions that are appropriate for their context. The AO protocol is an example implementation of this. +When given together with the message that it resulted from and a cryptographic assurance, a hashpath represents a commitment of correctness of an output: That `Message1(Message)` does in fact give rise to the given collection of keys. This output itself is addressable via a traditional cryptographic hash, or via a signature upon that hash. While AO-Core provides the framework for these security mechanisms and their mutual compatibility, it does not enforce any single set of choices, such that users and implementors can make design decisions that are appropriate for their context. The AO protocol is an example implementation of this. Notably, this structure does not require a computor of `Message3` to know the value of every key inside it. This is helpful, for example, in circumstances in which `Message3` may be extremely large, yet the user only wants to know the value of a single given key inside the message, or to compute a new message that only references a small number of its keys. Through this mechanism computation can be effectively 'sharded' among an arbitrarily large set of computation nodes as necessary. ## Redirection -Because messages in the Converge protocol are immutably referenced (both by their message IDs and their hashpaths), immutable 'links' can be made between them. For example, as `Message1.Hashpath(Message2.ID)` will always generate the same `M3`, an attestation of this linkage can be distributed amongst peers and used to 'shortcut' computation. Since many repeated computations typically occur in a distributed compute network, these linkages can be spread amongst peers as necessary in order to optimize computation times. As Converge is expressed as natively using the HTTP Semantics, these linkages are simply expressed as `308 Permanent Redirect`s -- allowing even browsers and other HTTP clients to follow them, without needing any awareness of the mechanics of Converge protocol directly. \ No newline at end of file +Because messages in the AO-Core protocol are immutably referenced (both by their message IDs and their hashpaths), immutable 'links' can be made between them. For example, as `Message1.Hashpath(Message2.ID)` will always generate the same `M3`, a commitment of this linkage can be distributed amongst peers and used to 'shortcut' computation. Since many repeated computations typically occur in a distributed compute network, these linkages can be spread amongst peers as necessary in order to optimize computation times. As AO-Core is expressed as natively using the HTTP Semantics, these linkages are simply expressed as `308 Permanent Redirect`s -- allowing even browsers and other HTTP clients to follow them, without needing any awareness of the mechanics of AO-Core protocol directly. \ No newline at end of file diff --git a/docs/misc/ao-core-web-apis.md b/docs/misc/ao-core-web-apis.md new file mode 100644 index 000000000..c57946d24 --- /dev/null +++ b/docs/misc/ao-core-web-apis.md @@ -0,0 +1,86 @@ +# Integrating AO-Core into UIs + +## Overview + +This guide provides a practical approach to using the `patch@1.0` AO-Core device to create RESTful-like APIs for AO processes. If you're familiar with dryruns, this method offers a more efficient alternative by making process data directly accessible via HTTP endpoints from any operational AO Mainnet HyperBEAM node, eliminating the need for repeated dryruns. Responses are cryptographically signed and linked to individual nodes. + +## Key Features + +- **HTTP Endpoints**: Access process data via HTTP, similar to RESTful APIs +- **Cryptographic Signatures**: Each response is signed, ensuring data integrity +- **No DryRuns**: Directly access and update process states without the overhead of dryruns +- **Real-time State Access**: Get the latest process state without waiting for dryrun computation + +## Implementation Steps + +### Initial State Synchronization + +Add an initial sync at the top of your process code to export the initial state: + +```lua +-- Sync state on spawn +InitialSync = InitialSync or 'INCOMPLETE' +if InitialSync == 'INCOMPLETE' then + Send({ + device = 'patch@1.0', + cache = { + table1 = { + [recordId1] = table1[recordId1] + }, + table2 = { + [recordId2] = table2[recordId2] + } + } + }) + InitialSync = 'COMPLETE' +end +``` + +### State Updates During Operation + +Incorporate patch messages wherever state changes occur. Example for an auction system: + +```lua +-- Inside any handler that modifies data +Handlers.add('update-data', function(msg) + -- Process your logic... + table1[recordId1].field = msg.newValue + table2[recordId2] = { + field1 = msg.value1, + field2 = msg.From + } + + -- Export the updated state + Send({ + device = 'patch@1.0', + cache = { + table1 = { + [recordId1] = table1[recordId1] + }, + table2 = { + [recordId2] = table2[recordId2] + } + } + }) + + -- Rest of handler logic... +end) +``` + +### Accessing Your API + +Access your process data via any HyperBEAM node. This provides immediate access to the latest state without the need for dryruns: + +- **Latest State**: `GET /YOUR_PROCESS_ID~process@1.0/now/cache` +- **Pre-computed State**: `GET /YOUR_PROCESS_ID~process@1.0/compute/cache` + +## Best Practices + +- **Selective Updates**: For large datasets, update only the altered data +- **Consistent State Updates**: Ensure all state-changing handlers include patch messages +- **Custom State Naming**: Name the state something other than `cache` if needed, and adjust HTTP requests accordingly +- **Migration Strategy**: Consider gradually transitioning from dryruns to this approach for existing applications + +## Important Note + +This approach leverages HyperBEAM milestone 3 functionality and is currently in preview. It is not recommended for applications that may lead to loss of value due to potential changes and existing bugs. If you're currently using dryruns, consider this as a more efficient alternative for state access, but maintain your existing dryrun-based validation where needed. \ No newline at end of file diff --git a/docs/converge-http-api.md b/docs/misc/ao-http-api.md similarity index 86% rename from docs/converge-http-api.md rename to docs/misc/ao-http-api.md index 0db03d966..536ef1ffe 100644 --- a/docs/converge-http-api.md +++ b/docs/misc/ao-http-api.md @@ -1,10 +1,10 @@ -# Converge HTTP API Design notes +# AO-Core HTTP API Design notes ### Date: 11 Jan, 2025. -The Converge protocol is designed to layer a computation system on top of +The AO-Core protocol is designed to layer a computation system on top of HTTP Semantics. As such, it offers syntax for HTTP requests that allow users to easily manipulate and traverse through the computation graph. This document -describes the semantics and syntax of the Converge HTTP API. +describes the semantics and syntax of the AO-Core HTTP API. ## Semantics and syntax @@ -12,7 +12,7 @@ describes the semantics and syntax of the Converge HTTP API. ``` GET /hashpath1/... ``` -- All keys after the base are interpreted as Converge messages individually, in +- All keys after the base are interpreted as AO-Core messages individually, in a chain. For example, `GET /hashpath1/key1/key2` is equivalent to: ``` GET /hashpath(hashpath1, key1)/key2 @@ -20,7 +20,7 @@ GET /hashpath(hashpath1, key1)/key2 - Each path segment is interpreted as a key to resolve upon the message for the resolution of the previous message. The `key` is taken as the full path for that message during that resolution. -- When Converge is not given a base hashpath (the first key is not a 43 character +- When AO-Core is not given a base hashpath (the first key is not a 43 character base64URL encoded string), the request message is assumed to be its own base message, with the path applied alone as the request (`Message2`). - Query parameters are treated as equivalent to headers. @@ -31,7 +31,7 @@ For example, `GET /hashpath1/key1+dict1=val1/key2` adds `dict1=val1` to the message for the resolution of the `key1` key. - `N.KeyName` in headers or query parameters is interpreted as `KeyName` in the message dictionary for the resolution of the `N`th key. -- `Key!DevName` in a path segment is interpreted as executing the message with +- `Key~DevName` in a path segment is interpreted as executing the message with the `Device` set to `DevName`. - A dictionary key of form `Key|Type` is interpreted as a type annotation for `Key`, which can be used to parse the value of the key using HTTP Structured @@ -92,9 +92,9 @@ curl -X POST http://host:port/Init/Compute \ -H "2.WASM-Function: fac" -H "2.WASM-Params: [10]" ``` -To fork an existing process to use a new device, we can use the `!` key operator: +To fork an existing process to use a new device, we can use the `~` key operator: ``` -curl -X POST http://host:port/Schedule!Process/2.0/(/ProcID/Now)+Method=POST +curl -X POST http://host:port/Schedule~Process/2.0/(/ProcID/Now)+Method=POST ``` To run multiple computations in the same request, with separate headers for @@ -112,9 +112,9 @@ curl http://host:port/Init+Device=WASM-64/1.0+Image=ID/Compute+ \ WASM-Function=fac+WASM-Params=[10]/Compute+WASM-Function=fac&WASM-Params=[11] ``` -Gather the node's known attestations (including its own) on a process output: +Gather the node's known commitments (including its own) on a process output: ``` -curl http://host:port/ProcID/Compute+Slot=1/Results/Attestations +curl http://host:port/ProcID/Compute+Slot=1/Results/Commitments ``` Dry-run a message on top of an AO process state: diff --git a/docs/misc/building-pre-post-processors.md b/docs/misc/building-pre-post-processors.md new file mode 100644 index 000000000..56fe7cf3d --- /dev/null +++ b/docs/misc/building-pre-post-processors.md @@ -0,0 +1,91 @@ +# Building Pre/Post-Processors in AO + +Pre/post-processors in AO allow you to intercept and potentially modify incoming requests before they are executed by the target device or process. This guide explains how to build a preprocessor, focusing on a common pattern: exempting certain request paths from being relayed or modified. + +## Core Concepts + +1. **Exempt Routes (`exempt-routes`):** A list of route templates defined in the node's configuration (`Msg1`). If an incoming request (`Msg2`) matches any of these templates, it should bypass the main preprocessor logic (e.g., relaying). +2. **Exemption Check (`is_exempt/3`):** A function that determines if a request should be exempt. It checks two things: + * An optional `is-exempt` message defined in the node's configuration (`Msg1`). If present, this message is resolved, and its result determines exemption. + * If `is-exempt` is not found, it matches the incoming request's path against the `exempt-routes` list using `dev_router:match/3`. +3. **Preprocessing Logic (`preprocess/3`):** The main function that receives the node configuration (`Msg1`), the incoming request (`Msg2`), and options (`Opts`). Based on the result of `is_exempt/3`, it either: + * **If Exempt:** Returns the original, parsed list of messages to be executed. This list is found in the `body` key of `Msg2`. + * **If Not Exempt:** Performs the main preprocessing action, such as rewriting the request to be relayed to another node. This often involves using the raw, unparsed request singleton found in the `request` key of `Msg2`. + +## Implementation Example (Pseudo-code) + +This pseudo-code illustrates the flow: + +```erlang +-define(DEFAULT_EXEMPT_ROUTES, [ + % Default routes that should bypass preprocessing + #{ <<"template">> => <<"/~meta@1.0/.*">> }, + #{ <<"template">> => <<"/~greenzone@1.0/.*">> } + % ... other default exempt routes +]). + +%% @doc Check if a request is exempt from preprocessing. +is_exempt(Msg1, Msg2, Opts) -> + case ao.get(<<"is-exempt">>, Msg1, Opts) of + not_found -> + % No explicit is-exempt message, check against exempt-routes + ExemptRoutes = + hb_opts:get( + exempt_routes, + ?DEFAULT_EXEMPT_ROUTES, + Msg1 + ), + Req = hb_ao:get(<<"request">>, Msg2, Opts), + {_, Matches} = + dev_router:match( + #{ <<"routes">> => ExemptRoutes }, + Req, + Msg1 % Use NodeMsg (Msg1) for Opts context if needed + ), + case Matches of + no_matching_route -> {ok, false}; % Not exempt + _ -> {ok, true} % Exempt + end; + IsExemptMsg -> + % Resolve the custom is-exempt message + ao.resolve(IsExemptMsg, Msg2, Opts) + end. + +%% @doc Preprocess an incoming request. +preprocess(Msg1, Msg2, Opts) -> + case is_exempt(Msg1, Msg2, Opts) of + {ok, true} -> + % Request is exempt. Return the original parsed message list. + % IMPORTANT: Use the 'body' key from Msg2. + {ok, hb_ao:get(<<"body">>, Msg2, Opts)}; + {ok, false} -> + % Request is not exempt. Perform preprocessing (e.g., relay). + % IMPORTANT: Use the 'request' key from Msg2 for the raw singleton. + {ok, + [ + #{ <<"device">> => <<"relay@1.0">> }, % Example: Relay device + #{ + <<"path">> => <<"call">>, + <<"target">> => <<"body">>, % Target the 'body' of the relay message + <<"body">> => + % Get the raw request singleton + hb_ao:get(<<"request">>, Msg2, Opts#{ hashpath => ignore }) + } + ] + }; + {error, Reason} -> + % Handle errors from is_exempt resolution + {error, Reason} + end. +``` + +## Key Considerations + +* **`body` vs. `request`:** The preprocessor receives the incoming request in two forms within `Msg2`: + * `body`: A **parsed list** of AO messages that represent the steps to be executed. Use this when you want to return the original execution plan (i.e., when exempting). + * `request`: The **raw, unparsed TABM singleton** message sent by the user. Use this when you need the original message structure, for example, to forward it unmodified in a relay request. +* **`dev_router:match/3`:** This function is used to match a request (`Req`) against a list of route templates (`Routes`). It's borrowed from the routing logic but is useful here for checking path-based exemptions. +* **Configuration:** The `exempt-routes` and the optional `is-exempt` message should be configured in the node's options (accessible via `Msg1` or `Opts`). +* **Error Handling:** Ensure proper error handling, especially when resolving the `is-exempt` message. + +By following this pattern, you can create flexible preprocessors that selectively apply logic based on configurable rules and request paths. \ No newline at end of file diff --git a/docs/misc/community/contributing-docs.md b/docs/misc/community/contributing-docs.md new file mode 100644 index 000000000..3565bbd98 --- /dev/null +++ b/docs/misc/community/contributing-docs.md @@ -0,0 +1,100 @@ +# Contributing Documentation + +This guide explains how to contribute documentation to the HyperBEAM project. Following these steps will help ensure your documentation is properly integrated into the official documentation. + +## Overview + +The HyperBEAM documentation is built using [MkDocs](https://www.mkdocs.org/) with the [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) theme. All documentation is written in Markdown and organized into logical sections within the `docs/` directory. + +## Contribution Process + +### 1. Fork the Repository + +First, fork the [HyperBEAM repository](https://github.com/permaweb/HyperBEAM) to your GitHub account. + +### 2. Choose the Right Location + +Review the existing documentation structure in `./docs/` to determine the appropriate location for your content. The documentation is organized into several main sections: + +- `overview/`: High-level concepts and architecture +- `installation-core/`: Setup and configuration guides +- `components/`: Detailed component documentation +- `usage/`: Tutorials and usage guides +- `resources/`: Reference materials and source code documentation +- `community/`: Contribution guidelines and community resources + +### 3. Create Your Documentation + +Create a new Markdown file (`.md`) in the appropriate directory. Follow these guidelines: + +- Use proper Markdown syntax +- Include clear headings and subheadings +- Add code blocks with appropriate language specification +- Link to related documentation +- For images: + - Upload images to Arweave using [ArDrive](https://ardrive.io/) or your preferred Arweave upload method + - Reference images using their Arweave transaction ID (txid) in the format: `https://arweave.net/` + - Example: `![Image Description](https://arweave.net/abc123...)` +- Follow the existing documentation style and format + +### 4. Update the Navigation + +Edit `mkdocs.yml` to add your documentation to the navigation: + +1. Open `mkdocs.yml` +2. Find the appropriate section under the `nav:` configuration +3. Add your entry following the existing indentation and format +4. Ensure the path to your documentation is correct + +### 5. Test Your Changes + +Set up a local development environment to test your changes: + +```bash +# Create and activate a virtual environment +python3 -m venv venv +source venv/bin/activate # (macOS/Linux) On Windows use `venv\Scripts\activate` + +# Install required packages +pip3 install mkdocs mkdocs-material + +# Run the build script +./docs/build-all.sh + +# Start a local server +cd mkdocs-site +python3 -m http.server 8000 +``` + +View your documentation at `http://127.0.0.1:8000/` to ensure everything renders correctly. + +### 6. Submit a Pull Request + +When your documentation is ready: + +1. Create a new branch for your changes +2. Commit your changes with a descriptive message +3. Submit a PR with: + - A clear title describing the documentation addition + - A detailed description explaining: + - The purpose of the new documentation + - Why it should be added to the official docs + - Any related issues or discussions + - Screenshots of the rendered documentation (if applicable) + +### 7. Review Process + +The HyperBEAM team will review your PR and may request changes. Be prepared to: + +- Address any feedback +- Make necessary adjustments +- Respond to questions about your contribution + +Once approved, your documentation will be merged into the main repository. + +## Additional Resources + +- [Community Guidelines](./guidelines.md) +- [Development Setup](./setup.md) +- [MkDocs Documentation](https://www.mkdocs.org/) +- [Material for MkDocs Documentation](https://squidfunk.github.io/mkdocs-material/) \ No newline at end of file diff --git a/docs/misc/community/guidelines.md b/docs/misc/community/guidelines.md new file mode 100644 index 000000000..1d019d970 --- /dev/null +++ b/docs/misc/community/guidelines.md @@ -0,0 +1,150 @@ +# Contribution Guidelines + +Thank you for your interest in contributing to HyperBEAM! This page outlines the process for contributing to the project and provides guidelines to ensure your contributions align with the project's goals and standards. + +## Code of Conduct + +By participating in this project, you agree to abide by our Code of Conduct. We expect all contributors to be respectful, inclusive, and considerate in all interactions. + +## Ways to Contribute + +There are many ways to contribute to HyperBEAM: + +- **Code contributions**: Implementing new features or fixing bugs +- **Documentation**: Improving or adding to the documentation +- **Testing**: Writing tests or manually testing functionality +- **Bug reports**: Reporting issues you encounter +- **Feature requests**: Suggesting new features or improvements +- **Community support**: Helping other users in forums or discussions + +## Getting Started + +1. **Fork the repository**: Create your own fork of the [HyperBEAM repository](https://github.com/permaweb/HyperBEAM). +2. **Set up your development environment**: Follow the setup instructions in the [Development Setup](setup.md) guide. +3. **Find an issue to work on**: Look for issues labeled "good first issue" or "help wanted" in the [GitHub issue tracker](https://github.com/permaweb/HyperBEAM/issues). +4. **Create a branch**: Create a new branch for your work with a descriptive name. + +## Development Workflow + +### Making Changes + +1. **Make your changes**: Implement your feature or fix the bug. +2. **Follow coding standards**: Ensure your code follows the project's [coding standards](#coding-standards). +3. **Write tests**: Add tests for your changes to ensure functionality and prevent regressions. +4. **Update documentation**: Update or add documentation to reflect your changes. + +### Submitting Changes + +1. **Commit your changes**: Use clear and descriptive commit messages. +2. **Push to your fork**: Push your changes to your GitHub fork. +3. **Create a pull request**: Submit a pull request from your fork to the main repository. +4. **Describe your changes**: In the pull request, describe what you've changed and why. +5. **Link related issues**: Link any related issues in your pull request description. + +### Code Review Process + +1. **Initial review**: A maintainer will review your pull request for basic compliance. +2. **Feedback**: You may receive feedback requesting changes or clarification. +3. **Iteration**: Make requested changes and push them to your branch. +4. **Approval**: Once approved, a maintainer will merge your changes. + +## Coding Standards + +### Erlang Code + +- Look at the existing code and match its style +- If you see a pattern in how things are written, follow it +- If you're not sure, check what the majority of the codebase does +- Write tests that match the existing test style +- Focus on writing working code, not debating style +- Remember: Cypherpunks write code! + +### JavaScript/Node.js Code + +- Follow the [Airbnb JavaScript Style Guide](https://github.com/airbnb/javascript) +- Use async/await for asynchronous operations +- Document functions with JSDoc comments +- Use meaningful variable and function names +- Write unit tests for all functionality + +### General Guidelines + +- Keep commits focused on a single change +- Write clear commit messages that explain *why* a change was made +- Avoid large, monolithic pull requests +- Add comments for complex logic +- Prioritize readability and maintainability + +## Documentation Guidelines + +- Use Markdown for all documentation +- Follow a consistent structure and style +- Include examples where appropriate +- Update documentation when changing functionality +- Ensure links work correctly +- Check spelling and grammar + +## Testing Guidelines + +- Write unit tests for all new functionality +- Ensure tests are deterministic (no flaky tests) +- Mock external dependencies +- Consider edge cases in your tests +- Aim for high test coverage +- Include integration tests where appropriate + +## Reporting Bugs + +When reporting bugs, please include: + +1. A clear, descriptive title +2. Steps to reproduce the issue +3. Expected behavior +4. Actual behavior +5. Environment details (OS, versions, etc.) +6. Screenshots or logs if applicable + +## Requesting Features + +When requesting features, please include: + +1. A clear, descriptive title +2. A detailed description of the feature +3. The problem it solves or benefit it provides +4. Any alternative solutions you've considered +5. Mockups or examples if applicable + +## Pull Request Template + +When creating a pull request, please use the following template: + +```markdown +## Description +[Describe the changes you've made] + +## Related Issue +[Link to the related issue] + +## Type of Change +- [ ] Bug fix +- [ ] New feature +- [ ] Documentation update +- [ ] Performance improvement +- [ ] Code refactoring +- [ ] Other (please describe) + +## Checklist +- [ ] I have read the contribution guidelines +- [ ] My code follows the project's coding standards +- [ ] I have added tests that prove my fix/feature works +- [ ] I have updated documentation to reflect my changes +- [ ] All new and existing tests pass +``` + +## Communication Channels + +- **GitHub Issues**: For bug reports and feature requests +- **Discord**: For general discussion and questions +- **Pull Requests**: For code review and discussion of implementations + +Thank you for contributing to HyperBEAM! \ No newline at end of file diff --git a/docs/misc/community/setup.md b/docs/misc/community/setup.md new file mode 100644 index 000000000..d56dc0de7 --- /dev/null +++ b/docs/misc/community/setup.md @@ -0,0 +1,160 @@ +# Development Setup + +This guide will help you set up your development environment for contributing to HyperBEAM. + +## Prerequisites + +Before starting, ensure you have the following installed: + +- **Ubuntu 22.04** (primary development platform) +- **Git** +- **Erlang/OTP 27** +- **Rebar3** +- **Rust** (for certain components) +- **A code editor** (Cursor, VSCode, Emacs, Vim, etc.) + +## Setting Up Your Development Environment + +### 1. Fork and Clone the Repository + +First, fork the HyperBEAM repository on GitHub, then clone your fork: + +```bash +git clone https://github.com/YOUR-USERNAME/HyperBEAM.git +cd HyperBEAM +``` + +### 2. Add the Upstream Remote + +Add the original repository as an upstream remote to keep your fork in sync: + +```bash +git remote add upstream https://github.com/permaweb/HyperBEAM.git +``` + +### 3. Install Dependencies + +#### HyperBEAM Core (Erlang) Dependencies + +```bash +# Install Erlang dependencies +rebar3 compile +``` + +### 4. Run Tests + +Verify that your setup is working correctly by running the tests: + +```bash +# Run Erlang tests +rebar3 eunit +``` + +## Development Workflow + +### 1. Keep Your Fork Updated + +Regularly sync your fork with the upstream repository: + +```bash +git fetch upstream +git checkout main +git merge upstream/main +``` + +### 2. Create a Feature Branch + +Create a branch for each feature or bugfix: + +```bash +git checkout -b feature/your-feature-name +``` + +or + +```bash +git checkout -b fix/bug-you-are-fixing +``` + +### 3. Development Cycle + +The typical development cycle is: + +1. Make changes to the code +2. Write tests for your changes +3. Run the tests to verify your changes +4. Commit your changes +5. Push to your fork +6. Create a pull request + +### 4. Running HyperBEAM Locally + +To run HyperBEAM for development: + +#### Start HyperBEAM + +```bash +rebar3 shell +``` + +In the Erlang shell: + +```erlang +application:ensure_all_started(hyperbeam). +``` + + +### 5. Debugging + +#### Erlang Debugging + +You can use the Erlang debugger or add logging: + +```erlang +?event({debug, Variable}). +% or for more context: +?event(module_name, {debug_label, Variable}). +% for more detailed output with explicit information: +?event(module_name, {debug_label, {explicit, Variable}}). +``` + +To filter logs for specific modules or files, you can prefix the rebar3 shell command with the `HB_PRINT` environment variable, providing a comma-separated list of module or file names: + +```bash +HB_PRINT=module_name1,module_name2,filename1 rebar3 shell +``` + +This will show debug logs only from the specified modules or files, making it easier to focus on relevant information during development and troubleshooting. + + +## Code Editor Setup + +### VS Code + +We recommend the following extensions for VS Code: + +- Erlang LS +- erlang +- Erlang/OTP + +## Common Issues and Solutions + +### Permission Issues + +If you encounter permission issues: + +```bash +sudo chown -R $(whoami) . +``` + +## Getting Help + +If you need help with your development setup: + +- Check existing issues on GitHub +- Ask for help in the Discord channel +- Create a new issue with the "question" label + +## Next Steps + +Once your development environment is set up, check out the [Contribution Guidelines](guidelines.md) for information on how to submit your changes. \ No newline at end of file diff --git a/docs/misc/components-compute-unit/configuration.md b/docs/misc/components-compute-unit/configuration.md new file mode 100644 index 000000000..69daeacfd --- /dev/null +++ b/docs/misc/components-compute-unit/configuration.md @@ -0,0 +1,118 @@ +# Compute Unit Configuration + +The Compute Unit (CU) supports numerous environment variables and configuration options. This document details the available options and recommended settings. + +## Configuration Methods + +The Compute Unit can be configured using: + +1. **Environment Variables**: Set directly in the shell or via a `.env` file +2. **Command Line Arguments**: Pass when starting the CU +3. **Configuration Files**: Use JSON configuration files + +## Essential Configuration Options + +### Core Settings + +| Variable | Description | Default | +|----------|-------------|---------| +| `UNIT_MODE` | Operating mode (set to "hbu" for HyperBEAM) | - | +| `HB_URL` | URL of your HyperBEAM instance | http://localhost:10000 | +| `PORT` | The port on which the CU server will listen | 6363 | +| `WALLET_FILE` | Path to your Arweave wallet JSON file | - | +| `NODE_CONFIG_ENV` | Configuration environment | "development" | + +### Network Settings + +| Variable | Description | Default | +|----------|-------------|---------| +| `GATEWAY_URL` | The url of the Arweave gateway | https://arweave.net | +| `ARWEAVE_URL` | URL for the Arweave HTTP API | GATEWAY_URL | +| `GRAPHQL_URL` | URL for the Arweave GraphQL server | ${GATEWAY_URL}/graphql | +| `CHECKPOINT_GRAPHQL_URL` | URL for querying Checkpoints | GRAPHQL_URL | +| `UPLOADER_URL` | URL for uploading Process Checkpoints | up.arweave.net | + +### WASM Execution Settings + +| Variable | Description | Default | +|----------|-------------|---------| +| `PROCESS_WASM_MEMORY_MAX_LIMIT` | Maximum memory limit for processes in bytes | 1GB | +| `PROCESS_WASM_COMPUTE_MAX_LIMIT` | Maximum compute limit for processes | 9 billion | +| `PROCESS_WASM_SUPPORTED_FORMATS` | Supported wasm module formats (comma-delimited) | wasm32-unknown-emscripten,wasm32-unknown-emscripten2 | +| `PROCESS_WASM_SUPPORTED_EXTENSIONS` | Supported wasm extensions (comma-delimited) | - | +| `WASM_EVALUATION_MAX_WORKERS` | Number of workers for message evaluation | CPU count - 1 | + +### Caching Settings + +| Variable | Description | Default | +|----------|-------------|---------| +| `WASM_BINARY_FILE_DIRECTORY` | Directory to cache wasm binaries | OS temp directory | +| `WASM_MODULE_CACHE_MAX_SIZE` | Maximum size of the wasm module cache | 5 modules | +| `WASM_INSTANCE_CACHE_MAX_SIZE` | Maximum size of the wasm instance cache | 5 instances | +| `PROCESS_MEMORY_CACHE_MAX_SIZE` | Maximum size of the process memory cache in bytes | - | +| `PROCESS_MEMORY_CACHE_TTL` | Time-to-live for memory cache entries | - | +| `PROCESS_MEMORY_CACHE_FILE_DIR` | Directory for drained process memory | OS temp directory | + +### Checkpoint Settings + +| Variable | Description | Default | +|----------|-------------|---------| +| `PROCESS_MEMORY_FILE_CHECKPOINTS_DIR` | Directory for file checkpoints | OS temp/file_checkpoints | +| `PROCESS_MEMORY_CACHE_CHECKPOINT_INTERVAL` | Checkpoint interval (0 to disable) | 0 | +| `PROCESS_CHECKPOINT_CREATION_THROTTLE` | Time between checkpoints for a process | 30 minutes | +| `DISABLE_PROCESS_CHECKPOINT_CREATION` | Disable Arweave checkpoint uploads | Enabled (must set to 'false' to enable) | +| `DISABLE_PROCESS_FILE_CHECKPOINT_CREATION` | Disable file checkpoint creation | Enabled (must set to 'false' to enable) | +| `EAGER_CHECKPOINT_ACCUMULATED_GAS_THRESHOLD` | Gas threshold for immediate checkpoint | - | + +### Performance and Monitoring + +| Variable | Description | Default | +|----------|-------------|---------| +| `MEM_MONITOR_INTERVAL` | Interval for logging memory usage | - | +| `BUSY_THRESHOLD` | Timeout for "busy" response to clients | 0 (disabled) | +| `ENABLE_METRICS_ENDPOINT` | Enable OpenTelemetry metrics endpoint | Disabled | +| `DEFAULT_LOG_LEVEL` | Logging level | debug | +| `LOG_CONFIG_PATH` | Path to log level configuration file | .loglevel | + +### Access Control + +| Variable | Description | Default | +|----------|-------------|---------| +| `RESTRICT_PROCESSES` | Process IDs to restrict (blacklist) | - | +| `ALLOW_PROCESSES` | Process IDs to allow (whitelist) | - | +| `ALLOW_OWNERS` | Process owners to allow (whitelist) | - | +| `PROCESS_CHECKPOINT_TRUSTED_OWNERS` | Wallets whose checkpoints are trusted | - | + +## Example .env File + +A minimal configuration file looks like this: + +```bash +UNIT_MODE=hbu +HB_URL=http://localhost:10000 +PORT=6363 +WALLET_FILE=./wallet.json +NODE_CONFIG_ENV="development" +``` + +## Logging Configuration + +The CU uses logging levels that conform to RFC5424 severity semantics: + +``` +{ + error: 0, // Errors that must be addressed immediately + warn: 1, // Warnings that should be monitored + info: 2, // Important operational information + http: 3, // HTTP request/response details + verbose: 4, // More detailed operational information + debug: 5, // Debugging information + silly: 6 // Very detailed debugging information +} +``` + +You can set the logging level with the `DEFAULT_LOG_LEVEL` environment variable or dynamically change it by creating or modifying a `.loglevel` file in the working directory. + +## Applying Configuration Changes + +For most configuration changes to take effect, you need to restart the Compute Unit service. When running in development mode with hot reloading, some configuration changes may require a full restart. \ No newline at end of file diff --git a/docs/misc/components-compute-unit/index.md b/docs/misc/components-compute-unit/index.md new file mode 100644 index 000000000..024e2bbac --- /dev/null +++ b/docs/misc/components-compute-unit/index.md @@ -0,0 +1,66 @@ +# Compute Unit Overview + +The ao Compute Unit (CU) is a spec-compliant implementation built with NodeJS that serves as the computational processing component in the ao ecosystem, handling WASM execution and state management. + +## What is the Compute Unit? + +The Compute Unit is responsible for executing WebAssembly modules and handling computational tasks within the ao ecosystem. It works in conjunction with HyperBEAM but runs as a separate process, providing the actual execution environment for ao processes. + +Key responsibilities include: +- Executing WebAssembly modules +- Managing process memory and state +- Handling process checkpointing +- Processing evaluation requests from HyperBEAM + +## Architecture + +The Compute Unit follows a Ports and Adapters Architecture (also known as Hexagonal Architecture): + +- **Business Logic**: Located in `src/domain`, this contains all core functionality +- **Driven Adapters**: Located in `effects`, these implement contracts for various platforms +- **Driving Adapter**: Also in `effects`, this exposes the public API + +This architecture separates business logic from external interfaces, making the system more maintainable and testable. + +## Project Structure + +- **domain**: Contains all business logic and public APIs + - **api**: Implements public interfaces + - **lib**: Contains business logic components + - **dal.js**: Defines contracts for driven adapters + +- **effects**: Contains implementations of external interfaces + - **ao-http**: Exposes the HTTP API consumed by other ao units + +## Technical Requirements + +The Compute Unit requires: + +- Node.ja +- Access to local file system for state persistence +- Network access to communicate with HyperBEAM +- An Arweave wallet for identity + +### System Requirements + +The ao Compute Unit is a stateless application that can be deployed to any containerized environment using its Dockerfile or directly with Node.js. It requires: + +- A containerization environment or Node.js runtime +- A filesystem to store files and an embedded database +- Ingress capability from the Internet +- Egress capability to other ao units and the Internet + +## Key Features + +- **WASM Execution**: Executes WebAssembly modules for ao processes +- **State Management**: Maintains process memory and state +- **Checkpointing**: Creates and manages checkpoints of process state +- **Configurable Limits**: Memory and compute limits can be adjusted +- **Event-Driven Architecture**: Processes messages asynchronously +- **Robust Logging**: Comprehensive logging with configurable levels + +## Next Steps + +- [Setup](setup.md): Learn how to install and run the Compute Unit +- [Configuration](configuration.md): Understand available configuration options +- [API Reference](api.md): Explore the Compute Unit's API \ No newline at end of file diff --git a/docs/misc/components-compute-unit/setup.md b/docs/misc/components-compute-unit/setup.md new file mode 100644 index 000000000..0bfce69e9 --- /dev/null +++ b/docs/misc/components-compute-unit/setup.md @@ -0,0 +1,144 @@ +# **Local CU Setup** + +This guide explains how to set up the local Compute Unit (CU) for HyperBEAM. + +## What is Local CU? + +The ao Compute Unit (CU) is a spec-compliant implementation built with NodeJS. It serves as the computational processing component in the ao ecosystem, handling WASM execution and state management. + +## Prerequisites + +* Node.js +* Git + +## Installation Steps + +### 1. Clone the Repository + +```bash +git clone https://github.com/permaweb/local-cu +cd local-cu +``` + +### 2. Install Dependencies + +```bash +npm i +``` + +### 3. Generate a Wallet (if needed) + +If you don't already have an Arweave wallet, you can generate one: + +```bash +npx --yes @permaweb/wallet > wallet.json +``` + +### 4. Configure Environment + +Create a `.env` file with the following minimal configuration: + +```bash +UNIT_MODE=hbu +HB_URL=http://localhost:10000 +PORT=6363 +WALLET_FILE=./wallet.json +NODE_CONFIG_ENV="development" +``` + +### 5. Start the Compute Unit + +```bash +npm start +``` + +For development with hot-reloading, you can use: + +```bash +npm run dev +``` + +## Environment Variables + +The CU supports numerous environment variables for configuration. Here are the key ones: + +### Essential Configuration + +* **WALLET/WALLET_FILE**: The JWK Interface stringified JSON or a file to load it from +* **PORT**: Which port the web server should listen on (defaults to 6363) +* **UNIT_MODE**: Set to "hbu" for HyperBEAM mode +* **HB_URL**: URL of your HyperBEAM instance + +### Gateway Configuration + +* **GATEWAY_URL**: The URL of the Arweave gateway (defaults to https://arweave.net) +* **ARWEAVE_URL**: The URL for the Arweave HTTP API (defaults to GATEWAY_URL) +* **GRAPHQL_URL**: The URL for the Arweave GraphQL server (defaults to ${GATEWAY_URL}/graphql) +* **UPLOADER_URL**: The URL of the uploader for Process Checkpoints (defaults to up.arweave.net) + +### Process Limits + +* **PROCESS_WASM_MEMORY_MAX_LIMIT**: Maximum memory limit for processes (defaults to 1GB) +* **PROCESS_WASM_COMPUTE_MAX_LIMIT**: Maximum compute limit for processes (defaults to 9 billion) +* **PROCESS_WASM_SUPPORTED_FORMATS**: Supported wasm module formats (comma-delimited) +* **PROCESS_WASM_SUPPORTED_EXTENSIONS**: Supported wasm extensions (comma-delimited) + +### Caching and Performance + +* **WASM_EVALUATION_MAX_WORKERS**: Number of workers for evaluating messages (defaults to CPU count - 1) +* **WASM_BINARY_FILE_DIRECTORY**: Directory to cache wasm binaries downloaded from Arweave +* **WASM_MODULE_CACHE_MAX_SIZE**: Maximum size of the in-memory wasm module cache +* **WASM_INSTANCE_CACHE_MAX_SIZE**: Maximum size of the in-memory wasm instance cache +* **PROCESS_MEMORY_CACHE_MAX_SIZE**: Maximum size of the process memory cache +* **PROCESS_MEMORY_CACHE_TTL**: Time-to-live for process memory cache entries + +### Checkpoint Configuration + +* **PROCESS_MEMORY_CACHE_CHECKPOINT_INTERVAL**: Interval for checkpointing processes (0 to disable) +* **PROCESS_CHECKPOINT_CREATION_THROTTLE**: Time to wait before creating another checkpoint for a process +* **DISABLE_PROCESS_CHECKPOINT_CREATION**: Whether to disable process checkpoint uploads to Arweave +* **DISABLE_PROCESS_FILE_CHECKPOINT_CREATION**: Whether to disable process checkpoint creation to the filesystem +* **EAGER_CHECKPOINT_ACCUMULATED_GAS_THRESHOLD**: Gas threshold for immediate checkpoint creation + +### Database Configuration + +* **DB_MODE**: Whether the database is embedded or remote (defaults to embedded) +* **DB_URL**: The name of the embedded database (defaults to ao-cache) + +### Other Settings + +* **ENABLE_METRICS_ENDPOINT**: Whether to enable the OpenTelemetry /metrics endpoint +* **DEFAULT_LOG_LEVEL**: The logging level to use (defaults to debug) +* **LOG_CONFIG_PATH**: Path to the file used to dynamically set logging level +* **BUSY_THRESHOLD**: Wait time before sending a "busy" response to clients +* **RESTRICT_PROCESSES**: List of process IDs to restrict (blacklist) +* **ALLOW_PROCESSES**: List of process IDs to allow (whitelist) +* **ALLOW_OWNERS**: List of process owners to allow (owner whitelist) + + +## Verification + +To verify that your CU is running correctly, you can check: + +```bash +curl http://localhost:6363 +``` + +You should receive a response confirming the CU is operational. + +### Dynamically Change Log Level + +If you need to change the log level while the CU is running, you can create or modify a `.loglevel` file in the working directory with the desired level (error, warn, info, http, verbose, debug, silly). + +### Manually Trigger Checkpointing + +To manually trigger checkpointing for all processes in memory: + +1. Obtain the process ID of the CU: `pgrep node` or `lsof -i :6363` +2. Send a SIGUSR2 signal: `kill -USR2 ` + +This will cause the CU to checkpoint all processes in its in-memory cache. + +## Next Steps + +After setting up the Compute Unit, see the [Configuration](configuration.md) page for more detailed configuration options and the [API Reference](api.md) for information on interacting with the CU. diff --git a/docs/gathering-metrics-locally.md b/docs/misc/gathering-metrics-locally.md similarity index 100% rename from docs/gathering-metrics-locally.md rename to docs/misc/gathering-metrics-locally.md diff --git a/docs/misc/getting-started-hyperpaths/index.md b/docs/misc/getting-started-hyperpaths/index.md new file mode 100644 index 000000000..ced7c14f3 --- /dev/null +++ b/docs/misc/getting-started-hyperpaths/index.md @@ -0,0 +1,28 @@ +# HyperPATHs + +## Overview + +HyperPATHs provides a comprehensive set of HTTP endpoints for interacting with HyperBEAM nodes and accessing process data. This section covers various ways to extract value and interact with HyperBEAM through HTTP requests. + +## Key Concepts + +- **HTTP Endpoints**: Access process data and node information through standardized HTTP endpoints +- **Cryptographic Signatures**: All responses are cryptographically signed for data integrity +- **State Management**: Various methods for accessing and updating process states +- **Node Interaction**: Tools for interacting with HyperBEAM nodes + +## Best Practices + +1. Always verify cryptographic signatures on responses +2. Use appropriate caching strategies for frequently accessed data +3. Implement proper error handling for network requests +4. Consider rate limits and performance implications +5. Keep sensitive data secure and use appropriate authentication methods + +## Common Use Cases + +- Real-time process state monitoring +- Data synchronization between processes +- Building web interfaces for AO processes +- Automated process management and monitoring +- Cross-process communication and data sharing \ No newline at end of file diff --git a/docs/hacking-on-hyperbeam.md b/docs/misc/hacking-on-hyperbeam.md similarity index 83% rename from docs/hacking-on-hyperbeam.md rename to docs/misc/hacking-on-hyperbeam.md index 5874de958..6df0f5a9d 100644 --- a/docs/hacking-on-hyperbeam.md +++ b/docs/misc/hacking-on-hyperbeam.md @@ -53,16 +53,16 @@ would like to analyze would HB's path management `dev_message` modules are doing while you run your tests, just execute: ``` - HB_PRINT=hb_path,hb_converge,converge_result rebar3 eunit --module=your_mod + HB_PRINT=hb_path,hb_ao,ao_result rebar3 eunit --module=your_mod ``` Some useful logging events are: ``` - converge_result: Outputs every Converge computation result (M1, M2, and M3). + ao_result: Outputs every AO-Core computation result (M1, M2, and M3). worker: Information about spawns and registrations of worker processes. - converge_core: Output information about progression through the core - computation loop of Converge. Produces a large volume of output. + ao_core: Output information about progression through the core + computation loop of AO-Core. Produces a large volume of output. ``` The HB printing system is reasonably intelligent. It has a custom @@ -92,4 +92,15 @@ If you would like to re-build HyperBEAM in-place, while it is running, just run `hb:build()`. This will invoke `rebar3` and build any modules changed since the last invocation. +## Profiling with eflame (building flamecharts) + +1. Benchmark a given function (in Erlang shell) with + `eflame:apply(hb, address, []).` + This will store traces under "stacks.out" in the project folder. + +2. Convert stacks.out into svg file (in shell): + `_build/default/lib/eflame/stack_to_flame.sh < stacks.out > hb_address_flame.svg` + +3. Open the svg file in browser. + Happy hacking! \ No newline at end of file diff --git a/docs/misc/index.md b/docs/misc/index.md new file mode 100644 index 000000000..9d7b25a59 --- /dev/null +++ b/docs/misc/index.md @@ -0,0 +1,10 @@ +# Guides Overview + +This section provides practical guides and tutorials for working with HyperBEAM and the Compute Unit. Whether you're just getting started or looking to build more advanced applications, these guides will help you get the most out of the platform. + +!!! warning + These guides are currently a work in progress. More detailed tutorials and examples will be added soon. + +## Available Guides + +- [HyperBEAM JavaScript Client Guide](js-client-guide.md): Learn how to interact with HyperBEAM nodes using the an example JavaScript client library. diff --git a/docs/misc/installation-core/hyperbeam-setup-config/api.md b/docs/misc/installation-core/hyperbeam-setup-config/api.md new file mode 100644 index 000000000..85c7e8f0f --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/api.md @@ -0,0 +1,152 @@ +# HyperBEAM API + +This document describes the HTTP API exposed by HyperBEAM for interacting with devices and processes. + +## API Overview + +HyperBEAM's API is built around the concept of messages and devices. Every interaction with HyperBEAM is done by sending HTTP requests to specific device endpoints. + +## Message Format + +In HyperBEAM, every piece of data is described as a message, which can be interpreted as a binary term or as a collection of named functions (a Map of functions). + +HTTP messages in HyperBEAM follow a standard format based on HTTP semantics as described in RFC specifications. + +## Core API Endpoints + +### Metadata Device (~meta@1.0) + +The ~meta@1.0 device provides information about the node and allows configuration changes. + +#### Get Node Information + +``` +GET /~meta@1.0/info +``` + +**Response**: Information about the node configuration, supported devices, and other metadata. + +#### Update Node Configuration + +``` +POST /~meta@1.0/info +Your-Config-Tag: Your-Config-Value +``` + +**Description**: Update the node's configuration. The headers provided in the message are interpreted as configuration options. + +### Relay Device (~relay@1.0) + +The ~relay@1.0 device is used to relay messages between nodes and the wider HTTP network. + +#### Send Message + +``` +POST /~relay@1.0/send +Content-Type: application/json + +{ + "destination": "target-process-id", + "message": "Your message content" +} +``` + +**Description**: Sends a message to the specified destination. + +### Process Device (~process@1.0) + +The ~process@1.0 device enables the creation and management of persistent, shared executions. + +#### Create Process + +``` +POST /~process@1.0/create +Content-Type: application/json + +{ + "module": "module-id", + "scheduler": "scheduler-id", + "parameters": {} +} +``` + +**Description**: Creates a new AO process with the specified module and scheduler. + +#### Push Message to Process + +``` +POST /~process@1.0/push +Content-Type: application/json + +{ + "process": "process-id", + "message": "Your message content" +} +``` + +**Description**: Pushes a message to the specified process's execution outbox. + +### WebAssembly Device (~wasm64@1.0) + +The ~wasm64@1.0 device executes WebAssembly code using the Web Assembly Micro-Runtime (WAMR). + +#### Execute WASM Module + +``` +POST /~wasm64@1.0/execute +Content-Type: application/json + +{ + "module": "module-id", + "function": "function-name", + "parameters": {} +} +``` + +**Description**: Executes the specified function in the WASM module with the provided parameters. + +## API Authentication + +Some API endpoints require authentication using a signed message. This is done using the RFC-9421 standard for HTTP Message Signatures. + +## Example Usage + +### Getting Node Information + +```bash +curl http://localhost:10000/~meta@1.0/info +``` + +### Creating a Process + +```bash +curl -X POST http://localhost:10000/~process@1.0/create \ + -H "Content-Type: application/json" \ + -d '{"module": "module-id", "scheduler": "scheduler-id"}' +``` + +### Sending a Message to a Process + +```bash +curl -X POST http://localhost:10000/~process@1.0/push \ + -H "Content-Type: application/json" \ + -d '{"process": "process-id", "message": "Hello, World!"}' +``` + +## Error Handling + +HyperBEAM returns standard HTTP status codes to indicate the success or failure of API requests. Common error codes include: + +- **400 Bad Request**: The request was malformed or invalid +- **401 Unauthorized**: Authentication is required +- **403 Forbidden**: The request is not allowed +- **404 Not Found**: The requested resource does not exist +- **500 Internal Server Error**: An error occurred on the server + +Error responses include JSON payloads with more detailed information about the error. + +## API Limitations + +- API rate limits may be enforced depending on node configuration +- Some operations may require payment depending on the pricing device configuration +- Large messages may be rejected depending on node configuration \ No newline at end of file diff --git a/docs/misc/installation-core/hyperbeam-setup-config/configuration.md b/docs/misc/installation-core/hyperbeam-setup-config/configuration.md new file mode 100644 index 000000000..93c452d42 --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/configuration.md @@ -0,0 +1,57 @@ +# HyperBEAM Configuration + +HyperBEAM can be configured using a variety of methods and options. This document provides an overview of the configuration system and links to specialized configuration topics. + +## Configuration System Overview + +HyperBEAM is a highly configurable node runtime for decentralized applications. Its configuration system allows operators to: + +- Define connection parameters +- Set up storage backends +- Configure routing rules +- Control execution behavior +- Optimize performance characteristics +- Enable debugging features + +## Configuration Documentation Sections + +For detailed information about specific aspects of HyperBEAM configuration, please refer to the following documentation: + +- [Configuration Methods](./configuration/configuration-methods.md) - Different ways to configure HyperBEAM +- [Configuration Options](./configuration/configuration-options.md) - Complete reference of all configuration options +- [Storage Configuration](./configuration/storage-configuration.md) - Setting up file systems, RocksDB, and other storage backends +- [Routing Configuration](./configuration/routing-configuration.md) - Configuring request routing and connectivity +- [Configuration Examples](./configuration/configuration-examples.md) - Common deployment scenarios and sample configurations + +## Getting Started + +If you're new to HyperBEAM, we recommend starting with a basic configuration file: + +1. Create a file named `config.flat` in your project directory +2. Add basic configuration: + ``` + port: 10000 + priv-key-location: /path/to/wallet.key + mode: debug + ``` +3. Start HyperBEAM with `rebar3 shell` + +HyperBEAM will automatically load your configuration and display the active settings in the startup log. + +## Core Configuration Priorities + +When multiple configuration methods are used simultaneously, HyperBEAM follows this precedence order: + +1. Environment variables (highest precedence) +2. Runtime configuration via HTTP +3. Command line arguments +4. Configuration file +5. Default values (lowest precedence) + +See [Configuration Methods](configuration-methods.md) for more details on these approaches. + +## Configuration Resources + +- [HyperBEAM GitHub Repository](https://github.com/hyperbeam-core/hyperbeam) +- [Quick Start Guide](../getting-started.md) +- [API Reference](../api-reference.md) diff --git a/docs/misc/installation-core/hyperbeam-setup-config/configuration/configuration-examples.md b/docs/misc/installation-core/hyperbeam-setup-config/configuration/configuration-examples.md new file mode 100644 index 000000000..5c78c856b --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/configuration/configuration-examples.md @@ -0,0 +1,49 @@ +# HyperBEAM Configuration Examples + +This document provides complete, practical configuration examples for common HyperBEAM deployment scenarios. Each example includes explanations and can be used as a starting point for your own configuration. + +## Basic Development Configuration + +This is a simple configuration for local development with debugging enabled: + +### Simple Options in config.flat + +For basic options, you can use a config.flat file: + +``` +port: 10000 +mode: debug +priv_key_location: ./wallet.json + +``` + +### Complete Configuration Using start_mainnet + +For a complete configuration including storage: + +```bash +rebar3 shell --eval " + hb:start_mainnet(#{ + port => 10001, + mode => debug, + priv_key_location => <<\"./wallet.json\">>, + + http_extra_opts => + #{ + force_message => true, + store => [{hb_store_fs, #{ prefix => \"local-cache\" }}, {hb_store_gateway, #{}}], + cache_control => [<<\"always\">>] + } + }). +" +``` + +**Key features**: + +- Development mode enabled +- Simple file system storage +- Extensive debugging options +- Local port 10001 + +!!! Note + More examples to come \ No newline at end of file diff --git a/docs/misc/installation-core/hyperbeam-setup-config/configuration/configuration-methods.md b/docs/misc/installation-core/hyperbeam-setup-config/configuration/configuration-methods.md new file mode 100644 index 000000000..b89890bfe --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/configuration/configuration-methods.md @@ -0,0 +1,119 @@ +# HyperBEAM Configuration Methods + +HyperBEAM offers multiple ways to set configuration options, each with different use cases. This document details these methods and explains when to use each one. + +## Available Configuration Methods + +HyperBEAM can be configured using these methods: + +1. **Configuration File** - Use a flat@1.0 encoded settings file +2. **Command Line Arguments** - Pass configuration when starting HyperBEAM **(Recommended)** +3. **Environment Variables** - Set options via environment variables + +!!! warning + The current flat@1.0 format has limitations in HyperBEAM. For now, it is recommended to use the `start_mainnet` approach for configuration. We plan to update config.flat in the future to allow for more complex configuration options. + +## Using a Configuration File + +The recommended way to configure HyperBEAM is through a flat@1.0 encoded settings file. By default, HyperBEAM looks for a file named `config.flat` in the current directory. + +### Basic Syntax + +The configuration file uses a simple `key: value` format: + +``` +port: 10000 +cache_lookup_hueristics: true +priv_key_location: /path/to/wallet.json +``` + +### Limitations of flat@1.0 Format + +**Important:** The flat@1.0 format has significant limitations in HyperBEAM: + +- Values can only be simple atoms (like `true`, `false`, `./wallet.json`) +- **DO NOT include complex data structures** (maps, lists, tuples) in the config.flat file. +- Attempting to use complex data in config.flat may result in parsing errors or silently failing configurations. + +For any configurations with complex data types, you **must** use `hb:start_mainnet/1` directly instead (see Command Line Arguments section). + +### Appropriate Values for config.flat + +In your config.flat file, stick to these types of values: + +``` +mode: debug +priv_key_location: /path/to/wallet.key +port: 10000 +``` + +### Complex Data Types - NOT for config.flat + +For complex structures like maps and lists, **do not use config.flat**. Instead, use the `hb:start_mainnet/1` approach: + +```bash +rebar3 shell --eval " + hb:start_mainnet(#{ + port => 10001, + http_extra_opts => #{ + force_message => true, + store => [{hb_store_fs, #{ prefix => \"local-cache\" }}, {hb_store_gateway, #{}}], + cache_control => [<<\"always\">>] + } + }). +" +``` + +### Loading Configuration Files + +HyperBEAM automatically loads `config.flat` when starting: + +```bash +rebar3 shell +``` + +## Command Line Arguments + +You can pass configuration options directly when starting HyperBEAM: + +```bash +rebar3 shell --eval "hb:start_mainnet(#{ port => 10001, priv_key_location => <<\"path/to/wallet.json\">> })." +``` + +### Recommended Approach for Complex Configurations + +Using `hb:start_mainnet/1` with a map of options is the **recommended approach** for any non-trivial configuration: + +- **Full Type Support**: You can use any Erlang data type directly, not just atoms and binaries. +- **Complex Data Structures**: Maps, lists, tuples, and other complex structures work without limitations. +- **Direct Validation**: Configuration errors are caught immediately at startup. +- **Runtime Flexibility**: Options can be computed or combined with other configurations at runtime. + +This approach is recommended for: + +- Any configurations with maps, lists, or other complex data structures +- Routing and storage configurations +- Testing different configurations without editing files +- Production deployments where reliability is critical + +## Environment Variables + +HyperBEAM recognizes these environment variables: + +| Variable | Corresponding Option | Example | +|----------|----------------------|---------| +| `HB_PORT` | `port` | `export HB_PORT=9001` | +| `HB_KEY` | `priv_key_location` | `export HB_KEY=/path/to/wallet.key` | +| `HB_CONFIG` | `hb_config_location` | `export HB_CONFIG=config.flat` | +| `HB_STORE` | `store` | `export HB_STORE=/path/to/store` | +| `HB_MODE` | `mode` | `export HB_MODE=debug` | +| `HB_PRINT` | `debug_print` | `export HB_PRINT=dev_meta` | + +## Configuration Precedence + +When multiple configuration methods are used, HyperBEAM follows this precedence order: + +1. Command line arguments (highest priority) +2. Configuration file +3. Environment variables +4. Default values (lowest priority) diff --git a/docs/misc/installation-core/hyperbeam-setup-config/configuration/configuration-options.md b/docs/misc/installation-core/hyperbeam-setup-config/configuration/configuration-options.md new file mode 100644 index 000000000..4ac2100c2 --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/configuration/configuration-options.md @@ -0,0 +1,113 @@ +# HyperBEAM Configuration Options Reference + +This document provides a comprehensive reference of all configuration options available in HyperBEAM, organized by functional category. + +## Core Configuration + +These options control fundamental HyperBEAM behavior. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `port` | Integer | 8734 | HTTP API port | +| `hb_config_location` | String | "config.flat" | Path to configuration file | +| `priv_key_location` | String | "hyperbeam-key.json" | Path to operator wallet key file | +| `mode` | Atom | debug | Execution mode (debug, prod) | + +## Server & Network Configuration + +These options control networking behavior and HTTP settings. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `host` | String | "localhost" | Choice of remote node for non-local tasks | +| `gateway` | String | "https://arweave.net" | Default gateway | +| `bundler_ans104` | String | "https://up.arweave.net:443" | Location of ANS-104 bundler | +| `protocol` | Atom | http2 | Protocol for HTTP requests (http1, http2, http3) | +| `http_client` | Atom | gun | HTTP client to use (gun, httpc) | +| `http_connect_timeout` | Integer | 5000 | HTTP connection timeout in milliseconds | +| `http_keepalive` | Integer | 120000 | HTTP keepalive time in milliseconds | +| `http_request_send_timeout` | Integer | 60000 | HTTP request send timeout in milliseconds | +| `relay_http_client` | Atom | httpc | HTTP client for the relay device | +| `http_extra_opts` | Map | See below | Additional HTTP options | + +### http_extra_opts Subcomponents + +| Subcomponent | Type | Default | Description | +|--------------|------|---------|-------------| +| `force_message` | Boolean | true | Whether to force a message format | +| `store` | List | See [Storage Configuration](storage-configuration.md) | Storage backends and their configurations | +| `cache_control` | List | [<<"always">>] | Cache control directives for HTTP requests | + +## Security & Identity + +These options control identity and security settings. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `trusted_device_signers` | List | [] | List of device signers the node should trust | +| `trusted` | Map | {} | Trusted entities | +| `scheduler_location_ttl` | Integer | 604800000 | TTL for scheduler registration (7 days in ms) | + +## Caching & Storage + +These options control caching behavior. For detailed storage configuration, see [Storage Configuration](storage-configuration.md). + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `cache_lookup_hueristics` | Boolean | false | Whether to use caching heuristics or always consult the local data store | +| `access_remote_cache_for_client` | Boolean | false | Whether to access data from remote caches for client requests | +| `store_all_signed` | Boolean | true | Whether the node should store all signed messages | +| `await_inprogress` | Atom/Boolean | named | Whether to await in-progress executions (false, named, true) | +| `cache_control` | List | ["no-cache", "no-store"] | Default cache control headers | + +## Execution & Processing + +These options control how HyperBEAM executes messages and processes. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `scheduling_mode` | Atom | local_confirmation | When to inform recipients about scheduled assignments (aggressive, local_confirmation, remote_confirmation) | +| `compute_mode` | Atom | lazy | Whether to execute more messages after returning a result (aggressive, lazy) | +| `process_workers` | Boolean | true | Whether the node should use persistent processes | +| `client_error_strategy` | Atom | throw | What to do if a client error occurs | +| `wasm_allow_aot` | Boolean | false | Allow ahead-of-time compilation for WASM | + +## Device Management + +These options control how HyperBEAM manages devices. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `preloaded_devices` | Map | (see code) | Devices for the node to use, overriding resolution via ID | +| `load_remote_devices` | Boolean | false | Whether to load devices from remote signers | +| `devices` | List | [] | Additional devices to load | + +## Routing & Connectivity + +See [Routing Configuration](routing-configuration.md) for detailed information on routing options. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `routes` | List | See Routing docs | Routing configuration for different request patterns | + +## Debug & Development + +These options control debugging and development features. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `debug_print` | Boolean/List | false | Debug printing control | +| `debug_stack_depth` | Integer | 40 | Maximum stack depth for debug printing | +| `debug_print_map_line_threshold` | Integer | 30 | Maximum lines for map printing | +| `debug_print_binary_max` | Integer | 60 | Maximum binary size for debug printing | +| `debug_print_indent` | Integer | 2 | Indentation for debug printing | +| `debug_print_trace` | Atom | short | Trace mode (short, false) | +| `short_trace_len` | Integer | 5 | Length of short traces | +| `debug_hide_metadata` | Boolean | true | Whether to hide metadata in debug output | +| `debug_ids` | Boolean | false | Whether to print IDs in debug output | +| `debug_hide_priv` | Boolean | true | Whether to hide private data in debug output | +| `stack_print_prefixes` | List | ["hb", "dev", "ar"] | Prefixes for stack printing | + +## Complete Option List + +For the most up-to-date list of configuration options, refer to the `default_message/0` function in the `hb_opts` module in the HyperBEAM source code. \ No newline at end of file diff --git a/docs/misc/installation-core/hyperbeam-setup-config/configuration/index.md b/docs/misc/installation-core/hyperbeam-setup-config/configuration/index.md new file mode 100644 index 000000000..c46b0e4ad --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/configuration/index.md @@ -0,0 +1,83 @@ +# HyperBEAM Configuration + +HyperBEAM can be configured using a variety of methods and options. This document provides an overview of the configuration system and links to specialized configuration topics. + +## Configuration System Overview + +HyperBEAM is a highly configurable node runtime for decentralized applications. Its configuration system allows operators to: + +- Define connection parameters +- Set up storage backends +- Configure routing rules +- Control execution behavior +- Optimize performance characteristics +- Enable debugging features + +## Configuration Documentation Sections + +For detailed information about specific aspects of HyperBEAM configuration, please refer to the following documentation: + +- [Configuration Methods](configuration-methods.md) - Different ways to configure HyperBEAM +- [Configuration Options](configuration-options.md) - Complete reference of all configuration options +- [Storage Configuration](storage-configuration.md) - Setting up file systems, RocksDB, and other storage backends +- [Routing Configuration](routing-configuration.md) - Configuring request routing and connectivity +- [Configuration Examples](configuration-examples.md) - Common deployment scenarios and sample configurations + +## Getting Started + +If you're new to HyperBEAM, you can start with a simple configuration file for basic settings: + +1. Create a file named `config.flat` in your project directory +2. Add **only** simple configuration values: + ``` + port: 10000 + priv_key_location: /path/to/wallet.json + ``` +3. Start HyperBEAM with `rebar3 shell` + +HyperBEAM will automatically load your configuration and display the active settings in the startup log. + +### Configuration File Limitations - IMPORTANT + +The flat@1.0 format used by `config.flat` has critical limitations: + +- **ONLY use simple atom values and binary values** +- **DO NOT include maps, lists, or any complex data structures** in config.flat +- Complex configurations in config.flat will either fail to parse or silently fail to apply correctly + +### Recommended Configuration Approach + +For any non-trivial configuration, especially those with complex data types, use the Erlang API directly: + +```bash +rebar3 shell --eval "hb:start_mainnet(#{ + port => 10001, + priv_key_location => <<\"./wallet.json\">>, + mode => debug, + + http_extra_opts => #{ + force_message => true, + store => {hb_store_fs, #{ prefix => \"local-storage\" }} + cache_control => [<<\"always\">>] + } +})." +``` + +This method allows you to use any Erlang data type and structure without limitations and is the **recommended approach for production deployments**. + +## Core Configuration Priorities + +When multiple configuration methods are used simultaneously, HyperBEAM follows this precedence order: + +1. Command line arguments (highest priority) +2. Configuration file +3. Environment variables +4. Default values (lowest priority) + +See [Configuration Methods](configuration-methods.md) for more details on these approaches. + +## Configuration Resources + +- [HyperBEAM GitHub Repository](https://github.com/permaweb/HyperBEAM) +- [Quick Start Guide](../../getting-started.md) +- [API Reference](../../api-reference.md) \ No newline at end of file diff --git a/docs/misc/installation-core/hyperbeam-setup-config/configuration/routing-configuration.md b/docs/misc/installation-core/hyperbeam-setup-config/configuration/routing-configuration.md new file mode 100644 index 000000000..a2f4a4359 --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/configuration/routing-configuration.md @@ -0,0 +1,64 @@ +# HyperBEAM Routing Configuration + +This document explains how to configure routing in HyperBEAM. + +## Routing System Overview + +HyperBEAM's routing system directs incoming requests to appropriate destinations based on path patterns. This allows you to route specific request patterns to different servers. + +## Routes Configuration Structure + +The `routes` configuration option accepts a list of route definitions. Each route is a map with the following components: + +| Component | Type | Description | +|-----------|------|-------------| +| `template` | Binary | Path pattern to match against incoming requests | +| `node` | Map | Single destination configuration | +| `nodes` | List | List of alternative destinations for load balancing | + +## Configuring Routes + +Due to the complex nature of routing configuration, you **must** use the `hb:start_mainnet/1` approach rather than config.flat. + +### Default Routes Example + +You can define multiple routes in order of priority: + +```bash +rebar3 shell --eval " + hb:start_mainnet(#{ + routes => [ + #{ + <<\"template\">> => <<\"/result/.*\">>, + <<\"node\">> => #{ <<\"prefix\">> => <<\"http://localhost:6363\">> } + }, + #{ + <<\"template\">> => <<\"/graphql\">>, + <<\"nodes\">> => + [ + #{ + <<\"prefix\">> => <<\"https://arweave-search.goldsky.com\">>, + <<\"opts\">> => #{ http_client => httpc } + }, + #{ + <<\"prefix\">> => <<\"https://arweave.net\">>, + <<\"opts\">> => #{ http_client => gun } + } + ] + }, + #{ + <<\"template\">> => <<\"/raw\">>, + <<\"node\">> => + #{ + <<\"prefix\">> => <<\"https://arweave.net\">>, + <<\"opts\">> => #{ http_client => gun } + } + } + ] + }). +" +``` + +## Route Order Importance + +Routes are evaluated in the order they appear in the configuration. When multiple routes could match a request, the first matching route in the list is used. Place more specific routes before general ones. diff --git a/docs/misc/installation-core/hyperbeam-setup-config/configuration/storage-configuration.md b/docs/misc/installation-core/hyperbeam-setup-config/configuration/storage-configuration.md new file mode 100644 index 000000000..5f33f875c --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/configuration/storage-configuration.md @@ -0,0 +1,58 @@ +# HyperBEAM Storage Configuration + +This document provides a basic overview of storage configuration in HyperBEAM. + +## Storage Backend Overview + +HyperBEAM supports multiple storage backends that can be used individually or in combination. When multiple backends are specified, HyperBEAM tries each in sequence until the requested data is found. + +## Configuring Storage Backends + +Storage backends are configured through the `http_extra_opts.store` setting. Due to the complexity of storage configuration, you **must** use the `hb:start_mainnet/1` approach rather than config.flat. + +## Basic Storage Configuration Examples + +### Simple File System Storage + +The most basic storage configuration uses the file system: + +```bash +rebar3 shell --eval " + hb:start_mainnet(#{ + http_extra_opts => #{ + force_message => true, + store => {hb_store_fs, #{ prefix => \"local-storage\" }} + cache_control => [<<\"always\">>] + } + }). +" +``` + +This configuration stores data in a directory named "local-storage". + +### Gateway Fallback Configuration + +To use local storage with a gateway fallback: + +```bash +rebar3 shell --eval " + hb:start_mainnet(#{ + http_extra_opts => #{ + force_message => true, + store => [{hb_store_fs, #{ prefix => \"mainnet-cache\" }}, {hb_store_gateway, #{}}], + cache_control => [<<\"always\">>] + } + }). +" +``` + +This configuration first looks for data in the local file system, then falls back to the Arweave gateway if not found locally. + +## Available Storage Backends + +HyperBEAM includes these storage backends: + +1. **File System Store (hb_store_fs)** - Uses the local file system +2. **RocksDB Store (hb_store_rocksdb)** - Uses RocksDB for efficient key-value storage +3. **Gateway Store (hb_store_gateway)** - Reads data from the Arweave gateway +4. **Remote Node Store (hb_store_remote_node)** - Reads data from another HyperBEAM node diff --git a/docs/misc/installation-core/hyperbeam-setup-config/index.md b/docs/misc/installation-core/hyperbeam-setup-config/index.md new file mode 100644 index 000000000..a698f0d54 --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/index.md @@ -0,0 +1,62 @@ +# HyperBEAM Overview + +HyperBEAM is a client implementation of the AO-Core protocol, written in Erlang. It serves as the 'node' software for the decentralized operating system that AO enables, abstracting hardware provisioning and details from the execution of individual programs. + +HyperBEAM node operators can offer the services of their machine to others inside the network by electing to execute any number of different devices, charging users for their computation as necessary. + +## Key Features + +- **Decentralized Execution**: Run AO processes in a decentralized manner +- **Message Passing**: Communicate between processes via asynchronous message passing +- **Scalable Architecture**: Built on Erlang's powerful concurrency model +- **Extensible Design**: Easy to add new devices and capabilities + +## Messages in HyperBEAM + +HyperBEAM describes every piece of data as a message, which can be interpreted as a binary term or as a collection of named functions (a Map of functions). + +Key properties of messages: +- Every message may specify a device which is interpreted by the AO-Core compatible system +- Executing a named function within a message results in another message +- Messages in AO-Core always beget further messages, giving rise to a vast computational space +- Keys may be lazily evaluated, allowing for efficient computation +- If a message does not explicitly specify a device, its implied device is a `message@1.0` + +## Devices + +HyperBEAM supports numerous devices, each enabling different services. There are approximately 25 different devices included in the preloaded_devices of a HyperBEAM node. + +### Key Preloaded Devices + +- **~meta@1.0**: Used to configure the node's hardware, supported devices, metering and payments information +- **~relay@1.0**: Used to relay messages between nodes and the wider HTTP network +- **~wasm64@1.0**: Used to execute WebAssembly code via WAMR +- **~json-iface@1.0**: Provides translation between JSON-encoded and HTTP message formats +- **~compute-lite@1.0**: A lightweight WASM executor wrapper for legacy AO processes +- **~snp@1.0**: Used for Trusted Execution Environment (TEE) operations +- **~p4@1.0**: Framework for node operators to sell usage of their hardware +- **~simple-pay@1.0**: Simple pricing device for flat-fee execution +- **~process@1.0**: Enables persistent, shared executions accessible by multiple users +- **~scheduler@1.0**: Assigns linear hashpaths to executions for deterministic ordering +- **~stack@1.0**: Executes an ordered set of devices over the same inputs + +## Components + +HyperBEAM consists of several core components: + +1. **Core Runtime**: The base system that manages process execution +2. **Device Registry**: System for registering and managing devices +3. **Message Router**: Handles message passing between processes and devices +4. **API Layer**: HTTP interfaces for interacting with the system + +## System Architecture + +HyperBEAM works in conjunction with the Compute Unit (CU), which handles the actual WASM execution. Together, they form a complete execution environment for AO processes. + +Each HyperBEAM node is configured using the `~meta@1.0` device, which provides an interface for specifying the node's supported devices, metering and payments information, amongst other configuration options. + +## Next Steps + +- [Setup HyperBEAM](setup.md): Instructions for installing and running HyperBEAM +- [Configuration](configuration.md): How to configure your HyperBEAM installation +- [Testing](testing.md): Run tests to verify your installation \ No newline at end of file diff --git a/docs/misc/installation-core/hyperbeam-setup-config/setup.md b/docs/misc/installation-core/hyperbeam-setup-config/setup.md new file mode 100644 index 000000000..5d497fe90 --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/setup.md @@ -0,0 +1,124 @@ +# **HyperBEAM Repository Setup** + +This guide provides step-by-step instructions for setting up and testing HyperBEAM. + +!!! info "TEE-based Computation" + If you intend to offer TEE-based computation of AO-Core devices, please see the [HyperBEAM OS repository](https://github.com/permaweb/hb-os) for details on configuration and deployment. Additional documentation on TEE setup and configuration will be added here in future updates. + +## **Prerequisites** + +Before you begin, ensure you have the following installed: + +- The Erlang runtime (OTP 27) +- Rebar3 +- Git + +## **1. Clone the HyperBEAM Repository** + +First, clone the `permaweb/HyperBEAM` repository from GitHub: + +```bash +git clone https://github.com/permaweb/HyperBEAM +``` + +Navigate to the project directory: + +```bash +cd HyperBEAM +``` + +## **2. Compile the Code with Rebar3** + +To compile the HyperBEAM code, you'll need to use Rebar3. Run the following command to compile the project: + +```bash +rebar3 compile +``` + +This will compile the necessary code to get HyperBEAM up and running. + +## **3. Run HyperBEAM with Shell** + +Once the code is compiled, you can start the HyperBEAM shell with Rebar3: + +```bash +rebar3 shell +``` + +This will start HyperBEAM using the default configuration inside the hb_opts.erl, +which preloads all devices and sets up default stores. All of these can be configured using +the config.flat file with any overrides you specify. + +To verify that your HyperBEAM node is running correctly, you can check: + +```bash +curl http://localhost:10000/~meta@1.0/info +``` +If you receive a response with node information, your HyperBEAM installation is working properly. + +## **4. Create and Run a HyperBEAM Release** + +For a more stable setup, especially when connecting to networks like mainnet or using specific features, it's recommended to create a release. + +### **a. Configure Your Node** + +HyperBEAM uses a `config.flat` file for configuration when running as a release. A sample file is included in the repository. + +1. Locate the `config.flat` file in the root of the HyperBEAM project directory. +2. Edit the file to specify your desired settings. For example, to set the port and specify your wallet key file: + + ``` + port: 10001 + priv_key_location: /path/to/your/wallet.json + # Add other configurations as needed + ``` + Ensure the `priv_key_location` points to the correct path of your Arweave wallet key file. + +### **b. Build the Release (with Optional Profiles)** + +You can build a standard release or include specific profiles for additional features (like `genesis_wasm`, `rocksdb`, `http3`). + +To build a standard release: +```bash +rebar3 release +``` + +To build a release with specific profiles (e.g., `rocksdb`): +```bash +rebar3 as rocksdb release +``` + +This command creates a self-contained release package in the `_build/default/rel/hb` directory. + +### **c. Run the Release** + +Navigate to the release directory and start the HyperBEAM node: + +```bash +cd _build/default/rel/hb +./bin/hb console +``` +Replace `console` with `start` to run it in the background. + +!!! note "Stopping the Node" + To stop a HyperBEAM node started with `./bin/hb start`, run `./bin/hb stop` from the release directory (`_build/default/rel/hb`). If started with `./bin/hb console`, press `Ctrl+C` in the terminal to stop it. + +### **d. Verify the Release Node** + +Once the node is running, verify it by checking the meta device info endpoint. Use the port you specified in your `config.flat` (e.g., 10001): + +```bash +curl http://localhost:10001/~meta@1.0/info +``` + +If you receive a response with node information, your HyperBEAM release is configured and running correctly. + +## **Next Steps** + +After setting up HyperBEAM, you should: + +1. [Configure your installation](configuration.md) to match your requirements +2. [Run tests](testing.md) to verify everything is working correctly +3. [Connect to the Compute Unit](../compute-unit/setup.md) to complete your setup + + diff --git a/docs/misc/installation-core/hyperbeam-setup-config/testing.md b/docs/misc/installation-core/hyperbeam-setup-config/testing.md new file mode 100644 index 000000000..3105c2ce0 --- /dev/null +++ b/docs/misc/installation-core/hyperbeam-setup-config/testing.md @@ -0,0 +1,53 @@ +# Testing HyperBEAM + +This guide covers how to test your HyperBEAM installation to ensure it's working correctly. + +## Unit Tests + +HyperBEAM comes with a suite of unit tests that can be run to verify the installation and functionality. + +### Running All Tests + +To run all unit tests for HyperBEAM, use the following Rebar3 command: + +```bash +rebar3 eunit +``` + +This will execute the EUnit tests and provide the results in your terminal. + +### Running Tests for a Specific Module + +To run tests for a specific module, use the following command: + +```bash +rebar3 eunit --module dev_meta +``` + +This will run the tests for the `dev_meta` module. + +### Running a Specific Test in a Module + +To run a specific test within a module, use the `--test` flag with the module name and test function. +For example, to run the `config_test` in the `dev_meta` module: + +```bash +rebar3 eunit --test dev_meta:config_test +``` + +## Troubleshooting Failed Tests + +If tests fail, check the following: + +1. Ensure all dependencies are installed correctly +2. Verify that HyperBEAM is properly configured +3. Look for error messages in the test output +4. Examine the HyperBEAM logs for more details + +### Common Issues + +- **Connection refused**: Ensure HyperBEAM is running on the expected port +- **Authentication errors**: Check your wallet configuration +- **Device not found**: Verify the device is included in your HyperBEAM configuration + +For specific error messages, refer to the [Troubleshooting Guide](../reference/troubleshooting.md). \ No newline at end of file diff --git a/docs/misc/installation-core/system-dependencies/index.md b/docs/misc/installation-core/system-dependencies/index.md new file mode 100644 index 000000000..e33f55a32 --- /dev/null +++ b/docs/misc/installation-core/system-dependencies/index.md @@ -0,0 +1,27 @@ +# Getting Started with HyperBEAM + +This section will guide you through the process of preparing your machine with the nessary dependencies in order to install and run HyperBEAM and the Compute Unit. + +## Installation Process Overview + +Setting up HyperBEAM involves several steps: + +1. **Check System Requirements** - Ensure your hardware and operating system meet the minimum requirements +2. **Install Dependencies** - Set up the necessary system packages and dependencies +3. **Setup HyperBEAM** - Clone, Compile, Configure, and Run the HyperBEAM +4. **SetUp the Compute Unit** - Clone, Compile, Configure, and Run the Local Compute Unit + +## Before You Begin + +Before starting the installation process, make sure to: + +- Have access to a terminal/command line with administrative privileges +- Have a stable internet connection for downloading packages +- Allocate sufficient time (approximately 30-60 minutes for a complete setup) + +## Next Steps + +Once you're ready to get started: + +1. First, check the [System Requirements](requirements.md) to ensure your system is compatible +2. Then, follow the [Installation Guide](installation/index.md) to set up all required components \ No newline at end of file diff --git a/docs/misc/installation-core/system-dependencies/installation/dependencies.md b/docs/misc/installation-core/system-dependencies/installation/dependencies.md new file mode 100644 index 000000000..762603eab --- /dev/null +++ b/docs/misc/installation-core/system-dependencies/installation/dependencies.md @@ -0,0 +1,26 @@ +# **Software Dependencies** + +HyperBEAM requires several software packages to be installed on your system. This document provides instructions for installing all required dependencies. + +## Base Packages + +Install all dependencies with the following command: + +```bash +sudo apt-get update && sudo apt-get install -y --no-install-recommends \ + build-essential cmake git pkg-config ncurses-dev \ + libssl-dev sudo curl ca-certificates +``` + +The following software packages are required: + +* **build-essential**: Contains basic build tools including gcc/g++ compilers +* **cmake**: Cross-platform build system +* **git**: Version control system +* **pkg-config**: Helper tool for compiling applications and libraries +* **ncurses-dev**: Development libraries for terminal-based interfaces +* **libssl-dev**: Development libraries for SSL (Secure Sockets Layer) +* **sudo**: Allows running programs with security privileges of another user +* **curl**: Command line tool for transferring data with URL syntax +* **ca-certificates**: Common CA certificates for SSL applications + diff --git a/docs/misc/installation-core/system-dependencies/installation/erlang.md b/docs/misc/installation-core/system-dependencies/installation/erlang.md new file mode 100644 index 000000000..35bfcd434 --- /dev/null +++ b/docs/misc/installation-core/system-dependencies/installation/erlang.md @@ -0,0 +1,35 @@ +# **Installing Erlang/OTP** + +HyperBEAM is built on Erlang/OTP, so you'll need to have Erlang installed on your system. + +## Building Erlang from Source + +For the best compatibility, we recommend building Erlang from source: + +```bash +git clone https://github.com/erlang/otp.git && \ + cd otp && git checkout maint-27 && \ + ./configure --without-wx --without-debugger --without-observer --without-et && \ + make -j$(nproc) && \ + sudo make install && \ + cd .. && rm -rf otp +``` + +This will: + +1. Clone the Erlang/OTP repository +2. Checkout the maintenance branch for version 27 +3. Configure the build to exclude GUI components (reducing dependencies) +4. Build Erlang using all available CPU cores +5. Install Erlang system-wide +6. Clean up the source directory + +## Verify Installation + +You can verify your Erlang installation with: + +```bash +erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' +``` + +This should output `27`, indicating the OTP release version. diff --git a/docs/misc/installation-core/system-dependencies/installation/index.md b/docs/misc/installation-core/system-dependencies/installation/index.md new file mode 100644 index 000000000..6a10950a1 --- /dev/null +++ b/docs/misc/installation-core/system-dependencies/installation/index.md @@ -0,0 +1,30 @@ +# Installation Overview + +HyperBEAM requires several dependencies to be installed on your system. This guide will walk you through the installation process for each component. + +## Installation Order + +For the best experience, we recommend installing prerequisites in this order: + +1. System dependencies (build tools, libraries) +2. Erlang/OTP (programming language for HyperBEAM) +3. Rebar3 (Erlang build tool) +4. Node.js (required for the Compute Unit) +5. Rust (required for the dev_snp_nif) + +## Component Guides + +Follow these guides in sequence to set up your environment: + +1. [System Dependencies](dependencies.md) - Basic system packages +2. [Erlang Installation](erlang.md) - Programming language for HyperBEAM +3. [Rebar3 Installation](rebar3.md) - Build tool for Erlang +4. [Node.js Installation](nodejs.md) - Required for the Compute Unit +5. [Rust Installation](rust.md) - Required for the dev_snp_nif + +## Next Steps + +After installing all the dependencies, you can proceed to: + +- [HyperBEAM Setup](../../hyperbeam/setup.md) +- [Compute Unit Setup](../../compute-unit/setup.md) \ No newline at end of file diff --git a/docs/misc/installation-core/system-dependencies/installation/nodejs.md b/docs/misc/installation-core/system-dependencies/installation/nodejs.md new file mode 100644 index 000000000..d23b1b8b3 --- /dev/null +++ b/docs/misc/installation-core/system-dependencies/installation/nodejs.md @@ -0,0 +1,23 @@ +# **Installing Node.js** + +Node.js is required for running a legacynet compatible local Compute Unit (CU) with the delegated-compute@1.0 device, if you plan to do so. This guide covers installing Node.js on Ubuntu 22.04. + +## Installing Node.js 22.x + +We recommend using Node.js version 22.x for optimal compatibility with the CU: + +```bash +curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - && \ +sudo apt-get install -y nodejs +``` + +## Verify Installation + +Verify that Node.js and npm are installed correctly: + +```bash +node -v +npm -v +``` + +These commands should display the installed versions of Node.js and npm respectively. diff --git a/docs/misc/installation-core/system-dependencies/installation/rebar3.md b/docs/misc/installation-core/system-dependencies/installation/rebar3.md new file mode 100644 index 000000000..f9f54f76f --- /dev/null +++ b/docs/misc/installation-core/system-dependencies/installation/rebar3.md @@ -0,0 +1,34 @@ +# **Installing Rebar3** + +Rebar3 is the Erlang build tool used by HyperBEAM for compilation and dependency management. + +## Building Rebar3 from Source + +To install Rebar3: + +```bash +git clone https://github.com/erlang/rebar3.git && cd rebar3 \ + ./bootstrap && \ + sudo mv rebar3 /usr/local/bin/ && \ + cd .. && rm -rf rebar3 +``` + +This will: + +1. Clone the Rebar3 repository +2. Bootstrap Rebar3 (build it) +3. Move the executable to your system path +4. Clean up the source directory + +## Verify Installation + +You can verify your Rebar3 installation with: + +```bash +rebar3 --version +``` + +This should display the version information for Rebar3. + +Example output: +`rebar 3.24.0+build.5437.ref5495da14 on Erlang/OTP 27 Erts 15.2`. \ No newline at end of file diff --git a/docs/misc/installation-core/system-dependencies/installation/rust.md b/docs/misc/installation-core/system-dependencies/installation/rust.md new file mode 100644 index 000000000..57a1f9585 --- /dev/null +++ b/docs/misc/installation-core/system-dependencies/installation/rust.md @@ -0,0 +1,32 @@ +# **Installing Rust and Cargo** + +Rust is required for certain components in the HyperBEAM ecosystem. + +## Installing Rust + +Install Rust using rustup: + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +``` + +## Load Rust Environment + +After installation, you'll need to load the Rust environment in your current shell: + +```bash +source "$HOME/.cargo/env" +``` + +To make this permanent, add this line to your shell profile (~/.bashrc, ~/.zshrc, etc.). + +## Verify Installation + +Verify that Rust and Cargo are installed correctly: + +```bash +rustc --version +cargo --version +``` + +These commands should display the installed versions of the Rust compiler and Cargo package manager. \ No newline at end of file diff --git a/docs/misc/installation-core/system-dependencies/requirements.md b/docs/misc/installation-core/system-dependencies/requirements.md new file mode 100644 index 000000000..3252e081e --- /dev/null +++ b/docs/misc/installation-core/system-dependencies/requirements.md @@ -0,0 +1,23 @@ +# **System Requirements** + +Before installing HyperBEAM, ensure your system meets these minimum requirements: + +## Hardware Requirements + +* **CPU**: 4+ cores recommended +* **RAM**: 8GB minimum, 16GB recommended +* **Disk**: 10GB free space minimum + +## Operating System + +HyperBEAM currently supports: + +* **Ubuntu 22.04 LTS** (recommended) + +!!! note "MacOS Support" + Support for other operating systems including macOS will be added in future releases. + +## Network + +* Stable internet connection +* Ability to open and forward ports (if you plan to run a public node) \ No newline at end of file diff --git a/docs/misc/intro.md b/docs/misc/intro.md new file mode 100644 index 000000000..da667ec0f --- /dev/null +++ b/docs/misc/intro.md @@ -0,0 +1,105 @@ +# AO + +Over the years, blockchain technology and decentralized systems have advanced significantly, yet they still face several fundamental challenges. Many traditional smart contract platforms struggle with scalability, meaning they cannot process many transactions quickly. They also have high costs for executing transactions and often come with limitations on how developers can program complex applications. + +Decentralized storage solutions have made it possible to store data permanently without relying on centralized servers. +However, these solutions primarily focus on storing data, not executing programs. +This means that while they provide reliable storage, they lack the computational layer necessary to create fully autonomous applications that can operate without ongoing human intervention. + +AO is a decentralized computing system built on Arweave, which provides permanent storage. +Unlike traditional blockchain platforms, AO is designed as a decentralized supercomputer capable of running thousands of processes in parallel. +This creates what is known as a hyper-parallel processing engine, allowing computations to happen at a much larger scale and with greater efficiency. + +AO is built to support autonomous agents—self-executing programs that run independently. These agents can respond to events, process data, and execute logic without requiring centralized control. Because AO combines Arweave’s permanent storage with a fully decentralized execution environment, it overcomes many of the limitations found in traditional blockchains, making it more scalable, cost-efficient, and resilient for decentralized applications. + +Decentralized systems like Bitcoin have introduced strong guarantees of trustlessness and permissionlessness, but they are not absolute. Users still need to trust that the majority of mining power follows the rules and that cryptographic security remains intact. +AO introduces a new approach where users do not need to trust any central authority or even other participants in the network. Instead, it relies entirely on cryptographic proofs and decentralized execution. The AO-Core protocol, which powers AO, is built on two key principles: +- Trustlessness – Anyone can participate in the network without having to trust other participants to act honestly. Everything is verifiable and executed based on predefined rules. +- Permissionlessness – No entity, individual, or group has the power to block another participant from accessing or using the network. Everyone has equal access to computation and storage resources. + +Unlike rigid blockchain models that enforce the same rules for all users, AO and AO-Core provide flexibility by allowing participants to choose from different levels of security, scalability, and computational resources. This way, users can collaborate while still maintaining control over their trade-offs. + +At the heart of AO is a unique concept: executable and isolated processes. These processes are self-contained pieces of code that are permanently stored on Arweave. Once stored, they can be executed at any time to produce specific results. + +One of the key advantages of AO is that multiple processes can run in parallel, independently of one another. Unlike traditional blockchains that require every node to process every transaction, AO allows computations to happen separately and simultaneously. + +Each process in AO can: +- Read and write data on Arweave. +- Act as an autonomous agent, continuously responding to new events. +- Communicate with other processes and nodes using messages. + +Instead of relying on a global consensus mechanism like Bitcoin or Ethereum, AO uses a more modular approach where messages trigger processes. These messages function like transactions or function calls in traditional systems, enabling communication and execution between nodes. +The execution of these processes is supported by modular layers, including transmission, scheduling, and execution components. This modular structure allows the network to scale efficiently and enables what is known as hyper-parallel computing, where thousands of computations can run simultaneously without interference. + +## AO-Core protocol: A new way to Process Data + +The AO-Core protocol is a foundational system that makes it easier to process and manage data across decentralized networks. It is designed to work with standard web technologies like HTTP, making it more accessible and flexible than traditional blockchain-based architectures. + +At its core, AO-Core helps users create, track, and combine pieces of information efficiently by organizing them into three main components: +- Messages – The smallest units of data and computation. +- Devices – Systems that process and store messages. +- Paths – The structures that connect and organize messages over time. + +By keeping messages small and lightweight, AO-Core reduces storage and computation costs. At the same time, it ensures scalability by distributing computations across multiple machines, increasing both speed and efficiency. Security is guaranteed through cryptographic proofs, ensuring that all data remains transparent and verifiable. + +### Messages: The Building Blocks of AO-Core + +At the heart of AO-Core’s decentralized system is the concept of Messages. Messages are the smallest units of data and computation, acting as the primary way that processes communicate and execute functions. + +Messages are processed by devices, stored permanently, and can be referenced at any time. This eliminates the need for centralized servers and ensures that every computation remains verifiable and traceable. + +Unlike traditional blockchains, where transactions are processed in a strict order, AO-Core allows messages to be processed in parallel. This is a critical innovation that enables AO’s hyper-parallel computing model. +Because messages are lightweight and cryptographically linked, they can be distributed across many devices without causing congestion. This makes AO-Core much more scalable than traditional blockchain-based smart contract platforms. + +Messages don’t just exist in isolation. Instead, they form a computation graph, where new messages reference previous ones to build complex logic and decision-making processes. +For example, one message might contain raw data, while another message applies a transformation to that data. A third message might then combine multiple previous messages to create a final result. +This graph-based structure eliminates the need for sequential processing and allows AO to operate in a truly decentralized and parallel manner. + +### Devices: Systems That Process Messages + +In the AO-Core protocol, a Device is a system responsible for processing and interpreting messages. Devices determine how messages are executed and play a key role in making AO a modular and flexible decentralized computing framework. + +Unlike traditional blockchains, where every node processes every transaction, AO-Core distributes tasks among different devices. This eliminates bottlenecks, improves efficiency, and enables AO to support a wide variety of use cases without enforcing a single computation model. + +Devices are not fixed; they can be added or upgraded over time. Each node in the network can select which devices to support based on their hardware capabilities and desired functionality. This decentralized approach allows nodes to contribute computing power and storage in a way that best suits their resources. + +Each device has specific logic that defines: +- How a message is processed. Some devices might execute code, while others might store or relay information. +- What kind of data or computation it handles. For example, some devices execute WebAssembly code, while others schedule tasks. +- How it interacts with other devices. Devices can communicate and depend on each other to perform more complex tasks. + +For example, a high-performance node with a powerful CPU may choose to run compute-intensive devices, while a lightweight node might only handle relay or scheduling tasks. This flexibility ensures scalability and prevents wasted resources. + +### Paths: Tracing the history of computation + +A Path in AO-Core is a way to link and structure messages over time, creating an organized and verifiable history of computations. + +Paths ensure that every transformation applied to a message is traceable, reproducible, and secure. Instead of storing each version of data separately, AO-Core keeps only the essential transformation steps, reducing storage costs while maintaining a full history of how data evolved. + +Each Path generates a HashPath, a cryptographic fingerprint that records every change applied to a message. This ensures once a message is recorded in a HashPath, it cannot be altered without detection. +Instead of storing every version of a message, AO-Core saves only the essential transformation steps. At the end, large-scale distributed computing becomes possible because only the necessary parts of a message are retrieved and processed, reducing network congestion. + +Paths function like version control systems (similar to Git), allowing AO-Core to maintain a structured and efficient way to track changes without wasting resources. + +## HyperBEAM + +HyperBEAM is the main implementation of the AO-Core protocol, written in Erlang, and serves as the backbone of AO’s decentralized operating system. By using Erlang and its BEAM runtime, HyperBEAM benefits from a battle-tested ecosystem designed for fault tolerance and distributed computing. +This architecture ensures key features such as process isolation, message-passing, and efficient task scheduling, all of which align with the core principles of AO. Erlang’s lightweight processes and actor-based concurrency model mirror AO’s execution requirements, where independent agents must operate in parallel, communicate asynchronously, and recover from failures automatically—ensuring a robust, decentralized computing environment. + +One of HyperBEAM’s most important capabilities is its abstraction and modular design, which allows programs to execute independently of the underlying hardware. HyperBEAM provides a hardware-agnostic execution layer, enabling computations to run across a distributed network of nodes. +This means that programs can function seamlessly across various infrastructures, ensuring scalability and resilience. By allowing nodes to dynamically select and execute different devices, HyperBEAM ensures adaptability while maintaining the robustness needed for a truly fault-tolerant and decentralized system. + +HyperBEAM plays a crucial role in coordinating node operations, allowing individual operators to contribute their machine’s resources to the network. Instead of enforcing a rigid infrastructure, AO enables flexible participation, where each node can expose specific services by running designated devices. +These devices act as modular execution units, providing different functionalities such as processing, communication, scheduling, and interaction with external networks. This decentralized approach ensures that AO can support a diverse set of computational needs, from handling autonomous agents to executing smart contract logic efficiently. + +To facilitate these operations, HyperBEAM includes several preloaded devices that serve distinct purposes. +- Then `~meta` device manages node configuration, defining hardware specifications and supported execution environments. +- Then `~relay` device ensures seamless communication both within the network and with external systems, acting as the bridge between AO’s decentralized computation and the broader digital ecosystem. +- Then `~process` device is responsible for maintaining persistent and shared executions, allowing long-running computations to be coordinated across multiple nodes. +- Then `~scheduling` device ensures deterministic execution ordering by leveraging hashpath-based computations, guaranteeing consistency across distributed processes. +- Then `~wasm64` device enables nodes to execute WebAssembly code, expanding the range of supported applications and ensuring compatibility with lightweight, portable execution environments. +etc. + +Through this modular approach, HyperBEAM enables AO to function as a highly flexible and extensible decentralized supercomputer, where computation is not restricted to a single paradigm but can evolve dynamically based on the network’s needs. + + diff --git a/docs/misc/js-client-guide.md b/docs/misc/js-client-guide.md new file mode 100644 index 000000000..867f838a1 --- /dev/null +++ b/docs/misc/js-client-guide.md @@ -0,0 +1,85 @@ +# JavaScript Client Guide for HyperBEAM + +This guide demonstrates how to integrate with HyperBEAM nodes using the an example JavaScript client library. + + +## Prerequisites + +- Node.js +- npm or yarn package manager + +## Project Setup + +First, create a new Node.js project: + +```bash +mkdir my-hyperbeam-project +cd my-hyperbeam-project +npm init -y +``` + +Next, manually edit your `package.json` file and add the `"type": "module"` property after the "main" line. This is required for using ES modules with import/export syntax. + +Then, install the required dependencies: + +```bash +npm install @permaweb/aoconnect +``` + +Download the HyperBEAM client implementation file and save it as `hyperbeam-client.js` in your project directory. + +[Download hyperbeam-client.js](../assets/hyperbeam-client.js){ .md-button .md-button--primary download="hyperbeam-client.js" style="display: block; width: 100%; text-align: center;" } + +Your final `package.json` should look similar to this: + +```json linenums="1" +{ + "name": "my-hyperbeam-project", + "version": "1.0.0", + "main": "index.js", + "type": "module", + "dependencies": { + "@permaweb/aoconnect": "^0.0.77" + } +} +``` + +## Starter + +```javascript linenums="1" +import { HyperBEAMClient } from './hyperbeam-client.js'; + +// Create a client instance +const client = new HyperBEAMClient({ + nodeUrl: 'http://localhost:10000', // HyperBEAM node URL + keyPath: 'wallet.json' // Path to your wallet key file +}); + +// Initialize the client +await client.initialize(); + +// Your Code Here +``` + +## Extending Stater Examples + +### Get Request ***~meta@1.0/info*** + +```javascript linenums="1" +// Make an unauthenticated GET request +const response = await client.get('/~meta@1.0/info', { debug: true }); +console.log(response); +``` + +### Post Request ***~meta@1.0/info*** + +```javascript linenums="1" +// Make an signed POST request +const data = { + "hello": "world", + "testValue": "example data" +}; +const infoResponse = await client.post('/~meta@1.0/info', data, { debug: true }); +console.log("Response:", infoResponse); +``` + diff --git a/docs/misc/overview/index.md b/docs/misc/overview/index.md new file mode 100644 index 000000000..554af770f --- /dev/null +++ b/docs/misc/overview/index.md @@ -0,0 +1,60 @@ +!!! warning "Platform Support" + This documentation is currently written specifically for **Ubuntu 22.04**. Support for macOS and other platforms will be added in future updates. + +## Overview + +HyperBEAM is a client implementation of the AO-Core protocol, written in Erlang. It enables a decentralized computing platform where programs run as independent processes, communicate via asynchronous message passing, and operate across a distributed network of nodes. + +For detailed technical information about HyperBEAM's architecture and functionality, see the [HyperBEAM Overview](hyperbeam/index.md). + +### What is AO-Core? + +AO-Core is a protocol built to enable decentralized computations, offering a series of universal primitives. Instead of enforcing a single, monolithic architecture, AO-Core provides a framework into which any number of different computational models, encapsulated as primitive and composable devices, can be attached. + +AO-Core's protocol is built upon the following primitives: + +- **Hashpaths**: A mechanism for referencing locations in a program's state-space prior to execution +- **Unified data structure**: For representing program states as HTTP documents +- **Attestation protocol**: For expressing attestations of states found at particular hashpaths +- **Meta-VM**: Allowing various state transformation programs (virtual machines and computational models, expressed in the form of devices) to be executed inside the AO-Core protocol + +## Installation Process Overview + +Setting up HyperBEAM involves several steps: + +1. **Check System Requirements** - Ensure your hardware and operating system meet the [minimum requirements](getting-started/requirements.md). +2. **Install System Dependencies** - Set up the necessary system packages via the [Installation Guide](getting-started/installation/index.md). +3. **Setup & Configure HyperBEAM** - Clone, Compile, Configure, and Run [HyperBEAM itself](hyperbeam/setup.md). +4. **Setup & Configure the Compute Unit** - Clone, Compile, Configure, and Run the [Local Compute Unit](compute-unit/setup.md). +5. **(Optional) Verify Installation** - Follow guides to ensure everything is working. (We might need to create or link to a verification guide here, e.g., `guides/verification.md`) + +### Before You Begin + +Before starting the installation process, make sure to: + +- Have access to a terminal/command line with administrative privileges. +- Have a stable internet connection for downloading packages. +- Allocate sufficient time (approximately 30-60 minutes for a complete setup). +- Review the [System Requirements](getting-started/requirements.md) first. + +## Documentation Structure + +This documentation is organized into the following main sections accessible via the top navigation: + +- **[Home](.)**: This page - overview and starting points. +- **[Installation & Core](getting-started/installation/index.md)**: Detailed steps for system dependencies and HyperBEAM setup/configuration. +- **[Components](compute-unit/index.md)**: Information on related components like the Compute Unit and TEE. +- **[Usage](guides/index.md)**: Practical guides and examples for using HyperBEAM. +- **[Resources](source-code-docs/index.md)**: Source code documentation and reference materials (Troubleshooting, Glossary, FAQ). +- **[Community](contribute/guidelines.md)**: How to contribute and get involved. + +## Community and Support + +- **GitHub HyperBEAM**: [permaweb/HyperBEAM](https://github.com/permaweb/HyperBEAM) +- **Github Local CU**: [permaweb/local-cu](https://github.com/permaweb/local-cu) +- **Discord**: [Join the community](https://discord.gg/V3yjzrBxPM) +- **Issues**: [File a bug report](https://github.com/permaweb/HyperBEAM/issues) + +## License + +HyperBEAM is open-source software licensed under the [MIT License](https://github.com/permaweb/HyperBEAM/blob/main/LICENSE.md). diff --git a/docs/misc/setting-up-selecting-devices.md b/docs/misc/setting-up-selecting-devices.md new file mode 100644 index 000000000..9268ad0c3 --- /dev/null +++ b/docs/misc/setting-up-selecting-devices.md @@ -0,0 +1,287 @@ +# Setting Up and Selecting Devices for HyperBEAM Nodes + +HyperBEAM is a client implementation of the AO-Core protocol that enables decentralized computations. As a node operator, you'll need to understand the various devices available and how to configure them for your specific use case. This guide will help you make informed decisions when setting up your HyperBEAM node. + +## What are Devices in HyperBEAM? + +In HyperBEAM, devices are modular components that provide specific functionalities to your node. They encapsulate different computational models that can be attached to the AO-Core protocol framework. Each device serves a specific purpose, from configuring your node to executing WebAssembly code, managing payments, or facilitating message relays. + +## Core Configuration: ~meta@1.0 Device + +The foundation of every HyperBEAM node is the `~meta@1.0` device, which is used to configure your node's hardware, supported devices, metering, and payment information. This is the first device you'll need to set up. + +### Setting Up the ~meta@1.0 Device + +You can initialize the ~meta@1.0 device via command line arguments when starting your node: + +```bash + +rebar3 shell --eval "hb:start_mainnet(#{ port => 9001, key_location => 'path/to/my/wallet.key' })." + +``` + +This command starts your node with a custom port (9001) and specifies the location of your Arweave wallet key file. You can also modify a running node's configuration by sending an HTTP Signed Message. + +## Choosing the Right Devices for Your Use Case + +HyperBEAM includes 25+ preloaded devices. Here's how to select the most appropriate ones based on your needs: + +### Basic Node Operation + +These devices form the foundation of a functional HyperBEAM node: + +- **~meta@1.0**: Core configuration device (required for all nodes) +- **~relay@1.0**: Handles message relaying between nodes and the HTTP network +- **dev_message**: The identity device that handles message processing + +### Computation Services + +If you want to offer computation services to the network: + +- **~wasm64@1.0**: Executes WebAssembly code using the Web Assembly Micro-Runtime +- **dev_stack**: Manages execution of a stack of devices in either fold or map mode +- **dev_scheduler**: Implements a simple scheduling scheme for handling process execution +- **~process@1.0**: Enables persistent, shared executions accessible by multiple users + +### Payment and Metering + +If you want to monetize your node's services: + +- **p4@1.0**: Core payment framework that works with pricing and ledger devices +- **~simple-pay@1.0**: Implements simple, flexible pricing for flat-fee execution +- **~faff@1.0**: Allows nodes to restrict access to specific users (useful for personal nodes) + +### Enhanced Security + +If security is a priority: + +- **~snp@1.0**: For generating and validating proofs that a node is executing inside a Trusted Execution Environment (TEE) +- **dev_codec_httpsig**: Implements HTTP Message Signatures as described in RFC-9421 + +### Legacynet Compatibility + +If you need to support older AO systems: + +- **~json-iface@1.0**: Translation layer between JSON-encoded messages and HyperBEAM's HTTP format +- **~compute-lite@1.0**: Lightweight device for executing legacynet AO processes +- **dev_genesis_wasm**: Provides an environment suitable for legacynet AO processes + +## Device Details and Functions + +### Communication and Relay Devices + +### ~relay@1.0 + +This device is responsible for relaying messages between nodes and other HTTP(S) endpoints. It can operate in either "call" or "cast" mode: + +- **Call mode**: Returns a response from the remote peer +- **Cast mode**: Returns immediately while the message is relayed asynchronously + +``` + +curl /~relay@.1.0/call?method=GET?0.path=https://www.arweave.net/ + +``` + +### dev_router + +Routes outbound messages from the node to appropriate network recipients via HTTP. It load-balances messages between downstream workers that perform the actual requests. Routes are defined in a precedence-ordered list of maps. + +### Computation Devices + +### ~wasm64@1.0 + +Executes WebAssembly code using the Web Assembly Micro-Runtime (WAMR). This device enables running WASM modules from any other device, supporting code written in Rust, C, and C++. + +Requirements: + +- Process definition +- WASM image +- Message with data to be processed + +### dev_stack + +Manages execution of multiple devices in sequence. It operates in two modes: + +- **Fold mode (default)**: Executes devices in order, passing state forward +- **Map mode**: Executes all devices and combines their results + +This is useful for creating complex execution patterns by combining different devices. + +### ~process@1.0 + +Enables persistent, shared executions that can be accessed by multiple users. Each user can add additional inputs to its hashpath. This device allows customization of execution and scheduler devices. + +Example process definition: + +``` + +Device: Process/1.0 +Scheduler-Device: Scheduler/1.0 +Execution-Device: Stack/1.0 +Execution-Stack: "Scheduler/1.0", "Cron/1.0", "WASM/1.0", "PoDA/1.0" +Cron-Frequency: 10-Minutes +WASM-Image: WASMImageID +PoDA: + Device: PoDA/1.0 + Authority: A + Authority: B + Authority: C + Quorum: 2 + +``` + +### Payment and Access Control Devices + +### p4@1.0 + +Core payment framework that works with pricing and ledger devices. It requires the following node message settings: + +- `p4_pricing-device`: Estimates request cost +- `p4_ledger-device`: Acts as payment ledger + +### ~simple-pay@1.0 + +Implements simple, flexible pricing for per-message, flat-fee execution. The device's ledger is stored in the node message at `simple_pay_ledger`. + +### ~faff@1.0 + +A "friends and family" pricing policy that allows users to process requests only if their addresses are in the node's allow-list. Useful for running a node for personal use or for a specific subset of users. + +### Security Devices + +### ~snp@1.0 + +Generates and validates proofs that a node is executing inside a Trusted Execution Environment (TEE). Nodes executing inside TEEs use an ephemeral key pair that provably exists only inside the TEE. + +### dev_codec_httpsig + +Implements HTTP Message Signatures as described in RFC-9421, providing a way to authenticate and verify the integrity of HTTP messages. + +### Utility Devices + +### dev_cron + +Inserts new messages into the schedule to allow processes to passively 'call' themselves without user interaction. Useful for creating automated, scheduled tasks. + +### dev_dedup + +Deduplicates messages sent to a process. It runs on the first pass of a `compute` key call if executed in a stack, preventing duplicate processing. + +### dev_monitor + +Allows flexible monitoring of a process execution. Adding this device to a process will call specified functions with the current process state during each pass. + +### dev_multipass + +Triggers repass events until a certain counter has been reached. Useful for stacks that need various execution passes to be completed in sequence across devices. + +### dev_patch + +Finds `PATCH` requests in the `results/outbox` of its message and applies them. Useful for processes whose computation needs to manipulate data outside of the `results` key. + +## Device Stacking Strategies + +One of the powerful features of HyperBEAM is the ability to stack devices to create complex computational models. Here are some effective stacking strategies: + +### Basic Process Execution Stack + +``` + +Execution-Device: Stack/1.0 +Execution-Stack: "Scheduler/1.0", "WASM/1.0" + +``` + +This simple stack handles scheduling and WASM execution for basic process needs. + +### Enhanced Security Stack + +``` + +Execution-Device: Stack/1.0 +Execution-Stack: "Scheduler/1.0", "WASM/1.0", "PoDA/1.0", "~snp@1.0" + +``` + +Adds proof-of-authority consensus and TEE validation for enhanced security. + +### Automated Process Stack + +``` + +Execution-Device: Stack/1.0 +Execution-Stack: "Scheduler/1.0", "Cron/1.0", "WASM/1.0" + +``` + +Adds the Cron device to enable automated, scheduled execution. + +### Analytics Stack + +``` + +Execution-Device: Stack/1.0 +Execution-Stack: "Scheduler/1.0", "WASM/1.0", "Monitor/1.0" + +``` + +Adds monitoring capability to track process execution. + +## Practical Setup Examples + +### Setting Up a Personal Node + +For a node intended for personal use only: + +```bash + +rebar3 shell --eval "hb:start_mainnet(#{ + port => 9001, + key_location => 'path/to/my/wallet.key', + p4_pricing-device => '~faff@1.0', + p4_ledger-device => '~faff@1.0', + faff_allow_list => ['my-wallet-address'] +})." + +``` + +### Setting Up a Public Computation Node + +For a node offering computation services to the network: + +```bash + +rebar3 shell --eval "hb:start_mainnet(#{ + port => 9001, + key_location => 'path/to/my/wallet.key', + p4_pricing-device => '~simple-pay@1.0', + p4_ledger-device => '~simple-pay@1.0', + simple_pay_price => 0.01, + preloaded_devices => ['~wasm64@1.0', '~process@1.0', 'dev_stack', 'dev_scheduler'] +})." + +``` + +### Setting Up a Secure TEE Node + +For a node running in a Trusted Execution Environment: + +```bash + +rebar3 shell --eval "hb:start_mainnet(#{ + port => 9001, + key_location => 'path/to/my/wallet.key', + p4_pricing-device => '~simple-pay@1.0', + p4_ledger-device => '~simple-pay@1.0', + simple_pay_price => 0.05, + preloaded_devices => ['~wasm64@1.0', '~process@1.0', 'dev_stack', 'dev_scheduler', '~snp@1.0'] +})." + +``` + +## Conclusion + +Setting up a HyperBEAM node involves understanding and configuring various devices based on your specific requirements. By selecting and stacking devices, you can create a node that meets your computational needs, security requirements, and monetization goals. + +For more detailed information about each device, refer to the HyperBEAM code repositories. Remember that the AO-Core protocol is designed to be flexible, allowing you to adapt your node to various use cases by combining different devices and computational models. diff --git a/docs/style-guide.md b/docs/misc/style-guide.md similarity index 100% rename from docs/style-guide.md rename to docs/misc/style-guide.md diff --git a/docs/resources/llms.md b/docs/resources/llms.md new file mode 100644 index 000000000..c0c120334 --- /dev/null +++ b/docs/resources/llms.md @@ -0,0 +1,20 @@ +# LLM Context Files + +This section provides access to specially formatted files intended for consumption by Large Language Models (LLMs) to provide context about the HyperBEAM documentation. + +1. **[LLM Summary (llms.txt)](../llms.txt)** + * **Content**: Contains a brief summary of the HyperBEAM documentation structure and a list of relative file paths for all markdown documents included in the build. + * **Usage**: Useful for providing an LLM with a high-level overview and the available navigation routes within the documentation. + +2. **[LLM Full Content (llms-full.txt)](../llms-full.txt)** + * **Content**: A single text file containing the complete, concatenated content of all markdown documents from the specified documentation directories (`begin`, `run`, `guides`, `devices`, `resources`). Each file's content is clearly demarcated. + * **Usage**: Ideal for feeding the entire documentation content into an LLM for comprehensive context, analysis, or question-answering based on the full documentation set. + +!!! note "Generation Process" + These files are automatically generated by the `docs/build-all.sh` script during the documentation build process. They consolidate information from the following directories: + + * `docs/begin` + * `docs/run` + * `docs/guides` + * `docs/devices` + * `docs/resources` diff --git a/docs/resources/reference/faq.md b/docs/resources/reference/faq.md new file mode 100644 index 000000000..af7bd79b3 --- /dev/null +++ b/docs/resources/reference/faq.md @@ -0,0 +1,117 @@ +# Frequently Asked Questions + +This page answers common questions about HyperBEAM, its components, and how to use them effectively. + +## General Questions + +### What is HyperBEAM? + +HyperBEAM is a client implementation of the AO-Core protocol written in Erlang. It serves as the node software for a decentralized operating system that allows operators to offer computational resources to users in the AO network. + +### How does HyperBEAM differ from other distributed systems? + +HyperBEAM focuses on true decentralization with asynchronous message passing between isolated processes. Unlike many distributed systems that rely on central coordination, HyperBEAM nodes can operate independently while still forming a cohesive network. Additionally, its Erlang foundation provides robust fault tolerance and concurrency capabilities. + +### What can I build with HyperBEAM? + +You can build a wide range of applications, including: + +- Decentralized applications (dApps) +- Distributed computation systems +- Peer-to-peer services +- Resilient microservices +- IoT device networks +- Decentralized storage solutions + +### Is HyperBEAM open source? + +Yes, HyperBEAM is open-source software licensed under the Business Source License License. + +### What is the current focus or phase of HyperBEAM development? + +The initial development phase focuses on integrating AO processes more deeply with HyperBEAM. A key part of this is phasing out the reliance on traditional "dryrun" simulations for reading process state. Instead, processes are encouraged to use the [~patch@1.0 device](../../resources/source-code/dev_patch.md) to expose specific parts of their state directly via HyperPATH GET requests. This allows for more efficient and direct state access, particularly for web interfaces and external integrations. You can learn more about this mechanism in the [Exposing Process State with the Patch Device](../../build/exposing-process-state.md) guide. + +## Installation and Setup + +### What are the system requirements for running HyperBEAM? + +Currently, HyperBEAM is primarily tested and documented for Ubuntu 22.04. Support for macOS and other platforms will be added in future updates. For detailed requirements, see the [System Requirements](../getting-started/requirements.md) page. + +### Can I run HyperBEAM in a container? + +While technically possible, running HyperBEAM in Docker containers or other containerization technologies is currently not recommended. The containerization approach may introduce additional complexity and potential performance issues. We recommend running HyperBEAM directly on the host system until container support is more thoroughly tested and optimized. + +### How do I update HyperBEAM to the latest version? + +To update HyperBEAM: + +1. Pull the latest code from the repository +2. Rebuild the application +3. Restart the HyperBEAM service + +Specific update instructions will vary depending on your installation method. + +### Can I run multiple HyperBEAM nodes on a single machine? + +Yes, you can run multiple HyperBEAM nodes on a single machine, but you'll need to configure them to use different ports and data directories to avoid conflicts. However, this is not recommended for production environments as each node should ideally have a unique IP address to properly participate in the network. Running multiple nodes on a single machine is primarily useful for development and testing purposes. + +## Architecture and Components + +### What is the difference between HyperBEAM and Compute Unit? + +- **HyperBEAM**: The Erlang-based node software that handles message routing, process management, and device coordination. +- **Compute Unit (CU)**: A NodeJS implementation that executes WebAssembly modules and handles computational tasks. + +Together, these components form a complete execution environment for AO processes. + +## Development and Usage + +### What programming languages can I use with HyperBEAM? + +You can use any programming language that compiles to WebAssembly (WASM) for creating modules that run on the Compute Unit. This includes languages like: + +- Lua +- Rust +- C/C++ +- And many others with WebAssembly support + +### How do I debug processes running in HyperBEAM? + +Debugging processes in HyperBEAM can be done through: + +1. Logging messages to the system log +2. Monitoring process state and message flow +3. Inspecting memory usage and performance metrics + +### Is there a limit to how many processes can run on a node? + +The practical limit depends on your hardware resources. Erlang is designed to handle millions of lightweight processes efficiently, but the actual number will be determined by: + +- Available memory +- CPU capacity +- Network bandwidth +- Storage speed +- The complexity of your processes + + +## Troubleshooting + +### What should I do if a node becomes unresponsive? + +If a node becomes unresponsive: + +1. Check the node's logs for error messages +2. Verify network connectivity +3. Ensure sufficient system resources +4. Restart the node if necessary +5. Check for configuration issues + +For persistent problems, consult the [Troubleshooting](troubleshooting.md) page. + +### Where can I get help if I encounter issues? + +If you encounter issues: + +- Check the [Troubleshooting](troubleshooting.md) guide +- Search or ask questions on [GitHub Issues](https://github.com/permaweb/HyperBEAM/issues) +- Join the community on [Discord](https://discord.gg/V3yjzrBxPM) \ No newline at end of file diff --git a/docs/resources/reference/glossary.md b/docs/resources/reference/glossary.md new file mode 100644 index 000000000..cdfe3355e --- /dev/null +++ b/docs/resources/reference/glossary.md @@ -0,0 +1,127 @@ +# Glossary + +This glossary provides definitions for terms and concepts used throughout the HyperBEAM documentation. For a comprehensive glossary of permaweb-specific terminology, check out the [permaweb glossary](#permaweb-glossary) section below. + +## AO-Core Protocol +The underlying protocol that HyperBEAM implements, enabling decentralized computing and communication between nodes. AO-Core provides a framework into which any number of different computational models, encapsulated as primitive devices, can be attached. + +## Asynchronous Message Passing +A communication paradigm where senders don't wait for receivers to be ready, allowing for non-blocking operations and better scalability. + +## Checkpoint +A saved state of a process that can be used to resume execution from a known point, used for persistence and recovery. + +## Compute Unit (CU) +The NodeJS component of HyperBEAM that executes WebAssembly modules and handles computational tasks. + +## Decentralized Execution +The ability to run processes across a distributed network without centralized control or coordination. + +## Device +A functional unit in HyperBEAM that provides specific capabilities to the system, such as storage, networking, or computational resources. + +## Erlang +The programming language used to implement the HyperBEAM core, known for its robustness and support for building distributed, fault-tolerant applications. + +## ~flat@1.0 +A format used for encoding settings files in HyperBEAM configuration, using HTTP header styling. + +## Hashpaths +A mechanism for referencing locations in a program's state-space prior to execution. These state-space links are represented as Merklized lists of programs inputs and initial states. + +## HyperBEAM +The Erlang-based node software that handles message routing, process management, and device coordination in the HyperBEAM ecosystem. + +## Message +A data structure used for communication between processes in the HyperBEAM system. Messages can be interpreted as a binary term or as a collection of named functions (a Map of functions). + +## Module +A unit of code that can be loaded and executed by the Compute Unit, typically in WebAssembly format. + +## Node +An instance of HyperBEAM running on a physical or virtual machine that participates in the distributed network. + +## ~p4@1.0 +A device that runs as a pre-processor and post-processor in HyperBEAM, enabling a framework for node operators to sell usage of their machine's hardware to execute AO-Core devices. + +## Process +An independent unit of computation in HyperBEAM with its own state and execution context. + +## Process ID +A unique identifier assigned to a process within the HyperBEAM system. + +## ~scheduler@1.0 +A device used to assign a linear hashpath to an execution, such that all users may access it with a deterministic ordering. + +## ~compute-lite@1.0 +A lightweight device wrapping a local WASM executor, used for executing legacynet AO processes inside HyperBEAM. + +## ~json-iface@1.0 +A device that offers a translation layer between the JSON-encoded message format used by legacy versions and HyperBEAM's native HTTP message format. + +## ~meta@1.0 +A device used to configure the node's hardware, supported devices, metering and payments information, amongst other configuration options. + +## ~process@1.0 +A device that enables users to create persistent, shared executions that can be accessed by any number of users, each of whom may add additional inputs to its hashpath. + +## ~relay@1.0 +A device used to relay messages between nodes and the wider HTTP network. It offers an interface for sending and receiving messages using a variety of execution strategies. + +## ~simple-pay@1.0 +A simple, flexible pricing device that can be used in conjunction with p4@1.0 to offer flat-fees for the execution of AO-Core messages. + +## ~snp@1.0 +A device used to generate and validate proofs that a node is executing inside a Trusted Execution Environment (TEE). + +## ~wasm64@1.0 +A device used to execute WebAssembly code, using the Web Assembly Micro-Runtime (WAMR) under-the-hood. + +## ~stack@1.0 +A device used to execute an ordered set of devices over the same inputs, allowing users to create complex combinations of other devices. + +## Trusted Execution Environment (TEE) +A secure area inside a processor that ensures the confidentiality and integrity of code and data loaded within it. Used in HyperBEAM for trust-minimized computation. + +## WebAssembly (WASM) +A binary instruction format that serves as a portable compilation target for programming languages, enabling deployment on the web and other environments. + +## Permaweb Glossary + +For a more comprehensive glossary of terms used in the permaweb, try the [Permaweb Glossary](https://glossary.arweave.net). Or use it below: + + + + +
+
+ +
+
+ +
+
\ No newline at end of file diff --git a/docs/resources/reference/troubleshooting.md b/docs/resources/reference/troubleshooting.md new file mode 100644 index 000000000..c0aa77cda --- /dev/null +++ b/docs/resources/reference/troubleshooting.md @@ -0,0 +1,104 @@ +# Troubleshooting Guide + +This guide addresses common issues you might encounter when working with HyperBEAM and the Compute Unit. + +## Installation Issues + +### Erlang Installation Fails + +**Symptoms**: Errors during Erlang compilation or installation + +**Solutions**: + +- Ensure all required dependencies are installed: `sudo apt-get install -y libssl-dev ncurses-dev make cmake gcc g++` +- Try configuring with fewer options: `./configure --without-wx --without-debugger --without-observer --without-et` +- Check disk space, as compilation requires several GB of free space + +### Rebar3 Bootstrap Fails + +**Symptoms**: Errors when running `./bootstrap` for Rebar3 + +**Solutions**: + +- Verify Erlang is correctly installed: `erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().'` +- Ensure you have the latest version of the repository: `git fetch && git reset --hard origin/master` +- Try manually downloading a precompiled Rebar3 binary + +## HyperBEAM Issues + +### HyperBEAM Won't Start + +**Symptoms**: Errors when running `rebar3 shell` or the HyperBEAM startup command + +**Solutions**: + +- Check for port conflicts: Another service might be using the configured port +- Verify the wallet key file exists and is accessible +- Examine Erlang crash dumps for detailed error information +- Ensure all required dependencies are installed + +### HyperBEAM Crashes During Operation + +**Symptoms**: Unexpected termination of the HyperBEAM process + +**Solutions**: + +- Check system resources (memory, disk space) +- Examine Erlang crash dumps for details +- Reduce memory limits if the system is resource-constrained +- Check for network connectivity issues if connecting to external services + +## Compute Unit Issues + +### Compute Unit Won't Start + +**Symptoms**: Errors when running `npm start` in the CU directory + +**Solutions**: + +- Verify Node.js is installed correctly: `node -v` +- Ensure all dependencies are installed: `npm i` +- Check that the wallet file exists and is correctly formatted +- Verify the `.env` file has all required settings + +### Memory Errors in Compute Unit + +**Symptoms**: Out of memory errors or excessive memory usage + +**Solutions**: + +- Adjust the `PROCESS_WASM_MEMORY_MAX_LIMIT` environment variable +- Enable garbage collection by setting an appropriate `GC_INTERVAL_MS` +- Monitor memory usage and adjust limits as needed +- If on a low-memory system, reduce concurrent process execution + +## Integration Issues + +### HyperBEAM Can't Connect to Compute Unit + +**Symptoms**: Connection errors in HyperBEAM logs when trying to reach the CU + +**Solutions**: + +- Verify the CU is running: `curl http://localhost:6363` +- Ensure there are no firewall rules blocking the connection +- Verify network configuration if components are on different machines + +### Process Execution Fails + +**Symptoms**: Errors when deploying or executing processes + +**Solutions**: + +- Check both HyperBEAM and CU logs for specific error messages +- Verify that the WASM module is correctly compiled and valid +- Test with a simple example process to isolate the issue +- Adjust memory limits if the process requires more resources + +## Getting Help + +If you're still experiencing issues after trying these troubleshooting steps: + +1. Check the [GitHub repository](https://github.com/permaweb/HyperBEAM) for known issues +2. Join the [Discord community](https://discord.gg/V3yjzrBxPM) for support +3. Open an issue on GitHub with detailed information about your problem \ No newline at end of file diff --git a/docs/resources/source-code/README.md b/docs/resources/source-code/README.md new file mode 100644 index 000000000..7e15fb008 --- /dev/null +++ b/docs/resources/source-code/README.md @@ -0,0 +1,115 @@ + + +# The hb application # + + +## Modules ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ar_bundles
ar_deep_hash
ar_rate_limiter
ar_timestamp
ar_tx
ar_wallet
dev_cache
dev_cacheviz
dev_codec_ans104
dev_codec_flat
dev_codec_httpsig
dev_codec_httpsig_conv
dev_codec_json
dev_codec_structured
dev_cron
dev_cu
dev_dedup
dev_delegated_compute
dev_faff
dev_genesis_wasm
dev_green_zone
dev_hook
dev_hyperbuddy
dev_json_iface
dev_local_name
dev_lookup
dev_lua
dev_lua_lib
dev_lua_test
dev_manifest
dev_message
dev_meta
dev_monitor
dev_multipass
dev_name
dev_node_process
dev_p4
dev_patch
dev_poda
dev_process
dev_process_cache
dev_process_worker
dev_push
dev_relay
dev_router
dev_scheduler
dev_scheduler_cache
dev_scheduler_formats
dev_scheduler_registry
dev_scheduler_server
dev_simple_pay
dev_snp
dev_snp_nif
dev_stack
dev_test
dev_volume
dev_wasi
dev_wasm
hb
hb_ao
hb_ao_test_vectors
hb_app
hb_beamr
hb_beamr_io
hb_cache
hb_cache_control
hb_cache_render
hb_client
hb_crypto
hb_debugger
hb_escape
hb_event
hb_examples
hb_features
hb_gateway_client
hb_http
hb_http_benchmark_tests
hb_http_client
hb_http_client_sup
hb_http_server
hb_json
hb_keccak
hb_logger
hb_message
hb_metrics_collector
hb_name
hb_opts
hb_path
hb_persistent
hb_private
hb_process_monitor
hb_router
hb_singleton
hb_store
hb_store_fs
hb_store_gateway
hb_store_remote_node
hb_store_rocksdb
hb_structured_fields
hb_sup
hb_test_utils
hb_tracer
hb_util
hb_volume
rsa_pss
+ diff --git a/docs/resources/source-code/ar_bundles.md b/docs/resources/source-code/ar_bundles.md new file mode 100644 index 000000000..704a92b71 --- /dev/null +++ b/docs/resources/source-code/ar_bundles.md @@ -0,0 +1,653 @@ +# [Module ar_bundles.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_bundles.erl) + + + + + + +## Function Index ## + + +
add_bundle_tags/1*
add_list_tags/1*
add_manifest_tags/2*
ar_bundles_test_/0*
assert_data_item/7*
check_size/2*Force that a binary is either empty or the given number of bytes.
check_type/2*Ensure that a value is of the given type.
data_item_signature_data/1Generate the data segment to be signed for a data item.
data_item_signature_data/2*
decode_avro_name/3*
decode_avro_tags/2*Decode Avro blocks (for tags) from binary.
decode_avro_value/4*
decode_bundle_header/2*
decode_bundle_header/3*
decode_bundle_items/2*
decode_optional_field/1*
decode_signature/1*Decode the signature from a binary format.
decode_tags/1Decode tags from a binary format using Apache Avro.
decode_vint/3*
decode_zigzag/1*Decode a VInt encoded ZigZag integer from binary.
deserialize/1Convert binary data back to a #tx record.
deserialize/2
encode_avro_string/1*Encode a string for Avro using ZigZag and VInt encoding.
encode_optional_field/1*Encode an optional field (target, anchor) with a presence byte.
encode_signature_type/1*Only RSA 4096 is currently supported.
encode_tags/1Encode tags into a binary format using Apache Avro.
encode_tags_size/2*
encode_vint/1*Encode a ZigZag integer to VInt binary format.
encode_vint/2*
encode_zigzag/1*Encode an integer using ZigZag encoding.
enforce_valid_tx/1*Take an item and ensure that it is of valid form.
finalize_bundle_data/1*
find/2Find an item in a bundle-map/list and return it.
find_single_layer/2*An internal helper for finding an item in a single-layer of a bundle.
format/1
format/2
format_binary/1*
format_data/2*
format_line/2*
format_line/3*
hd/1Return the first item in a bundle-map/list.
id/1Return the ID of an item -- either signed or unsigned as specified.
id/2
is_signed/1Check if an item is signed.
manifest/1
manifest_item/1Return the manifest item in a bundle-map/list.
map/1Convert an item containing a map or list into an Erlang map.
maybe_map_to_list/1*
maybe_unbundle/1*
maybe_unbundle_map/1*
member/2Check if an item exists in a bundle-map/list.
new_item/4Create a new data item.
new_manifest/1*
normalize/1
normalize_data/1*Ensure that a data item (potentially containing a map or list) has a standard, serialized form.
normalize_data_size/1*Reset the data size of a data item.
ok_or_throw/3*Throw an error if the given value is not ok.
parse_manifest/1
print/1
reset_ids/1Re-calculate both of the IDs for an item.
run_test/0*
serialize/1Convert a #tx record to its binary representation.
serialize/2
serialize_bundle_data/2*
sign_item/2Sign a data item.
signer/1Return the address of the signer of an item, if it is signed.
test_basic_member_id/0*
test_bundle_map/0*
test_bundle_with_one_item/0*
test_bundle_with_two_items/0*
test_deep_member/0*
test_empty_bundle/0*
test_extremely_large_bundle/0*
test_no_tags/0*
test_recursive_bundle/0*
test_serialize_deserialize_deep_signed_bundle/0*
test_unsigned_data_item_id/0*
test_unsigned_data_item_normalization/0*
test_with_tags/0*
test_with_zero_length_tag/0*
to_serialized_pair/1*
type/1
unbundle/1*
unbundle_list/1*
update_ids/1*Take an item and ensure that both the unsigned and signed IDs are +appropriately set.
utf8_encoded/1*Encode a UTF-8 string to binary.
verify_data_item_id/1*Verify the data item's ID matches the signature.
verify_data_item_signature/1*Verify the data item's signature.
verify_data_item_tags/1*Verify the validity of the data item's tags.
verify_item/1Verify the validity of a data item.
+ + + + +## Function Details ## + + + +### add_bundle_tags/1 * ### + +`add_bundle_tags(Tags) -> any()` + + + +### add_list_tags/1 * ### + +`add_list_tags(Tags) -> any()` + + + +### add_manifest_tags/2 * ### + +`add_manifest_tags(Tags, ManifestID) -> any()` + + + +### ar_bundles_test_/0 * ### + +`ar_bundles_test_() -> any()` + + + +### assert_data_item/7 * ### + +`assert_data_item(KeyType, Owner, Target, Anchor, Tags, Data, DataItem) -> any()` + + + +### check_size/2 * ### + +`check_size(Bin, Sizes) -> any()` + +Force that a binary is either empty or the given number of bytes. + + + +### check_type/2 * ### + +`check_type(Value, X2) -> any()` + +Ensure that a value is of the given type. + + + +### data_item_signature_data/1 ### + +`data_item_signature_data(RawItem) -> any()` + +Generate the data segment to be signed for a data item. + + + +### data_item_signature_data/2 * ### + +`data_item_signature_data(RawItem, X2) -> any()` + + + +### decode_avro_name/3 * ### + +`decode_avro_name(NameSize, Rest, Count) -> any()` + + + +### decode_avro_tags/2 * ### + +`decode_avro_tags(Binary, Count) -> any()` + +Decode Avro blocks (for tags) from binary. + + + +### decode_avro_value/4 * ### + +`decode_avro_value(ValueSize, Name, Rest, Count) -> any()` + + + +### decode_bundle_header/2 * ### + +`decode_bundle_header(Count, Bin) -> any()` + + + +### decode_bundle_header/3 * ### + +`decode_bundle_header(Count, ItemsBin, Header) -> any()` + + + +### decode_bundle_items/2 * ### + +`decode_bundle_items(RestItems, ItemsBin) -> any()` + + + +### decode_optional_field/1 * ### + +`decode_optional_field(X1) -> any()` + + + +### decode_signature/1 * ### + +`decode_signature(Other) -> any()` + +Decode the signature from a binary format. Only RSA 4096 is currently supported. +Note: the signature type '1' corresponds to RSA 4096 - but it is is written in +little-endian format which is why we match on `<<1, 0>>`. + + + +### decode_tags/1 ### + +`decode_tags(X1) -> any()` + +Decode tags from a binary format using Apache Avro. + + + +### decode_vint/3 * ### + +`decode_vint(X1, Result, Shift) -> any()` + + + +### decode_zigzag/1 * ### + +`decode_zigzag(Binary) -> any()` + +Decode a VInt encoded ZigZag integer from binary. + + + +### deserialize/1 ### + +`deserialize(Binary) -> any()` + +Convert binary data back to a #tx record. + + + +### deserialize/2 ### + +`deserialize(Item, X2) -> any()` + + + +### encode_avro_string/1 * ### + +`encode_avro_string(String) -> any()` + +Encode a string for Avro using ZigZag and VInt encoding. + + + +### encode_optional_field/1 * ### + +`encode_optional_field(Field) -> any()` + +Encode an optional field (target, anchor) with a presence byte. + + + +### encode_signature_type/1 * ### + +`encode_signature_type(X1) -> any()` + +Only RSA 4096 is currently supported. +Note: the signature type '1' corresponds to RSA 4096 -- but it is is written in +little-endian format which is why we encode to `<<1, 0>>`. + + + +### encode_tags/1 ### + +`encode_tags(Tags) -> any()` + +Encode tags into a binary format using Apache Avro. + + + +### encode_tags_size/2 * ### + +`encode_tags_size(Tags, EncodedTags) -> any()` + + + +### encode_vint/1 * ### + +`encode_vint(ZigZag) -> any()` + +Encode a ZigZag integer to VInt binary format. + + + +### encode_vint/2 * ### + +`encode_vint(ZigZag, Acc) -> any()` + + + +### encode_zigzag/1 * ### + +`encode_zigzag(Int) -> any()` + +Encode an integer using ZigZag encoding. + + + +### enforce_valid_tx/1 * ### + +`enforce_valid_tx(List) -> any()` + +Take an item and ensure that it is of valid form. Useful for ensuring +that a message is viable for serialization/deserialization before execution. +This function should throw simple, easy to follow errors to aid devs in +debugging issues. + + + +### finalize_bundle_data/1 * ### + +`finalize_bundle_data(Processed) -> any()` + + + +### find/2 ### + +`find(Key, Map) -> any()` + +Find an item in a bundle-map/list and return it. + + + +### find_single_layer/2 * ### + +`find_single_layer(UnsignedID, TX) -> any()` + +An internal helper for finding an item in a single-layer of a bundle. +Does not recurse! You probably want `find/2` in most cases. + + + +### format/1 ### + +`format(Item) -> any()` + + + +### format/2 ### + +`format(Item, Indent) -> any()` + + + +### format_binary/1 * ### + +`format_binary(Bin) -> any()` + + + +### format_data/2 * ### + +`format_data(Item, Indent) -> any()` + + + +### format_line/2 * ### + +`format_line(Str, Indent) -> any()` + + + +### format_line/3 * ### + +`format_line(RawStr, Fmt, Ind) -> any()` + + + +### hd/1 ### + +`hd(Tx) -> any()` + +Return the first item in a bundle-map/list. + + + +### id/1 ### + +`id(Item) -> any()` + +Return the ID of an item -- either signed or unsigned as specified. +If the item is unsigned and the user requests the signed ID, we return +the atom `not_signed`. In all other cases, we return the ID of the item. + + + +### id/2 ### + +`id(Item, Type) -> any()` + + + +### is_signed/1 ### + +`is_signed(Item) -> any()` + +Check if an item is signed. + + + +### manifest/1 ### + +`manifest(Map) -> any()` + + + +### manifest_item/1 ### + +`manifest_item(Tx) -> any()` + +Return the manifest item in a bundle-map/list. + + + +### map/1 ### + +`map(Tx) -> any()` + +Convert an item containing a map or list into an Erlang map. + + + +### maybe_map_to_list/1 * ### + +`maybe_map_to_list(Item) -> any()` + + + +### maybe_unbundle/1 * ### + +`maybe_unbundle(Item) -> any()` + + + +### maybe_unbundle_map/1 * ### + +`maybe_unbundle_map(Bundle) -> any()` + + + +### member/2 ### + +`member(Key, Item) -> any()` + +Check if an item exists in a bundle-map/list. + + + +### new_item/4 ### + +`new_item(Target, Anchor, Tags, Data) -> any()` + +Create a new data item. Should only be used for testing. + + + +### new_manifest/1 * ### + +`new_manifest(Index) -> any()` + + + +### normalize/1 ### + +`normalize(Item) -> any()` + + + +### normalize_data/1 * ### + +`normalize_data(Bundle) -> any()` + +Ensure that a data item (potentially containing a map or list) has a standard, serialized form. + + + +### normalize_data_size/1 * ### + +`normalize_data_size(Item) -> any()` + +Reset the data size of a data item. Assumes that the data is already normalized. + + + +### ok_or_throw/3 * ### + +`ok_or_throw(TX, X2, Error) -> any()` + +Throw an error if the given value is not ok. + + + +### parse_manifest/1 ### + +`parse_manifest(Item) -> any()` + + + +### print/1 ### + +`print(Item) -> any()` + + + +### reset_ids/1 ### + +`reset_ids(Item) -> any()` + +Re-calculate both of the IDs for an item. This is a wrapper +function around `update_id/1` that ensures both IDs are set from +scratch. + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### serialize/1 ### + +`serialize(TX) -> any()` + +Convert a #tx record to its binary representation. + + + +### serialize/2 ### + +`serialize(TX, X2) -> any()` + + + +### serialize_bundle_data/2 * ### + +`serialize_bundle_data(Map, Manifest) -> any()` + + + +### sign_item/2 ### + +`sign_item(RawItem, X2) -> any()` + +Sign a data item. + + + +### signer/1 ### + +`signer(Tx) -> any()` + +Return the address of the signer of an item, if it is signed. + + + +### test_basic_member_id/0 * ### + +`test_basic_member_id() -> any()` + + + +### test_bundle_map/0 * ### + +`test_bundle_map() -> any()` + + + +### test_bundle_with_one_item/0 * ### + +`test_bundle_with_one_item() -> any()` + + + +### test_bundle_with_two_items/0 * ### + +`test_bundle_with_two_items() -> any()` + + + +### test_deep_member/0 * ### + +`test_deep_member() -> any()` + + + +### test_empty_bundle/0 * ### + +`test_empty_bundle() -> any()` + + + +### test_extremely_large_bundle/0 * ### + +`test_extremely_large_bundle() -> any()` + + + +### test_no_tags/0 * ### + +`test_no_tags() -> any()` + + + +### test_recursive_bundle/0 * ### + +`test_recursive_bundle() -> any()` + + + +### test_serialize_deserialize_deep_signed_bundle/0 * ### + +`test_serialize_deserialize_deep_signed_bundle() -> any()` + + + +### test_unsigned_data_item_id/0 * ### + +`test_unsigned_data_item_id() -> any()` + + + +### test_unsigned_data_item_normalization/0 * ### + +`test_unsigned_data_item_normalization() -> any()` + + + +### test_with_tags/0 * ### + +`test_with_tags() -> any()` + + + +### test_with_zero_length_tag/0 * ### + +`test_with_zero_length_tag() -> any()` + + + +### to_serialized_pair/1 * ### + +`to_serialized_pair(Item) -> any()` + + + +### type/1 ### + +`type(Item) -> any()` + + + +### unbundle/1 * ### + +`unbundle(Item) -> any()` + + + +### unbundle_list/1 * ### + +`unbundle_list(Item) -> any()` + + + +### update_ids/1 * ### + +`update_ids(Item) -> any()` + +Take an item and ensure that both the unsigned and signed IDs are +appropriately set. This function is structured to fall through all cases +of poorly formed items, recursively ensuring its correctness for each case +until the item has a coherent set of IDs. +The cases in turn are: +- The item has no unsigned_id. This is never valid. +- The item has the default signature and ID. This is valid. +- The item has the default signature but a non-default ID. Reset the ID. +- The item has a signature. We calculate the ID from the signature. +- Valid: The item is fully formed and has both an unsigned and signed ID. + + + +### utf8_encoded/1 * ### + +`utf8_encoded(String) -> any()` + +Encode a UTF-8 string to binary. + + + +### verify_data_item_id/1 * ### + +`verify_data_item_id(DataItem) -> any()` + +Verify the data item's ID matches the signature. + + + +### verify_data_item_signature/1 * ### + +`verify_data_item_signature(DataItem) -> any()` + +Verify the data item's signature. + + + +### verify_data_item_tags/1 * ### + +`verify_data_item_tags(DataItem) -> any()` + +Verify the validity of the data item's tags. + + + +### verify_item/1 ### + +`verify_item(DataItem) -> any()` + +Verify the validity of a data item. + diff --git a/docs/resources/source-code/ar_deep_hash.md b/docs/resources/source-code/ar_deep_hash.md new file mode 100644 index 000000000..b6fc9f9c4 --- /dev/null +++ b/docs/resources/source-code/ar_deep_hash.md @@ -0,0 +1,41 @@ +# [Module ar_deep_hash.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_deep_hash.erl) + + + + + + +## Function Index ## + + +
hash/1
hash_bin/1*
hash_bin_or_list/1*
hash_list/2*
+ + + + +## Function Details ## + + + +### hash/1 ### + +`hash(List) -> any()` + + + +### hash_bin/1 * ### + +`hash_bin(Bin) -> any()` + + + +### hash_bin_or_list/1 * ### + +`hash_bin_or_list(Bin) -> any()` + + + +### hash_list/2 * ### + +`hash_list(List, Acc) -> any()` + diff --git a/docs/resources/source-code/ar_rate_limiter.md b/docs/resources/source-code/ar_rate_limiter.md new file mode 100644 index 000000000..7e0a132b0 --- /dev/null +++ b/docs/resources/source-code/ar_rate_limiter.md @@ -0,0 +1,93 @@ +# [Module ar_rate_limiter.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_rate_limiter.erl) + + + + +__Behaviours:__ [`gen_server`](gen_server.md). + + + +## Function Index ## + + +
cut_trace/4*
handle_call/3
handle_cast/2
handle_info/2
init/1
off/0Turn rate limiting off.
on/0Turn rate limiting on.
start_link/1
terminate/2
throttle/3Hang until it is safe to make another request to the given Peer with the +given Path.
throttle2/3*
+ + + + +## Function Details ## + + + +### cut_trace/4 * ### + +`cut_trace(N, Trace, Now, Opts) -> any()` + + + +### handle_call/3 ### + +`handle_call(Request, From, State) -> any()` + + + +### handle_cast/2 ### + +`handle_cast(Cast, State) -> any()` + + + +### handle_info/2 ### + +`handle_info(Message, State) -> any()` + + + +### init/1 ### + +`init(Opts) -> any()` + + + +### off/0 ### + +`off() -> any()` + +Turn rate limiting off. + + + +### on/0 ### + +`on() -> any()` + +Turn rate limiting on. + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + + + +### terminate/2 ### + +`terminate(Reason, State) -> any()` + + + +### throttle/3 ### + +`throttle(Peer, Path, Opts) -> any()` + +Hang until it is safe to make another request to the given Peer with the +given Path. The limits are configured in include/ar_blacklist_middleware.hrl. + + + +### throttle2/3 * ### + +`throttle2(Peer, Path, Opts) -> any()` + diff --git a/docs/resources/source-code/ar_timestamp.md b/docs/resources/source-code/ar_timestamp.md new file mode 100644 index 000000000..d6694cf1d --- /dev/null +++ b/docs/resources/source-code/ar_timestamp.md @@ -0,0 +1,59 @@ +# [Module ar_timestamp.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_timestamp.erl) + + + + + + +## Function Index ## + + +
cache/1*Cache the current timestamp from Arweave.
get/0Get the current timestamp from the server, starting the server if it +isn't already running.
refresher/1*Refresh the timestamp cache periodically.
spawn_server/0*Spawn a new server and its refresher.
start/0Check if the server is already running, and if not, start it.
+ + + + +## Function Details ## + + + +### cache/1 * ### + +`cache(Current) -> any()` + +Cache the current timestamp from Arweave. + + + +### get/0 ### + +`get() -> any()` + +Get the current timestamp from the server, starting the server if it +isn't already running. + + + +### refresher/1 * ### + +`refresher(TSServer) -> any()` + +Refresh the timestamp cache periodically. + + + +### spawn_server/0 * ### + +`spawn_server() -> any()` + +Spawn a new server and its refresher. + + + +### start/0 ### + +`start() -> any()` + +Check if the server is already running, and if not, start it. + diff --git a/docs/resources/source-code/ar_tx.md b/docs/resources/source-code/ar_tx.md new file mode 100644 index 000000000..1ff1243c0 --- /dev/null +++ b/docs/resources/source-code/ar_tx.md @@ -0,0 +1,107 @@ +# [Module ar_tx.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_tx.erl) + + + + +The module with utilities for transaction creation, signing, and verification. + + + +## Function Index ## + + +
collect_validation_results/2*
do_verify/2*Verify transaction.
json_struct_to_tx/1
new/4Create a new transaction.
new/5
sign/2Cryptographically sign (claim ownership of) a transaction.
signature_data_segment/1*Generate the data segment to be signed for a given TX.
tx_to_json_struct/1
verify/1Verify whether a transaction is valid.
verify_hash/1*Verify that the transaction's ID is a hash of its signature.
verify_signature/2*Verify the transaction's signature.
verify_tx_id/2Verify the given transaction actually has the given identifier.
+ + + + +## Function Details ## + + + +### collect_validation_results/2 * ### + +`collect_validation_results(TXID, Checks) -> any()` + + + +### do_verify/2 * ### + +`do_verify(TX, VerifySignature) -> any()` + +Verify transaction. + + + +### json_struct_to_tx/1 ### + +`json_struct_to_tx(TXStruct) -> any()` + + + +### new/4 ### + +`new(Dest, Reward, Qty, Last) -> any()` + +Create a new transaction. + + + +### new/5 ### + +`new(Dest, Reward, Qty, Last, SigType) -> any()` + + + +### sign/2 ### + +`sign(TX, X2) -> any()` + +Cryptographically sign (claim ownership of) a transaction. + + + +### signature_data_segment/1 * ### + +`signature_data_segment(TX) -> any()` + +Generate the data segment to be signed for a given TX. + + + +### tx_to_json_struct/1 ### + +`tx_to_json_struct(Tx) -> any()` + + + +### verify/1 ### + +`verify(TX) -> any()` + +Verify whether a transaction is valid. + + + +### verify_hash/1 * ### + +`verify_hash(Tx) -> any()` + +Verify that the transaction's ID is a hash of its signature. + + + +### verify_signature/2 * ### + +`verify_signature(TX, X2) -> any()` + +Verify the transaction's signature. + + + +### verify_tx_id/2 ### + +`verify_tx_id(ExpectedID, Tx) -> any()` + +Verify the given transaction actually has the given identifier. + diff --git a/docs/resources/source-code/ar_wallet.md b/docs/resources/source-code/ar_wallet.md new file mode 100644 index 000000000..da8b2069e --- /dev/null +++ b/docs/resources/source-code/ar_wallet.md @@ -0,0 +1,160 @@ +# [Module ar_wallet.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/ar_wallet.erl) + + + + + + +## Function Index ## + + +
compress_ecdsa_pubkey/1*
hash_address/1*
hmac/1
hmac/2
load_key/1Read the keyfile for the key with the given address from disk.
load_keyfile/1Extract the public and private key from a keyfile.
new/0
new/1
new_keyfile/2Generate a new wallet public and private key, with a corresponding keyfile.
sign/2Sign some data with a private key.
sign/3sign some data, hashed using the provided DigestType.
to_address/1Generate an address from a public key.
to_address/2
to_ecdsa_address/1*
to_rsa_address/1*
verify/3Verify that a signature is correct.
verify/4
wallet_filepath/1*
wallet_filepath/3*
wallet_filepath2/1*
wallet_name/3*
+ + + + +## Function Details ## + + + +### compress_ecdsa_pubkey/1 * ### + +`compress_ecdsa_pubkey(X1) -> any()` + + + +### hash_address/1 * ### + +`hash_address(PubKey) -> any()` + + + +### hmac/1 ### + +`hmac(Data) -> any()` + + + +### hmac/2 ### + +`hmac(Data, DigestType) -> any()` + + + +### load_key/1 ### + +`load_key(Addr) -> any()` + +Read the keyfile for the key with the given address from disk. +Return not_found if arweave_keyfile_[addr].json or [addr].json is not found +in [data_dir]/?WALLET_DIR. + + + +### load_keyfile/1 ### + +`load_keyfile(File) -> any()` + +Extract the public and private key from a keyfile. + + + +### new/0 ### + +`new() -> any()` + + + +### new/1 ### + +`new(KeyType) -> any()` + + + +### new_keyfile/2 ### + +`new_keyfile(KeyType, WalletName) -> any()` + +Generate a new wallet public and private key, with a corresponding keyfile. +The provided key is used as part of the file name. + + + +### sign/2 ### + +`sign(Key, Data) -> any()` + +Sign some data with a private key. + + + +### sign/3 ### + +`sign(X1, Data, DigestType) -> any()` + +sign some data, hashed using the provided DigestType. + + + +### to_address/1 ### + +`to_address(Pubkey) -> any()` + +Generate an address from a public key. + + + +### to_address/2 ### + +`to_address(PubKey, X2) -> any()` + + + +### to_ecdsa_address/1 * ### + +`to_ecdsa_address(PubKey) -> any()` + + + +### to_rsa_address/1 * ### + +`to_rsa_address(PubKey) -> any()` + + + +### verify/3 ### + +`verify(Key, Data, Sig) -> any()` + +Verify that a signature is correct. + + + +### verify/4 ### + +`verify(X1, Data, Sig, DigestType) -> any()` + + + +### wallet_filepath/1 * ### + +`wallet_filepath(Wallet) -> any()` + + + +### wallet_filepath/3 * ### + +`wallet_filepath(WalletName, PubKey, KeyType) -> any()` + + + +### wallet_filepath2/1 * ### + +`wallet_filepath2(Wallet) -> any()` + + + +### wallet_name/3 * ### + +`wallet_name(WalletName, PubKey, KeyType) -> any()` + diff --git a/docs/resources/source-code/dev_cache.md b/docs/resources/source-code/dev_cache.md new file mode 100644 index 000000000..f94697515 --- /dev/null +++ b/docs/resources/source-code/dev_cache.md @@ -0,0 +1,130 @@ +# [Module dev_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cache.erl) + + + + +A device that looks up an ID from a local store and returns it, +honoring the `accept` key to return the correct format. + + + +## Description ## +The cache also +supports writing messages to the store, if the node message has the +writer's address in its `cache_writers` key. + +## Function Index ## + + +
cache_write_binary_test/0*Ensure that we can write direct binaries to the cache.
cache_write_message_test/0*Test that the cache can be written to and read from using the hb_cache +API.
is_trusted_writer/2*Verify that the request originates from a trusted writer.
link/3Link a source to a destination in the cache.
read/3Read data from the cache.
read_from_cache/2*Read data from the cache via HTTP.
setup_test_env/0*Create a test environment with a local store and node.
write/3Write data to the cache.
write_single/2*Helper function to write a single data item to the cache.
write_to_cache/3*Write data to the cache via HTTP.
+ + + + +## Function Details ## + + + +### cache_write_binary_test/0 * ### + +`cache_write_binary_test() -> any()` + +Ensure that we can write direct binaries to the cache. + + + +### cache_write_message_test/0 * ### + +`cache_write_message_test() -> any()` + +Test that the cache can be written to and read from using the hb_cache +API. + + + +### is_trusted_writer/2 * ### + +`is_trusted_writer(Req, Opts) -> any()` + +Verify that the request originates from a trusted writer. +Checks that the single signer of the request is present in the list +of trusted cache writer addresses specified in the options. + + + +### link/3 ### + +`link(Base, Req, Opts) -> any()` + +Link a source to a destination in the cache. + + + +### read/3 ### + +`read(M1, M2, Opts) -> any()` + +Read data from the cache. +Retrieves data corresponding to a key from a local store. +The key is extracted from the incoming message under <<"target">>. +The options map may include store configuration. +If the "accept" header is set to <<"application/aos-2">>, the result is +converted to a JSON structure and encoded. + + + +### read_from_cache/2 * ### + +`read_from_cache(Node, Path) -> any()` + +Read data from the cache via HTTP. +Constructs a GET request using the provided path, sends it to the node, +and returns the response. + + + +### setup_test_env/0 * ### + +`setup_test_env() -> any()` + +Create a test environment with a local store and node. +Ensures that the required application is started, configures a local +file-system store, resets the store for a clean state, creates a wallet +for signing requests, and starts a node with the store and trusted cache +writer configuration. + + + +### write/3 ### + +`write(M1, M2, Opts) -> any()` + +Write data to the cache. +Processes a write request by first verifying that the request comes from a +trusted writer (as defined by the `cache_writers` configuration in the +options). The write type is determined from the message ("single" or "batch") +and the data is stored accordingly. + + + +### write_single/2 * ### + +`write_single(Msg, Opts) -> any()` + +Helper function to write a single data item to the cache. +Extracts the body, location, and operation from the message. +Depending on the type of data (map or binary) or if a link operation is +requested, it writes the data to the store using the appropriate function. + + + +### write_to_cache/3 * ### + +`write_to_cache(Node, Data, Wallet) -> any()` + +Write data to the cache via HTTP. +Constructs a write request message with the provided data, signs it with the +given wallet, sends it to the node, and verifies that the response indicates +a successful write. + diff --git a/docs/resources/source-code/dev_cacheviz.md b/docs/resources/source-code/dev_cacheviz.md new file mode 100644 index 000000000..60dd68313 --- /dev/null +++ b/docs/resources/source-code/dev_cacheviz.md @@ -0,0 +1,40 @@ +# [Module dev_cacheviz.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cacheviz.erl) + + + + +A device that generates renders (or renderable dot output) of a node's +cache. + + + +## Function Index ## + + +
dot/3Output the dot representation of the cache, or a specific path within +the cache set by the target key in the request.
svg/3Output the SVG representation of the cache, or a specific path within +the cache set by the target key in the request.
+ + + + +## Function Details ## + + + +### dot/3 ### + +`dot(X1, Req, Opts) -> any()` + +Output the dot representation of the cache, or a specific path within +the cache set by the `target` key in the request. + + + +### svg/3 ### + +`svg(Base, Req, Opts) -> any()` + +Output the SVG representation of the cache, or a specific path within +the cache set by the `target` key in the request. + diff --git a/docs/resources/source-code/dev_codec_ans104.md b/docs/resources/source-code/dev_codec_ans104.md new file mode 100644 index 000000000..a597b00be --- /dev/null +++ b/docs/resources/source-code/dev_codec_ans104.md @@ -0,0 +1,199 @@ +# [Module dev_codec_ans104.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_ans104.erl) + + + + +Codec for managing transformations from `ar_bundles`-style Arweave TX +records to and from TABMs. + + + +## Function Index ## + + +
commit/3Sign a message using the priv_wallet key in the options.
committed/3Return a list of committed keys from an ANS-104 message.
committed_from_trusted_keys/3*
content_type/1Return the content type for the codec.
deduplicating_from_list/1*Deduplicate a list of key-value pairs by key, generating a list of +values for each normalized key if there are duplicates.
deserialize/1Deserialize a binary ans104 message to a TABM.
do_from/1*
duplicated_tag_name_test/0*
encoded_tags_to_map/1*Convert an ANS-104 encoded tag list into a HyperBEAM-compatible map.
from/1Convert a #tx record into a message map recursively.
from_maintains_tag_name_case_test/0*
id/1Return the ID of a message.
normal_tags/1*Check whether a list of key-value pairs contains only normalized keys.
normal_tags_test/0*
only_committed_maintains_target_test/0*
quantity_field_is_ignored_in_from_test/0*
quantity_key_encoded_as_tag_test/0*
restore_tag_name_case_from_cache_test/0*
serialize/1Serialize a message or TX to a binary.
signed_duplicated_tag_name_test/0*
simple_to_conversion_test/0*
tag_map_to_encoded_tags/1*Convert a HyperBEAM-compatible map into an ANS-104 encoded tag list, +recreating the original order of the tags.
to/1Internal helper to translate a message to its #tx record representation, +which can then be used by ar_bundles to serialize the message.
verify/3Verify an ANS-104 commitment.
+ + + + +## Function Details ## + + + +### commit/3 ### + +`commit(Msg, Req, Opts) -> any()` + +Sign a message using the `priv_wallet` key in the options. + + + +### committed/3 ### + +`committed(Msg, Req, Opts) -> any()` + +Return a list of committed keys from an ANS-104 message. + + + +### committed_from_trusted_keys/3 * ### + +`committed_from_trusted_keys(Msg, TrustedKeys, Opts) -> any()` + + + +### content_type/1 ### + +`content_type(X1) -> any()` + +Return the content type for the codec. + + + +### deduplicating_from_list/1 * ### + +`deduplicating_from_list(Tags) -> any()` + +Deduplicate a list of key-value pairs by key, generating a list of +values for each normalized key if there are duplicates. + + + +### deserialize/1 ### + +`deserialize(Binary) -> any()` + +Deserialize a binary ans104 message to a TABM. + + + +### do_from/1 * ### + +`do_from(RawTX) -> any()` + + + +### duplicated_tag_name_test/0 * ### + +`duplicated_tag_name_test() -> any()` + + + +### encoded_tags_to_map/1 * ### + +`encoded_tags_to_map(Tags) -> any()` + +Convert an ANS-104 encoded tag list into a HyperBEAM-compatible map. + + + +### from/1 ### + +`from(Binary) -> any()` + +Convert a #tx record into a message map recursively. + + + +### from_maintains_tag_name_case_test/0 * ### + +`from_maintains_tag_name_case_test() -> any()` + + + +### id/1 ### + +`id(Msg) -> any()` + +Return the ID of a message. + + + +### normal_tags/1 * ### + +`normal_tags(Tags) -> any()` + +Check whether a list of key-value pairs contains only normalized keys. + + + +### normal_tags_test/0 * ### + +`normal_tags_test() -> any()` + + + +### only_committed_maintains_target_test/0 * ### + +`only_committed_maintains_target_test() -> any()` + + + +### quantity_field_is_ignored_in_from_test/0 * ### + +`quantity_field_is_ignored_in_from_test() -> any()` + + + +### quantity_key_encoded_as_tag_test/0 * ### + +`quantity_key_encoded_as_tag_test() -> any()` + + + +### restore_tag_name_case_from_cache_test/0 * ### + +`restore_tag_name_case_from_cache_test() -> any()` + + + +### serialize/1 ### + +`serialize(Msg) -> any()` + +Serialize a message or TX to a binary. + + + +### signed_duplicated_tag_name_test/0 * ### + +`signed_duplicated_tag_name_test() -> any()` + + + +### simple_to_conversion_test/0 * ### + +`simple_to_conversion_test() -> any()` + + + +### tag_map_to_encoded_tags/1 * ### + +`tag_map_to_encoded_tags(TagMap) -> any()` + +Convert a HyperBEAM-compatible map into an ANS-104 encoded tag list, +recreating the original order of the tags. + + + +### to/1 ### + +`to(Binary) -> any()` + +Internal helper to translate a message to its #tx record representation, +which can then be used by ar_bundles to serialize the message. We call the +message's device in order to get the keys that we will be checkpointing. We +do this recursively to handle nested messages. The base case is that we hit +a binary, which we return as is. + + + +### verify/3 ### + +`verify(Msg, Req, Opts) -> any()` + +Verify an ANS-104 commitment. + diff --git a/docs/resources/source-code/dev_codec_flat.md b/docs/resources/source-code/dev_codec_flat.md new file mode 100644 index 000000000..912442777 --- /dev/null +++ b/docs/resources/source-code/dev_codec_flat.md @@ -0,0 +1,115 @@ +# [Module dev_codec_flat.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_flat.erl) + + + + +A codec for turning TABMs into/from flat Erlang maps that have +(potentially multi-layer) paths as their keys, and a normal TABM binary as +their value. + + + +## Function Index ## + + +
binary_passthrough_test/0*
commit/3
committed/3
deep_nesting_test/0*
deserialize/1
empty_map_test/0*
from/1Convert a flat map to a TABM.
inject_at_path/3*
multiple_paths_test/0*
nested_conversion_test/0*
path_list_test/0*
serialize/1
simple_conversion_test/0*
to/1Convert a TABM to a flat map.
verify/3
+ + + + +## Function Details ## + + + +### binary_passthrough_test/0 * ### + +`binary_passthrough_test() -> any()` + + + +### commit/3 ### + +`commit(Msg, Req, Opts) -> any()` + + + +### committed/3 ### + +`committed(Msg, Req, Opts) -> any()` + + + +### deep_nesting_test/0 * ### + +`deep_nesting_test() -> any()` + + + +### deserialize/1 ### + +`deserialize(Bin) -> any()` + + + +### empty_map_test/0 * ### + +`empty_map_test() -> any()` + + + +### from/1 ### + +`from(Bin) -> any()` + +Convert a flat map to a TABM. + + + +### inject_at_path/3 * ### + +`inject_at_path(Rest, Value, Map) -> any()` + + + +### multiple_paths_test/0 * ### + +`multiple_paths_test() -> any()` + + + +### nested_conversion_test/0 * ### + +`nested_conversion_test() -> any()` + + + +### path_list_test/0 * ### + +`path_list_test() -> any()` + + + +### serialize/1 ### + +`serialize(Map) -> any()` + + + +### simple_conversion_test/0 * ### + +`simple_conversion_test() -> any()` + + + +### to/1 ### + +`to(Bin) -> any()` + +Convert a TABM to a flat map. + + + +### verify/3 ### + +`verify(Msg, Req, Opts) -> any()` + diff --git a/docs/resources/source-code/dev_codec_httpsig.md b/docs/resources/source-code/dev_codec_httpsig.md new file mode 100644 index 000000000..974533e6a --- /dev/null +++ b/docs/resources/source-code/dev_codec_httpsig.md @@ -0,0 +1,636 @@ +# [Module dev_codec_httpsig.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_httpsig.erl) + + + + +This module implements HTTP Message Signatures as described in RFC-9421 +(https://datatracker.ietf.org/doc/html/rfc9421), as an AO-Core device. + + + +## Description ## +It implements the codec standard (from/1, to/1), as well as the optional +commitment functions (id/3, sign/3, verify/3). The commitment functions +are found in this module, while the codec functions are relayed to the +`dev_codec_httpsig_conv` module. + + +## Data Types ## + + + + +### authority_state() ### + + +

+authority_state() = #{component_identifiers => [component_identifier()], sig_params => signature_params(), key => binary()}
+
+ + + + +### component_identifier() ### + + +

+component_identifier() = {item, {string, binary()}, {binary(), integer() | boolean() | {string | token | binary, binary()}}}
+
+ + + + +### fields() ### + + +

+fields() = #{binary() | atom() | string() => binary() | atom() | string()}
+
+ + + + +### request_message() ### + + +

+request_message() = #{url => binary(), method => binary(), headers => fields(), trailers => fields(), is_absolute_form => boolean()}
+
+ + + + +### response_message() ### + + +

+response_message() = #{status => integer(), headers => fields(), trailers => fields()}
+
+ + + + +### signature_params() ### + + +

+signature_params() = #{atom() | binary() | string() => binary() | integer()}
+
+ + + +## Function Index ## + + +
add_content_digest/1If the body key is present, replace it with a content-digest.
add_derived_specifiers/1Normalize key parameters to ensure their names are correct.
add_sig_params/2*Add the signature parameters to the authority state.
address_to_sig_name/1*Convert an address to a signature name that is short, unique to the +address, and lowercase.
authority/3*A helper to validate and produce an "Authority" State.
bin/1*
commit/3Main entrypoint for signing a HTTP Message, using the standardized format.
committed/3Return the list of committed keys from a message.
committed_from_body/1*Return the list of committed keys from a message that are derived from +the body components.
committed_id_test/0*
derive_component/3*Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, specifically a "Derived" Component within the Messages Context, +and return the normalized form of the identifier, along with the extracted +encoded value.
derive_component/4*
derive_component_error_query_param_no_name_test/0*
derive_component_error_req_param_on_request_target_test/0*
derive_component_error_status_req_target_test/0*
do_committed/4*
extract_dictionary_field_value/2*Extract a value from a Structured Field, and return the normalized field, +along with the encoded value.
extract_field/3*Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, specifically a field on a Message within the Messages Context, +and return the normalized form of the identifier, along with the extracted +encoded value.
extract_field_value/2*Extract values from the field and return the normalized field, +along with encoded value.
find_byte_sequence_param/1*
find_id/1*Find the ID of the message, which is the hmac of the fields referenced in +the signature and signature input.
find_key_param/1*
find_name_param/1*
find_request_param/1*
find_sf_param/3*Given a parameter Name, extract the Parameter value from the HTTP +Structured Field data structure.
find_strict_format_param/1*
find_trailer_param/1*
from/1
hmac/1*Generate the ID of the message, with the current signature and signature +input as the components for the hmac.
id/3
identifier_to_component/3*Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, and return the normalized form of the identifier, along with the +extracted encoded value.
join_signature_base/2*
join_signature_base_test/0*
lower_bin/1*
multicommitted_id_test/0*
normalize_component_identifiers/1*Takes a list of keys that will be used in the signature inputs and +ensures that they have deterministic sorting, as well as the coorect +component identifiers if applicable.
public_keys/1
remove_derived_specifiers/1Remove derived specifiers from a list of component identifiers.
reset_hmac/1Ensure that the commitments and hmac are properly encoded.
sf_encode/1*Attempt to encode the data structure into an HTTP Structured Field.
sf_encode/2*
sf_item/1*Attempt to parse the provided value into an HTTP Structured Field Item.
sf_parse/1*Attempt to parse the binary into a data structure that represents +an HTTP Structured Field.
sf_parse/2*
sf_signature_param/1*construct the structured field Parameter for the signature parameter, +checking whether the parameter name is valid according RFC-9421.
sf_signature_params/2*construct the structured field List for the +"signature-params-line" part of the signature base.
sig_name_from_dict/1*
sign_auth/3*using the provided Authority and Request/Response Messages Context, +create a Name, Signature and SignatureInput that can be used to additional +signatures to a corresponding HTTP Message.
signature_base/3*create the signature base that will be signed in order to create the +Signature and SignatureInput.
signature_components_line/3*Given a list of Component Identifiers and a Request/Response Message +context, create the "signature-base-line" portion of the signature base.
signature_params_line/2*construct the "signature-params-line" part of the signature base.
signature_params_line_test/0*
to/1
trim_and_normalize/1*
trim_ws/1*Recursively trim space characters from the beginning of the binary.
trim_ws_end/2*
trim_ws_test/0*
upper_bin/1*
validate_large_message_from_http_test/0*Ensure that we can validate a signature on an extremely large and complex +message that is sent over HTTP, signed with the codec.
verify/3Verify different forms of httpsig committed messages.
verify_auth/2*same verify/3, but with an empty Request Message Context.
verify_auth/3*Given the signature name, and the Request/Response Message Context +verify the named signature by constructing the signature base and comparing.
+ + + + +## Function Details ## + + + +### add_content_digest/1 ### + +`add_content_digest(Msg) -> any()` + +If the `body` key is present, replace it with a content-digest. + + + +### add_derived_specifiers/1 ### + +`add_derived_specifiers(ComponentIdentifiers) -> any()` + +Normalize key parameters to ensure their names are correct. + + + +### add_sig_params/2 * ### + +`add_sig_params(Authority, X2) -> any()` + +Add the signature parameters to the authority state + + + +### address_to_sig_name/1 * ### + +

+address_to_sig_name(Address::binary()) -> binary()
+
+
+ +Convert an address to a signature name that is short, unique to the +address, and lowercase. + + + +### authority/3 * ### + +

+authority(ComponentIdentifiers::[binary() | component_identifier()], SigParams::#{binary() => binary() | integer()}, PubKey::{}) -> authority_state()
+
+
+ +A helper to validate and produce an "Authority" State + + + +### bin/1 * ### + +`bin(Item) -> any()` + + + +### commit/3 ### + +`commit(MsgToSign, Req, Opts) -> any()` + +Main entrypoint for signing a HTTP Message, using the standardized format. + + + +### committed/3 ### + +`committed(RawMsg, Req, Opts) -> any()` + +Return the list of committed keys from a message. The message will have +had the `commitments` key removed and the signature inputs added to the +root. Subsequently, we can parse that to get the list of committed keys. + + + +### committed_from_body/1 * ### + +`committed_from_body(Msg) -> any()` + +Return the list of committed keys from a message that are derived from +the body components. + + + +### committed_id_test/0 * ### + +`committed_id_test() -> any()` + + + +### derive_component/3 * ### + +`derive_component(Identifier, Req, Res) -> any()` + +Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, specifically a "Derived" Component within the Messages Context, +and return the normalized form of the identifier, along with the extracted +encoded value. + +This implements a portion of RFC-9421 +See https://datatracker.ietf.org/doc/html/rfc9421#name-derived-components + + + +### derive_component/4 * ### + +`derive_component(X1, Req, Res, Subject) -> any()` + + + +### derive_component_error_query_param_no_name_test/0 * ### + +`derive_component_error_query_param_no_name_test() -> any()` + + + +### derive_component_error_req_param_on_request_target_test/0 * ### + +`derive_component_error_req_param_on_request_target_test() -> any()` + + + +### derive_component_error_status_req_target_test/0 * ### + +`derive_component_error_status_req_target_test() -> any()` + + + +### do_committed/4 * ### + +`do_committed(SigInputStr, Msg, Req, Opts) -> any()` + + + +### extract_dictionary_field_value/2 * ### + +`extract_dictionary_field_value(StructuredField, Key) -> any()` + +Extract a value from a Structured Field, and return the normalized field, +along with the encoded value + + + +### extract_field/3 * ### + +`extract_field(X1, Req, Res) -> any()` + +Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, specifically a field on a Message within the Messages Context, +and return the normalized form of the identifier, along with the extracted +encoded value. + +This implements a portion of RFC-9421 +See https://datatracker.ietf.org/doc/html/rfc9421#name-http-fields + + + +### extract_field_value/2 * ### + +`extract_field_value(RawFields, X2) -> any()` + +Extract values from the field and return the normalized field, +along with encoded value + + + +### find_byte_sequence_param/1 * ### + +`find_byte_sequence_param(Params) -> any()` + + + +### find_id/1 * ### + +`find_id(Msg) -> any()` + +Find the ID of the message, which is the hmac of the fields referenced in +the signature and signature input. If the message already has a signature-input, +directly, it is treated differently: We relabel it as `x-signature-input` to +avoid key collisions. + + + +### find_key_param/1 * ### + +`find_key_param(Params) -> any()` + + + +### find_name_param/1 * ### + +`find_name_param(Params) -> any()` + + + +### find_request_param/1 * ### + +`find_request_param(Params) -> any()` + + + +### find_sf_param/3 * ### + +`find_sf_param(Name, Params, Default) -> any()` + +Given a parameter Name, extract the Parameter value from the HTTP +Structured Field data structure. + +If no value is found, then false is returned + + + +### find_strict_format_param/1 * ### + +`find_strict_format_param(Params) -> any()` + + + +### find_trailer_param/1 * ### + +`find_trailer_param(Params) -> any()` + + + +### from/1 ### + +`from(Msg) -> any()` + + + +### hmac/1 * ### + +`hmac(Msg) -> any()` + +Generate the ID of the message, with the current signature and signature +input as the components for the hmac. + + + +### id/3 ### + +`id(Msg, Params, Opts) -> any()` + + + +### identifier_to_component/3 * ### + +`identifier_to_component(Identifier, Req, Res) -> any()` + +Given a Component Identifier and a Request/Response Messages Context +extract the value represented by the Component Identifier, from the Messages +Context, and return the normalized form of the identifier, along with the +extracted encoded value. + +Generally speaking, a Component Identifier may reference a "Derived" Component, +a Message Field, or a sub-component of a Message Field. + +Since a Component Identifier is itself a Structured Field, it may also specify +parameters, which are used to describe behavior such as which Message to +derive a field or sub-component of the field, and how to encode the value as +part of the signature base. + + + +### join_signature_base/2 * ### + +`join_signature_base(ComponentsLine, ParamsLine) -> any()` + + + +### join_signature_base_test/0 * ### + +`join_signature_base_test() -> any()` + + + +### lower_bin/1 * ### + +`lower_bin(Item) -> any()` + + + +### multicommitted_id_test/0 * ### + +`multicommitted_id_test() -> any()` + + + +### normalize_component_identifiers/1 * ### + +`normalize_component_identifiers(ComponentIdentifiers) -> any()` + +Takes a list of keys that will be used in the signature inputs and +ensures that they have deterministic sorting, as well as the coorect +component identifiers if applicable. + + + +### public_keys/1 ### + +`public_keys(Commitment) -> any()` + + + +### remove_derived_specifiers/1 ### + +`remove_derived_specifiers(ComponentIdentifiers) -> any()` + +Remove derived specifiers from a list of component identifiers. + + + +### reset_hmac/1 ### + +`reset_hmac(RawMsg) -> any()` + +Ensure that the commitments and hmac are properly encoded + + + +### sf_encode/1 * ### + +`sf_encode(StructuredField) -> any()` + +Attempt to encode the data structure into an HTTP Structured Field. +This is the inverse of sf_parse. + + + +### sf_encode/2 * ### + +`sf_encode(Serializer, StructuredField) -> any()` + + + +### sf_item/1 * ### + +`sf_item(SfItem) -> any()` + +Attempt to parse the provided value into an HTTP Structured Field Item + + + +### sf_parse/1 * ### + +`sf_parse(Raw) -> any()` + +Attempt to parse the binary into a data structure that represents +an HTTP Structured Field. + +Lacking some sort of "hint", there isn't a way to know which "kind" of +Structured Field the binary is, apriori. So we simply try each parser, +and return the first invocation that doesn't result in an error. + +If no parser is successful, then we return an error tuple + + + +### sf_parse/2 * ### + +`sf_parse(Rest, Raw) -> any()` + + + +### sf_signature_param/1 * ### + +`sf_signature_param(X1) -> any()` + +construct the structured field Parameter for the signature parameter, +checking whether the parameter name is valid according RFC-9421 + +See https://datatracker.ietf.org/doc/html/rfc9421#section-2.3-3 + + + +### sf_signature_params/2 * ### + +`sf_signature_params(ComponentIdentifiers, SigParams) -> any()` + +construct the structured field List for the +"signature-params-line" part of the signature base. + +Can be parsed into a binary by simply passing to hb_structured_fields:list/1 + +See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.3.2.4 + + + +### sig_name_from_dict/1 * ### + +`sig_name_from_dict(DictBin) -> any()` + + + +### sign_auth/3 * ### + +

+sign_auth(Authority::authority_state(), Req::request_message(), Res::response_message()) -> {ok, {binary(), binary(), binary()}}
+
+
+ +using the provided Authority and Request/Response Messages Context, +create a Name, Signature and SignatureInput that can be used to additional +signatures to a corresponding HTTP Message + + + +### signature_base/3 * ### + +`signature_base(Authority, Req, Res) -> any()` + +create the signature base that will be signed in order to create the +Signature and SignatureInput. + +This implements a portion of RFC-9421 see: +https://datatracker.ietf.org/doc/html/rfc9421#name-creating-the-signature-base + + + +### signature_components_line/3 * ### + +`signature_components_line(ComponentIdentifiers, Req, Res) -> any()` + +Given a list of Component Identifiers and a Request/Response Message +context, create the "signature-base-line" portion of the signature base + + + +### signature_params_line/2 * ### + +`signature_params_line(ComponentIdentifiers, SigParams) -> any()` + +construct the "signature-params-line" part of the signature base. + +See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.3.2.4 + + + +### signature_params_line_test/0 * ### + +`signature_params_line_test() -> any()` + + + +### to/1 ### + +`to(Msg) -> any()` + + + +### trim_and_normalize/1 * ### + +`trim_and_normalize(Bin) -> any()` + + + +### trim_ws/1 * ### + +`trim_ws(Bin) -> any()` + +Recursively trim space characters from the beginning of the binary + + + +### trim_ws_end/2 * ### + +`trim_ws_end(Value, N) -> any()` + + + +### trim_ws_test/0 * ### + +`trim_ws_test() -> any()` + + + +### upper_bin/1 * ### + +`upper_bin(Item) -> any()` + + + +### validate_large_message_from_http_test/0 * ### + +`validate_large_message_from_http_test() -> any()` + +Ensure that we can validate a signature on an extremely large and complex +message that is sent over HTTP, signed with the codec. + + + +### verify/3 ### + +`verify(MsgToVerify, Req, Opts) -> any()` + +Verify different forms of httpsig committed messages. `dev_message:verify` +already places the keys from the commitment message into the root of the +message. + + + +### verify_auth/2 * ### + +`verify_auth(Verifier, Msg) -> any()` + +same verify/3, but with an empty Request Message Context + + + +### verify_auth/3 * ### + +`verify_auth(X1, Req, Res) -> any()` + +Given the signature name, and the Request/Response Message Context +verify the named signature by constructing the signature base and comparing + diff --git a/docs/resources/source-code/dev_codec_httpsig_conv.md b/docs/resources/source-code/dev_codec_httpsig_conv.md new file mode 100644 index 000000000..17f5f28ca --- /dev/null +++ b/docs/resources/source-code/dev_codec_httpsig_conv.md @@ -0,0 +1,241 @@ +# [Module dev_codec_httpsig_conv.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_httpsig_conv.erl) + + + + +A codec for the that marshals TABM encoded messages to and from the +"HTTP" message structure. + + + +## Description ## + +Every HTTP message is an HTTP multipart message. +See https://datatracker.ietf.org/doc/html/rfc7578 + +For each TABM Key: + +The Key/Value Pair will be encoded according to the following rules: +"signatures" -> {SignatureInput, Signature} header Tuples, each encoded +as a Structured Field Dictionary +"body" -> +- if a map, then recursively encode as its own HyperBEAM message +- otherwise encode as a normal field +_ -> encode as a normal field + +Each field will be mapped to the HTTP Message according to the following +rules: +"body" -> always encoded part of the body as with Content-Disposition +type of "inline" +_ -> +- If the byte size of the value is less than the ?MAX_TAG_VALUE, +then encode as a header, also attempting to encode as a +structured field. +- Otherwise encode the value as a part in the multipart response + + +## Function Index ## + + +
boundary_from_parts/1*Generate a unique, reproducible boundary for the +multipart body, however we cannot use the id of the message as +the boundary, as the id is not known until the message is +encoded.
commitments_from_signature/4*Populate the /commitments key on the TABM with the dictionary of +signatures and their corresponding inputs.
do_to/2*
encode_body_keys/1*Encode a list of body parts into a binary.
encode_body_part/3*Encode a multipart body part to a flat binary.
encode_http_msg/1*Encode a HTTP message into a binary.
extract_hashpaths/1*Extract all keys labelled hashpath* from the commitments, and add them +to the HTTP message as hashpath* keys.
field_to_http/3*All maps are encoded into the body of the HTTP message +to be further encoded later.
from/1Convert a HTTP Message into a TABM.
from_body/4*
from_body_parts/3*
group_ids/1*Group all elements with: +1.
group_maps/1*Merge maps at the same level, if possible.
group_maps/3*
group_maps_flat_compatible_test/0*The grouped maps encoding is a subset of the flat encoding, +where on keys with maps values are flattened.
group_maps_test/0*
hashpaths_from_message/1*
inline_key/1*given a message, returns a binary tuple: +- A list of pairs to add to the msg, if any +- the field name for the inlined key.
to/1Convert a TABM into an HTTP Message.
to/2*
ungroup_ids/1*Decode the ao-ids key into a map.
+ + + + +## Function Details ## + + + +### boundary_from_parts/1 * ### + +`boundary_from_parts(PartList) -> any()` + +Generate a unique, reproducible boundary for the +multipart body, however we cannot use the id of the message as +the boundary, as the id is not known until the message is +encoded. Subsequently, we generate each body part individually, +concatenate them, and apply a SHA2-256 hash to the result. +This ensures that the boundary is unique, reproducible, and +secure. + + + +### commitments_from_signature/4 * ### + +`commitments_from_signature(Map, HPs, RawSig, RawSigInput) -> any()` + +Populate the `/commitments` key on the TABM with the dictionary of +signatures and their corresponding inputs. + + + +### do_to/2 * ### + +`do_to(Binary, Opts) -> any()` + + + +### encode_body_keys/1 * ### + +`encode_body_keys(PartList) -> any()` + +Encode a list of body parts into a binary. + + + +### encode_body_part/3 * ### + +`encode_body_part(PartName, BodyPart, InlineKey) -> any()` + +Encode a multipart body part to a flat binary. + + + +### encode_http_msg/1 * ### + +`encode_http_msg(Httpsig) -> any()` + +Encode a HTTP message into a binary. + + + +### extract_hashpaths/1 * ### + +`extract_hashpaths(Map) -> any()` + +Extract all keys labelled `hashpath*` from the commitments, and add them +to the HTTP message as `hashpath*` keys. + + + +### field_to_http/3 * ### + +`field_to_http(Httpsig, X2, Opts) -> any()` + +All maps are encoded into the body of the HTTP message +to be further encoded later. + + + +### from/1 ### + +`from(Bin) -> any()` + +Convert a HTTP Message into a TABM. +HTTP Structured Field is encoded into it's equivalent TABM encoding. + + + +### from_body/4 * ### + +`from_body(TABM, InlinedKey, ContentType, Body) -> any()` + + + +### from_body_parts/3 * ### + +`from_body_parts(TABM, InlinedKey, Rest) -> any()` + + + +### group_ids/1 * ### + +`group_ids(Map) -> any()` + +Group all elements with: +1. A key that ?IS_ID returns true for, and +2. A value that is immediate +into a combined SF dict-_like_ structure. If not encoded, these keys would +be sent as headers and lower-cased, losing their comparability against the +original keys. The structure follows all SF dict rules, except that it allows +for keys to contain capitals. The HyperBEAM SF parser will accept these keys, +but standard RFC 8741 parsers will not. Subsequently, the resulting `ao-cased` +key is not added to the `ao-types` map. + + + +### group_maps/1 * ### + +`group_maps(Map) -> any()` + +Merge maps at the same level, if possible. + + + +### group_maps/3 * ### + +`group_maps(Map, Parent, Top) -> any()` + + + +### group_maps_flat_compatible_test/0 * ### + +`group_maps_flat_compatible_test() -> any()` + +The grouped maps encoding is a subset of the flat encoding, +where on keys with maps values are flattened. + +So despite needing a special encoder to produce it +We can simply apply the flat encoder to it to get back +the original message. + +The test asserts that is indeed the case. + + + +### group_maps_test/0 * ### + +`group_maps_test() -> any()` + + + +### hashpaths_from_message/1 * ### + +`hashpaths_from_message(Msg) -> any()` + + + +### inline_key/1 * ### + +`inline_key(Msg) -> any()` + +given a message, returns a binary tuple: +- A list of pairs to add to the msg, if any +- the field name for the inlined key + +In order to preserve the field name of the inlined +part, an additional field may need to be added + + + +### to/1 ### + +`to(Bin) -> any()` + +Convert a TABM into an HTTP Message. The HTTP Message is a simple Erlang Map +that can translated to a given web server Response API + + + +### to/2 * ### + +`to(TABM, Opts) -> any()` + + + +### ungroup_ids/1 * ### + +`ungroup_ids(Msg) -> any()` + +Decode the `ao-ids` key into a map. + diff --git a/docs/resources/source-code/dev_codec_json.md b/docs/resources/source-code/dev_codec_json.md new file mode 100644 index 000000000..9f69c1b45 --- /dev/null +++ b/docs/resources/source-code/dev_codec_json.md @@ -0,0 +1,82 @@ +# [Module dev_codec_json.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_json.erl) + + + + +A simple JSON codec for HyperBEAM's message format. + + + +## Description ## +Takes a +message as TABM and returns an encoded JSON string representation. +This codec utilizes the httpsig@1.0 codec for signing and verifying. + +## Function Index ## + + +
commit/3
committed/1
content_type/1Return the content type for the codec.
deserialize/3Deserialize the JSON string found at the given path.
from/1Decode a JSON string to a message.
serialize/3Serialize a message to a JSON string.
to/1Encode a message to a JSON string.
verify/3
+ + + + +## Function Details ## + + + +### commit/3 ### + +`commit(Msg, Req, Opts) -> any()` + + + +### committed/1 ### + +`committed(Msg) -> any()` + + + +### content_type/1 ### + +`content_type(X1) -> any()` + +Return the content type for the codec. + + + +### deserialize/3 ### + +`deserialize(Base, Req, Opts) -> any()` + +Deserialize the JSON string found at the given path. + + + +### from/1 ### + +`from(Map) -> any()` + +Decode a JSON string to a message. + + + +### serialize/3 ### + +`serialize(Base, Msg, Opts) -> any()` + +Serialize a message to a JSON string. + + + +### to/1 ### + +`to(Msg) -> any()` + +Encode a message to a JSON string. + + + +### verify/3 ### + +`verify(Msg, Req, Opts) -> any()` + diff --git a/docs/resources/source-code/dev_codec_structured.md b/docs/resources/source-code/dev_codec_structured.md new file mode 100644 index 000000000..1b28f9735 --- /dev/null +++ b/docs/resources/source-code/dev_codec_structured.md @@ -0,0 +1,106 @@ +# [Module dev_codec_structured.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_codec_structured.erl) + + + + +A device implementing the codec interface (to/1, from/1) for +HyperBEAM's internal, richly typed message format. + + + +## Description ## + +This format mirrors HTTP Structured Fields, aside from its limitations of +compound type depths, as well as limited floating point representations. + +As with all AO-Core codecs, its target format (the format it expects to +receive in the `to/1` function, and give in `from/1`) is TABM. + +For more details, see the HTTP Structured Fields (RFC-9651) specification. + +## Function Index ## + + +
commit/3
committed/3
decode_value/2Convert non-binary values to binary for serialization.
encode_value/1Convert a term to a binary representation, emitting its type for +serialization as a separate tag.
from/1Convert a rich message into a 'Type-Annotated-Binary-Message' (TABM).
implicit_keys/1Find the implicit keys of a TABM.
list_encoding_test/0*
parse_ao_types/1*Parse the ao-types field of a TABM and return a map of keys and their +types.
to/1Convert a TABM into a native HyperBEAM message.
verify/3
+ + + + +## Function Details ## + + + +### commit/3 ### + +`commit(Msg, Req, Opts) -> any()` + + + +### committed/3 ### + +`committed(Msg, Req, Opts) -> any()` + + + +### decode_value/2 ### + +`decode_value(Type, Value) -> any()` + +Convert non-binary values to binary for serialization. + + + +### encode_value/1 ### + +`encode_value(Value) -> any()` + +Convert a term to a binary representation, emitting its type for +serialization as a separate tag. + + + +### from/1 ### + +`from(Bin) -> any()` + +Convert a rich message into a 'Type-Annotated-Binary-Message' (TABM). + + + +### implicit_keys/1 ### + +`implicit_keys(Req) -> any()` + +Find the implicit keys of a TABM. + + + +### list_encoding_test/0 * ### + +`list_encoding_test() -> any()` + + + +### parse_ao_types/1 * ### + +`parse_ao_types(Msg) -> any()` + +Parse the `ao-types` field of a TABM and return a map of keys and their +types + + + +### to/1 ### + +`to(Bin) -> any()` + +Convert a TABM into a native HyperBEAM message. + + + +### verify/3 ### + +`verify(Msg, Req, Opts) -> any()` + diff --git a/docs/resources/source-code/dev_cron.md b/docs/resources/source-code/dev_cron.md new file mode 100644 index 000000000..264bf9c5a --- /dev/null +++ b/docs/resources/source-code/dev_cron.md @@ -0,0 +1,127 @@ +# [Module dev_cron.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cron.erl) + + + + +A device that inserts new messages into the schedule to allow processes +to passively 'call' themselves without user interaction. + + + +## Function Index ## + + +
every/3Exported function for scheduling a recurring message.
every_worker_loop/4*
every_worker_loop_test/0*This test verifies that a recurring task can be scheduled and executed.
info/1Exported function for getting device info.
info/3
once/3Exported function for scheduling a one-time message.
once_executed_test/0*This test verifies that a one-time task can be scheduled and executed.
once_worker/3*Internal function for scheduling a one-time message.
parse_time/1*Parse a time string into milliseconds.
stop/3Exported function for stopping a scheduled task.
stop_every_test/0*This test verifies that a recurring task can be stopped by +calling the stop function with the task ID.
stop_once_test/0*
test_worker/0*This is a helper function that is used to test the cron device.
test_worker/1*
+ + + + +## Function Details ## + + + +### every/3 ### + +`every(Msg1, Msg2, Opts) -> any()` + +Exported function for scheduling a recurring message. + + + +### every_worker_loop/4 * ### + +`every_worker_loop(CronPath, Req, Opts, IntervalMillis) -> any()` + + + +### every_worker_loop_test/0 * ### + +`every_worker_loop_test() -> any()` + +This test verifies that a recurring task can be scheduled and executed. + + + +### info/1 ### + +`info(X1) -> any()` + +Exported function for getting device info. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + + + +### once/3 ### + +`once(Msg1, Msg2, Opts) -> any()` + +Exported function for scheduling a one-time message. + + + +### once_executed_test/0 * ### + +`once_executed_test() -> any()` + +This test verifies that a one-time task can be scheduled and executed. + + + +### once_worker/3 * ### + +`once_worker(Path, Req, Opts) -> any()` + +Internal function for scheduling a one-time message. + + + +### parse_time/1 * ### + +`parse_time(BinString) -> any()` + +Parse a time string into milliseconds. + + + +### stop/3 ### + +`stop(Msg1, Msg2, Opts) -> any()` + +Exported function for stopping a scheduled task. + + + +### stop_every_test/0 * ### + +`stop_every_test() -> any()` + +This test verifies that a recurring task can be stopped by +calling the stop function with the task ID. + + + +### stop_once_test/0 * ### + +`stop_once_test() -> any()` + + + +### test_worker/0 * ### + +`test_worker() -> any()` + +This is a helper function that is used to test the cron device. +It is used to increment a counter and update the state of the worker. + + + +### test_worker/1 * ### + +`test_worker(State) -> any()` + diff --git a/docs/resources/source-code/dev_cu.md b/docs/resources/source-code/dev_cu.md new file mode 100644 index 000000000..6442e165e --- /dev/null +++ b/docs/resources/source-code/dev_cu.md @@ -0,0 +1,29 @@ +# [Module dev_cu.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_cu.erl) + + + + + + +## Function Index ## + + +
execute/2
push/2
+ + + + +## Function Details ## + + + +### execute/2 ### + +`execute(CarrierMsg, S) -> any()` + + + +### push/2 ### + +`push(Msg, S) -> any()` + diff --git a/docs/resources/source-code/dev_dedup.md b/docs/resources/source-code/dev_dedup.md new file mode 100644 index 000000000..7578408fe --- /dev/null +++ b/docs/resources/source-code/dev_dedup.md @@ -0,0 +1,53 @@ +# [Module dev_dedup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_dedup.erl) + + + + +A device that deduplicates messages send to a process. + + + +## Description ## +Only runs on the first pass of the `compute` key call if executed +in a stack. Currently the device stores its list of already seen +items in memory, but at some point it will likely make sense to +drop them in the cache. + +## Function Index ## + + +
dedup_test/0*
dedup_with_multipass_test/0*
handle/4*Forward the keys function to the message device, handle all others +with deduplication.
info/1
+ + + + +## Function Details ## + + + +### dedup_test/0 * ### + +`dedup_test() -> any()` + + + +### dedup_with_multipass_test/0 * ### + +`dedup_with_multipass_test() -> any()` + + + +### handle/4 * ### + +`handle(Key, M1, M2, Opts) -> any()` + +Forward the keys function to the message device, handle all others +with deduplication. We only act on the first pass. + + + +### info/1 ### + +`info(M1) -> any()` + diff --git a/docs/resources/source-code/dev_delegated_compute.md b/docs/resources/source-code/dev_delegated_compute.md new file mode 100644 index 000000000..ffb32c430 --- /dev/null +++ b/docs/resources/source-code/dev_delegated_compute.md @@ -0,0 +1,60 @@ +# [Module dev_delegated_compute.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_delegated_compute.erl) + + + + +Simple wrapper module that enables compute on remote machines, +implementing the JSON-Iface. + + + +## Description ## +This can be used either as a standalone, to +bring trusted results into the local node, or as the `Execution-Device` of +an AO process. + +## Function Index ## + + +
compute/3
do_compute/3*Execute computation on a remote machine via relay and the JSON-Iface.
init/3Initialize or normalize the compute-lite device.
normalize/3
snapshot/3
+ + + + +## Function Details ## + + + +### compute/3 ### + +`compute(Msg1, Msg2, Opts) -> any()` + + + +### do_compute/3 * ### + +`do_compute(ProcID, Msg2, Opts) -> any()` + +Execute computation on a remote machine via relay and the JSON-Iface. + + + +### init/3 ### + +`init(Msg1, Msg2, Opts) -> any()` + +Initialize or normalize the compute-lite device. For now, we don't +need to do anything special here. + + + +### normalize/3 ### + +`normalize(Msg1, Msg2, Opts) -> any()` + + + +### snapshot/3 ### + +`snapshot(Msg1, Msg2, Opts) -> any()` + diff --git a/docs/resources/source-code/dev_faff.md b/docs/resources/source-code/dev_faff.md new file mode 100644 index 000000000..90020b84a --- /dev/null +++ b/docs/resources/source-code/dev_faff.md @@ -0,0 +1,54 @@ +# [Module dev_faff.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_faff.erl) + + + + +A module that implements a 'friends and family' pricing policy. + + + +## Description ## + +It will allow users to process requests only if their addresses are +in the allow-list for the node. + +Fundamentally against the spirit of permissionlessness, but it is useful if +you are running a node for your own purposes and would not like to allow +others to make use of it -- even for a fee. It also serves as a useful +example of how to implement a custom pricing policy, as it implements stubs +for both the pricing and ledger P4 APIs. + +## Function Index ## + + +
debit/3Debit the user's account if the request is allowed.
estimate/3Decide whether or not to service a request from a given address.
is_admissible/2*Check whether all of the signers of the request are in the allow-list.
+ + + + +## Function Details ## + + + +### debit/3 ### + +`debit(X1, Req, NodeMsg) -> any()` + +Debit the user's account if the request is allowed. + + + +### estimate/3 ### + +`estimate(X1, Msg, NodeMsg) -> any()` + +Decide whether or not to service a request from a given address. + + + +### is_admissible/2 * ### + +`is_admissible(Msg, NodeMsg) -> any()` + +Check whether all of the signers of the request are in the allow-list. + diff --git a/docs/resources/source-code/dev_genesis_wasm.md b/docs/resources/source-code/dev_genesis_wasm.md new file mode 100644 index 000000000..d4839976b --- /dev/null +++ b/docs/resources/source-code/dev_genesis_wasm.md @@ -0,0 +1,108 @@ +# [Module dev_genesis_wasm.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_genesis_wasm.erl) + + + + +A device that mimics an environment suitable for `legacynet` AO +processes, using HyperBEAM infrastructure. + + + +## Description ## +This allows existing `legacynet` +AO process definitions to be used in HyperBEAM. + +## Function Index ## + + +
collect_events/1*Collect events from the port and log them.
collect_events/2*
compute/3All the delegated-compute@1.0 device to execute the request.
ensure_started/1*Ensure the local genesis-wasm@1.0 is live.
init/3Initialize the device.
is_genesis_wasm_server_running/1*Check if the genesis-wasm server is running, using the cached process ID +if available.
log_server_events/1*Log lines of output from the genesis-wasm server.
normalize/3Normalize the device.
snapshot/3Snapshot the device.
status/1*Check if the genesis-wasm server is running by requesting its status +endpoint.
+ + + + +## Function Details ## + + + +### collect_events/1 * ### + +`collect_events(Port) -> any()` + +Collect events from the port and log them. + + + +### collect_events/2 * ### + +`collect_events(Port, Acc) -> any()` + + + +### compute/3 ### + +`compute(Msg, Msg2, Opts) -> any()` + +All the `delegated-compute@1.0` device to execute the request. We then apply +the `patch@1.0` device, applying any state patches that the AO process may have +requested. + + + +### ensure_started/1 * ### + +`ensure_started(Opts) -> any()` + +Ensure the local `genesis-wasm@1.0` is live. If it not, start it. + + + +### init/3 ### + +`init(Msg, Msg2, Opts) -> any()` + +Initialize the device. + + + +### is_genesis_wasm_server_running/1 * ### + +`is_genesis_wasm_server_running(Opts) -> any()` + +Check if the genesis-wasm server is running, using the cached process ID +if available. + + + +### log_server_events/1 * ### + +`log_server_events(Bin) -> any()` + +Log lines of output from the genesis-wasm server. + + + +### normalize/3 ### + +`normalize(Msg, Msg2, Opts) -> any()` + +Normalize the device. + + + +### snapshot/3 ### + +`snapshot(Msg, Msg2, Opts) -> any()` + +Snapshot the device. + + + +### status/1 * ### + +`status(Opts) -> any()` + +Check if the genesis-wasm server is running by requesting its status +endpoint. + diff --git a/docs/resources/source-code/dev_green_zone.md b/docs/resources/source-code/dev_green_zone.md new file mode 100644 index 000000000..6ff9b42e7 --- /dev/null +++ b/docs/resources/source-code/dev_green_zone.md @@ -0,0 +1,399 @@ +# [Module dev_green_zone.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_green_zone.erl) + + + + +The green zone device, which provides secure communication and identity +management between trusted nodes. + + + +## Description ## +It handles node initialization, joining existing green zones, key exchange, +and node identity cloning. All operations are protected by hardware +commitment and encryption. + +## Function Index ## + + +
add_trusted_node/4*Adds a node to the trusted nodes list with its commitment report.
become/3Clones the identity of a target node in the green zone.
calculate_node_message/3*Generate the node message that should be set prior to joining +a green zone.
decrypt_zone_key/2*Decrypts an AES key using the node's RSA private key.
default_zone_required_opts/1*Provides the default required options for a green zone.
encrypt_payload/2*Encrypts an AES key with a node's RSA public key.
finalize_become/5*
info/1Controls which functions are exposed via the device API.
info/3Provides information about the green zone device and its API.
init/3Initialize the green zone for a node.
join/3Initiates the join process for a node to enter an existing green zone.
join_peer/5*Processes a join request to a specific peer node.
key/3Encrypts and provides the node's private key for secure sharing.
maybe_set_zone_opts/4*Adopts configuration from a peer when joining a green zone.
rsa_wallet_integration_test/0*Test RSA operations with the existing wallet structure.
try_mount_encrypted_volume/2*Attempts to mount an encrypted volume using the green zone AES key.
validate_join/3*Validates an incoming join request from another node.
validate_peer_opts/2*Validates that a peer's configuration matches required options.
+ + + + +## Function Details ## + + + +### add_trusted_node/4 * ### + +

+add_trusted_node(NodeAddr::binary(), Report::map(), RequesterPubKey::term(), Opts::map()) -> ok
+
+
+ +`NodeAddr`: The joining node's address
`Report`: The commitment report provided by the joining node
`RequesterPubKey`: The joining node's public key
`Opts`: A map of configuration options
+ +returns: ok + +Adds a node to the trusted nodes list with its commitment report. + +This function updates the trusted nodes configuration: +1. Retrieves the current trusted nodes map +2. Adds the new node with its report and public key +3. Updates the node configuration with the new trusted nodes list + + + +### become/3 ### + +

+become(M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options
+ +returns: `{ok, Map}` on success with confirmation details, or +`{error, Binary}` if the node is not part of a green zone or +identity adoption fails. + +Clones the identity of a target node in the green zone. + +This function performs the following operations: +1. Retrieves target node location and ID from the configuration +2. Verifies that the local node has a valid shared AES key +3. Requests the target node's encrypted key via its key endpoint +4. Verifies the response is from the expected peer +5. Decrypts the target node's private key using the shared AES key +6. Updates the local node's wallet with the target node's identity + +Required configuration in Opts map: +- green_zone_peer_location: Target node's address +- green_zone_peer_id: Target node's unique identifier +- priv_green_zone_aes: The shared AES key for the green zone + + + +### calculate_node_message/3 * ### + +`calculate_node_message(RequiredOpts, Req, List) -> any()` + +Generate the node message that should be set prior to joining +a green zone. + +This function takes a required opts message, a request message, and an +`adopt-config` value. The `adopt-config` value can be a boolean, a list of +fields that should be included in the node message from the request, or a +binary string of fields to include, separated by commas. + + + +### decrypt_zone_key/2 * ### + +

+decrypt_zone_key(EncZoneKey::binary(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`EncZoneKey`: The encrypted zone AES key (Base64 encoded or binary)
`Opts`: A map of configuration options
+ +returns: {ok, DecryptedKey} on success with the decrypted AES key + +Decrypts an AES key using the node's RSA private key. + +This function handles decryption of the zone key: +1. Decodes the encrypted key if it's in Base64 format +2. Extracts the RSA private key components from the wallet +3. Creates an RSA private key record +4. Performs private key decryption on the encrypted key + + + +### default_zone_required_opts/1 * ### + +

+default_zone_required_opts(Opts::map()) -> map()
+
+
+ +`Opts`: A map of configuration options from which to derive defaults
+ +returns: A map of required configuration options for the green zone + +Provides the default required options for a green zone. + +This function defines the baseline security requirements for nodes in a green zone: +1. Restricts loading of remote devices and only allows trusted signers +2. Limits to preloaded devices from the initiating machine +3. Enforces specific store configuration +4. Prevents route changes from the defaults +5. Requires matching hooks across all peers +6. Disables message scheduling to prevent conflicts +7. Enforces a permanent state to prevent further configuration changes + + + +### encrypt_payload/2 * ### + +

+encrypt_payload(AESKey::binary(), RequesterPubKey::term()) -> binary()
+
+
+ +`AESKey`: The shared AES key (256-bit binary)
`RequesterPubKey`: The node's public RSA key
+ +returns: The encrypted AES key + +Encrypts an AES key with a node's RSA public key. + +This function securely encrypts the shared key for transmission: +1. Extracts the RSA public key components +2. Creates an RSA public key record +3. Performs public key encryption on the AES key + + + +### finalize_become/5 * ### + +`finalize_become(KeyResp, NodeLocation, NodeID, GreenZoneAES, Opts) -> any()` + + + +### info/1 ### + +`info(X1) -> any()` + +Controls which functions are exposed via the device API. + +This function defines the security boundary for the green zone device by +explicitly listing which functions are available through the API. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + +Provides information about the green zone device and its API. + +This function returns detailed documentation about the device, including: +1. A high-level description of the device's purpose +2. Version information +3. Available API endpoints with their parameters and descriptions + + + +### init/3 ### + +

+init(M1::term(), M2::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options
+ +returns: `{ok, Binary}` on success with confirmation message, or +`{error, Binary}` on failure with error message. + +Initialize the green zone for a node. + +This function performs the following operations: +1. Validates the node's history to ensure this is a valid initialization +2. Retrieves or creates a required configuration for the green zone +3. Ensures a wallet (keypair) exists or creates a new one +4. Generates a new 256-bit AES key for secure communication +5. Updates the node's configuration with these cryptographic identities + +Config options in Opts map: +- green_zone_required_config: (Optional) Custom configuration requirements +- priv_wallet: (Optional) Existing wallet to use instead of creating a new one +- priv_green_zone_aes: (Optional) Existing AES key, if already part of a zone + + + +### join/3 ### + +

+join(M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`M1`: The join request message with target peer information
`M2`: Additional request details, may include adoption preferences
`Opts`: A map of configuration options for join operations
+ +returns: `{ok, Map}` on success with join response details, or +`{error, Binary}` on failure with error message. + +Initiates the join process for a node to enter an existing green zone. + +This function performs the following operations depending on the state: +1. Validates the node's history to ensure proper initialization +2. Checks for target peer information (location and ID) +3. If target peer is specified: +a. Generates a commitment report for the peer +b. Prepares and sends a POST request to the target peer +c. Verifies the response and decrypts the returned zone key +d. Updates local configuration with the shared AES key +4. If no peer is specified, processes the join request locally + +Config options in Opts map: +- green_zone_peer_location: Target peer's address +- green_zone_peer_id: Target peer's unique identifier +- green_zone_adopt_config: +(Optional) Whether to adopt peer's configuration (default: true) + + + +### join_peer/5 * ### + +

+join_peer(PeerLocation::binary(), PeerID::binary(), M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, map() | binary()}
+
+
+ +`PeerLocation`: The target peer's address
`PeerID`: The target peer's unique identifier
`M2`: May contain ShouldMount flag to enable encrypted volume mounting
+ +returns: `{ok, Map}` on success with confirmation message, or +`{error, Map|Binary}` on failure with error details + +Processes a join request to a specific peer node. + +This function handles the client-side join flow when connecting to a peer: +1. Verifies the node is not already in a green zone +2. Optionally adopts configuration from the target peer +3. Generates a hardware-backed commitment report +4. Sends a POST request to the peer's join endpoint +5. Verifies the response signature +6. Decrypts the returned AES key +7. Updates local configuration with the shared key +8. Optionally mounts an encrypted volume using the shared key + + + +### key/3 ### + +

+key(M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options
+ +returns: `{ok, Map}` containing the encrypted key and IV on success, or +`{error, Binary}` if the node is not part of a green zone + +Encrypts and provides the node's private key for secure sharing. + +This function performs the following operations: +1. Retrieves the shared AES key and the node's wallet +2. Verifies that the node is part of a green zone (has a shared AES key) +3. Generates a random initialization vector (IV) for encryption +4. Encrypts the node's private key using AES-256-GCM with the shared key +5. Returns the encrypted key and IV for secure transmission + +Required configuration in Opts map: +- priv_green_zone_aes: The shared AES key for the green zone +- priv_wallet: The node's wallet containing the private key to encrypt + + + +### maybe_set_zone_opts/4 * ### + +

+maybe_set_zone_opts(PeerLocation::binary(), PeerID::binary(), Req::map(), InitOpts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`PeerLocation`: The location of the peer node to join
`PeerID`: The ID of the peer node to join
`Req`: The request message with adoption preferences
`InitOpts`: A map of initial configuration options
+ +returns: `{ok, Map}` with updated configuration on success, or +`{error, Binary}` if configuration retrieval fails + +Adopts configuration from a peer when joining a green zone. + +This function handles the conditional adoption of peer configuration: +1. Checks if adoption is enabled (default: true) +2. Requests required configuration from the peer +3. Verifies the authenticity of the configuration +4. Creates a node message with appropriate settings +5. Updates the local node configuration + +Config options: +- green_zone_adopt_config: Controls configuration adoption (boolean, list, or binary) + + + +### rsa_wallet_integration_test/0 * ### + +`rsa_wallet_integration_test() -> any()` + +Test RSA operations with the existing wallet structure. + +This test function verifies that encryption and decryption using the RSA keys +from the wallet work correctly. It creates a new wallet, encrypts a test +message with the RSA public key, and then decrypts it with the RSA private +key, asserting that the decrypted message matches the original. + + + +### try_mount_encrypted_volume/2 * ### + +`try_mount_encrypted_volume(AESKey, Opts) -> any()` + +Attempts to mount an encrypted volume using the green zone AES key. + +This function handles the complete process of secure storage setup by +delegating to the dev_volume module, which provides a unified interface +for volume management. + +The encryption key used for the volume is the same AES key used for green zone +communication, ensuring that only nodes in the green zone can access the data. + + + +### validate_join/3 * ### + +

+validate_join(M1::term(), Req::map(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`M1`: Ignored parameter
`Req`: The join request containing commitment report and public key
`Opts`: A map of configuration options
+ +returns: `{ok, Map}` on success with encrypted AES key, or +`{error, Binary}` on failure with error message + +Validates an incoming join request from another node. + +This function handles the server-side join flow when receiving a connection +request: +1. Validates the peer's configuration meets required standards +2. Extracts the commitment report and public key from the request +3. Verifies the hardware-backed commitment report +4. Adds the joining node to the trusted nodes list +5. Encrypts the shared AES key with the peer's public key +6. Returns the encrypted key to the requesting node + + + +### validate_peer_opts/2 * ### + +

+validate_peer_opts(Req::map(), Opts::map()) -> boolean()
+
+
+ +`Req`: The request message containing the peer's configuration
`Opts`: A map of the local node's configuration options
+ +returns: true if the peer's configuration is valid, false otherwise + +Validates that a peer's configuration matches required options. + +This function ensures the peer node meets configuration requirements: +1. Retrieves the local node's required configuration +2. Gets the peer's options from its message +3. Adds required configuration to peer's required options list +4. Verifies the peer's node history is valid +5. Checks that the peer's options match the required configuration + diff --git a/docs/resources/source-code/dev_hook.md b/docs/resources/source-code/dev_hook.md new file mode 100644 index 000000000..477df1e5f --- /dev/null +++ b/docs/resources/source-code/dev_hook.md @@ -0,0 +1,166 @@ +# [Module dev_hook.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_hook.erl) + + + + +A generalized interface for `hooking` into HyperBEAM nodes. + + + +## Description ## + +This module allows users to define `hooks` that are executed at various +points in the lifecycle of nodes and message evaluations. + +Hooks are maintained in the `node message` options, under the key `on` +key. Each `hook` may have zero or many `handlers` which their request is +executed against. A new `handler` of a hook can be registered by simply +adding a new key to that message. If multiple hooks need to be executed for +a single event, the key's value can be set to a list of hooks. + +`hook`s themselves do not need to be added explicitly. Any device can add +a hook by simply executing `dev_hook:on(HookName, Req, Opts)`. This +function is does not affect the hashpath of a message and is not exported on +the device`s API, such that it is not possible to call it directly with +AO-Core resolution. + +All handlers are expressed in the form of a message, upon which the hook's +request is evaluated: + +AO(HookMsg, Req, Opts) => {Status, Result} + +The `Status` and `Result` of the evaluation can be used at the `hook` caller's +discretion. If multiple handlers are to be executed for a single `hook`, the +result of each is used as the input to the next, on the assumption that the +status of the previous is `ok`. If a non-`ok` status is encountered, the +evaluation is halted and the result is returned to the caller. This means +that in most cases, hooks take the form of chainable pipelines of functions, +passing the most pertinent data in the `body` key of both the request and +result. Hook definitions can also set the `hook/result` key to `ignore`, if +the result of the execution should be discarded and the prior value (the +input to the hook) should be used instead. The `hook/commit-request` key can +also be set to `true` if the request should be committed by the node before +execution of the hook. + +The default HyperBEAM node implements several useful hooks. They include: + +start: Executed when the node starts. +Req/body: The node's initial configuration. +Result/body: The node's possibly updated configuration. +request: Executed when a request is received via the HTTP API. +Req/body: The sequence of messages that the node will evaluate. +Req/request: The raw, unparsed singleton request. +Result/body: The sequence of messages that the node will evaluate. +step: Executed after each message in a sequence has been evaluated. +Req/body: The result of the evaluation. +Result/body: The result of the evaluation. +response: Executed when a response is sent via the HTTP API. +Req/body: The result of the evaluation. +Req/request: The raw, unparsed singleton request that was used to +generate the response. +Result/body: The message to be sent in response to the request. + +Additionally, this module implements a traditional device API, allowing the +node operator to register hooks to the node and find those that are +currently active. + +## Function Index ## + + +
execute_handler/4*Execute a single handler +Handlers are expressed as messages that can be resolved via AO.
execute_handlers/4*Execute a list of handlers in sequence.
find/2Get all handlers for a specific hook from the node message options.
find/3
halt_on_error_test/0*Test that pipeline execution halts on error.
info/1Device API information.
multiple_handlers_test/0*Test that multiple handlers form a pipeline.
no_handlers_test/0*Test that hooks with no handlers return the original request.
on/3Execute a named hook with the provided request and options +This function finds all handlers for the hook and evaluates them in sequence.
single_handler_test/0*Test that a single handler is executed correctly.
+ + + + +## Function Details ## + + + +### execute_handler/4 * ### + +`execute_handler(HookName, Handler, Req, Opts) -> any()` + +Execute a single handler +Handlers are expressed as messages that can be resolved via AO. + + + +### execute_handlers/4 * ### + +`execute_handlers(HookName, Rest, Req, Opts) -> any()` + +Execute a list of handlers in sequence. +The result of each handler is used as input to the next handler. +If a handler returns a non-ok status, execution is halted. + + + +### find/2 ### + +`find(HookName, Opts) -> any()` + +Get all handlers for a specific hook from the node message options. +Handlers are stored in the `on` key of this message. The `find/2` variant of +this function only takes a hook name and node message, and is not called +directly via the device API. Instead it is used by `on/3` and other internal +functionality to find handlers when necessary. The `find/3` variant can, +however, be called directly via the device API. + + + +### find/3 ### + +`find(Base, Req, Opts) -> any()` + + + +### halt_on_error_test/0 * ### + +`halt_on_error_test() -> any()` + +Test that pipeline execution halts on error + + + +### info/1 ### + +`info(X1) -> any()` + +Device API information + + + +### multiple_handlers_test/0 * ### + +`multiple_handlers_test() -> any()` + +Test that multiple handlers form a pipeline + + + +### no_handlers_test/0 * ### + +`no_handlers_test() -> any()` + +Test that hooks with no handlers return the original request + + + +### on/3 ### + +`on(HookName, Req, Opts) -> any()` + +Execute a named hook with the provided request and options +This function finds all handlers for the hook and evaluates them in sequence. +The result of each handler is used as input to the next handler. + + + +### single_handler_test/0 * ### + +`single_handler_test() -> any()` + +Test that a single handler is executed correctly + diff --git a/docs/resources/source-code/dev_hyperbuddy.md b/docs/resources/source-code/dev_hyperbuddy.md new file mode 100644 index 000000000..da40fdaaf --- /dev/null +++ b/docs/resources/source-code/dev_hyperbuddy.md @@ -0,0 +1,60 @@ +# [Module dev_hyperbuddy.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_hyperbuddy.erl) + + + + +A device that renders a REPL-like interface for AO-Core via HTML. + + + +## Function Index ## + + +
format/3Employ HyperBEAM's internal pretty printer to format a message.
info/0Export an explicit list of files via http.
metrics/3The main HTML page for the REPL device.
return_file/1*Read a file from disk and serve it as a static HTML page.
serve/4*Serve a file from the priv directory.
+ + + + +## Function Details ## + + + +### format/3 ### + +`format(Base, X2, X3) -> any()` + +Employ HyperBEAM's internal pretty printer to format a message. + + + +### info/0 ### + +`info() -> any()` + +Export an explicit list of files via http. + + + +### metrics/3 ### + +`metrics(X1, Req, Opts) -> any()` + +The main HTML page for the REPL device. + + + +### return_file/1 * ### + +`return_file(Name) -> any()` + +Read a file from disk and serve it as a static HTML page. + + + +### serve/4 * ### + +`serve(Key, M1, M2, Opts) -> any()` + +Serve a file from the priv directory. Only serves files that are explicitly +listed in the `routes` field of the `info/0` return value. + diff --git a/docs/resources/source-code/dev_json_iface.md b/docs/resources/source-code/dev_json_iface.md new file mode 100644 index 000000000..dca4dd7b4 --- /dev/null +++ b/docs/resources/source-code/dev_json_iface.md @@ -0,0 +1,254 @@ +# [Module dev_json_iface.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_json_iface.erl) + + + + +A device that provides a way for WASM execution to interact with +the HyperBEAM (and AO) systems, using JSON as a shared data representation. + + + +## Description ## + +The interface is easy to use. It works as follows: + +1. The device is given a message that contains a process definition, WASM +environment, and a message that contains the data to be processed, +including the image to be used in part of `execute{pass=1}`. +2. The device is called with `execute{pass=2}`, which reads the result of +the process execution from the WASM environment and adds it to the +message. + +The device has the following requirements and interface: + +``` + + M1/Computed when /Pass == 1 -> + Assumes: + M1/priv/wasm/instance + M1/Process + M2/Message + M2/Assignment/Block-Height + Generates: + /wasm/handler + /wasm/params + Side-effects: + Writes the process and message as JSON representations into the + WASM environment. + M1/Computed when M2/Pass == 2 -> + Assumes: + M1/priv/wasm/instance + M2/Results + M2/Process + Generates: + /Results/Outbox + /Results/Data +``` + + +## Function Index ## + + +
aos_stack_benchmark_test_/0*
basic_aos_call_test_/0*
compute/3On first pass prepare the call, on second pass get the results.
denormalize_message/1*Normalize a message for AOS-compatibility.
env_read/3*Read the results out of the execution environment.
env_write/5*Write the message and process into the execution environment.
generate_aos_msg/2
generate_stack/1
generate_stack/2
header_case_string/1*
init/3Initialize the device.
json_to_message/2Translates a compute result -- either from a WASM execution using the +JSON-Iface, or from a Legacy CU -- and transforms it into a result message.
maybe_list_to_binary/1*
message_to_json_struct/1
message_to_json_struct/2*
normalize_results/1*Normalize the results of an evaluation.
postprocess_outbox/3*Post-process messages in the outbox to add the correct from-process +and from-image tags.
prep_call/3*Prepare the WASM environment for execution by writing the process string and +the message as JSON representations into the WASM environment.
prepare_header_case_tags/1*Convert a message without an original-tags field into a list of +key-value pairs, with the keys in HTTP header-case.
prepare_tags/1*Prepare the tags of a message as a key-value list, for use in the +construction of the JSON-Struct message.
preprocess_results/2*After the process returns messages from an evaluation, the +signing node needs to add some tags to each message and spawn such that +the target process knows these messages are created by a process.
results/3*Read the computed results out of the WASM environment, assuming that +the environment has been set up by prep_call/3 and that the WASM executor +has been called with computed{pass=1}.
safe_to_id/1*
tags_to_map/1*Convert a message with tags into a map of their key-value pairs.
test_init/0*
+ + + + +## Function Details ## + + + +### aos_stack_benchmark_test_/0 * ### + +`aos_stack_benchmark_test_() -> any()` + + + +### basic_aos_call_test_/0 * ### + +`basic_aos_call_test_() -> any()` + + + +### compute/3 ### + +`compute(M1, M2, Opts) -> any()` + +On first pass prepare the call, on second pass get the results. + + + +### denormalize_message/1 * ### + +`denormalize_message(Message) -> any()` + +Normalize a message for AOS-compatibility. + + + +### env_read/3 * ### + +`env_read(M1, M2, Opts) -> any()` + +Read the results out of the execution environment. + + + +### env_write/5 * ### + +`env_write(ProcessStr, MsgStr, Base, Req, Opts) -> any()` + +Write the message and process into the execution environment. + + + +### generate_aos_msg/2 ### + +`generate_aos_msg(ProcID, Code) -> any()` + + + +### generate_stack/1 ### + +`generate_stack(File) -> any()` + + + +### generate_stack/2 ### + +`generate_stack(File, Mode) -> any()` + + + +### header_case_string/1 * ### + +`header_case_string(Key) -> any()` + + + +### init/3 ### + +`init(M1, M2, Opts) -> any()` + +Initialize the device. + + + +### json_to_message/2 ### + +`json_to_message(JSON, Opts) -> any()` + +Translates a compute result -- either from a WASM execution using the +JSON-Iface, or from a `Legacy` CU -- and transforms it into a result message. + + + +### maybe_list_to_binary/1 * ### + +`maybe_list_to_binary(List) -> any()` + + + +### message_to_json_struct/1 ### + +`message_to_json_struct(RawMsg) -> any()` + + + +### message_to_json_struct/2 * ### + +`message_to_json_struct(RawMsg, Features) -> any()` + + + +### normalize_results/1 * ### + +`normalize_results(Msg) -> any()` + +Normalize the results of an evaluation. + + + +### postprocess_outbox/3 * ### + +`postprocess_outbox(Msg, Proc, Opts) -> any()` + +Post-process messages in the outbox to add the correct `from-process` +and `from-image` tags. + + + +### prep_call/3 * ### + +`prep_call(M1, M2, Opts) -> any()` + +Prepare the WASM environment for execution by writing the process string and +the message as JSON representations into the WASM environment. + + + +### prepare_header_case_tags/1 * ### + +`prepare_header_case_tags(TABM) -> any()` + +Convert a message without an `original-tags` field into a list of +key-value pairs, with the keys in HTTP header-case. + + + +### prepare_tags/1 * ### + +`prepare_tags(Msg) -> any()` + +Prepare the tags of a message as a key-value list, for use in the +construction of the JSON-Struct message. + + + +### preprocess_results/2 * ### + +`preprocess_results(Msg, Opts) -> any()` + +After the process returns messages from an evaluation, the +signing node needs to add some tags to each message and spawn such that +the target process knows these messages are created by a process. + + + +### results/3 * ### + +`results(M1, M2, Opts) -> any()` + +Read the computed results out of the WASM environment, assuming that +the environment has been set up by `prep_call/3` and that the WASM executor +has been called with `computed{pass=1}`. + + + +### safe_to_id/1 * ### + +`safe_to_id(ID) -> any()` + + + +### tags_to_map/1 * ### + +`tags_to_map(Msg) -> any()` + +Convert a message with tags into a map of their key-value pairs. + + + +### test_init/0 * ### + +`test_init() -> any()` + diff --git a/docs/resources/source-code/dev_local_name.md b/docs/resources/source-code/dev_local_name.md new file mode 100644 index 000000000..c536e6cec --- /dev/null +++ b/docs/resources/source-code/dev_local_name.md @@ -0,0 +1,130 @@ +# [Module dev_local_name.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_local_name.erl) + + + + +A device for registering and looking up local names. + + + +## Description ## +This device uses +the node message to store a local cache of its known names, and the typical +non-volatile storage of the node message to store the names long-term. + +## Function Index ## + + +
default_lookup/4*Handle all other requests by delegating to the lookup function.
direct_register/2Register a name without checking if the caller is an operator.
find_names/1*Returns a message containing all known names.
generate_test_opts/0*
http_test/0*
info/1Export only the lookup and register functions.
load_names/1*Loads all known names from the cache and returns the new node message +with those names loaded into it.
lookup/3Takes a key argument and returns the value of the name, if it exists.
lookup_opts_name_test/0*
no_names_test/0*
register/3Takes a key and value argument and registers the name.
register_test/0*
unauthorized_test/0*
update_names/2*Updates the node message with the new names.
+ + + + +## Function Details ## + + + +### default_lookup/4 * ### + +`default_lookup(Key, X2, Req, Opts) -> any()` + +Handle all other requests by delegating to the lookup function. + + + +### direct_register/2 ### + +`direct_register(Req, Opts) -> any()` + +Register a name without checking if the caller is an operator. Exported +for use by other devices, but not publicly available. + + + +### find_names/1 * ### + +`find_names(Opts) -> any()` + +Returns a message containing all known names. + + + +### generate_test_opts/0 * ### + +`generate_test_opts() -> any()` + + + +### http_test/0 * ### + +`http_test() -> any()` + + + +### info/1 ### + +`info(Opts) -> any()` + +Export only the `lookup` and `register` functions. + + + +### load_names/1 * ### + +`load_names(Opts) -> any()` + +Loads all known names from the cache and returns the new `node message` +with those names loaded into it. + + + +### lookup/3 ### + +`lookup(X1, Req, Opts) -> any()` + +Takes a `key` argument and returns the value of the name, if it exists. + + + +### lookup_opts_name_test/0 * ### + +`lookup_opts_name_test() -> any()` + + + +### no_names_test/0 * ### + +`no_names_test() -> any()` + + + +### register/3 ### + +`register(X1, Req, Opts) -> any()` + +Takes a `key` and `value` argument and registers the name. The caller +must be the node operator in order to register a name. + + + +### register_test/0 * ### + +`register_test() -> any()` + + + +### unauthorized_test/0 * ### + +`unauthorized_test() -> any()` + + + +### update_names/2 * ### + +`update_names(LocalNames, Opts) -> any()` + +Updates the node message with the new names. Further HTTP requests will +use this new message, removing the need to look up the names from non-volatile +storage. + diff --git a/docs/resources/source-code/dev_lookup.md b/docs/resources/source-code/dev_lookup.md new file mode 100644 index 000000000..6ed34a647 --- /dev/null +++ b/docs/resources/source-code/dev_lookup.md @@ -0,0 +1,52 @@ +# [Module dev_lookup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lookup.erl) + + + + +A device that looks up an ID from a local store and returns it, honoring +the `accept` key to return the correct format. + + + +## Function Index ## + + +
aos2_message_lookup_test/0*
binary_lookup_test/0*
http_lookup_test/0*
message_lookup_test/0*
read/3Fetch a resource from the cache using "target" ID extracted from the message.
+ + + + +## Function Details ## + + + +### aos2_message_lookup_test/0 * ### + +`aos2_message_lookup_test() -> any()` + + + +### binary_lookup_test/0 * ### + +`binary_lookup_test() -> any()` + + + +### http_lookup_test/0 * ### + +`http_lookup_test() -> any()` + + + +### message_lookup_test/0 * ### + +`message_lookup_test() -> any()` + + + +### read/3 ### + +`read(M1, M2, Opts) -> any()` + +Fetch a resource from the cache using "target" ID extracted from the message + diff --git a/docs/resources/source-code/dev_lua.md b/docs/resources/source-code/dev_lua.md new file mode 100644 index 000000000..9101078aa --- /dev/null +++ b/docs/resources/source-code/dev_lua.md @@ -0,0 +1,305 @@ +# [Module dev_lua.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lua.erl) + + + + +A device that calls a Lua module upon a request and returns the result. + + + +## Function Index ## + + +
ao_core_resolution_from_lua_test/0*Run an AO-Core resolution from the Lua environment.
ao_core_sandbox_test/0*Run an AO-Core resolution from the Lua environment.
aos_authority_not_trusted_test/0*
aos_process_benchmark_test_/0*Benchmark the performance of Lua executions.
compute/4*Call the Lua script with the given arguments.
decode/1Decode a Lua result into a HyperBEAM structured@1.0 message.
decode_params/2*Decode a list of Lua references, as found in a stack trace, into a +list of Erlang terms.
decode_stacktrace/2*Parse a Lua stack trace into a list of messages.
decode_stacktrace/3*
direct_benchmark_test/0*Benchmark the performance of Lua executions.
encode/1Encode a HyperBEAM structured@1.0 message into a Lua term.
ensure_initialized/3*Initialize the Lua VM if it is not already initialized.
error_response_test/0*
find_modules/2*Find the script in the base message, either by ID or by string.
functions/3Return a list of all functions in the Lua environment.
generate_lua_process/1*Generate a Lua process message.
generate_stack/1*Generate a stack message for the Lua process.
generate_test_message/1*Generate a test message for a Lua process.
info/1All keys that are not directly available in the base message are +resolved by calling the Lua function in the module of the same name.
init/3Initialize the device state, loading the script into memory if it is +a reference.
initialize/3*Initialize a new Lua state with a given base message and module.
invoke_aos_test/0*
invoke_non_compute_key_test/0*Call a non-compute key on a Lua device message and ensure that the +function of the same name in the script is called.
load_modules/2*Load a list of modules for installation into the Lua VM.
load_modules/3*
load_modules_by_id_test/0*
lua_http_hook_test/0*Use a Lua module as a hook on the HTTP server via ~meta@1.0.
multiple_modules_test/0*
normalize/3Restore the Lua state from a snapshot, if it exists.
process_response/2*Process a response to a Luerl invocation.
pure_lua_process_benchmark_test_/0*
pure_lua_process_test/0*Call a process whose execution-device is set to lua@5.3a.
sandbox/3*Sandbox (render inoperable) a set of Lua functions.
sandboxed_failure_test/0*
simple_invocation_test/0*
snapshot/3Snapshot the Lua state from a live computation.
+ + + + +## Function Details ## + + + +### ao_core_resolution_from_lua_test/0 * ### + +`ao_core_resolution_from_lua_test() -> any()` + +Run an AO-Core resolution from the Lua environment. + + + +### ao_core_sandbox_test/0 * ### + +`ao_core_sandbox_test() -> any()` + +Run an AO-Core resolution from the Lua environment. + + + +### aos_authority_not_trusted_test/0 * ### + +`aos_authority_not_trusted_test() -> any()` + + + +### aos_process_benchmark_test_/0 * ### + +`aos_process_benchmark_test_() -> any()` + +Benchmark the performance of Lua executions. + + + +### compute/4 * ### + +`compute(Key, RawBase, Req, Opts) -> any()` + +Call the Lua script with the given arguments. + + + +### decode/1 ### + +`decode(EncMsg) -> any()` + +Decode a Lua result into a HyperBEAM `structured@1.0` message. + + + +### decode_params/2 * ### + +`decode_params(Rest, State) -> any()` + +Decode a list of Lua references, as found in a stack trace, into a +list of Erlang terms. + + + +### decode_stacktrace/2 * ### + +`decode_stacktrace(StackTrace, State0) -> any()` + +Parse a Lua stack trace into a list of messages. + + + +### decode_stacktrace/3 * ### + +`decode_stacktrace(Rest, State, Acc) -> any()` + + + +### direct_benchmark_test/0 * ### + +`direct_benchmark_test() -> any()` + +Benchmark the performance of Lua executions. + + + +### encode/1 ### + +`encode(Map) -> any()` + +Encode a HyperBEAM `structured@1.0` message into a Lua term. + + + +### ensure_initialized/3 * ### + +`ensure_initialized(Base, Req, Opts) -> any()` + +Initialize the Lua VM if it is not already initialized. Optionally takes +the script as a Binary string. If not provided, the module will be loaded +from the base message. + + + +### error_response_test/0 * ### + +`error_response_test() -> any()` + + + +### find_modules/2 * ### + +`find_modules(Base, Opts) -> any()` + +Find the script in the base message, either by ID or by string. + + + +### functions/3 ### + +`functions(Base, Req, Opts) -> any()` + +Return a list of all functions in the Lua environment. + + + +### generate_lua_process/1 * ### + +`generate_lua_process(File) -> any()` + +Generate a Lua process message. + + + +### generate_stack/1 * ### + +`generate_stack(File) -> any()` + +Generate a stack message for the Lua process. + + + +### generate_test_message/1 * ### + +`generate_test_message(Process) -> any()` + +Generate a test message for a Lua process. + + + +### info/1 ### + +`info(Base) -> any()` + +All keys that are not directly available in the base message are +resolved by calling the Lua function in the module of the same name. +Additionally, we exclude the `keys`, `set`, `encode` and `decode` functions +which are `message@1.0` core functions, and Lua public utility functions. + + + +### init/3 ### + +`init(Base, Req, Opts) -> any()` + +Initialize the device state, loading the script into memory if it is +a reference. + + + +### initialize/3 * ### + +`initialize(Base, Modules, Opts) -> any()` + +Initialize a new Lua state with a given base message and module. + + + +### invoke_aos_test/0 * ### + +`invoke_aos_test() -> any()` + + + +### invoke_non_compute_key_test/0 * ### + +`invoke_non_compute_key_test() -> any()` + +Call a non-compute key on a Lua device message and ensure that the +function of the same name in the script is called. + + + +### load_modules/2 * ### + +`load_modules(Modules, Opts) -> any()` + +Load a list of modules for installation into the Lua VM. + + + +### load_modules/3 * ### + +`load_modules(Rest, Opts, Acc) -> any()` + + + +### load_modules_by_id_test/0 * ### + +`load_modules_by_id_test() -> any()` + + + +### lua_http_hook_test/0 * ### + +`lua_http_hook_test() -> any()` + +Use a Lua module as a hook on the HTTP server via `~meta@1.0`. + + + +### multiple_modules_test/0 * ### + +`multiple_modules_test() -> any()` + + + +### normalize/3 ### + +`normalize(Base, Req, RawOpts) -> any()` + +Restore the Lua state from a snapshot, if it exists. + + + +### process_response/2 * ### + +`process_response(X1, Priv) -> any()` + +Process a response to a Luerl invocation. Returns the typical AO-Core +HyperBEAM response format. + + + +### pure_lua_process_benchmark_test_/0 * ### + +`pure_lua_process_benchmark_test_() -> any()` + + + +### pure_lua_process_test/0 * ### + +`pure_lua_process_test() -> any()` + +Call a process whose `execution-device` is set to `lua@5.3a`. + + + +### sandbox/3 * ### + +`sandbox(State, Map, Opts) -> any()` + +Sandbox (render inoperable) a set of Lua functions. Each function is +referred to as if it is a path in AO-Core, with its value being what to +return to the caller. For example, 'os.exit' would be referred to as +referred to as `os/exit`. If preferred, a list rather than a map may be +provided, in which case the functions all return `sandboxed`. + + + +### sandboxed_failure_test/0 * ### + +`sandboxed_failure_test() -> any()` + + + +### simple_invocation_test/0 * ### + +`simple_invocation_test() -> any()` + + + +### snapshot/3 ### + +`snapshot(Base, Req, Opts) -> any()` + +Snapshot the Lua state from a live computation. Normalizes its `priv` +state element, then serializes the state to a binary. + diff --git a/docs/resources/source-code/dev_lua_lib.md b/docs/resources/source-code/dev_lua_lib.md new file mode 100644 index 000000000..016e90a0b --- /dev/null +++ b/docs/resources/source-code/dev_lua_lib.md @@ -0,0 +1,86 @@ +# [Module dev_lua_lib.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lua_lib.erl) + + + + +A module for providing AO library functions to the Lua environment. + + + +## Description ## + +This module contains the implementation of the functions, each by the name +that should be used in the `ao` table in the Lua environment. Every export +is imported into the Lua environment. + +Each function adheres closely to the Luerl calling convention, adding the +appropriate node message as a third argument: + +fun(Args, State, NodeMsg) -> {ResultTerms, NewState} + +As Lua allows for multiple return values, each function returns a list of +terms to grant to the caller. Matching the tuple convention used by AO-Core, +the first term is typically the status, and the second term is the result. + +## Function Index ## + + +
convert_as/1*Converts any as terms from Lua to their HyperBEAM equivalents.
event/3Allows Lua scripts to signal events using the HyperBEAM hosts internal +event system.
install/3Install the library into the given Lua environment.
resolve/3A wrapper function for performing AO-Core resolutions.
return/2*Helper function for returning a result from a Lua function.
set/3Wrapper for hb_ao's set functionality.
+ + + + +## Function Details ## + + + +### convert_as/1 * ### + +`convert_as(Other) -> any()` + +Converts any `as` terms from Lua to their HyperBEAM equivalents. + + + +### event/3 ### + +`event(X1, ExecState, Opts) -> any()` + +Allows Lua scripts to signal events using the HyperBEAM hosts internal +event system. + + + +### install/3 ### + +`install(Base, State, Opts) -> any()` + +Install the library into the given Lua environment. + + + +### resolve/3 ### + +`resolve(Msgs, ExecState, ExecOpts) -> any()` + +A wrapper function for performing AO-Core resolutions. Offers both the +single-message (using `hb_singleton:from/1` to parse) and multiple-message +(using `hb_ao:resolve_many/2`) variants. + + + +### return/2 * ### + +`return(Result, ExecState) -> any()` + +Helper function for returning a result from a Lua function. + + + +### set/3 ### + +`set(X1, ExecState, ExecOpts) -> any()` + +Wrapper for `hb_ao`'s `set` functionality. + diff --git a/docs/resources/source-code/dev_lua_test.md b/docs/resources/source-code/dev_lua_test.md new file mode 100644 index 000000000..7aed2bb75 --- /dev/null +++ b/docs/resources/source-code/dev_lua_test.md @@ -0,0 +1,92 @@ +# [Module dev_lua_test.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_lua_test.erl) + + + + + + +## Function Index ## + + +
exec_test/2*Generate an EUnit test for a given function.
exec_test_/0*Main entrypoint for Lua tests.
new_state/1*Create a new Lua environment for a given script.
parse_spec/1Parse a string representation of test descriptions received from the +command line via the LUA_TESTS environment variable.
suite/2*Generate an EUnit test suite for a given Lua script.
terminates_with/2*Check if a string terminates with a given suffix.
+ + + + +## Function Details ## + + + +### exec_test/2 * ### + +`exec_test(State, Function) -> any()` + +Generate an EUnit test for a given function. + + + +### exec_test_/0 * ### + +`exec_test_() -> any()` + +Main entrypoint for Lua tests. + + + +### new_state/1 * ### + +`new_state(File) -> any()` + +Create a new Lua environment for a given script. + + + +### parse_spec/1 ### + +`parse_spec(Str) -> any()` + +Parse a string representation of test descriptions received from the +command line via the `LUA_TESTS` environment variable. + +Supported syntax in loose BNF/RegEx: + +Definitions := (ModDef,)+ +ModDef := ModName(TestDefs)? +ModName := ModuleInLUA_SCRIPTS|(FileName[.lua])? +TestDefs := (:TestDef)+ +TestDef := TestName + +File names ending in `.lua` are assumed to be relative paths from the current +working directory. Module names lacking the `.lua` extension are assumed to +be modules found in the `LUA_SCRIPTS` environment variable (defaulting to +`scripts/`). + +For example, to run a single test one could call the following: + +LUA_TESTS=~/src/LuaScripts/test.yourTest rebar3 lua-tests + +To specify that one would like to run all of the tests in the +`scripts/test.lua` file and two tests from the `scripts/test2.lua` file, the +user could provide the following test definition: + +LUA_TESTS="test,scripts/test2.userTest1|userTest2" rebar3 lua-tests + + + +### suite/2 * ### + +`suite(File, Funcs) -> any()` + +Generate an EUnit test suite for a given Lua script. If the `Funcs` is +the atom `tests` we find all of the global functions in the script, then +filter for those ending in `_test` in a similar fashion to Eunit. + + + +### terminates_with/2 * ### + +`terminates_with(String, Suffix) -> any()` + +Check if a string terminates with a given suffix. + diff --git a/docs/resources/source-code/dev_manifest.md b/docs/resources/source-code/dev_manifest.md new file mode 100644 index 000000000..4ee517450 --- /dev/null +++ b/docs/resources/source-code/dev_manifest.md @@ -0,0 +1,49 @@ +# [Module dev_manifest.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_manifest.erl) + + + + +An Arweave path manifest resolution device. + + + +## Description ## +Follows the v1 schema: +https://specs.ar.io/?tx=lXLd0OPwo-dJLB_Amz5jgIeDhiOkjXuM3-r0H_aiNj0 + +## Function Index ## + + +
info/0Use the route/4 function as the handler for all requests, aside +from keys and set, which are handled by the default resolver.
manifest/3*Find and deserialize a manifest from the given base.
route/4*Route a request to the associated data via its manifest.
+ + + + +## Function Details ## + + + +### info/0 ### + +`info() -> any()` + +Use the `route/4` function as the handler for all requests, aside +from `keys` and `set`, which are handled by the default resolver. + + + +### manifest/3 * ### + +`manifest(Base, Req, Opts) -> any()` + +Find and deserialize a manifest from the given base. + + + +### route/4 * ### + +`route(Key, M1, M2, Opts) -> any()` + +Route a request to the associated data via its manifest. + diff --git a/docs/resources/source-code/dev_message.md b/docs/resources/source-code/dev_message.md new file mode 100644 index 000000000..714013bc9 --- /dev/null +++ b/docs/resources/source-code/dev_message.md @@ -0,0 +1,325 @@ +# [Module dev_message.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_message.erl) + + + + +The identity device: For non-reserved keys, it simply returns a key +from the message as it is found in the message's underlying Erlang map. + + + +## Description ## +Private keys (`priv[.*]`) are not included. +Reserved keys are: `id`, `commitments`, `committers`, `keys`, `path`, +`set`, `remove`, `get`, and `verify`. Their function comments describe the +behaviour of the device when these keys are set. + +## Function Index ## + + +
calculate_ids/3*
cannot_get_private_keys_test/0*
case_insensitive_get/2*Key matching should be case insensitive, following RFC-9110, so we +implement a case-insensitive key lookup rather than delegating to +maps:get/2.
case_insensitive_get_test/0*
commit/3Commit to a message, using the commitment-device key to specify the +device that should be used to commit to the message.
commitment_ids_from_committers/2*Returns a list of commitment IDs in a commitments map that are relevant +for a list of given committer addresses.
commitment_ids_from_request/3*Implements a standardized form of specifying commitment IDs for a +message request.
committed/3Return the list of committed keys from a message.
committers/1Return the committers of a message that are present in the given request.
committers/2
committers/3
deep_unset_test/0*
exec_for_commitment/5*Execute a function for a single commitment in the context of its +parent message.
get/2Return the value associated with the key as it exists in the message's +underlying Erlang map.
get/3
get_keys_mod_test/0*
id/1Return the ID of a message, using the committers list if it exists.
id/2
id/3
id_device/1*Locate the ID device of a message.
info/0Return the info for the identity device.
is_private_mod_test/0*
key_from_device_test/0*
keys/1Get the public keys of a message.
keys_from_device_test/0*
private_keys_are_filtered_test/0*
remove/2Remove a key or keys from a message.
remove_test/0*
run_test/0*
set/3Deep merge keys in a message.
set_conflicting_keys_test/0*
set_ignore_undefined_test/0*
set_path/3Special case of set/3 for setting the path key.
unset_with_set_test/0*
verify/3Verify a message.
verify_test/0*
with_relevant_commitments/3*Return a message with only the relevant commitments for a given request.
+ + + + +## Function Details ## + + + +### calculate_ids/3 * ### + +`calculate_ids(Base, Req, NodeOpts) -> any()` + + + +### cannot_get_private_keys_test/0 * ### + +`cannot_get_private_keys_test() -> any()` + + + +### case_insensitive_get/2 * ### + +`case_insensitive_get(Key, Msg) -> any()` + +Key matching should be case insensitive, following RFC-9110, so we +implement a case-insensitive key lookup rather than delegating to +`maps:get/2`. Encode the key to a binary if it is not already. + + + +### case_insensitive_get_test/0 * ### + +`case_insensitive_get_test() -> any()` + + + +### commit/3 ### + +`commit(Self, Req, Opts) -> any()` + +Commit to a message, using the `commitment-device` key to specify the +device that should be used to commit to the message. If the key is not set, +the default device (`httpsig@1.0`) is used. + + + +### commitment_ids_from_committers/2 * ### + +`commitment_ids_from_committers(CommitterAddrs, Commitments) -> any()` + +Returns a list of commitment IDs in a commitments map that are relevant +for a list of given committer addresses. + + + +### commitment_ids_from_request/3 * ### + +`commitment_ids_from_request(Base, Req, Opts) -> any()` + +Implements a standardized form of specifying commitment IDs for a +message request. The caller may specify a list of committers (by address) +or a list of commitment IDs directly. They may specify both, in which case +the returned list will be the union of the two lists. In each case, they +may specify `all` or `none` for each group. If no specifiers are provided, +the default is `all` for commitments -- also implying `all` for committers. + + + +### committed/3 ### + +`committed(Self, Req, Opts) -> any()` + +Return the list of committed keys from a message. + + + +### committers/1 ### + +`committers(Base) -> any()` + +Return the committers of a message that are present in the given request. + + + +### committers/2 ### + +`committers(Base, Req) -> any()` + + + +### committers/3 ### + +`committers(X1, X2, NodeOpts) -> any()` + + + +### deep_unset_test/0 * ### + +`deep_unset_test() -> any()` + + + +### exec_for_commitment/5 * ### + +`exec_for_commitment(Func, Base, Commitment, Req, Opts) -> any()` + +Execute a function for a single commitment in the context of its +parent message. +Note: Assumes that the `commitments` key has already been removed from the +message if applicable. + + + +### get/2 ### + +`get(Key, Msg) -> any()` + +Return the value associated with the key as it exists in the message's +underlying Erlang map. First check the public keys, then check case- +insensitively if the key is a binary. + + + +### get/3 ### + +`get(Key, Msg, Msg2) -> any()` + + + +### get_keys_mod_test/0 * ### + +`get_keys_mod_test() -> any()` + + + +### id/1 ### + +`id(Base) -> any()` + +Return the ID of a message, using the `committers` list if it exists. +If the `committers` key is `all`, return the ID including all known +commitments -- `none` yields the ID without any commitments. If the +`committers` key is a list/map, return the ID including only the specified +commitments. + +The `id-device` key in the message can be used to specify the device that +should be used to calculate the ID. If it is not set, the default device +(`httpsig@1.0`) is used. + +Note: This function _does not_ use AO-Core's `get/3` function, as it +as it would require significant computation. We may want to change this +if/when non-map message structures are created. + + + +### id/2 ### + +`id(Base, Req) -> any()` + + + +### id/3 ### + +`id(Base, Req, NodeOpts) -> any()` + + + +### id_device/1 * ### + +`id_device(X1) -> any()` + +Locate the ID device of a message. The ID device is determined the +`device` set in _all_ of the commitments. If no commitments are present, +the default device (`httpsig@1.0`) is used. + + + +### info/0 ### + +`info() -> any()` + +Return the info for the identity device. + + + +### is_private_mod_test/0 * ### + +`is_private_mod_test() -> any()` + + + +### key_from_device_test/0 * ### + +`key_from_device_test() -> any()` + + + +### keys/1 ### + +`keys(Msg) -> any()` + +Get the public keys of a message. + + + +### keys_from_device_test/0 * ### + +`keys_from_device_test() -> any()` + + + +### private_keys_are_filtered_test/0 * ### + +`private_keys_are_filtered_test() -> any()` + + + +### remove/2 ### + +`remove(Message1, X2) -> any()` + +Remove a key or keys from a message. + + + +### remove_test/0 * ### + +`remove_test() -> any()` + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### set/3 ### + +`set(Message1, NewValuesMsg, Opts) -> any()` + +Deep merge keys in a message. Takes a map of key-value pairs and sets +them in the message, overwriting any existing values. + + + +### set_conflicting_keys_test/0 * ### + +`set_conflicting_keys_test() -> any()` + + + +### set_ignore_undefined_test/0 * ### + +`set_ignore_undefined_test() -> any()` + + + +### set_path/3 ### + +`set_path(Message1, X2, Opts) -> any()` + +Special case of `set/3` for setting the `path` key. This cannot be set +using the normal `set` function, as the `path` is a reserved key, necessary +for AO-Core to know the key to evaluate in requests. + + + +### unset_with_set_test/0 * ### + +`unset_with_set_test() -> any()` + + + +### verify/3 ### + +`verify(Self, Req, Opts) -> any()` + +Verify a message. By default, all commitments are verified. The +`committers` key in the request can be used to specify that only the +commitments from specific committers should be verified. Similarly, specific +commitments can be specified using the `commitments` key. + + + +### verify_test/0 * ### + +`verify_test() -> any()` + + + +### with_relevant_commitments/3 * ### + +`with_relevant_commitments(Base, Req, Opts) -> any()` + +Return a message with only the relevant commitments for a given request. +See `commitment_ids_from_request/3` for more information on the request format. + diff --git a/docs/resources/source-code/dev_meta.md b/docs/resources/source-code/dev_meta.md new file mode 100644 index 000000000..a7f2d5d51 --- /dev/null +++ b/docs/resources/source-code/dev_meta.md @@ -0,0 +1,292 @@ +# [Module dev_meta.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_meta.erl) + + + + +The hyperbeam meta device, which is the default entry point +for all messages processed by the machine. + + + +## Description ## +This device executes a +AO-Core singleton request, after first applying the node's +pre-processor, if set. The pre-processor can halt the request by +returning an error, or return a modified version if it deems necessary -- +the result of the pre-processor is used as the request for the AO-Core +resolver. Additionally, a post-processor can be set, which is executed after +the AO-Core resolver has returned a result. + +## Function Index ## + + +
add_dynamic_keys/1*Add dynamic keys to the node message.
adopt_node_message/2Attempt to adopt changes to a node message.
authorized_set_node_msg_succeeds_test/0*Test that we can set the node message if the request is signed by the +owner of the node.
build/3Emits the version number and commit hash of the HyperBEAM node source, +if available.
buildinfo_test/0*Test that version information is available and returned correctly.
claim_node_test/0*Test that we can claim the node correctly and set the node message after.
config_test/0*Test that we can get the node message.
embed_status/1*Wrap the result of a device call in a status.
filter_node_msg/1*Remove items from the node message that are not encodable into a +message.
halt_request_test/0*Test that we can halt a request if the hook returns an error.
handle/2Normalize and route messages downstream based on their path.
handle_initialize/2*
handle_resolve/3*Handle an AO-Core request, which is a list of messages.
info/1Ensure that the helper function adopt_node_message/2 is not exported.
info/3Get/set the node message.
is/2Check if the request in question is signed by a given role on the node.
is/3
maybe_sign/2*Sign the result of a device call if the node is configured to do so.
message_to_status/1*Get the HTTP status code from a transaction (if it exists).
modify_request_test/0*Test that a hook can modify a request.
permanent_node_message_test/0*Test that a permanent node message cannot be changed.
priv_inaccessible_test/0*Test that we can't get the node message if the requested key is private.
request_response_hooks_test/0*
resolve_hook/4*Execute a hook from the node message upon the user's request.
status_code/1*Calculate the appropriate HTTP status code for an AO-Core result.
unauthorized_set_node_msg_fails_test/0*Test that we can't set the node message if the request is not signed by +the owner of the node.
uninitialized_node_test/0*Test that an uninitialized node will not run computation.
update_node_message/2*Validate that the request is signed by the operator of the node, then +allow them to update the node message.
+ + + + +## Function Details ## + + + +### add_dynamic_keys/1 * ### + +`add_dynamic_keys(NodeMsg) -> any()` + +Add dynamic keys to the node message. + + + +### adopt_node_message/2 ### + +`adopt_node_message(Request, NodeMsg) -> any()` + +Attempt to adopt changes to a node message. + + + +### authorized_set_node_msg_succeeds_test/0 * ### + +`authorized_set_node_msg_succeeds_test() -> any()` + +Test that we can set the node message if the request is signed by the +owner of the node. + + + +### build/3 ### + +`build(X1, X2, NodeMsg) -> any()` + +Emits the version number and commit hash of the HyperBEAM node source, +if available. + +We include the short hash separately, as the length of this hash may change in +the future, depending on the git version/config used to build the node. +Subsequently, rather than embedding the `git-short-hash-length`, for the +avoidance of doubt, we include the short hash separately, as well as its long +hash. + + + +### buildinfo_test/0 * ### + +`buildinfo_test() -> any()` + +Test that version information is available and returned correctly. + + + +### claim_node_test/0 * ### + +`claim_node_test() -> any()` + +Test that we can claim the node correctly and set the node message after. + + + +### config_test/0 * ### + +`config_test() -> any()` + +Test that we can get the node message. + + + +### embed_status/1 * ### + +`embed_status(X1) -> any()` + +Wrap the result of a device call in a status. + + + +### filter_node_msg/1 * ### + +`filter_node_msg(Msg) -> any()` + +Remove items from the node message that are not encodable into a +message. + + + +### halt_request_test/0 * ### + +`halt_request_test() -> any()` + +Test that we can halt a request if the hook returns an error. + + + +### handle/2 ### + +`handle(NodeMsg, RawRequest) -> any()` + +Normalize and route messages downstream based on their path. Messages +with a `Meta` key are routed to the `handle_meta/2` function, while all +other messages are routed to the `handle_resolve/2` function. + + + +### handle_initialize/2 * ### + +`handle_initialize(Rest, NodeMsg) -> any()` + + + +### handle_resolve/3 * ### + +`handle_resolve(Req, Msgs, NodeMsg) -> any()` + +Handle an AO-Core request, which is a list of messages. We apply +the node's pre-processor to the request first, and then resolve the request +using the node's AO-Core implementation if its response was `ok`. +After execution, we run the node's `response` hook on the result of +the request before returning the result it grants back to the user. + + + +### info/1 ### + +`info(X1) -> any()` + +Ensure that the helper function `adopt_node_message/2` is not exported. +The naming of this method carefully avoids a clash with the exported `info/3` +function. We would like the node information to be easily accessible via the +`info` endpoint, but AO-Core also uses `info` as the name of the function +that grants device information. The device call takes two or fewer arguments, +so we are safe to use the name for both purposes in this case, as the user +info call will match the three-argument version of the function. If in the +future the `request` is added as an argument to AO-Core's internal `info` +function, we will need to find a different approach. + + + +### info/3 ### + +`info(X1, Request, NodeMsg) -> any()` + +Get/set the node message. If the request is a `POST`, we check that the +request is signed by the owner of the node. If not, we return the node message +as-is, aside all keys that are private (according to `hb_private`). + + + +### is/2 ### + +`is(Request, NodeMsg) -> any()` + +Check if the request in question is signed by a given `role` on the node. +The `role` can be one of `operator` or `initiator`. + + + +### is/3 ### + +`is(X1, Request, NodeMsg) -> any()` + + + +### maybe_sign/2 * ### + +`maybe_sign(Res, NodeMsg) -> any()` + +Sign the result of a device call if the node is configured to do so. + + + +### message_to_status/1 * ### + +`message_to_status(Item) -> any()` + +Get the HTTP status code from a transaction (if it exists). + + + +### modify_request_test/0 * ### + +`modify_request_test() -> any()` + +Test that a hook can modify a request. + + + +### permanent_node_message_test/0 * ### + +`permanent_node_message_test() -> any()` + +Test that a permanent node message cannot be changed. + + + +### priv_inaccessible_test/0 * ### + +`priv_inaccessible_test() -> any()` + +Test that we can't get the node message if the requested key is private. + + + +### request_response_hooks_test/0 * ### + +`request_response_hooks_test() -> any()` + + + +### resolve_hook/4 * ### + +`resolve_hook(HookName, InitiatingRequest, Body, NodeMsg) -> any()` + +Execute a hook from the node message upon the user's request. The +invocation of the hook provides a request of the following form: + +``` + + /path => request | response + /request => the original request singleton + /body => parsed sequence of messages to process | the execution result +``` + + + +### status_code/1 * ### + +`status_code(X1) -> any()` + +Calculate the appropriate HTTP status code for an AO-Core result. +The order of precedence is: +1. The status code from the message. +2. The HTTP representation of the status code. +3. The default status code. + + + +### unauthorized_set_node_msg_fails_test/0 * ### + +`unauthorized_set_node_msg_fails_test() -> any()` + +Test that we can't set the node message if the request is not signed by +the owner of the node. + + + +### uninitialized_node_test/0 * ### + +`uninitialized_node_test() -> any()` + +Test that an uninitialized node will not run computation. + + + +### update_node_message/2 * ### + +`update_node_message(Request, NodeMsg) -> any()` + +Validate that the request is signed by the operator of the node, then +allow them to update the node message. + diff --git a/docs/resources/source-code/dev_monitor.md b/docs/resources/source-code/dev_monitor.md new file mode 100644 index 000000000..f092fb0a0 --- /dev/null +++ b/docs/resources/source-code/dev_monitor.md @@ -0,0 +1,53 @@ +# [Module dev_monitor.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_monitor.erl) + + + + + + +## Function Index ## + + +
add_monitor/2
end_of_schedule/1
execute/2
init/3
signal/2*
uses/0
+ + + + +## Function Details ## + + + +### add_monitor/2 ### + +`add_monitor(Mon, State) -> any()` + + + +### end_of_schedule/1 ### + +`end_of_schedule(State) -> any()` + + + +### execute/2 ### + +`execute(Message, State) -> any()` + + + +### init/3 ### + +`init(State, X2, InitState) -> any()` + + + +### signal/2 * ### + +`signal(State, Signal) -> any()` + + + +### uses/0 ### + +`uses() -> any()` + diff --git a/docs/resources/source-code/dev_multipass.md b/docs/resources/source-code/dev_multipass.md new file mode 100644 index 000000000..1cf3fec8a --- /dev/null +++ b/docs/resources/source-code/dev_multipass.md @@ -0,0 +1,46 @@ +# [Module dev_multipass.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_multipass.erl) + + + + +A device that triggers repass events until a certain counter has been +reached. + + + +## Description ## +This is useful for certain types of stacks that need various +execution passes to be completed in sequence across devices. + +## Function Index ## + + +
basic_multipass_test/0*
handle/4*Forward the keys function to the message device, handle all others +with deduplication.
info/1
+ + + + +## Function Details ## + + + +### basic_multipass_test/0 * ### + +`basic_multipass_test() -> any()` + + + +### handle/4 * ### + +`handle(Key, M1, M2, Opts) -> any()` + +Forward the keys function to the message device, handle all others +with deduplication. We only act on the first pass. + + + +### info/1 ### + +`info(M1) -> any()` + diff --git a/docs/resources/source-code/dev_name.md b/docs/resources/source-code/dev_name.md new file mode 100644 index 000000000..9edd22fca --- /dev/null +++ b/docs/resources/source-code/dev_name.md @@ -0,0 +1,97 @@ +# [Module dev_name.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_name.erl) + + + + +A device for resolving names to their corresponding values, through the +use of a `resolver` interface. + + + +## Description ## +Each `resolver` is a message that can be +given a `key` and returns an associated value. The device will attempt to +match the key against each resolver in turn, and return the value of the +first resolver that matches. + +## Function Index ## + + +
execute_resolver/3*Execute a resolver with the given key and return its value.
info/1Configure the default key to proxy to the resolver/4 function.
load_and_execute_test/0*Test that we can resolve messages from a name loaded with the device.
match_resolver/3*Find the first resolver that matches the key and return its value.
message_lookup_device_resolver/1*
multiple_resolvers_test/0*
no_resolvers_test/0*
resolve/4*Resolve a name to its corresponding value.
single_resolver_test/0*
+ + + + +## Function Details ## + + + +### execute_resolver/3 * ### + +`execute_resolver(Key, Resolver, Opts) -> any()` + +Execute a resolver with the given key and return its value. + + + +### info/1 ### + +`info(X1) -> any()` + +Configure the `default` key to proxy to the `resolver/4` function. +Exclude the `keys` and `set` keys from being processed by this device, as +these are needed to modify the base message itself. + + + +### load_and_execute_test/0 * ### + +`load_and_execute_test() -> any()` + +Test that we can resolve messages from a name loaded with the device. + + + +### match_resolver/3 * ### + +`match_resolver(Key, Resolvers, Opts) -> any()` + +Find the first resolver that matches the key and return its value. + + + +### message_lookup_device_resolver/1 * ### + +`message_lookup_device_resolver(Msg) -> any()` + + + +### multiple_resolvers_test/0 * ### + +`multiple_resolvers_test() -> any()` + + + +### no_resolvers_test/0 * ### + +`no_resolvers_test() -> any()` + + + +### resolve/4 * ### + +`resolve(Key, X2, Req, Opts) -> any()` + +Resolve a name to its corresponding value. The name is given by the key +called. For example, `GET /~name@1.0/hello&load=false` grants the value of +`hello`. If the `load` key is set to `true`, the value is treated as a +pointer and its contents is loaded from the cache. For example, +`GET /~name@1.0/reference` yields the message at the path specified by the +`reference` key. + + + +### single_resolver_test/0 * ### + +`single_resolver_test() -> any()` + diff --git a/docs/resources/source-code/dev_node_process.md b/docs/resources/source-code/dev_node_process.md new file mode 100644 index 000000000..70a546cd9 --- /dev/null +++ b/docs/resources/source-code/dev_node_process.md @@ -0,0 +1,97 @@ +# [Module dev_node_process.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_node_process.erl) + + + + +A device that implements the singleton pattern for processes specific +to an individual node. + + + +## Description ## + +This device uses the `local-name@1.0` device to +register processes with names locally, persistenting them across reboots. + +Definitions of singleton processes are expected to be found with their +names in the `node_processes` section of the node message. + +## Function Index ## + + +
augment_definition/2*Augment the given process definition with the node's address.
generate_test_opts/0*Helper function to generate a test environment and its options.
generate_test_opts/1*
info/1Register a default handler for the device.
lookup/4*Lookup a process by name.
lookup_execute_test/0*Test that a process can be spawned, executed upon, and its result retrieved.
lookup_no_spawn_test/0*
lookup_spawn_test/0*
spawn_register/2*Spawn a new process according to the process definition found in the +node message, and register it with the given name.
+ + + + +## Function Details ## + + + +### augment_definition/2 * ### + +`augment_definition(BaseDef, Opts) -> any()` + +Augment the given process definition with the node's address. + + + +### generate_test_opts/0 * ### + +`generate_test_opts() -> any()` + +Helper function to generate a test environment and its options. + + + +### generate_test_opts/1 * ### + +`generate_test_opts(Defs) -> any()` + + + +### info/1 ### + +`info(Opts) -> any()` + +Register a default handler for the device. Inherits `keys` and `set` +from the default device. + + + +### lookup/4 * ### + +`lookup(Name, Base, Req, Opts) -> any()` + +Lookup a process by name. + + + +### lookup_execute_test/0 * ### + +`lookup_execute_test() -> any()` + +Test that a process can be spawned, executed upon, and its result retrieved. + + + +### lookup_no_spawn_test/0 * ### + +`lookup_no_spawn_test() -> any()` + + + +### lookup_spawn_test/0 * ### + +`lookup_spawn_test() -> any()` + + + +### spawn_register/2 * ### + +`spawn_register(Name, Opts) -> any()` + +Spawn a new process according to the process definition found in the +node message, and register it with the given name. + diff --git a/docs/resources/source-code/dev_p4.md b/docs/resources/source-code/dev_p4.md new file mode 100644 index 000000000..081a4814a --- /dev/null +++ b/docs/resources/source-code/dev_p4.md @@ -0,0 +1,139 @@ +# [Module dev_p4.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_p4.erl) + + + + +The HyperBEAM core payment ledger. + + + +## Description ## + +This module allows the operator to +specify another device that can act as a pricing mechanism for transactions +on the node, as well as orchestrating a payment ledger to calculate whether +the node should fulfil services for users. + +The device requires the following node message settings in order to function: + +- `p4_pricing-device`: The device that will estimate the cost of a request. +- `p4_ledger-device`: The device that will act as a payment ledger. + +The pricing device should implement the following keys: + +``` +GET /estimate?type=pre|post&body=[...]&request=RequestMessageGET /price?type=pre|post&body=[...]&request=RequestMessage +``` + +The `body` key is used to pass either the request or response messages to the +device. The `type` key is used to specify whether the inquiry is for a request +(pre) or a response (post) object. Requests carry lists of messages that will +be executed, while responses carry the results of the execution. The `price` +key may return `infinity` if the node will not serve a user under any +circumstances. Else, the value returned by the `price` key will be passed to +the ledger device as the `amount` key. + +A ledger device should implement the following keys: + +``` +POST /credit?message=PaymentMessage&request=RequestMessagePOST /debit?amount=PriceMessage&request=RequestMessageGET /balance?request=RequestMessage +``` + +The `type` key is optional and defaults to `pre`. If `type` is set to `post`, +the debit must be applied to the ledger, whereas the `pre` type is used to +check whether the debit would succeed before execution. + +## Function Index ## + + +
balance/3Get the balance of a user in the ledger.
faff_test/0*Simple test of p4's capabilities with the faff@1.0 device.
is_chargable_req/2*The node operator may elect to make certain routes non-chargable, using +the routes syntax also used to declare routes in router@1.0.
lua_pricing_test/0*Ensure that Lua modules can be used as pricing and ledger devices.
non_chargable_route_test/0*Test that a non-chargable route is not charged for.
request/3Estimate the cost of a transaction and decide whether to proceed with +a request.
response/3Postprocess the request after it has been fulfilled.
test_opts/1*
test_opts/2*
test_opts/3*
+ + + + +## Function Details ## + + + +### balance/3 ### + +`balance(X1, Req, NodeMsg) -> any()` + +Get the balance of a user in the ledger. + + + +### faff_test/0 * ### + +`faff_test() -> any()` + +Simple test of p4's capabilities with the `faff@1.0` device. + + + +### is_chargable_req/2 * ### + +`is_chargable_req(Req, NodeMsg) -> any()` + +The node operator may elect to make certain routes non-chargable, using +the `routes` syntax also used to declare routes in `router@1.0`. + + + +### lua_pricing_test/0 * ### + +`lua_pricing_test() -> any()` + +Ensure that Lua modules can be used as pricing and ledger devices. Our +modules come in two parts: +- A `process` module which is executed as a persistent `local-process` on the +node, and which maintains the state of the ledger. +- A `client` module, which is executed as a `p4@1.0` device, marshalling +requests to the `process` module. + + + +### non_chargable_route_test/0 * ### + +`non_chargable_route_test() -> any()` + +Test that a non-chargable route is not charged for. + + + +### request/3 ### + +`request(State, Raw, NodeMsg) -> any()` + +Estimate the cost of a transaction and decide whether to proceed with +a request. The default behavior if `pricing-device` or `p4_balances` are +not set is to proceed, so it is important that a user initialize them. + + + +### response/3 ### + +`response(State, RawResponse, NodeMsg) -> any()` + +Postprocess the request after it has been fulfilled. + + + +### test_opts/1 * ### + +`test_opts(Opts) -> any()` + + + +### test_opts/2 * ### + +`test_opts(Opts, PricingDev) -> any()` + + + +### test_opts/3 * ### + +`test_opts(Opts, PricingDev, LedgerDev) -> any()` + diff --git a/docs/resources/source-code/dev_patch.md b/docs/resources/source-code/dev_patch.md new file mode 100644 index 000000000..92cf5373c --- /dev/null +++ b/docs/resources/source-code/dev_patch.md @@ -0,0 +1,125 @@ +# [Module dev_patch.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_patch.erl) + + + + +A device that can be used to reorganize a message: Moving data from +one path inside it to another. + + + +## Description ## + +This device's function runs in two modes: + +1. When using `all` to move all data at the path given in `from` to the +path given in `to`. +2. When using `patches` to move all submessages in the source to the target, +_if_ they have a `method` key of `PATCH` or a `device` key of `patch@1.0`. + +Source and destination paths may be prepended by `base:` or `req:` keys to +indicate that they are relative to either of the message`s that the +computation is being performed on. + +The search order for finding the source and destination keys is as follows, +where `X` is either `from` or `to`: + +1. The `patch-X` key of the execution message. +2. The `X` key of the execution message. +3. The `patch-X` key of the request message. +4. The `X` key of the request message. + +Additionally, this device implements the standard computation device keys, +allowing it to be used as an element of an execution stack pipeline, etc. + +## Function Index ## + + +
all/3Get the value found at the patch-from key of the message, or the +from key if the former is not present.
all_mode_test/0*
compute/3
init/3Necessary hooks for compliance with the execution-device standard.
move/4*Unified executor for the all and patches modes.
normalize/3
patch_to_submessage_test/0*
patches/3Find relevant PATCH messages in the given source key of the execution +and request messages, and apply them to the given destination key of the +request.
req_prefix_test/0*
snapshot/3
uninitialized_patch_test/0*
+ + + + +## Function Details ## + + + +### all/3 ### + +`all(Msg1, Msg2, Opts) -> any()` + +Get the value found at the `patch-from` key of the message, or the +`from` key if the former is not present. Remove it from the message and set +the new source to the value found. + + + +### all_mode_test/0 * ### + +`all_mode_test() -> any()` + + + +### compute/3 ### + +`compute(Msg1, Msg2, Opts) -> any()` + + + +### init/3 ### + +`init(Msg1, Msg2, Opts) -> any()` + +Necessary hooks for compliance with the `execution-device` standard. + + + +### move/4 * ### + +`move(Mode, Msg1, Msg2, Opts) -> any()` + +Unified executor for the `all` and `patches` modes. + + + +### normalize/3 ### + +`normalize(Msg1, Msg2, Opts) -> any()` + + + +### patch_to_submessage_test/0 * ### + +`patch_to_submessage_test() -> any()` + + + +### patches/3 ### + +`patches(Msg1, Msg2, Opts) -> any()` + +Find relevant `PATCH` messages in the given source key of the execution +and request messages, and apply them to the given destination key of the +request. + + + +### req_prefix_test/0 * ### + +`req_prefix_test() -> any()` + + + +### snapshot/3 ### + +`snapshot(Msg1, Msg2, Opts) -> any()` + + + +### uninitialized_patch_test/0 * ### + +`uninitialized_patch_test() -> any()` + diff --git a/docs/resources/source-code/dev_poda.md b/docs/resources/source-code/dev_poda.md new file mode 100644 index 000000000..91427a8ea --- /dev/null +++ b/docs/resources/source-code/dev_poda.md @@ -0,0 +1,129 @@ +# [Module dev_poda.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_poda.erl) + + + + +A simple exemplar decentralized proof of authority consensus algorithm +for AO processes. + + + +## Description ## + +This device is split into two flows, spanning three +actions. + +Execution flow: +1. Initialization. +2. Validation of incoming messages before execution. +Commitment flow: +1. Adding commitments to results, either on a CU or MU. + +## Function Index ## + + +
add_commitments/2*
commit_to_results/2*
execute/3
extract_opts/1*
find_process/2*Find the process that this message is targeting, in order to +determine which commitments to add.
init/2
is_user_signed/1Determines if a user committed.
pfiltermap/2*Helper function for parallel execution of commitment +gathering.
push/2Hook used by the MU pathway (currently) to add commitments to an +outbound message if the computation requests it.
return_error/2*
validate/2*
validate_commitment/3*
validate_stage/3*
validate_stage/4*
+ + + + +## Function Details ## + + + +### add_commitments/2 * ### + +`add_commitments(NewMsg, S) -> any()` + + + +### commit_to_results/2 * ### + +`commit_to_results(Msg, S) -> any()` + + + +### execute/3 ### + +`execute(Outer, S, Opts) -> any()` + + + +### extract_opts/1 * ### + +`extract_opts(Params) -> any()` + + + +### find_process/2 * ### + +`find_process(Item, X2) -> any()` + +Find the process that this message is targeting, in order to +determine which commitments to add. + + + +### init/2 ### + +`init(S, Params) -> any()` + + + +### is_user_signed/1 ### + +`is_user_signed(Tx) -> any()` + +Determines if a user committed + + + +### pfiltermap/2 * ### + +`pfiltermap(Pred, List) -> any()` + +Helper function for parallel execution of commitment +gathering. + + + +### push/2 ### + +`push(Item, S) -> any()` + +Hook used by the MU pathway (currently) to add commitments to an +outbound message if the computation requests it. + + + +### return_error/2 * ### + +`return_error(S, Reason) -> any()` + + + +### validate/2 * ### + +`validate(Msg, Opts) -> any()` + + + +### validate_commitment/3 * ### + +`validate_commitment(Msg, Comm, Opts) -> any()` + + + +### validate_stage/3 * ### + +`validate_stage(X1, Msg, Opts) -> any()` + + + +### validate_stage/4 * ### + +`validate_stage(X1, Tx, Content, Opts) -> any()` + diff --git a/docs/resources/source-code/dev_process.md b/docs/resources/source-code/dev_process.md new file mode 100644 index 000000000..f9a4074ea --- /dev/null +++ b/docs/resources/source-code/dev_process.md @@ -0,0 +1,453 @@ +# [Module dev_process.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_process.erl) + + + + +This module contains the device implementation of AO processes +in AO-Core. + + + +## Description ## + +The core functionality of the module is in 'routing' requests +for different functionality (scheduling, computing, and pushing messages) +to the appropriate device. This is achieved by swapping out the device +of the process message with the necessary component in order to run the +execution, then swapping it back before returning. Computation is supported +as a stack of devices, customizable by the user, while the scheduling +device is (by default) a single device. + +This allows the devices to share state as needed. Additionally, after each +computation step the device caches the result at a path relative to the +process definition itself, such that the process message's ID can act as an +immutable reference to the process's growing list of interactions. See +`dev_process_cache` for details. + +The external API of the device is as follows: + +``` + + GET /ID/Schedule: Returns the messages in the schedule + POST /ID/Schedule: Adds a message to the schedule + GET /ID/Compute/[IDorSlotNum]: Returns the state of the process after + applying a message + GET /ID/Now: Returns the /Results key of the latest + computed message +``` + +An example process definition will look like this: + +``` + + Device: Process/1.0 + Scheduler-Device: Scheduler/1.0 + Execution-Device: Stack/1.0 + Execution-Stack: "Scheduler/1.0", "Cron/1.0", "WASM/1.0", "PoDA/1.0" + Cron-Frequency: 10-Minutes + WASM-Image: WASMImageID + PoDA: + Device: PoDA/1.0 + Authority: A + Authority: B + Authority: C + Quorum: 2 +``` + +Runtime options: +Cache-Frequency: The number of assignments that will be computed +before the full (restorable) state should be cached. +Cache-Keys: A list of the keys that should be cached for all +assignments, in addition to `/Results`. + +## Function Index ## + + +
aos_browsable_state_test_/0*
aos_compute_test_/0*
aos_persistent_worker_benchmark_test_/0*
aos_state_access_via_http_test_/0*
aos_state_patch_test_/0*
as_process/2Change the message to for that has the device set as this module.
compute/3Compute the result of an assignment applied to the process state, if it +is the next message.
compute_slot/5*Compute a single slot for a process, given an initialized state.
compute_to_slot/5*Continually get and apply the next assignment from the scheduler until +we reach the target slot that the user has requested.
default_device/3*Returns the default device for a given piece of functionality.
default_device_index/1*
dev_test_process/0Generate a device that has a stack of two dev_tests for +execution.
do_test_restore/0
ensure_loaded/3*Ensure that the process message we have in memory is live and +up-to-date.
ensure_process_key/2Helper function to store a copy of the process key in the message.
get_scheduler_slot_test/0*
http_wasm_process_by_id_test/0*
info/1When the info key is called, we should return the process exports.
init/0
init/3*Before computation begins, a boot phase is required.
next/3*
now/3Returns the known state of the process at either the current slot, or +the latest slot in the cache depending on the process_now_from_cache option.
now_results_test_/0*
persistent_process_test/0*
prior_results_accessible_test_/0*
process_id/3Returns the process ID of the current process.
push/3Recursively push messages to the scheduler until we find a message +that does not lead to any further messages being scheduled.
recursive_path_resolution_test/0*
restore_test_/0*Manually test state restoration without using the cache.
run_as/4*Run a message against Msg1, with the device being swapped out for +the device found at Key.
schedule/3Wraps functions in the Scheduler device.
schedule_aos_call/2
schedule_aos_call/3
schedule_on_process_test/0*
schedule_test_message/2*
schedule_test_message/3*
schedule_wasm_call/3*
schedule_wasm_call/4*
simple_wasm_persistent_worker_benchmark_test/0*
slot/3
snapshot/3
store_result/5*Store the resulting state in the cache, potentially with the snapshot +key.
test_aos_process/0Generate a process message with a random number, and the +dev_wasm device for execution.
test_aos_process/1
test_aos_process/2*
test_base_process/0*Generate a process message with a random number, and no +executor.
test_base_process/1*
test_device_compute_test/0*
test_wasm_process/1
test_wasm_process/2*
wasm_compute_from_id_test/0*
wasm_compute_test/0*
+ + + + +## Function Details ## + + + +### aos_browsable_state_test_/0 * ### + +`aos_browsable_state_test_() -> any()` + + + +### aos_compute_test_/0 * ### + +`aos_compute_test_() -> any()` + + + +### aos_persistent_worker_benchmark_test_/0 * ### + +`aos_persistent_worker_benchmark_test_() -> any()` + + + +### aos_state_access_via_http_test_/0 * ### + +`aos_state_access_via_http_test_() -> any()` + + + +### aos_state_patch_test_/0 * ### + +`aos_state_patch_test_() -> any()` + + + +### as_process/2 ### + +`as_process(Msg1, Opts) -> any()` + +Change the message to for that has the device set as this module. +In situations where the key that is `run_as` returns a message with a +transformed device, this is useful. + + + +### compute/3 ### + +`compute(Msg1, Msg2, Opts) -> any()` + +Compute the result of an assignment applied to the process state, if it +is the next message. + + + +### compute_slot/5 * ### + +`compute_slot(ProcID, State, RawInputMsg, ReqMsg, Opts) -> any()` + +Compute a single slot for a process, given an initialized state. + + + +### compute_to_slot/5 * ### + +`compute_to_slot(ProcID, Msg1, Msg2, TargetSlot, Opts) -> any()` + +Continually get and apply the next assignment from the scheduler until +we reach the target slot that the user has requested. + + + +### default_device/3 * ### + +`default_device(Msg1, Key, Opts) -> any()` + +Returns the default device for a given piece of functionality. Expects +the `process/variant` key to be set in the message. The `execution-device` +_must_ be set in all processes aside those marked with `ao.TN.1` variant. +This is in order to ensure that post-mainnet processes do not default to +using infrastructure that should not be present on nodes in the future. + + + +### default_device_index/1 * ### + +`default_device_index(X1) -> any()` + + + +### dev_test_process/0 ### + +`dev_test_process() -> any()` + +Generate a device that has a stack of two `dev_test`s for +execution. This should generate a message state has doubled +`Already-Seen` elements for each assigned slot. + + + +### do_test_restore/0 ### + +`do_test_restore() -> any()` + + + +### ensure_loaded/3 * ### + +`ensure_loaded(Msg1, Msg2, Opts) -> any()` + +Ensure that the process message we have in memory is live and +up-to-date. + + + +### ensure_process_key/2 ### + +`ensure_process_key(Msg1, Opts) -> any()` + +Helper function to store a copy of the `process` key in the message. + + + +### get_scheduler_slot_test/0 * ### + +`get_scheduler_slot_test() -> any()` + + + +### http_wasm_process_by_id_test/0 * ### + +`http_wasm_process_by_id_test() -> any()` + + + +### info/1 ### + +`info(Msg1) -> any()` + +When the info key is called, we should return the process exports. + + + +### init/0 ### + +`init() -> any()` + + + +### init/3 * ### + +`init(Msg1, Msg2, Opts) -> any()` + +Before computation begins, a boot phase is required. This phase +allows devices on the execution stack to initialize themselves. We set the +`Initialized` key to `True` to indicate that the process has been +initialized. + + + +### next/3 * ### + +`next(Msg1, Msg2, Opts) -> any()` + + + +### now/3 ### + +`now(RawMsg1, Msg2, Opts) -> any()` + +Returns the known state of the process at either the current slot, or +the latest slot in the cache depending on the `process_now_from_cache` option. + + + +### now_results_test_/0 * ### + +`now_results_test_() -> any()` + + + +### persistent_process_test/0 * ### + +`persistent_process_test() -> any()` + + + +### prior_results_accessible_test_/0 * ### + +`prior_results_accessible_test_() -> any()` + + + +### process_id/3 ### + +`process_id(Msg1, Msg2, Opts) -> any()` + +Returns the process ID of the current process. + + + +### push/3 ### + +`push(Msg1, Msg2, Opts) -> any()` + +Recursively push messages to the scheduler until we find a message +that does not lead to any further messages being scheduled. + + + +### recursive_path_resolution_test/0 * ### + +`recursive_path_resolution_test() -> any()` + + + +### restore_test_/0 * ### + +`restore_test_() -> any()` + +Manually test state restoration without using the cache. + + + +### run_as/4 * ### + +`run_as(Key, Msg1, Msg2, Opts) -> any()` + +Run a message against Msg1, with the device being swapped out for +the device found at `Key`. After execution, the device is swapped back +to the original device if the device is the same as we left it. + + + +### schedule/3 ### + +`schedule(Msg1, Msg2, Opts) -> any()` + +Wraps functions in the Scheduler device. + + + +### schedule_aos_call/2 ### + +`schedule_aos_call(Msg1, Code) -> any()` + + + +### schedule_aos_call/3 ### + +`schedule_aos_call(Msg1, Code, Opts) -> any()` + + + +### schedule_on_process_test/0 * ### + +`schedule_on_process_test() -> any()` + + + +### schedule_test_message/2 * ### + +`schedule_test_message(Msg1, Text) -> any()` + + + +### schedule_test_message/3 * ### + +`schedule_test_message(Msg1, Text, MsgBase) -> any()` + + + +### schedule_wasm_call/3 * ### + +`schedule_wasm_call(Msg1, FuncName, Params) -> any()` + + + +### schedule_wasm_call/4 * ### + +`schedule_wasm_call(Msg1, FuncName, Params, Opts) -> any()` + + + +### simple_wasm_persistent_worker_benchmark_test/0 * ### + +`simple_wasm_persistent_worker_benchmark_test() -> any()` + + + +### slot/3 ### + +`slot(Msg1, Msg2, Opts) -> any()` + + + +### snapshot/3 ### + +`snapshot(RawMsg1, Msg2, Opts) -> any()` + + + +### store_result/5 * ### + +`store_result(ProcID, Slot, Msg3, Msg2, Opts) -> any()` + +Store the resulting state in the cache, potentially with the snapshot +key. + + + +### test_aos_process/0 ### + +`test_aos_process() -> any()` + +Generate a process message with a random number, and the +`dev_wasm` device for execution. + + + +### test_aos_process/1 ### + +`test_aos_process(Opts) -> any()` + + + +### test_aos_process/2 * ### + +`test_aos_process(Opts, Stack) -> any()` + + + +### test_base_process/0 * ### + +`test_base_process() -> any()` + +Generate a process message with a random number, and no +executor. + + + +### test_base_process/1 * ### + +`test_base_process(Opts) -> any()` + + + +### test_device_compute_test/0 * ### + +`test_device_compute_test() -> any()` + + + +### test_wasm_process/1 ### + +`test_wasm_process(WASMImage) -> any()` + + + +### test_wasm_process/2 * ### + +`test_wasm_process(WASMImage, Opts) -> any()` + + + +### wasm_compute_from_id_test/0 * ### + +`wasm_compute_from_id_test() -> any()` + + + +### wasm_compute_test/0 * ### + +`wasm_compute_test() -> any()` + diff --git a/docs/resources/source-code/dev_process_cache.md b/docs/resources/source-code/dev_process_cache.md new file mode 100644 index 000000000..59f195d20 --- /dev/null +++ b/docs/resources/source-code/dev_process_cache.md @@ -0,0 +1,117 @@ +# [Module dev_process_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_process_cache.erl) + + + + +A wrapper around the hb_cache module that provides a more +convenient interface for reading the result of a process at a given slot or +message ID. + + + +## Function Index ## + + +
find_latest_outputs/1*Test for retrieving the latest computed output for a process.
first_with_path/4*Find the latest assignment with the requested path suffix.
first_with_path/5*
latest/2Retrieve the latest slot for a given process.
latest/3
latest/4
path/3*Calculate the path of a result, given a process ID and a slot.
path/4*
process_cache_suite_test_/0*
read/2Read the result of a process at a given slot.
read/3
test_write_and_read_output/1*Test for writing multiple computed outputs, then getting them by +their slot number and by their signed and unsigned IDs.
write/4Write a process computation result to the cache.
+ + + + +## Function Details ## + + + +### find_latest_outputs/1 * ### + +`find_latest_outputs(Opts) -> any()` + +Test for retrieving the latest computed output for a process. + + + +### first_with_path/4 * ### + +`first_with_path(ProcID, RequiredPath, Slots, Opts) -> any()` + +Find the latest assignment with the requested path suffix. + + + +### first_with_path/5 * ### + +`first_with_path(ProcID, Required, Rest, Opts, Store) -> any()` + + + +### latest/2 ### + +`latest(ProcID, Opts) -> any()` + +Retrieve the latest slot for a given process. Optionally state a limit +on the slot number to search for, as well as a required path that the slot +must have. + + + +### latest/3 ### + +`latest(ProcID, RequiredPath, Opts) -> any()` + + + +### latest/4 ### + +`latest(ProcID, RawRequiredPath, Limit, Opts) -> any()` + + + +### path/3 * ### + +`path(ProcID, Ref, Opts) -> any()` + +Calculate the path of a result, given a process ID and a slot. + + + +### path/4 * ### + +`path(ProcID, Ref, PathSuffix, Opts) -> any()` + + + +### process_cache_suite_test_/0 * ### + +`process_cache_suite_test_() -> any()` + + + +### read/2 ### + +`read(ProcID, Opts) -> any()` + +Read the result of a process at a given slot. + + + +### read/3 ### + +`read(ProcID, SlotRef, Opts) -> any()` + + + +### test_write_and_read_output/1 * ### + +`test_write_and_read_output(Opts) -> any()` + +Test for writing multiple computed outputs, then getting them by +their slot number and by their signed and unsigned IDs. + + + +### write/4 ### + +`write(ProcID, Slot, Msg, Opts) -> any()` + +Write a process computation result to the cache. + diff --git a/docs/resources/source-code/dev_process_worker.md b/docs/resources/source-code/dev_process_worker.md new file mode 100644 index 000000000..4e34029e3 --- /dev/null +++ b/docs/resources/source-code/dev_process_worker.md @@ -0,0 +1,104 @@ +# [Module dev_process_worker.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_process_worker.erl) + + + + +A long-lived process worker that keeps state in memory between +calls. + + + +## Description ## +Implements the interface of `hb_ao` to receive and respond +to computation requests regarding a process as a singleton. + +## Function Index ## + + +
await/5Await a resolution from a worker executing the process@1.0 device.
group/3Returns a group name for a request.
grouper_test/0*
info_test/0*
notify_compute/4Notify any waiters for a specific slot of the computed results.
notify_compute/5*
process_to_group_name/2*
send_notification/4*
server/3Spawn a new worker process.
stop/1Stop a worker process.
test_init/0*
+ + + + +## Function Details ## + + + +### await/5 ### + +`await(Worker, GroupName, Msg1, Msg2, Opts) -> any()` + +Await a resolution from a worker executing the `process@1.0` device. + + + +### group/3 ### + +`group(Msg1, Msg2, Opts) -> any()` + +Returns a group name for a request. The worker is responsible for all +computation work on the same process on a single node, so we use the +process ID as the group name. + + + +### grouper_test/0 * ### + +`grouper_test() -> any()` + + + +### info_test/0 * ### + +`info_test() -> any()` + + + +### notify_compute/4 ### + +`notify_compute(GroupName, SlotToNotify, Msg3, Opts) -> any()` + +Notify any waiters for a specific slot of the computed results. + + + +### notify_compute/5 * ### + +`notify_compute(GroupName, SlotToNotify, Msg3, Opts, Count) -> any()` + + + +### process_to_group_name/2 * ### + +`process_to_group_name(Msg1, Opts) -> any()` + + + +### send_notification/4 * ### + +`send_notification(Listener, GroupName, SlotToNotify, Msg3) -> any()` + + + +### server/3 ### + +`server(GroupName, Msg1, Opts) -> any()` + +Spawn a new worker process. This is called after the end of the first +execution of `hb_ao:resolve/3`, so the state we are given is the +already current. + + + +### stop/1 ### + +`stop(Worker) -> any()` + +Stop a worker process. + + + +### test_init/0 * ### + +`test_init() -> any()` + diff --git a/docs/resources/source-code/dev_push.md b/docs/resources/source-code/dev_push.md new file mode 100644 index 000000000..f966c2457 --- /dev/null +++ b/docs/resources/source-code/dev_push.md @@ -0,0 +1,196 @@ +# [Module dev_push.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_push.erl) + + + + +`push@1.0` takes a message or slot number, evaluates it, and recursively +pushes the resulting messages to other processes. + + + +## Description ## +The `push`ing mechanism +continues until the there are no remaining messages to push. + +## Function Index ## + + +
additional_keys/3*Set the necessary keys in order for the recipient to know where the +message came from.
do_push/3*Push a message or slot number, including its downstream results.
extract/2*Return either the target or the hint.
find_type/2*
full_push_test_/0*
is_async/3*Determine if the push is asynchronous.
multi_process_push_test_/0*
normalize_message/2*Augment the message with from-* keys, if it doesn't already have them.
parse_redirect/1*
ping_pong_script/1*
push/3Push either a message or an assigned slot number.
push_prompts_encoding_change_test/0*
push_result_message/4*Push a downstream message result.
push_with_mode/3*
push_with_redirect_hint_test_disabled/0*
remote_schedule_result/3*
reply_script/0*
schedule_initial_message/3*Push a message or a process, prior to pushing the resulting slot number.
schedule_result/4*Add the necessary keys to the message to be scheduled, then schedule it.
schedule_result/5*
split_target/1*Split the target into the process ID and the optional query string.
target_process/2*Find the target process ID for a message to push.
+ + + + +## Function Details ## + + + +### additional_keys/3 * ### + +`additional_keys(Origin, ToSched, Opts) -> any()` + +Set the necessary keys in order for the recipient to know where the +message came from. + + + +### do_push/3 * ### + +`do_push(Process, Assignment, Opts) -> any()` + +Push a message or slot number, including its downstream results. + + + +### extract/2 * ### + +`extract(X1, Raw) -> any()` + +Return either the `target` or the `hint`. + + + +### find_type/2 * ### + +`find_type(Req, Opts) -> any()` + + + +### full_push_test_/0 * ### + +`full_push_test_() -> any()` + + + +### is_async/3 * ### + +`is_async(Process, Req, Opts) -> any()` + +Determine if the push is asynchronous. + + + +### multi_process_push_test_/0 * ### + +`multi_process_push_test_() -> any()` + + + +### normalize_message/2 * ### + +`normalize_message(MsgToPush, Opts) -> any()` + +Augment the message with from-* keys, if it doesn't already have them. + + + +### parse_redirect/1 * ### + +`parse_redirect(Location) -> any()` + + + +### ping_pong_script/1 * ### + +`ping_pong_script(Limit) -> any()` + + + +### push/3 ### + +`push(Base, Req, Opts) -> any()` + +Push either a message or an assigned slot number. If a `Process` is +provided in the `body` of the request, it will be scheduled (initializing +it if it does not exist). Otherwise, the message specified by the given +`slot` key will be pushed. + +Optional parameters: +`/result-depth`: The depth to which the full contents of the result +will be included in the response. Default: 1, returning +the full result of the first message, but only the 'tree' +of downstream messages. +`/push-mode`: Whether or not the push should be done asynchronously. +Default: `sync`, pushing synchronously. + + + +### push_prompts_encoding_change_test/0 * ### + +`push_prompts_encoding_change_test() -> any()` + + + +### push_result_message/4 * ### + +`push_result_message(TargetProcess, MsgToPush, Origin, Opts) -> any()` + +Push a downstream message result. The `Origin` map contains information +about the origin of the message: The process that originated the message, +the slot number from which it was sent, and the outbox key of the message, +and the depth to which downstream results should be included in the message. + + + +### push_with_mode/3 * ### + +`push_with_mode(Process, Req, Opts) -> any()` + + + +### push_with_redirect_hint_test_disabled/0 * ### + +`push_with_redirect_hint_test_disabled() -> any()` + + + +### remote_schedule_result/3 * ### + +`remote_schedule_result(Location, SignedReq, Opts) -> any()` + + + +### reply_script/0 * ### + +`reply_script() -> any()` + + + +### schedule_initial_message/3 * ### + +`schedule_initial_message(Base, Req, Opts) -> any()` + +Push a message or a process, prior to pushing the resulting slot number. + + + +### schedule_result/4 * ### + +`schedule_result(TargetProcess, MsgToPush, Origin, Opts) -> any()` + +Add the necessary keys to the message to be scheduled, then schedule it. +If the remote scheduler does not support the given codec, it will be +downgraded and re-signed. + + + +### schedule_result/5 * ### + +`schedule_result(TargetProcess, MsgToPush, Codec, Origin, Opts) -> any()` + + + +### split_target/1 * ### + +`split_target(RawTarget) -> any()` + +Split the target into the process ID and the optional query string. + + + +### target_process/2 * ### + +`target_process(MsgToPush, Opts) -> any()` + +Find the target process ID for a message to push. + diff --git a/docs/resources/source-code/dev_relay.md b/docs/resources/source-code/dev_relay.md new file mode 100644 index 000000000..2828b72eb --- /dev/null +++ b/docs/resources/source-code/dev_relay.md @@ -0,0 +1,84 @@ +# [Module dev_relay.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_relay.erl) + + + + +This module implements the relay device, which is responsible for +relaying messages between nodes and other HTTP(S) endpoints. + + + +## Description ## + +It can be called in either `call` or `cast` mode. In `call` mode, it +returns a `{ok, Result}` tuple, where `Result` is the response from the +remote peer to the message sent. In `cast` mode, the invocation returns +immediately, and the message is relayed asynchronously. No response is given +and the device returns `{ok, <<"OK">>}`. + +Example usage: + +``` + + curl /~relay@.1.0/call?method=GET?0.path=https://www.arweave.net/ +``` + + +## Function Index ## + + +
call/3Execute a call request using a node's routes.
call_get_test/0*
cast/3Execute a request in the same way as call/3, but asynchronously.
request/3Preprocess a request to check if it should be relayed to a different node.
request_hook_reroute_to_nearest_test/0*Test that the preprocess/3 function re-routes a request to remote +peers, according to the node's routing table.
+ + + + +## Function Details ## + + + +### call/3 ### + +`call(M1, RawM2, Opts) -> any()` + +Execute a `call` request using a node's routes. + +Supports the following options: +- `target`: The target message to relay. Defaults to the original message. +- `relay-path`: The path to relay the message to. Defaults to the original path. +- `method`: The method to use for the request. Defaults to the original method. +- `requires-sign`: Whether the request requires signing before dispatching. +Defaults to `false`. + + + +### call_get_test/0 * ### + +`call_get_test() -> any()` + + + +### cast/3 ### + +`cast(M1, M2, Opts) -> any()` + +Execute a request in the same way as `call/3`, but asynchronously. Always +returns `<<"OK">>`. + + + +### request/3 ### + +`request(Msg1, Msg2, Opts) -> any()` + +Preprocess a request to check if it should be relayed to a different node. + + + +### request_hook_reroute_to_nearest_test/0 * ### + +`request_hook_reroute_to_nearest_test() -> any()` + +Test that the `preprocess/3` function re-routes a request to remote +peers, according to the node's routing table. + diff --git a/docs/resources/source-code/dev_router.md b/docs/resources/source-code/dev_router.md new file mode 100644 index 000000000..c513e1b3d --- /dev/null +++ b/docs/resources/source-code/dev_router.md @@ -0,0 +1,431 @@ +# [Module dev_router.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_router.erl) + + + + +A device that routes outbound messages from the node to their +appropriate network recipients via HTTP. + + + +## Description ## + +All messages are initially +routed to a single process per node, which then load-balances them +between downstream workers that perform the actual requests. + +The routes for the router are defined in the `routes` key of the `Opts`, +as a precidence-ordered list of maps. The first map that matches the +message will be used to determine the route. + +Multiple nodes can be specified as viable for a single route, with the +`Choose` key determining how many nodes to choose from the list (defaulting +to 1). The `Strategy` key determines the load distribution strategy, +which can be one of `Random`, `By-Base`, or `Nearest`. The route may also +define additional parallel execution parameters, which are used by the +`hb_http` module to manage control of requests. + +The structure of the routes should be as follows: + +``` + + Node?: The node to route the message to. + Nodes?: A list of nodes to route the message to. + Strategy?: The load distribution strategy to use. + Choose?: The number of nodes to choose from the list. + Template?: A message template to match the message against, either as a + map or a path regex. +``` + + +## Function Index ## + + +
add_route_test/0*
apply_route/2*Apply a node map's rules for transforming the path of the message.
apply_routes/3*Generate a uri key for each node in a route.
binary_to_bignum/1*Cast a human-readable or native-encoded ID to a big integer.
by_base_determinism_test/0*Ensure that By-Base always chooses the same node for the same +hashpath.
choose/5*Implements the load distribution strategies if given a cluster.
choose_1_test/1*
choose_n_test/1*
device_call_from_singleton_test/0*
dynamic_route_provider_test/0*
dynamic_router_test/0*Example of a Lua module being used as the route_provider for a +HyperBEAM node.
dynamic_routing_by_performance/0*
dynamic_routing_by_performance_test_/0*Demonstrates routing tables being dynamically created and adjusted +according to the real-time performance of nodes.
explicit_route_test/0*
extract_base/2*Extract the base message ID from a request message.
field_distance/2*Calculate the minimum distance between two numbers +(either progressing backwards or forwards), assuming a +256-bit field.
find_target_path/2*Find the target path to route for a request message.
generate_hashpaths/1*
generate_nodes/1*
get_routes_test/0*
info/1Exported function for getting device info, controls which functions are +exposed via the device API.
info/3HTTP info response providing information about this device.
load_routes/1*Load the current routes for the node.
local_dynamic_router_test/0*Example of a Lua module being used as the route_provider for a +HyperBEAM node.
local_process_route_provider_test/0*
lowest_distance/1*Find the node with the lowest distance to the given hashpath.
lowest_distance/2*
match/3Find the first matching template in a list of known routes.
match_routes/3*
match_routes/4*
preprocess/3Preprocess a request to check if it should be relayed to a different node.
register/3
relay_nearest_test/0*
route/2Find the appropriate route for the given message.
route/3
route_provider_test/0*
route_regex_matches_test/0*
route_template_message_matches_test/0*
routes/3Device function that returns all known routes.
simulate/4*
simulation_distribution/2*
simulation_occurences/2*
strategy_suite_test_/0*
template_matches/3*Check if a message matches a message template or path regex.
unique_nodes/1*
unique_test/1*
weighted_random_strategy_test/0*
within_norms/3*
+ + + + +## Function Details ## + + + +### add_route_test/0 * ### + +`add_route_test() -> any()` + + + +### apply_route/2 * ### + +`apply_route(Msg, Route) -> any()` + +Apply a node map's rules for transforming the path of the message. +Supports the following keys: +- `opts`: A map of options to pass to the request. +- `prefix`: The prefix to add to the path. +- `suffix`: The suffix to add to the path. +- `replace`: A regex to replace in the path. + + + +### apply_routes/3 * ### + +`apply_routes(Msg, R, Opts) -> any()` + +Generate a `uri` key for each node in a route. + + + +### binary_to_bignum/1 * ### + +`binary_to_bignum(Bin) -> any()` + +Cast a human-readable or native-encoded ID to a big integer. + + + +### by_base_determinism_test/0 * ### + +`by_base_determinism_test() -> any()` + +Ensure that `By-Base` always chooses the same node for the same +hashpath. + + + +### choose/5 * ### + +`choose(N, X2, Hashpath, Nodes, Opts) -> any()` + +Implements the load distribution strategies if given a cluster. + + + +### choose_1_test/1 * ### + +`choose_1_test(Strategy) -> any()` + + + +### choose_n_test/1 * ### + +`choose_n_test(Strategy) -> any()` + + + +### device_call_from_singleton_test/0 * ### + +`device_call_from_singleton_test() -> any()` + + + +### dynamic_route_provider_test/0 * ### + +`dynamic_route_provider_test() -> any()` + + + +### dynamic_router_test/0 * ### + +`dynamic_router_test() -> any()` + +Example of a Lua module being used as the `route_provider` for a +HyperBEAM node. The module utilized in this example dynamically adjusts the +likelihood of routing to a given node, depending upon price and performance. +also include preprocessing support for routing + + + +### dynamic_routing_by_performance/0 * ### + +`dynamic_routing_by_performance() -> any()` + + + +### dynamic_routing_by_performance_test_/0 * ### + +`dynamic_routing_by_performance_test_() -> any()` + +Demonstrates routing tables being dynamically created and adjusted +according to the real-time performance of nodes. This test utilizes the +`dynamic-router` script to manage routes and recalculate weights based on the +reported performance. + + + +### explicit_route_test/0 * ### + +`explicit_route_test() -> any()` + + + +### extract_base/2 * ### + +`extract_base(RawPath, Opts) -> any()` + +Extract the base message ID from a request message. Produces a single +binary ID that can be used for routing decisions. + + + +### field_distance/2 * ### + +`field_distance(A, B) -> any()` + +Calculate the minimum distance between two numbers +(either progressing backwards or forwards), assuming a +256-bit field. + + + +### find_target_path/2 * ### + +`find_target_path(Msg, Opts) -> any()` + +Find the target path to route for a request message. + + + +### generate_hashpaths/1 * ### + +`generate_hashpaths(Runs) -> any()` + + + +### generate_nodes/1 * ### + +`generate_nodes(N) -> any()` + + + +### get_routes_test/0 * ### + +`get_routes_test() -> any()` + + + +### info/1 ### + +`info(X1) -> any()` + +Exported function for getting device info, controls which functions are +exposed via the device API. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + +HTTP info response providing information about this device + + + +### load_routes/1 * ### + +`load_routes(Opts) -> any()` + +Load the current routes for the node. Allows either explicit routes from +the node message's `routes` key, or dynamic routes generated by resolving the +`route_provider` message. + + + +### local_dynamic_router_test/0 * ### + +`local_dynamic_router_test() -> any()` + +Example of a Lua module being used as the `route_provider` for a +HyperBEAM node. The module utilized in this example dynamically adjusts the +likelihood of routing to a given node, depending upon price and performance. + + + +### local_process_route_provider_test/0 * ### + +`local_process_route_provider_test() -> any()` + + + +### lowest_distance/1 * ### + +`lowest_distance(Nodes) -> any()` + +Find the node with the lowest distance to the given hashpath. + + + +### lowest_distance/2 * ### + +`lowest_distance(Nodes, X) -> any()` + + + +### match/3 ### + +`match(Base, Req, Opts) -> any()` + +Find the first matching template in a list of known routes. Allows the +path to be specified by either the explicit `path` (for internal use by this +module), or `route-path` for use by external devices and users. + + + +### match_routes/3 * ### + +`match_routes(ToMatch, Routes, Opts) -> any()` + + + +### match_routes/4 * ### + +`match_routes(ToMatch, Routes, Keys, Opts) -> any()` + + + +### preprocess/3 ### + +`preprocess(Msg1, Msg2, Opts) -> any()` + +Preprocess a request to check if it should be relayed to a different node. + + + +### register/3 ### + +`register(M1, M2, Opts) -> any()` + + + +### relay_nearest_test/0 * ### + +`relay_nearest_test() -> any()` + + + +### route/2 ### + +`route(Msg, Opts) -> any()` + +Find the appropriate route for the given message. If we are able to +resolve to a single host+path, we return that directly. Otherwise, we return +the matching route (including a list of nodes under `nodes`) from the list of +routes. + +If we have a route that has multiple resolving nodes, check +the load distribution strategy and choose a node. Supported strategies: + +``` + + All: Return all nodes (default). + Random: Distribute load evenly across all nodes, non-deterministically. + By-Base: According to the base message's hashpath. + By-Weight: According to the node's weight key. + Nearest: According to the distance of the node's wallet address to the + base message's hashpath. +``` + +`By-Base` will ensure that all traffic for the same hashpath is routed to the +same node, minimizing work duplication, while `Random` ensures a more even +distribution of the requests. + +Can operate as a `~router@1.0` device, which will ignore the base message, +routing based on the Opts and request message provided, or as a standalone +function, taking only the request message and the `Opts` map. + + + +### route/3 ### + +`route(X1, Msg, Opts) -> any()` + + + +### route_provider_test/0 * ### + +`route_provider_test() -> any()` + + + +### route_regex_matches_test/0 * ### + +`route_regex_matches_test() -> any()` + + + +### route_template_message_matches_test/0 * ### + +`route_template_message_matches_test() -> any()` + + + +### routes/3 ### + +`routes(M1, M2, Opts) -> any()` + +Device function that returns all known routes. + + + +### simulate/4 * ### + +`simulate(Runs, ChooseN, Nodes, Strategy) -> any()` + + + +### simulation_distribution/2 * ### + +`simulation_distribution(SimRes, Nodes) -> any()` + + + +### simulation_occurences/2 * ### + +`simulation_occurences(SimRes, Nodes) -> any()` + + + +### strategy_suite_test_/0 * ### + +`strategy_suite_test_() -> any()` + + + +### template_matches/3 * ### + +`template_matches(ToMatch, Template, Opts) -> any()` + +Check if a message matches a message template or path regex. + + + +### unique_nodes/1 * ### + +`unique_nodes(Simulation) -> any()` + + + +### unique_test/1 * ### + +`unique_test(Strategy) -> any()` + + + +### weighted_random_strategy_test/0 * ### + +`weighted_random_strategy_test() -> any()` + + + +### within_norms/3 * ### + +`within_norms(SimRes, Nodes, TestSize) -> any()` + diff --git a/docs/resources/source-code/dev_scheduler.md b/docs/resources/source-code/dev_scheduler.md new file mode 100644 index 000000000..e9a1cb10d --- /dev/null +++ b/docs/resources/source-code/dev_scheduler.md @@ -0,0 +1,552 @@ +# [Module dev_scheduler.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler.erl) + + + + +A simple scheduler scheme for AO. + + + +## Description ## +This device expects a message of the form: +Process: `#{ id, Scheduler: #{ Authority } }` + +``` + + It exposes the following keys for scheduling:#{ method: GET, path: <<"/info">> } -> + Returns information about the scheduler.#{ method: GET, path: <<"/slot">> } -> slot(Msg1, Msg2, Opts) + Returns the current slot for a process.#{ method: GET, path: <<"/schedule">> } -> get_schedule(Msg1, Msg2, Opts) + Returns the schedule for a process in a cursor-traversable format.#{ method: POST, path: <<"/schedule">> } -> post_schedule(Msg1, Msg2, Opts) + Schedules a new message for a process, or starts a new scheduler + for the given message. +``` + + +## Function Index ## + + +
benchmark_suite/2*
benchmark_suite_test_/0*
cache_remote_schedule/2*Cache a schedule received from a remote scheduler.
check_lookahead_and_local_cache/4*Check if we have a result from a lookahead worker or from our local +cache.
checkpoint/1Returns the current state of the scheduler.
do_get_remote_schedule/6*Get a schedule from a remote scheduler, unless we already have already +read all of the assignments from the local cache.
do_post_schedule/4*Post schedule the message.
filter_json_assignments/3*Filter JSON assignment results from a remote legacy scheduler.
find_message_to_schedule/3*Search the given base and request message pair to find the message to +schedule.
find_remote_scheduler/3*Use the SchedulerLocation to the remote path and return a redirect.
find_server/3*Locate the correct scheduling server for a given process.
find_server/4*
find_target_id/3*Find the schedule ID from a given request.
generate_local_schedule/5*Generate a GET /schedule response for a process.
generate_redirect/3*Generate a redirect message to a scheduler.
get_hint/2*If a hint is present in the string, return it.
get_local_assignments/4*Get the assignments for a process, and whether the request was truncated.
get_local_schedule_test/0*
get_location/3*Search for the location of the scheduler in the scheduler-location +cache.
get_remote_schedule/5*Get a schedule from a remote scheduler, but first read all of the +assignments from the local cache that we already know about.
get_schedule/3*Generate and return a schedule for a process, optionally between +two slots -- labelled as from and to.
http_get_json_schedule_test_/0*
http_get_legacy_schedule_as_aos2_test_/0*
http_get_legacy_schedule_slot_range_test_/0*
http_get_legacy_schedule_test_/0*
http_get_legacy_slot_test_/0*
http_get_schedule/4*
http_get_schedule/5*
http_get_schedule_redirect_test/0*
http_get_schedule_test_/0*
http_get_slot/2*
http_init/0*
http_init/1*
http_post_legacy_schedule_test_/0*
http_post_schedule_sign/4*
http_post_schedule_test/0*
info/0This device uses a default_handler to route requests to the correct +function.
location/3Router for record requests.
many_clients/1*
message_cached_assignments/2*Non-device exported helper to get the cached assignments held in a +process.
next/3Load the schedule for a process into the cache, then return the next +assignment.
node_from_redirect/2*Get the node URL from a redirect.
post_legacy_schedule/4*
post_location/3*Generate a new scheduler location record and register it.
post_remote_schedule/4*
post_schedule/3*Schedules a new message on the SU.
read_local_assignments/4*Get the assignments for a process.
redirect_from_graphql_test/0*
redirect_to_hint_test/0*
register_location_on_boot_test/0*Test that a scheduler location is registered on boot.
register_new_process_test/0*
register_scheduler_test/0*
remote_slot/3*Get the current slot from a remote scheduler.
remote_slot/4*Get the current slot from a remote scheduler, based on the variant of +the process's scheduler.
router/4The default handler for the scheduler device.
schedule/3A router for choosing between getting the existing schedule, or +scheduling a new message.
schedule_message_and_get_slot_test/0*
single_resolution/1*
slot/3Returns information about the current slot for a process.
spawn_lookahead_worker/3*Spawn a new Erlang process to fetch the next assignments from the local +cache, if we have them available.
start/0Helper to ensure that the environment is started.
status/3Returns information about the entire scheduler.
status_test/0*
test_process/0Generate a _transformed_ process message, not as they are generated +by users.
test_process/1*
without_hint/1*Take a process ID or target with a potential hint and return just the +process ID.
+ + + + +## Function Details ## + + + +### benchmark_suite/2 * ### + +`benchmark_suite(Port, Base) -> any()` + + + +### benchmark_suite_test_/0 * ### + +`benchmark_suite_test_() -> any()` + + + +### cache_remote_schedule/2 * ### + +`cache_remote_schedule(Schedule, Opts) -> any()` + +Cache a schedule received from a remote scheduler. + + + +### check_lookahead_and_local_cache/4 * ### + +`check_lookahead_and_local_cache(Msg1, ProcID, TargetSlot, Opts) -> any()` + +Check if we have a result from a lookahead worker or from our local +cache. If we have a result in the local cache, we may also start a new +lookahead worker to fetch the next assignments if we have them locally, +ahead of time. This can be enabled/disabled with the `scheduler_lookahead` +option. + + + +### checkpoint/1 ### + +`checkpoint(State) -> any()` + +Returns the current state of the scheduler. + + + +### do_get_remote_schedule/6 * ### + +`do_get_remote_schedule(ProcID, LocalAssignments, From, To, Redirect, Opts) -> any()` + +Get a schedule from a remote scheduler, unless we already have already +read all of the assignments from the local cache. + + + +### do_post_schedule/4 * ### + +`do_post_schedule(ProcID, PID, Msg2, Opts) -> any()` + +Post schedule the message. `Msg2` by this point has been refined to only +committed keys, and to only include the `target` message that is to be +scheduled. + + + +### filter_json_assignments/3 * ### + +`filter_json_assignments(JSONRes, To, From) -> any()` + +Filter JSON assignment results from a remote legacy scheduler. + + + +### find_message_to_schedule/3 * ### + +`find_message_to_schedule(Msg1, Msg2, Opts) -> any()` + +Search the given base and request message pair to find the message to +schedule. The precidence order for search is as follows: +1. `Msg2/body` +2. `Msg2` + + + +### find_remote_scheduler/3 * ### + +`find_remote_scheduler(ProcID, Scheduler, Opts) -> any()` + +Use the SchedulerLocation to the remote path and return a redirect. + + + +### find_server/3 * ### + +`find_server(ProcID, Msg1, Opts) -> any()` + +Locate the correct scheduling server for a given process. + + + +### find_server/4 * ### + +`find_server(ProcID, Msg1, ToSched, Opts) -> any()` + + + +### find_target_id/3 * ### + +`find_target_id(Msg1, Msg2, Opts) -> any()` + +Find the schedule ID from a given request. The precidence order for +search is as follows: +[1. `ToSched/id` -- in the case of `POST schedule`, handled locally] +2. `Msg2/target` +3. `Msg2/id` when `Msg2` has `type: Process` +4. `Msg1/process/id` +5. `Msg1/id` when `Msg1` has `type: Process` +6. `Msg2/id` + + + +### generate_local_schedule/5 * ### + +`generate_local_schedule(Format, ProcID, From, To, Opts) -> any()` + +Generate a `GET /schedule` response for a process. + + + +### generate_redirect/3 * ### + +`generate_redirect(ProcID, SchedulerLocation, Opts) -> any()` + +Generate a redirect message to a scheduler. + + + +### get_hint/2 * ### + +`get_hint(Str, Opts) -> any()` + +If a hint is present in the string, return it. Else, return not_found. + + + +### get_local_assignments/4 * ### + +`get_local_assignments(ProcID, From, RequestedTo, Opts) -> any()` + +Get the assignments for a process, and whether the request was truncated. + + + +### get_local_schedule_test/0 * ### + +`get_local_schedule_test() -> any()` + + + +### get_location/3 * ### + +`get_location(Msg1, Req, Opts) -> any()` + +Search for the location of the scheduler in the scheduler-location +cache. If an address is provided, we search for the location of that +specific scheduler. Otherwise, we return the location record for the current +node's scheduler, if it has been established. + + + +### get_remote_schedule/5 * ### + +`get_remote_schedule(RawProcID, From, To, Redirect, Opts) -> any()` + +Get a schedule from a remote scheduler, but first read all of the +assignments from the local cache that we already know about. + + + +### get_schedule/3 * ### + +`get_schedule(Msg1, Msg2, Opts) -> any()` + +Generate and return a schedule for a process, optionally between +two slots -- labelled as `from` and `to`. If the schedule is not local, +we redirect to the remote scheduler or proxy based on the node opts. + + + +### http_get_json_schedule_test_/0 * ### + +`http_get_json_schedule_test_() -> any()` + + + +### http_get_legacy_schedule_as_aos2_test_/0 * ### + +`http_get_legacy_schedule_as_aos2_test_() -> any()` + + + +### http_get_legacy_schedule_slot_range_test_/0 * ### + +`http_get_legacy_schedule_slot_range_test_() -> any()` + + + +### http_get_legacy_schedule_test_/0 * ### + +`http_get_legacy_schedule_test_() -> any()` + + + +### http_get_legacy_slot_test_/0 * ### + +`http_get_legacy_slot_test_() -> any()` + + + +### http_get_schedule/4 * ### + +`http_get_schedule(N, PMsg, From, To) -> any()` + + + +### http_get_schedule/5 * ### + +`http_get_schedule(N, PMsg, From, To, Format) -> any()` + + + +### http_get_schedule_redirect_test/0 * ### + +`http_get_schedule_redirect_test() -> any()` + + + +### http_get_schedule_test_/0 * ### + +`http_get_schedule_test_() -> any()` + + + +### http_get_slot/2 * ### + +`http_get_slot(N, PMsg) -> any()` + + + +### http_init/0 * ### + +`http_init() -> any()` + + + +### http_init/1 * ### + +`http_init(Opts) -> any()` + + + +### http_post_legacy_schedule_test_/0 * ### + +`http_post_legacy_schedule_test_() -> any()` + + + +### http_post_schedule_sign/4 * ### + +`http_post_schedule_sign(Node, Msg, ProcessMsg, Wallet) -> any()` + + + +### http_post_schedule_test/0 * ### + +`http_post_schedule_test() -> any()` + + + +### info/0 ### + +`info() -> any()` + +This device uses a default_handler to route requests to the correct +function. + + + +### location/3 ### + +`location(Msg1, Msg2, Opts) -> any()` + +Router for `record` requests. Expects either a `POST` or `GET` request. + + + +### many_clients/1 * ### + +`many_clients(Opts) -> any()` + + + +### message_cached_assignments/2 * ### + +`message_cached_assignments(Msg, Opts) -> any()` + +Non-device exported helper to get the cached assignments held in a +process. + + + +### next/3 ### + +`next(Msg1, Msg2, Opts) -> any()` + +Load the schedule for a process into the cache, then return the next +assignment. Assumes that Msg1 is a `dev_process` or similar message, having +a `Current-Slot` key. It stores a local cache of the schedule in the +`priv/To-Process` key. + + + +### node_from_redirect/2 * ### + +`node_from_redirect(Redirect, Opts) -> any()` + +Get the node URL from a redirect. + + + +### post_legacy_schedule/4 * ### + +`post_legacy_schedule(ProcID, OnlyCommitted, Node, Opts) -> any()` + + + +### post_location/3 * ### + +`post_location(Msg1, RawReq, Opts) -> any()` + +Generate a new scheduler location record and register it. We both send +the new scheduler-location to the given registry, and return it to the caller. + + + +### post_remote_schedule/4 * ### + +`post_remote_schedule(RawProcID, Redirect, OnlyCommitted, Opts) -> any()` + + + +### post_schedule/3 * ### + +`post_schedule(Msg1, Msg2, Opts) -> any()` + +Schedules a new message on the SU. Searches Msg1 for the appropriate ID, +then uses the wallet address of the scheduler to determine if the message is +for this scheduler. If so, it schedules the message and returns the assignment. + + + +### read_local_assignments/4 * ### + +`read_local_assignments(ProcID, From, To, Opts) -> any()` + +Get the assignments for a process. + + + +### redirect_from_graphql_test/0 * ### + +`redirect_from_graphql_test() -> any()` + + + +### redirect_to_hint_test/0 * ### + +`redirect_to_hint_test() -> any()` + + + +### register_location_on_boot_test/0 * ### + +`register_location_on_boot_test() -> any()` + +Test that a scheduler location is registered on boot. + + + +### register_new_process_test/0 * ### + +`register_new_process_test() -> any()` + + + +### register_scheduler_test/0 * ### + +`register_scheduler_test() -> any()` + + + +### remote_slot/3 * ### + +`remote_slot(ProcID, Redirect, Opts) -> any()` + +Get the current slot from a remote scheduler. + + + +### remote_slot/4 * ### + +`remote_slot(X1, ProcID, Node, Opts) -> any()` + +Get the current slot from a remote scheduler, based on the variant of +the process's scheduler. + + + +### router/4 ### + +`router(X1, Msg1, Msg2, Opts) -> any()` + +The default handler for the scheduler device. + + + +### schedule/3 ### + +`schedule(Msg1, Msg2, Opts) -> any()` + +A router for choosing between getting the existing schedule, or +scheduling a new message. + + + +### schedule_message_and_get_slot_test/0 * ### + +`schedule_message_and_get_slot_test() -> any()` + + + +### single_resolution/1 * ### + +`single_resolution(Opts) -> any()` + + + +### slot/3 ### + +`slot(M1, M2, Opts) -> any()` + +Returns information about the current slot for a process. + + + +### spawn_lookahead_worker/3 * ### + +`spawn_lookahead_worker(ProcID, Slot, Opts) -> any()` + +Spawn a new Erlang process to fetch the next assignments from the local +cache, if we have them available. + + + +### start/0 ### + +`start() -> any()` + +Helper to ensure that the environment is started. + + + +### status/3 ### + +`status(M1, M2, Opts) -> any()` + +Returns information about the entire scheduler. + + + +### status_test/0 * ### + +`status_test() -> any()` + + + +### test_process/0 ### + +`test_process() -> any()` + +Generate a _transformed_ process message, not as they are generated +by users. See `dev_process` for examples of AO process messages. + + + +### test_process/1 * ### + +`test_process(Wallet) -> any()` + + + +### without_hint/1 * ### + +`without_hint(Target) -> any()` + +Take a process ID or target with a potential hint and return just the +process ID. + diff --git a/docs/resources/source-code/dev_scheduler_cache.md b/docs/resources/source-code/dev_scheduler_cache.md new file mode 100644 index 000000000..2ddd9866a --- /dev/null +++ b/docs/resources/source-code/dev_scheduler_cache.md @@ -0,0 +1,65 @@ +# [Module dev_scheduler_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_cache.erl) + + + + + + +## Function Index ## + + +
latest/2Get the latest assignment from the cache.
list/2Get the assignments for a process.
read/3Get an assignment message from the cache.
read_location/2Read the latest known scheduler location for an address.
write/2Write an assignment message into the cache.
write_location/2Write the latest known scheduler location for an address.
+ + + + +## Function Details ## + + + +### latest/2 ### + +`latest(ProcID, Opts) -> any()` + +Get the latest assignment from the cache. + + + +### list/2 ### + +`list(ProcID, Opts) -> any()` + +Get the assignments for a process. + + + +### read/3 ### + +`read(ProcID, Slot, Opts) -> any()` + +Get an assignment message from the cache. + + + +### read_location/2 ### + +`read_location(Address, Opts) -> any()` + +Read the latest known scheduler location for an address. + + + +### write/2 ### + +`write(Assignment, Opts) -> any()` + +Write an assignment message into the cache. + + + +### write_location/2 ### + +`write_location(LocMsg, Opts) -> any()` + +Write the latest known scheduler location for an address. + diff --git a/docs/resources/source-code/dev_scheduler_formats.md b/docs/resources/source-code/dev_scheduler_formats.md new file mode 100644 index 000000000..4c356b752 --- /dev/null +++ b/docs/resources/source-code/dev_scheduler_formats.md @@ -0,0 +1,120 @@ +# [Module dev_scheduler_formats.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_formats.erl) + + + + +This module is used by dev_scheduler in order to produce outputs that +are compatible with various forms of AO clients. + + + +## Description ## + +It features two main formats: + +- `application/json` +- `application/http` + +The `application/json` format is a legacy format that is not recommended for +new integrations of the AO protocol. + +## Function Index ## + + +
aos2_normalize_data/1*The hb_gateway_client module expects all JSON structures to at least +have a data field.
aos2_normalize_types/1Normalize an AOS2 formatted message to ensure that all field NAMES and +types are correct.
aos2_to_assignment/2Create and normalize an assignment from an AOS2-style JSON structure.
aos2_to_assignments/3Convert an AOS2-style JSON structure to a normalized HyperBEAM +assignments response.
assignment_to_aos2/2*Convert an assignment to an AOS2-compatible JSON structure.
assignments_to_aos2/4
assignments_to_bundle/4Generate a GET /schedule response for a process as HTTP-sig bundles.
assignments_to_bundle/5*
cursor/2*Generate a cursor for an assignment.
format_opts/1*For all scheduler format operations, we do not calculate hashpaths, +perform cache lookups, or await inprogress results.
+ + + + +## Function Details ## + + + +### aos2_normalize_data/1 * ### + +`aos2_normalize_data(JSONStruct) -> any()` + +The `hb_gateway_client` module expects all JSON structures to at least +have a `data` field. This function ensures that. + + + +### aos2_normalize_types/1 ### + +`aos2_normalize_types(Msg) -> any()` + +Normalize an AOS2 formatted message to ensure that all field NAMES and +types are correct. This involves converting field names to integers and +specific field names to their canonical form. +NOTE: This will result in a message that is not verifiable! It is, however, +necessary for gaining compatibility with the AOS2-style scheduling API. + + + +### aos2_to_assignment/2 ### + +`aos2_to_assignment(A, RawOpts) -> any()` + +Create and normalize an assignment from an AOS2-style JSON structure. +NOTE: This method is destructive to the verifiability of the assignment. + + + +### aos2_to_assignments/3 ### + +`aos2_to_assignments(ProcID, Body, RawOpts) -> any()` + +Convert an AOS2-style JSON structure to a normalized HyperBEAM +assignments response. + + + +### assignment_to_aos2/2 * ### + +`assignment_to_aos2(Assignment, RawOpts) -> any()` + +Convert an assignment to an AOS2-compatible JSON structure. + + + +### assignments_to_aos2/4 ### + +`assignments_to_aos2(ProcID, Assignments, More, RawOpts) -> any()` + + + +### assignments_to_bundle/4 ### + +`assignments_to_bundle(ProcID, Assignments, More, Opts) -> any()` + +Generate a `GET /schedule` response for a process as HTTP-sig bundles. + + + +### assignments_to_bundle/5 * ### + +`assignments_to_bundle(ProcID, Assignments, More, TimeInfo, RawOpts) -> any()` + + + +### cursor/2 * ### + +`cursor(Assignment, RawOpts) -> any()` + +Generate a cursor for an assignment. This should be the slot number, at +least in the case of mainnet `ao.N.1` assignments. In the case of legacynet +(`ao.TN.1`) assignments, we may want to use the assignment ID. + + + +### format_opts/1 * ### + +`format_opts(Opts) -> any()` + +For all scheduler format operations, we do not calculate hashpaths, +perform cache lookups, or await inprogress results. + diff --git a/docs/resources/source-code/dev_scheduler_registry.md b/docs/resources/source-code/dev_scheduler_registry.md new file mode 100644 index 000000000..4415c8ff7 --- /dev/null +++ b/docs/resources/source-code/dev_scheduler_registry.md @@ -0,0 +1,95 @@ +# [Module dev_scheduler_registry.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_registry.erl) + + + + + + +## Function Index ## + + +
create_and_find_process_test/0*
create_multiple_processes_test/0*
find/1Find a process associated with the processor ID in the local registry +If the process is not found, it will not create a new one.
find/2Find a process associated with the processor ID in the local registry +If the process is not found and GenIfNotHosted is true, it attemps to create a new one.
find/3Same as find/2 but with additional options passed when spawning a new process (if needed).
find_non_existent_process_test/0*
get_all_processes_test/0*
get_processes/0Return a list of all currently registered ProcID.
get_wallet/0
maybe_new_proc/3*
start/0
+ + + + +## Function Details ## + + + +### create_and_find_process_test/0 * ### + +`create_and_find_process_test() -> any()` + + + +### create_multiple_processes_test/0 * ### + +`create_multiple_processes_test() -> any()` + + + +### find/1 ### + +`find(ProcID) -> any()` + +Find a process associated with the processor ID in the local registry +If the process is not found, it will not create a new one + + + +### find/2 ### + +`find(ProcID, GenIfNotHosted) -> any()` + +Find a process associated with the processor ID in the local registry +If the process is not found and `GenIfNotHosted` is true, it attemps to create a new one + + + +### find/3 ### + +`find(ProcID, GenIfNotHosted, Opts) -> any()` + +Same as `find/2` but with additional options passed when spawning a new process (if needed) + + + +### find_non_existent_process_test/0 * ### + +`find_non_existent_process_test() -> any()` + + + +### get_all_processes_test/0 * ### + +`get_all_processes_test() -> any()` + + + +### get_processes/0 ### + +`get_processes() -> any()` + +Return a list of all currently registered ProcID. + + + +### get_wallet/0 ### + +`get_wallet() -> any()` + + + +### maybe_new_proc/3 * ### + +`maybe_new_proc(ProcID, GenIfNotHosted, Opts) -> any()` + + + +### start/0 ### + +`start() -> any()` + diff --git a/docs/resources/source-code/dev_scheduler_server.md b/docs/resources/source-code/dev_scheduler_server.md new file mode 100644 index 000000000..ab8f015fd --- /dev/null +++ b/docs/resources/source-code/dev_scheduler_server.md @@ -0,0 +1,102 @@ +# [Module dev_scheduler_server.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_scheduler_server.erl) + + + + +A long-lived server that schedules messages for a process. + + + +## Description ## +It acts as a deliberate 'bottleneck' to prevent the server accidentally +assigning multiple messages to the same slot. + +## Function Index ## + + +
assign/3*Assign a message to the next slot.
do_assign/3*Generate and store the actual assignment message.
info/1Get the current slot from the scheduling server.
maybe_inform_recipient/5*
new_proc_test_/0*Test the basic functionality of the server.
next_hashchain/2*Create the next element in a chain of hashes that links this and prior +assignments.
schedule/2Call the appropriate scheduling server to assign a message.
server/1*The main loop of the server.
start/2Start a scheduling server for a given computation.
stop/1
+ + + + +## Function Details ## + + + +### assign/3 * ### + +`assign(State, Message, ReplyPID) -> any()` + +Assign a message to the next slot. + + + +### do_assign/3 * ### + +`do_assign(State, Message, ReplyPID) -> any()` + +Generate and store the actual assignment message. + + + +### info/1 ### + +`info(ProcID) -> any()` + +Get the current slot from the scheduling server. + + + +### maybe_inform_recipient/5 * ### + +`maybe_inform_recipient(Mode, ReplyPID, Message, Assignment, State) -> any()` + + + +### new_proc_test_/0 * ### + +`new_proc_test_() -> any()` + +Test the basic functionality of the server. + + + +### next_hashchain/2 * ### + +`next_hashchain(HashChain, Message) -> any()` + +Create the next element in a chain of hashes that links this and prior +assignments. + + + +### schedule/2 ### + +`schedule(AOProcID, Message) -> any()` + +Call the appropriate scheduling server to assign a message. + + + +### server/1 * ### + +`server(State) -> any()` + +The main loop of the server. Simply waits for messages to assign and +returns the current slot. + + + +### start/2 ### + +`start(ProcID, Opts) -> any()` + +Start a scheduling server for a given computation. + + + +### stop/1 ### + +`stop(ProcID) -> any()` + diff --git a/docs/resources/source-code/dev_simple_pay.md b/docs/resources/source-code/dev_simple_pay.md new file mode 100644 index 000000000..b65230680 --- /dev/null +++ b/docs/resources/source-code/dev_simple_pay.md @@ -0,0 +1,100 @@ +# [Module dev_simple_pay.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_simple_pay.erl) + + + + +A simple device that allows the operator to specify a price for a +request and then charge the user for it, on a per message basis. + + + +## Description ## +The device's ledger is stored in the node message at `simple_pay_ledger`, +and can be topped-up by either the operator, or an external device. The +price is specified in the node message at `simple_pay_price`. +This device acts as both a pricing device and a ledger device, by p4's +definition. + +## Function Index ## + + +
balance/3Get the balance of a user in the ledger.
debit/3Preprocess a request by checking the ledger and charging the user.
estimate/3Estimate the cost of a request by counting the number of messages in +the request, then multiplying by the per-message price.
get_balance/2*Get the balance of a user in the ledger.
get_balance_and_top_up_test/0*
is_operator/2*Check if the request is from the operator.
set_balance/3*Adjust a user's balance, normalizing their wallet ID first.
test_opts/1*
topup/3Top up the user's balance in the ledger.
+ + + + +## Function Details ## + + + +### balance/3 ### + +`balance(X1, RawReq, NodeMsg) -> any()` + +Get the balance of a user in the ledger. + + + +### debit/3 ### + +`debit(X1, RawReq, NodeMsg) -> any()` + +Preprocess a request by checking the ledger and charging the user. We +can charge the user at this stage because we know statically what the price +will be + + + +### estimate/3 ### + +`estimate(X1, EstimateReq, NodeMsg) -> any()` + +Estimate the cost of a request by counting the number of messages in +the request, then multiplying by the per-message price. The operator does +not pay for their own requests. + + + +### get_balance/2 * ### + +`get_balance(Signer, NodeMsg) -> any()` + +Get the balance of a user in the ledger. + + + +### get_balance_and_top_up_test/0 * ### + +`get_balance_and_top_up_test() -> any()` + + + +### is_operator/2 * ### + +`is_operator(Req, NodeMsg) -> any()` + +Check if the request is from the operator. + + + +### set_balance/3 * ### + +`set_balance(Signer, Amount, NodeMsg) -> any()` + +Adjust a user's balance, normalizing their wallet ID first. + + + +### test_opts/1 * ### + +`test_opts(Ledger) -> any()` + + + +### topup/3 ### + +`topup(X1, Req, NodeMsg) -> any()` + +Top up the user's balance in the ledger. + diff --git a/docs/resources/source-code/dev_snp.md b/docs/resources/source-code/dev_snp.md new file mode 100644 index 000000000..9eb969850 --- /dev/null +++ b/docs/resources/source-code/dev_snp.md @@ -0,0 +1,104 @@ +# [Module dev_snp.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_snp.erl) + + + + +This device offers an interface for validating AMD SEV-SNP commitments, +as well as generating them, if called in an appropriate environment. + + + +## Function Index ## + + +
execute_is_trusted/3*Ensure that all of the software hashes are trusted.
generate/3Generate an commitment report and emit it as a message, including all of +the necessary data to generate the nonce (ephemeral node address + node +message ID), as well as the expected measurement (firmware, kernel, and VMSAs +hashes).
generate_nonce/2*Generate the nonce to use in the commitment report.
is_debug/1*Ensure that the node's debug policy is disabled.
real_node_test/0*
report_data_matches/3*Ensure that the report data matches the expected report data.
trusted/3Validates if a given message parameter matches a trusted value from the SNP trusted list +Returns {ok, true} if the message is trusted, {ok, false} otherwise.
verify/3Verify an commitment report message; validating the identity of a +remote node, its ephemeral private address, and the integrity of the report.
+ + + + +## Function Details ## + + + +### execute_is_trusted/3 * ### + +`execute_is_trusted(M1, Msg, NodeOpts) -> any()` + +Ensure that all of the software hashes are trusted. The caller may set +a specific device to use for the `is-trusted` key. The device must then +implement the `trusted` resolver. + + + +### generate/3 ### + +`generate(M1, M2, Opts) -> any()` + +Generate an commitment report and emit it as a message, including all of +the necessary data to generate the nonce (ephemeral node address + node +message ID), as well as the expected measurement (firmware, kernel, and VMSAs +hashes). + + + +### generate_nonce/2 * ### + +`generate_nonce(RawAddress, RawNodeMsgID) -> any()` + +Generate the nonce to use in the commitment report. + + + +### is_debug/1 * ### + +`is_debug(Report) -> any()` + +Ensure that the node's debug policy is disabled. + + + +### real_node_test/0 * ### + +`real_node_test() -> any()` + + + +### report_data_matches/3 * ### + +`report_data_matches(Address, NodeMsgID, ReportData) -> any()` + +Ensure that the report data matches the expected report data. + + + +### trusted/3 ### + +`trusted(Msg1, Msg2, NodeOpts) -> any()` + +Validates if a given message parameter matches a trusted value from the SNP trusted list +Returns {ok, true} if the message is trusted, {ok, false} otherwise + + + +### verify/3 ### + +`verify(M1, M2, NodeOpts) -> any()` + +Verify an commitment report message; validating the identity of a +remote node, its ephemeral private address, and the integrity of the report. +The checks that must be performed to validate the report are: +1. Verify the address and the node message ID are the same as the ones +used to generate the nonce. +2. Verify the address that signed the message is the same as the one used +to generate the nonce. +3. Verify that the debug flag is disabled. +4. Verify that the firmware, kernel, and OS (VMSAs) hashes, part of the +measurement, are trusted. +5. Verify the measurement is valid. +6. Verify the report's certificate chain to hardware root of trust. + diff --git a/docs/resources/source-code/dev_snp_nif.md b/docs/resources/source-code/dev_snp_nif.md new file mode 100644 index 000000000..34d8e95fd --- /dev/null +++ b/docs/resources/source-code/dev_snp_nif.md @@ -0,0 +1,83 @@ +# [Module dev_snp_nif.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_snp_nif.erl) + + + + + + +## Function Index ## + + +
check_snp_support/0
compute_launch_digest/1
compute_launch_digest_test/0*
generate_attestation_report/2
generate_attestation_report_test/0*
init/0*
not_loaded/1*
verify_measurement/2
verify_measurement_test/0*
verify_signature/1
verify_signature_test/0*
+ + + + +## Function Details ## + + + +### check_snp_support/0 ### + +`check_snp_support() -> any()` + + + +### compute_launch_digest/1 ### + +`compute_launch_digest(Args) -> any()` + + + +### compute_launch_digest_test/0 * ### + +`compute_launch_digest_test() -> any()` + + + +### generate_attestation_report/2 ### + +`generate_attestation_report(UniqueData, VMPL) -> any()` + + + +### generate_attestation_report_test/0 * ### + +`generate_attestation_report_test() -> any()` + + + +### init/0 * ### + +`init() -> any()` + + + +### not_loaded/1 * ### + +`not_loaded(Line) -> any()` + + + +### verify_measurement/2 ### + +`verify_measurement(Report, Expected) -> any()` + + + +### verify_measurement_test/0 * ### + +`verify_measurement_test() -> any()` + + + +### verify_signature/1 ### + +`verify_signature(Report) -> any()` + + + +### verify_signature_test/0 * ### + +`verify_signature_test() -> any()` + diff --git a/docs/resources/source-code/dev_stack.md b/docs/resources/source-code/dev_stack.md new file mode 100644 index 000000000..d00bc24cf --- /dev/null +++ b/docs/resources/source-code/dev_stack.md @@ -0,0 +1,360 @@ +# [Module dev_stack.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_stack.erl) + + + + +A device that contains a stack of other devices, and manages their +execution. + + + +## Description ## + +It can run in two modes: fold (the default), and map. + +In fold mode, it runs upon input messages in the order of their keys. A +stack maintains and passes forward a state (expressed as a message) as it +progresses through devices. + +For example, a stack of devices as follows: + +``` + + Device -> Stack + Device-Stack/1/Name -> Add-One-Device + Device-Stack/2/Name -> Add-Two-Device +``` + +When called with the message: + +``` + + #{ Path = "FuncName", binary => <<"0">> } +``` + +Will produce the output: + +``` + + #{ Path = "FuncName", binary => <<"3">> } + {ok, #{ bin => <<"3">> }} +``` + +In map mode, the stack will run over all the devices in the stack, and +combine their results into a single message. Each of the devices' +output values have a key that is the device's name in the `Device-Stack` +(its number if the stack is a list). + +You can switch between fold and map modes by setting the `Mode` key in the +`Msg2` to either `Fold` or `Map`, or set it globally for the stack by +setting the `Mode` key in the `Msg1` message. The key in `Msg2` takes +precedence over the key in `Msg1`. + +The key that is called upon the device stack is the same key that is used +upon the devices that are contained within it. For example, in the above +scenario we resolve FuncName on the stack, leading FuncName to be called on +Add-One-Device and Add-Two-Device. + +A device stack responds to special statuses upon responses as follows: + +`skip`: Skips the rest of the device stack for the current pass. + +`pass`: Causes the stack to increment its pass number and re-execute +the stack from the first device, maintaining the state +accumulated so far. Only available in fold mode. + +In all cases, the device stack will return the accumulated state to the +caller as the result of the call to the stack. + +The dev_stack adds additional metadata to the message in order to track +the state of its execution as it progresses through devices. These keys +are as follows: + +`Stack-Pass`: The number of times the stack has reset and re-executed +from the first device for the current message. + +`Input-Prefix`: The prefix that the device should use for its outputs +and inputs. + +`Output-Prefix`: The device that was previously executed. + +All counters used by the stack are initialized to 1. + +Additionally, as implemented in HyperBEAM, the device stack will honor a +number of options that are passed to it as keys in the message. Each of +these options is also passed through to the devices contained within the +stack during execution. These options include: + +`Error-Strategy`: Determines how the stack handles errors from devices. +See `maybe_error/5` for more information. + +`Allow-Multipass`: Determines whether the stack is allowed to automatically +re-execute from the first device when the `pass` tag is returned. See +`maybe_pass/3` for more information. + +Under-the-hood, dev_stack uses a `default` handler to resolve all calls to +devices, aside `set/2` which it calls itself to mutate the message's `device` +key in order to change which device is currently being executed. This method +allows dev_stack to ensure that the message's HashPath is always correct, +even as it delegates calls to other devices. An example flow for a `dev_stack` +execution is as follows: + +``` + + /Msg1/AlicesExcitingKey -> + dev_stack:execute -> + /Msg1/Set?device=/Device-Stack/1 -> + /Msg2/AlicesExcitingKey -> + /Msg3/Set?device=/Device-Stack/2 -> + /Msg4/AlicesExcitingKey + ... -> + /MsgN/Set?device=[This-Device] -> + returns {ok, /MsgN+1} -> + /MsgN+1 +``` + +In this example, the `device` key is mutated a number of times, but the +resulting HashPath remains correct and verifiable. + +## Function Index ## + + +
benchmark_test/0*
example_device_for_stack_test/0*
generate_append_device/1
generate_append_device/2*
increment_pass/2*Helper to increment the pass number.
info/1
input_and_output_prefixes_test/0*
input_output_prefixes_passthrough_test/0*
input_prefix/3Return the input prefix for the stack.
many_devices_test/0*
maybe_error/5*
no_prefix_test/0*
not_found_test/0*
output_prefix/3Return the output prefix for the stack.
output_prefix_test/0*
pass_test/0*
prefix/3Return the default prefix for the stack.
reinvocation_test/0*
resolve_fold/3*The main device stack execution engine.
resolve_fold/4*
resolve_map/3*Map over the devices in the stack, accumulating the output in a single +message of keys and values, where keys are the same as the keys in the +original message (typically a number).
router/3*
router/4The device stack key router.
simple_map_test/0*
simple_stack_execute_test/0*
skip_test/0*
test_prefix_msg/0*
transform/3*Return Message1, transformed such that the device named Key from the +Device-Stack key in the message takes the place of the original Device +key.
transform_external_call_device_test/0*Ensure we can generate a transformer message that can be called to +return a version of msg1 with only that device attached.
transform_internal_call_device_test/0*Test that the transform function can be called correctly internally +by other functions in the module.
transformer_message/2*Return a message which, when given a key, will transform the message +such that the device named Key from the Device-Stack key in the message +takes the place of the original Device key.
+ + + + +## Function Details ## + + + +### benchmark_test/0 * ### + +`benchmark_test() -> any()` + + + +### example_device_for_stack_test/0 * ### + +`example_device_for_stack_test() -> any()` + + + +### generate_append_device/1 ### + +`generate_append_device(Separator) -> any()` + + + +### generate_append_device/2 * ### + +`generate_append_device(Separator, Status) -> any()` + + + +### increment_pass/2 * ### + +`increment_pass(Message, Opts) -> any()` + +Helper to increment the pass number. + + + +### info/1 ### + +`info(Msg) -> any()` + + + +### input_and_output_prefixes_test/0 * ### + +`input_and_output_prefixes_test() -> any()` + + + +### input_output_prefixes_passthrough_test/0 * ### + +`input_output_prefixes_passthrough_test() -> any()` + + + +### input_prefix/3 ### + +`input_prefix(Msg1, Msg2, Opts) -> any()` + +Return the input prefix for the stack. + + + +### many_devices_test/0 * ### + +`many_devices_test() -> any()` + + + +### maybe_error/5 * ### + +`maybe_error(Message1, Message2, DevNum, Info, Opts) -> any()` + + + +### no_prefix_test/0 * ### + +`no_prefix_test() -> any()` + + + +### not_found_test/0 * ### + +`not_found_test() -> any()` + + + +### output_prefix/3 ### + +`output_prefix(Msg1, Msg2, Opts) -> any()` + +Return the output prefix for the stack. + + + +### output_prefix_test/0 * ### + +`output_prefix_test() -> any()` + + + +### pass_test/0 * ### + +`pass_test() -> any()` + + + +### prefix/3 ### + +`prefix(Msg1, Msg2, Opts) -> any()` + +Return the default prefix for the stack. + + + +### reinvocation_test/0 * ### + +`reinvocation_test() -> any()` + + + +### resolve_fold/3 * ### + +`resolve_fold(Message1, Message2, Opts) -> any()` + +The main device stack execution engine. See the moduledoc for more +information. + + + +### resolve_fold/4 * ### + +`resolve_fold(Message1, Message2, DevNum, Opts) -> any()` + + + +### resolve_map/3 * ### + +`resolve_map(Message1, Message2, Opts) -> any()` + +Map over the devices in the stack, accumulating the output in a single +message of keys and values, where keys are the same as the keys in the +original message (typically a number). + + + +### router/3 * ### + +`router(Message1, Message2, Opts) -> any()` + + + +### router/4 ### + +`router(Key, Message1, Message2, Opts) -> any()` + +The device stack key router. Sends the request to `resolve_stack`, +except for `set/2` which is handled by the default implementation in +`dev_message`. + + + +### simple_map_test/0 * ### + +`simple_map_test() -> any()` + + + +### simple_stack_execute_test/0 * ### + +`simple_stack_execute_test() -> any()` + + + +### skip_test/0 * ### + +`skip_test() -> any()` + + + +### test_prefix_msg/0 * ### + +`test_prefix_msg() -> any()` + + + +### transform/3 * ### + +`transform(Msg1, Key, Opts) -> any()` + +Return Message1, transformed such that the device named `Key` from the +`Device-Stack` key in the message takes the place of the original `Device` +key. This transformation allows dev_stack to correctly track the HashPath +of the message as it delegates execution to devices contained within it. + + + +### transform_external_call_device_test/0 * ### + +`transform_external_call_device_test() -> any()` + +Ensure we can generate a transformer message that can be called to +return a version of msg1 with only that device attached. + + + +### transform_internal_call_device_test/0 * ### + +`transform_internal_call_device_test() -> any()` + +Test that the transform function can be called correctly internally +by other functions in the module. + + + +### transformer_message/2 * ### + +`transformer_message(Msg1, Opts) -> any()` + +Return a message which, when given a key, will transform the message +such that the device named `Key` from the `Device-Stack` key in the message +takes the place of the original `Device` key. This allows users to call +a single device from the stack: + +/Msg1/Transform/DeviceName/keyInDevice -> +keyInDevice executed on DeviceName against Msg1. + diff --git a/docs/resources/source-code/dev_test.md b/docs/resources/source-code/dev_test.md new file mode 100644 index 000000000..854ac1d7d --- /dev/null +++ b/docs/resources/source-code/dev_test.md @@ -0,0 +1,134 @@ +# [Module dev_test.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_test.erl) + + + + + + +## Function Index ## + + +
compute/3Example implementation of a compute handler.
compute_test/0*
delay/3Does nothing, just sleeps Req/duration or 750 ms and returns the +appropriate form in order to be used as a hook.
device_with_function_key_module_test/0*Tests the resolution of a default function.
increment_counter/3Find a test worker's PID and send it an increment message.
info/1Exports a default_handler function that can be used to test the +handler resolution mechanism.
info/3Exports a default_handler function that can be used to test the +handler resolution mechanism.
init/3Example init/3 handler.
mul/2Example implementation of an imported function for a WASM +executor.
restore/3Example restore/3 handler.
restore_test/0*
snapshot/3Do nothing when asked to snapshot.
test_func/1
update_state/3Find a test worker's PID and send it an update message.
+ + + + +## Function Details ## + + + +### compute/3 ### + +`compute(Msg1, Msg2, Opts) -> any()` + +Example implementation of a `compute` handler. Makes a running list of +the slots that have been computed in the state message and places the new +slot number in the results key. + + + +### compute_test/0 * ### + +`compute_test() -> any()` + + + +### delay/3 ### + +`delay(Msg1, Req, Opts) -> any()` + +Does nothing, just sleeps `Req/duration or 750` ms and returns the +appropriate form in order to be used as a hook. + + + +### device_with_function_key_module_test/0 * ### + +`device_with_function_key_module_test() -> any()` + +Tests the resolution of a default function. + + + +### increment_counter/3 ### + +`increment_counter(Msg1, Msg2, Opts) -> any()` + +Find a test worker's PID and send it an increment message. + + + +### info/1 ### + +`info(X1) -> any()` + +Exports a default_handler function that can be used to test the +handler resolution mechanism. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + +Exports a default_handler function that can be used to test the +handler resolution mechanism. + + + +### init/3 ### + +`init(Msg, Msg2, Opts) -> any()` + +Example `init/3` handler. Sets the `Already-Seen` key to an empty list. + + + +### mul/2 ### + +`mul(Msg1, Msg2) -> any()` + +Example implementation of an `imported` function for a WASM +executor. + + + +### restore/3 ### + +`restore(Msg, Msg2, Opts) -> any()` + +Example `restore/3` handler. Sets the hidden key `Test/Started` to the +value of `Current-Slot` and checks whether the `Already-Seen` key is valid. + + + +### restore_test/0 * ### + +`restore_test() -> any()` + + + +### snapshot/3 ### + +`snapshot(Msg1, Msg2, Opts) -> any()` + +Do nothing when asked to snapshot. + + + +### test_func/1 ### + +`test_func(X1) -> any()` + + + +### update_state/3 ### + +`update_state(Msg, Msg2, Opts) -> any()` + +Find a test worker's PID and send it an update message. + diff --git a/docs/resources/source-code/dev_volume.md b/docs/resources/source-code/dev_volume.md new file mode 100644 index 000000000..210df23d9 --- /dev/null +++ b/docs/resources/source-code/dev_volume.md @@ -0,0 +1,271 @@ +# [Module dev_volume.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_volume.erl) + + + + +Secure Volume Management for HyperBEAM Nodes. + + + +## Description ## + +This module handles encrypted storage operations for HyperBEAM, providing +a robust and secure approach to data persistence. It manages the complete +lifecycle of encrypted volumes from detection to creation, formatting, and +mounting. + +Key responsibilities: +- Volume detection and initialization +- Encrypted partition creation and formatting +- Secure mounting using cryptographic keys +- Store path reconfiguration to use mounted volumes +- Automatic handling of various system states +(new device, existing partition, etc.) + +The primary entry point is the `mount/3` function, which orchestrates the +entire process based on the provided configuration parameters. This module +works alongside `hb_volume` which provides the low-level operations for +device manipulation. + +Security considerations: +- Ensures data at rest is protected through LUKS encryption +- Provides proper volume sanitization and secure mounting +- IMPORTANT: This module only applies configuration set in node options and +does NOT accept disk operations via HTTP requests. It cannot format arbitrary +disks as all operations are safeguarded by host operating system permissions +enforced upon the HyperBEAM environment. + +## Function Index ## + + +
check_base_device/8*Check if the base device exists and if it does, check if the partition exists.
check_partition/8*Check if the partition exists.
create_and_mount_partition/8*Create, format and mount a new partition.
decrypt_volume_key/2*Decrypts an encrypted volume key using the node's private key.
format_and_mount/6*Format and mount a newly created partition.
info/1Exported function for getting device info, controls which functions are +exposed via the device API.
info/3HTTP info response providing information about this device.
mount/3Handles the complete process of secure encrypted volume mounting.
mount_existing_partition/6*Mount an existing partition.
mount_formatted_partition/6*Mount a newly formatted partition.
public_key/3Returns the node's public key for secure key exchange.
update_node_config/2*Update the node's configuration with the new store.
update_store_path/2*Update the store path to use the mounted volume.
+ + + + +## Function Details ## + + + +### check_base_device/8 * ### + +

+check_base_device(Device::term(), Partition::term(), PartitionType::term(), VolumeName::term(), MountPoint::term(), StorePath::term(), Key::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Device`: The base device to check.
`Partition`: The partition to check.
`PartitionType`: The type of partition to check.
`VolumeName`: The name of the volume to check.
`MountPoint`: The mount point to check.
`StorePath`: The store path to check.
`Key`: The key to check.
`Opts`: The options to check.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Check if the base device exists and if it does, check if the partition exists. + + + +### check_partition/8 * ### + +

+check_partition(Device::term(), Partition::term(), PartitionType::term(), VolumeName::term(), MountPoint::term(), StorePath::term(), Key::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Device`: The base device to check.
`Partition`: The partition to check.
`PartitionType`: The type of partition to check.
`VolumeName`: The name of the volume to check.
`MountPoint`: The mount point to check.
`StorePath`: The store path to check.
`Key`: The key to check.
`Opts`: The options to check.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Check if the partition exists. If it does, attempt to mount it. +If it doesn't exist, create it, format it with encryption and mount it. + + + +### create_and_mount_partition/8 * ### + +

+create_and_mount_partition(Device::term(), Partition::term(), PartitionType::term(), Key::term(), MountPoint::term(), VolumeName::term(), StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Device`: The device to create the partition on.
`Partition`: The partition to create.
`PartitionType`: The type of partition to create.
`Key`: The key to create the partition with.
`MountPoint`: The mount point to mount the partition to.
`VolumeName`: The name of the volume to mount.
`StorePath`: The store path to mount.
`Opts`: The options to mount.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Create, format and mount a new partition. + + + +### decrypt_volume_key/2 * ### + +

+decrypt_volume_key(EncryptedKeyBase64::binary(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options.
+ +returns: `{ok, DecryptedKey}` on successful decryption, or +`{error, Binary}` if decryption fails. + +Decrypts an encrypted volume key using the node's private key. + +This function takes an encrypted key (typically sent by a client who encrypted +it with the node's public key) and decrypts it using the node's private RSA key. + + + +### format_and_mount/6 * ### + +

+format_and_mount(Partition::term(), Key::term(), MountPoint::term(), VolumeName::term(), StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Partition`: The partition to format and mount.
`Key`: The key to format and mount the partition with.
`MountPoint`: The mount point to mount the partition to.
`VolumeName`: The name of the volume to mount.
`StorePath`: The store path to mount.
`Opts`: The options to mount.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Format and mount a newly created partition. + + + +### info/1 ### + +`info(X1) -> any()` + +Exported function for getting device info, controls which functions are +exposed via the device API. + + + +### info/3 ### + +`info(Msg1, Msg2, Opts) -> any()` + +HTTP info response providing information about this device + + + +### mount/3 ### + +

+mount(M1::term(), M2::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`M1`: Base message for context.
`M2`: Request message with operation details.
`Opts`: A map of configuration options for volume operations.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Handles the complete process of secure encrypted volume mounting. + +This function performs the following operations depending on the state: +1. Validates the encryption key is present +2. Checks if the base device exists +3. Checks if the partition exists on the device +4. If the partition exists, attempts to mount it +5. If the partition doesn't exist, creates it, formats it with encryption +and mounts it +6. Updates the node's store configuration to use the mounted volume + +Config options in Opts map: +- volume_key: (Required) The encryption key +- volume_device: Base device path +- volume_partition: Partition path +- volume_partition_type: Filesystem type +- volume_name: Name for encrypted volume +- volume_mount_point: Where to mount +- volume_store_path: Store path on volume + + + +### mount_existing_partition/6 * ### + +

+mount_existing_partition(Partition::term(), Key::term(), MountPoint::term(), VolumeName::term(), StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Partition`: The partition to mount.
`Key`: The key to mount.
`MountPoint`: The mount point to mount.
`VolumeName`: The name of the volume to mount.
`StorePath`: The store path to mount.
`Opts`: The options to mount.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Mount an existing partition. + + + +### mount_formatted_partition/6 * ### + +

+mount_formatted_partition(Partition::term(), Key::term(), MountPoint::term(), VolumeName::term(), StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`Partition`: The partition to mount.
`Key`: The key to mount the partition with.
`MountPoint`: The mount point to mount the partition to.
`VolumeName`: The name of the volume to mount.
`StorePath`: The store path to mount.
`Opts`: The options to mount.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Mount a newly formatted partition. + + + +### public_key/3 ### + +

+public_key(M1::term(), M2::term(), Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`Opts`: A map of configuration options.
+ +returns: `{ok, Map}` containing the node's public key on success, or +`{error, Binary}` if the node's wallet is not available. + +Returns the node's public key for secure key exchange. + +This function retrieves the node's wallet and extracts the public key +for encryption purposes. It allows users to securely exchange encryption keys +by first encrypting their volume key with the node's public key. + +The process ensures that sensitive keys are never transmitted in plaintext. +The encrypted key can then be securely sent to the node, which will decrypt it +using its private key before using it for volume encryption. + + + +### update_node_config/2 * ### + +

+update_node_config(NewStore::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`NewStore`: The new store to update the node's configuration with.
`Opts`: The options to update the node's configuration with.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Update the node's configuration with the new store. + + + +### update_store_path/2 * ### + +

+update_store_path(StorePath::term(), Opts::map()) -> {ok, binary()} | {error, binary()}
+
+
+ +`StorePath`: The store path to update.
`Opts`: The options to update.
+ +returns: `{ok, Binary}` on success with operation result message, or +`{error, Binary}` on failure with error message. + +Update the store path to use the mounted volume. + diff --git a/docs/resources/source-code/dev_wasi.md b/docs/resources/source-code/dev_wasi.md new file mode 100644 index 000000000..e3f2ce045 --- /dev/null +++ b/docs/resources/source-code/dev_wasi.md @@ -0,0 +1,149 @@ +# [Module dev_wasi.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_wasi.erl) + + + + +A virtual filesystem device. + + + +## Description ## +Implements a file-system-as-map structure, which is traversible externally. +Each file is a binary and each directory is an AO-Core message. +Additionally, this module adds a series of WASI-preview-1 compatible +functions for accessing the filesystem as imported functions by WASM +modules. + +## Function Index ## + + +
basic_aos_exec_test/0*
clock_time_get/3
compute/1
fd_read/3Read from a file using the WASI-p1 standard interface.
fd_read/5*
fd_write/3WASM stdlib implementation of fd_write, using the WASI-p1 standard +interface.
fd_write/5*
gen_test_aos_msg/1*
gen_test_env/0*
generate_wasi_stack/3*
init/0*
init/3On-boot, initialize the virtual file system with: +- Empty stdio files +- WASI-preview-1 compatible functions for accessing the filesystem +- File descriptors for those files.
parse_iovec/2*Parse an iovec in WASI-preview-1 format.
path_open/3Adds a file descriptor to the state message.
stdout/1Return the stdout buffer from a state message.
vfs_is_serializable_test/0*
wasi_stack_is_serializable_test/0*
+ + + + +## Function Details ## + + + +### basic_aos_exec_test/0 * ### + +`basic_aos_exec_test() -> any()` + + + +### clock_time_get/3 ### + +`clock_time_get(Msg1, Msg2, Opts) -> any()` + + + +### compute/1 ### + +`compute(Msg1) -> any()` + + + +### fd_read/3 ### + +`fd_read(Msg1, Msg2, Opts) -> any()` + +Read from a file using the WASI-p1 standard interface. + + + +### fd_read/5 * ### + +`fd_read(S, Instance, X3, BytesRead, Opts) -> any()` + + + +### fd_write/3 ### + +`fd_write(Msg1, Msg2, Opts) -> any()` + +WASM stdlib implementation of `fd_write`, using the WASI-p1 standard +interface. + + + +### fd_write/5 * ### + +`fd_write(S, Instance, X3, BytesWritten, Opts) -> any()` + + + +### gen_test_aos_msg/1 * ### + +`gen_test_aos_msg(Command) -> any()` + + + +### gen_test_env/0 * ### + +`gen_test_env() -> any()` + + + +### generate_wasi_stack/3 * ### + +`generate_wasi_stack(File, Func, Params) -> any()` + + + +### init/0 * ### + +`init() -> any()` + + + +### init/3 ### + +`init(M1, M2, Opts) -> any()` + +On-boot, initialize the virtual file system with: +- Empty stdio files +- WASI-preview-1 compatible functions for accessing the filesystem +- File descriptors for those files. + + + +### parse_iovec/2 * ### + +`parse_iovec(Instance, Ptr) -> any()` + +Parse an iovec in WASI-preview-1 format. + + + +### path_open/3 ### + +`path_open(Msg1, Msg2, Opts) -> any()` + +Adds a file descriptor to the state message. +path_open(M, Instance, [FDPtr, LookupFlag, PathPtr|_]) -> + + + +### stdout/1 ### + +`stdout(M) -> any()` + +Return the stdout buffer from a state message. + + + +### vfs_is_serializable_test/0 * ### + +`vfs_is_serializable_test() -> any()` + + + +### wasi_stack_is_serializable_test/0 * ### + +`wasi_stack_is_serializable_test() -> any()` + diff --git a/docs/resources/source-code/dev_wasm.md b/docs/resources/source-code/dev_wasm.md new file mode 100644 index 000000000..174d97760 --- /dev/null +++ b/docs/resources/source-code/dev_wasm.md @@ -0,0 +1,233 @@ +# [Module dev_wasm.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/dev_wasm.erl) + + + + +A device that executes a WASM image on messages using the Memory-64 +preview standard. + + + +## Description ## + +In the backend, this device uses `beamr`: An Erlang wrapper +for WAMR, the WebAssembly Micro Runtime. + +The device has the following requirements and interface: + +``` + + M1/Init -> + Assumes: + M1/process + M1/[Prefix]/image + Generates: + /priv/[Prefix]/instance + /priv/[Prefix]/import-resolver + Side-effects: + Creates a WASM executor loaded in memory of the HyperBEAM node. + M1/Compute -> + Assumes: + M1/priv/[Prefix]/instance + M1/priv/[Prefix]/import-resolver + M1/process + M2/message + M2/message/function OR M1/function + M2/message/parameters OR M1/parameters + Generates: + /results/[Prefix]/type + /results/[Prefix]/output + Side-effects: + Calls the WASM executor with the message and process. + M1/[Prefix]/state -> + Assumes: + M1/priv/[Prefix]/instance + Generates: + Raw binary WASM state +``` + + +## Function Index ## + + +
basic_execution_64_test/0*
basic_execution_test/0*
benchmark_test/0*
cache_wasm_image/1
cache_wasm_image/2
compute/3Call the WASM executor with a message that has been prepared by a prior +pass.
default_import_resolver/3*Take a BEAMR import call and resolve it using hb_ao.
import/3Handle standard library calls by: +1.
imported_function_test/0*
info/2Export all functions aside the instance/3 function.
init/0*
init/3Boot a WASM image on the image stated in the process/image field of +the message.
init_test/0*
input_prefix_test/0*
instance/3Get the WASM instance from the message.
normalize/3Normalize the message to have an open WASM instance, but no literal +State key.
process_prefixes_test/0*Test that realistic prefixing for a dev_process works -- +including both inputs (from Process/) and outputs (to the +Device-Key) work.
snapshot/3Serialize the WASM state to a binary.
state_export_and_restore_test/0*
terminate/3Tear down the WASM executor.
test_run_wasm/4*
undefined_import_stub/3*Log the call to the standard library as an event, and write the +call details into the message.
+ + + + +## Function Details ## + + + +### basic_execution_64_test/0 * ### + +`basic_execution_64_test() -> any()` + + + +### basic_execution_test/0 * ### + +`basic_execution_test() -> any()` + + + +### benchmark_test/0 * ### + +`benchmark_test() -> any()` + + + +### cache_wasm_image/1 ### + +`cache_wasm_image(Image) -> any()` + + + +### cache_wasm_image/2 ### + +`cache_wasm_image(Image, Opts) -> any()` + + + +### compute/3 ### + +`compute(RawM1, M2, Opts) -> any()` + +Call the WASM executor with a message that has been prepared by a prior +pass. + + + +### default_import_resolver/3 * ### + +`default_import_resolver(Msg1, Msg2, Opts) -> any()` + +Take a BEAMR import call and resolve it using `hb_ao`. + + + +### import/3 ### + +`import(Msg1, Msg2, Opts) -> any()` + +Handle standard library calls by: +1. Adding the right prefix to the path from BEAMR. +2. Adding the state to the message at the stdlib path. +3. Resolving the adjusted-path-Msg2 against the added-state-Msg1. +4. If it succeeds, return the new state from the message. +5. If it fails with `not_found`, call the stub handler. + + + +### imported_function_test/0 * ### + +`imported_function_test() -> any()` + + + +### info/2 ### + +`info(Msg1, Opts) -> any()` + +Export all functions aside the `instance/3` function. + + + +### init/0 * ### + +`init() -> any()` + + + +### init/3 ### + +`init(M1, M2, Opts) -> any()` + +Boot a WASM image on the image stated in the `process/image` field of +the message. + + + +### init_test/0 * ### + +`init_test() -> any()` + + + +### input_prefix_test/0 * ### + +`input_prefix_test() -> any()` + + + +### instance/3 ### + +`instance(M1, M2, Opts) -> any()` + +Get the WASM instance from the message. Note that this function is exported +such that other devices can use it, but it is excluded from calls from AO-Core +resolution directly. + + + +### normalize/3 ### + +`normalize(RawM1, M2, Opts) -> any()` + +Normalize the message to have an open WASM instance, but no literal +`State` key. Ensure that we do not change the hashpath during this process. + + + +### process_prefixes_test/0 * ### + +`process_prefixes_test() -> any()` + +Test that realistic prefixing for a `dev_process` works -- +including both inputs (from `Process/`) and outputs (to the +Device-Key) work + + + +### snapshot/3 ### + +`snapshot(M1, M2, Opts) -> any()` + +Serialize the WASM state to a binary. + + + +### state_export_and_restore_test/0 * ### + +`state_export_and_restore_test() -> any()` + + + +### terminate/3 ### + +`terminate(M1, M2, Opts) -> any()` + +Tear down the WASM executor. + + + +### test_run_wasm/4 * ### + +`test_run_wasm(File, Func, Params, AdditionalMsg) -> any()` + + + +### undefined_import_stub/3 * ### + +`undefined_import_stub(Msg1, Msg2, Opts) -> any()` + +Log the call to the standard library as an event, and write the +call details into the message. + diff --git a/docs/resources/source-code/edoc-info b/docs/resources/source-code/edoc-info new file mode 100644 index 000000000..1163d1833 --- /dev/null +++ b/docs/resources/source-code/edoc-info @@ -0,0 +1,24 @@ +%% encoding: UTF-8 +{application,hb}. +{modules,[ar_bundles,ar_deep_hash,ar_rate_limiter,ar_timestamp,ar_tx, + ar_wallet,dev_cache,dev_cacheviz,dev_codec_ans104,dev_codec_flat, + dev_codec_httpsig,dev_codec_httpsig_conv,dev_codec_json, + dev_codec_structured,dev_cron,dev_cu,dev_dedup, + dev_delegated_compute,dev_faff,dev_genesis_wasm,dev_green_zone, + dev_hook,dev_hyperbuddy,dev_json_iface,dev_local_name,dev_lookup, + dev_lua,dev_lua_lib,dev_lua_test,dev_manifest,dev_message,dev_meta, + dev_monitor,dev_multipass,dev_name,dev_node_process,dev_p4, + dev_patch,dev_poda,dev_process,dev_process_cache,dev_process_worker, + dev_push,dev_relay,dev_router,dev_scheduler,dev_scheduler_cache, + dev_scheduler_formats,dev_scheduler_registry,dev_scheduler_server, + dev_simple_pay,dev_snp,dev_snp_nif,dev_stack,dev_test,dev_volume, + dev_wasi,dev_wasm,hb,hb_ao,hb_ao_test_vectors,hb_app,hb_beamr, + hb_beamr_io,hb_cache,hb_cache_control,hb_cache_render,hb_client, + hb_crypto,hb_debugger,hb_escape,hb_event,hb_examples,hb_features, + hb_gateway_client,hb_http,hb_http_benchmark_tests,hb_http_client, + hb_http_client_sup,hb_http_server,hb_json,hb_keccak,hb_logger, + hb_message,hb_metrics_collector,hb_name,hb_opts,hb_path, + hb_persistent,hb_private,hb_process_monitor,hb_router,hb_singleton, + hb_store,hb_store_fs,hb_store_gateway,hb_store_remote_node, + hb_store_rocksdb,hb_structured_fields,hb_sup,hb_test_utils, + hb_tracer,hb_util,hb_volume,rsa_pss]}. diff --git a/docs/resources/source-code/erlang.png b/docs/resources/source-code/erlang.png new file mode 100644 index 000000000..987a618e2 Binary files /dev/null and b/docs/resources/source-code/erlang.png differ diff --git a/docs/resources/source-code/hb.md b/docs/resources/source-code/hb.md new file mode 100644 index 000000000..8fc383d1d --- /dev/null +++ b/docs/resources/source-code/hb.md @@ -0,0 +1,271 @@ +# [Module hb.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb.erl) + + + + +Hyperbeam is a decentralized node implementing the AO-Core protocol +on top of Arweave. + + + +## Description ## + +This protocol offers a computation layer for executing arbitrary logic on +top of the network's data. + +Arweave is built to offer a robust, permanent storage layer for static data +over time. It can be seen as a globally distributed key-value store that +allows users to lookup IDs to retrieve data at any point in time: + +`Arweave(ID) => Message` + +Hyperbeam adds another layer of functionality on top of Arweave's protocol: +Allowing users to store and retrieve not only arbitrary bytes, but also to +perform execution of computation upon that data: + +`Hyperbeam(Message1, Message2) => Message3` + +When Hyperbeam executes a message, it will return a new message containing +the result of that execution, as well as signed commitments of its +correctness. If the computation that is executed is deterministic, recipients +of the new message are able to verify that the computation was performed +correctly. The new message may be stored back to Arweave if desired, +forming a permanent, verifiable, and decentralized log of computation. + +The mechanisms described above form the basis of a decentralized and +verifiable compute engine without any relevant protocol-enforced +scalability limits. It is an implementation of a global, shared +supercomputer. + +Hyperbeam can be used for an extremely large variety of applications, from +serving static Arweave data with signed commitments of correctness, to +executing smart contracts that have _built-in_ HTTP APIs. The Hyperbeam +node implementation implements AO, an Actor-Oriented process-based +environment for orchestrating computation over Arweave messages in order to +facilitate the execution of more traditional, consensus-based smart +contracts. + +The core abstractions of the Hyperbeam node are broadly as follows: + +1. The `hb` and `hb_opts` modules manage the node's configuration, +environment variables, and debugging tools. + +2. The `hb_http` and `hb_http_server` modules manage all HTTP-related +functionality. `hb_http_server` handles turning received HTTP requests +into messages and applying those messages with the appropriate devices. +`hb_http` handles making requests and responding with messages. `cowboy` +is used to implement the underlying HTTP server. + +3. `hb_ao` implements the computation logic of the node: A mechanism +for resolving messages to other messages, via the application of logic +implemented in `devices`. `hb_ao` also manages the loading of Erlang +modules for each device into the node's environment. There are many +different default devices implemented in the hyperbeam node, using the +namespace `dev_*`. Some of the critical components are: + +- `dev_message`: The default handler for all messages that do not +specify their own device. The message device is also used to resolve +keys that are not implemented by the device specified in a message, +unless otherwise signalled. + +- `dev_stack`: The device responsible for creating and executing stacks +of other devices on messages that request it. There are many uses for +this device, one of which is the resolution of AO processes. + +- `dev_p4`: The device responsible for managing payments for the services +provided by the node. + +4. `hb_store`, `hb_cache` and the store implementations forms a layered +system for managing the node's access to persistent storage. `hb_cache` +is used as a resolution mechanism for reading and writing messages, while +`hb_store` provides an abstraction over the underlying persistent key-value +byte storage mechanisms. Example `hb_store` mechanisms can be found in +`hb_store_fs` and `hb_store_remote_node`. + +5. `ar_*` modules implement functionality related to the base-layer Arweave +protocol and are largely unchanged from their counterparts in the Arweave +node codebase presently maintained by the Digital History Association +(@dha-team/Arweave). + +You can find documentation of a similar form to this note in each of the core +modules of the hyperbeam node. + +## Function Index ## + + +
address/0Get the address of a wallet.
address/1*
benchmark/2Run a function as many times as possible in a given amount of time.
benchmark/3Run multiple instances of a function in parallel for a given amount of time.
build/0Utility function to hot-recompile and load the hyperbeam environment.
debug_wait/4Utility function to wait for a given amount of time, printing a debug +message to the console first.
do_start_simple_pay/1*
init/0Initialize system-wide settings for the hyperbeam node.
no_prod/3Utility function to throw an error if the current mode is prod and +non-prod ready code is being executed.
now/0Utility function to get the current time in milliseconds.
profile/1Utility function to start a profiling session and run a function, +then analyze the results.
read/1Debugging function to read a message from the cache.
read/2
start_mainnet/0Start a mainnet server without payments.
start_mainnet/1
start_simple_pay/0Start a server with a simple-pay@1.0 pre-processor.
start_simple_pay/1
start_simple_pay/2
topup/3Helper for topping up a user's balance on a simple-pay node.
topup/4
wallet/0
wallet/1
+ + + + +## Function Details ## + + + +### address/0 ### + +`address() -> any()` + +Get the address of a wallet. Defaults to the address of the wallet +specified by the `priv_key_location` configuration key. It can also take a +wallet tuple as an argument. + + + +### address/1 * ### + +`address(Wallet) -> any()` + + + +### benchmark/2 ### + +`benchmark(Fun, TLen) -> any()` + +Run a function as many times as possible in a given amount of time. + + + +### benchmark/3 ### + +`benchmark(Fun, TLen, Procs) -> any()` + +Run multiple instances of a function in parallel for a given amount of time. + + + +### build/0 ### + +`build() -> any()` + +Utility function to hot-recompile and load the hyperbeam environment. + + + +### debug_wait/4 ### + +`debug_wait(T, Mod, Func, Line) -> any()` + +Utility function to wait for a given amount of time, printing a debug +message to the console first. + + + +### do_start_simple_pay/1 * ### + +`do_start_simple_pay(Opts) -> any()` + + + +### init/0 ### + +`init() -> any()` + +Initialize system-wide settings for the hyperbeam node. + + + +### no_prod/3 ### + +`no_prod(X, Mod, Line) -> any()` + +Utility function to throw an error if the current mode is prod and +non-prod ready code is being executed. You can find these in the codebase +by looking for ?NO_PROD calls. + + + +### now/0 ### + +`now() -> any()` + +Utility function to get the current time in milliseconds. + + + +### profile/1 ### + +`profile(Fun) -> any()` + +Utility function to start a profiling session and run a function, +then analyze the results. Obviously -- do not use in production. + + + +### read/1 ### + +`read(ID) -> any()` + +Debugging function to read a message from the cache. +Specify either a scope atom (local or remote) or a store tuple +as the second argument. + + + +### read/2 ### + +`read(ID, ScopeAtom) -> any()` + + + +### start_mainnet/0 ### + +`start_mainnet() -> any()` + +Start a mainnet server without payments. + + + +### start_mainnet/1 ### + +`start_mainnet(Port) -> any()` + + + +### start_simple_pay/0 ### + +`start_simple_pay() -> any()` + +Start a server with a `simple-pay@1.0` pre-processor. + + + +### start_simple_pay/1 ### + +`start_simple_pay(Addr) -> any()` + + + +### start_simple_pay/2 ### + +`start_simple_pay(Addr, Port) -> any()` + + + +### topup/3 ### + +`topup(Node, Amount, Recipient) -> any()` + +Helper for topping up a user's balance on a simple-pay node. + + + +### topup/4 ### + +`topup(Node, Amount, Recipient, Wallet) -> any()` + + + +### wallet/0 ### + +`wallet() -> any()` + + + +### wallet/1 ### + +`wallet(Location) -> any()` + diff --git a/docs/resources/source-code/hb_ao.md b/docs/resources/source-code/hb_ao.md new file mode 100644 index 000000000..a01a66fb5 --- /dev/null +++ b/docs/resources/source-code/hb_ao.md @@ -0,0 +1,533 @@ +# [Module hb_ao.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_ao.erl) + + + + +This module is the root of the device call logic of the +AO-Core protocol in HyperBEAM. + + + +## Description ## + +At the implementation level, every message is simply a collection of keys, +dictated by its `Device`, that can be resolved in order to yield their +values. Each key may return another message or a raw value: + +`ao(Message1, Message2) -> {Status, Message3}` + +Under-the-hood, `AO-Core(Message1, Message2)` leads to the evaluation of +`DeviceMod:PathPart(Message1, Message2)`, which defines the user compute +to be performed. If `Message1` does not specify a device, `dev_message` is +assumed. The key to resolve is specified by the `Path` field of the message. + +After each output, the `HashPath` is updated to include the `Message2` +that was executed upon it. + +Because each message implies a device that can resolve its keys, as well +as generating a merkle tree of the computation that led to the result, +you can see AO-Core protocol as a system for cryptographically chaining +the execution of `combinators`. See `docs/ao-core-protocol.md` for more +information about AO-Core. + +The `Fun(Message1, Message2)` pattern is repeated throughout the HyperBEAM +codebase, sometimes with `MessageX` replaced with `MX` or `MsgX` for brevity. + +Message3 can be either a new message or a raw output value (a binary, integer, +float, atom, or list of such values). + +Devices can be expressed as either modules or maps. They can also be +referenced by an Arweave ID, which can be used to load a device from +the network (depending on the value of the `load_remote_devices` and +`trusted_device_signers` environment settings). + +HyperBEAM device implementations are defined as follows: + +``` + + DevMod:ExportedFunc : Key resolution functions. All are assumed to be + device keys (thus, present in every message that + uses it) unless specified by DevMod:info(). + Each function takes a set of parameters + of the form DevMod:KeyHandler(Msg1, Msg2, Opts). + Each of these arguments can be ommitted if not + needed. Non-exported functions are not assumed + to be device keys. + DevMod:info : Optional. Returns a map of options for the device. All + options are optional and assumed to be the defaults if + not specified. This function can accept a Message1 as + an argument, allowing it to specify its functionality + based on a specific message if appropriate. + info/exports : Overrides the export list of the Erlang module, such that + only the functions in this list are assumed to be device + keys. Defaults to all of the functions that DevMod + exports in the Erlang environment. + info/excludes : A list of keys that should not be resolved by the device, + despite being present in the Erlang module exports list. + info/handler : A function that should be used to handle _all_ keys for + messages using the device. + info/default : A function that should be used to handle all keys that + are not explicitly implemented by the device. Defaults to + the dev_message device, which contains general keys for + interacting with messages. + info/default_mod : A different device module that should be used to + handle all keys that are not explicitly implemented + by the device. Defaults to the dev_message device. + info/grouper : A function that returns the concurrency 'group' name for + an execution. Executions with the same group name will + be executed by sending a message to the associated process + and waiting for a response. This allows you to control + concurrency of execution and to allow executions to share + in-memory state as applicable. Default: A derivation of + Msg1+Msg2. This means that concurrent calls for the same + output will lead to only a single execution. + info/worker : A function that should be run as the 'server' loop of + the executor for interactions using the device. + The HyperBEAM resolver also takes a number of runtime options that change + the way that the environment operates:update_hashpath: Whether to add the Msg2 to HashPath for the Msg3. + Default: true.add_key: Whether to add the key to the start of the arguments. + Default: . +``` + + +## Function Index ## + + +
deep_set/4Recursively search a map, resolving keys, and set the value of the key +at the given path.
default_module/0*The default device is the identity device, which simply returns the +value associated with any key as it exists in its Erlang map.
device_set/4*Call the device's set function.
device_set/5*
do_resolve_many/2*
ensure_loaded/2*Ensure that the message is loaded from the cache if it is an ID.
error_execution/5*Handle an error in a device call.
error_infinite/3*Catch all return if we are in an infinite loop.
error_invalid_intermediate_status/5*
error_invalid_message/3*Catch all return if the message is invalid.
find_exported_function/5Find the function with the highest arity that has the given name, if it +exists.
force_message/2
get/2Shortcut for resolving a key in a message without its status if it is +ok.
get/3
get/4
get_first/2take a sequence of base messages and paths, then return the value of the +first message that can be resolved using a path.
get_first/3
info/2Get the info map for a device, optionally giving it a message if the +device's info function is parameterized by one.
info/3*
info_handler_to_fun/4*Parse a handler key given by a device's info.
internal_opts/1*The execution options that are used internally by this module +when calling itself.
is_exported/2*
is_exported/4Check if a device is guarding a key via its exports list.
keys/1Shortcut to get the list of keys from a message.
keys/2
keys/3
load_device/2Load a device module from its name or a message ID.
maybe_force_message/2*Force the result of a device call into a message if the result is not +requested by the Opts.
message_to_device/2Extract the device module from a message.
message_to_fun/3Calculate the Erlang function that should be called to get a value for +a given key from a device.
normalize_key/1Convert a key to a binary in normalized form.
normalize_key/2
normalize_keys/1Ensure that a message is processable by the AO-Core resolver: No lists.
remove/2Remove a key from a message, using its underlying device.
remove/3
resolve/2Get the value of a message's key by running its associated device +function.
resolve/3
resolve_many/2Resolve a list of messages in sequence.
resolve_stage/4*
resolve_stage/5*
resolve_stage/6*
set/2Shortcut for setting a key in the message using its underlying device.
set/3
set/4
subresolve/4*Execute a sub-resolution.
truncate_args/2Truncate the arguments of a function to the number of arguments it +actually takes.
verify_device_compatibility/2*Verify that a device is compatible with the current machine.
+ + + + +## Function Details ## + + + +### deep_set/4 ### + +`deep_set(Msg, Rest, Value, Opts) -> any()` + +Recursively search a map, resolving keys, and set the value of the key +at the given path. This function has special cases for handling `set` calls +where the path is an empty list (`/`). In this case, if the value is an +immediate, non-complex term, we can set it directly. Otherwise, we use the +device's `set` function to set the value. + + + +### default_module/0 * ### + +`default_module() -> any()` + +The default device is the identity device, which simply returns the +value associated with any key as it exists in its Erlang map. It should also +implement the `set` key, which returns a `Message3` with the values changed +according to the `Message2` passed to it. + + + +### device_set/4 * ### + +`device_set(Msg, Key, Value, Opts) -> any()` + +Call the device's `set` function. + + + +### device_set/5 * ### + +`device_set(Msg, Key, Value, Mode, Opts) -> any()` + + + +### do_resolve_many/2 * ### + +`do_resolve_many(MsgList, Opts) -> any()` + + + +### ensure_loaded/2 * ### + +`ensure_loaded(MsgID, Opts) -> any()` + +Ensure that the message is loaded from the cache if it is an ID. If is +not loadable or already present, we raise an error. + + + +### error_execution/5 * ### + +`error_execution(ExecGroup, Msg2, Whence, X4, Opts) -> any()` + +Handle an error in a device call. + + + +### error_infinite/3 * ### + +`error_infinite(Msg1, Msg2, Opts) -> any()` + +Catch all return if we are in an infinite loop. + + + +### error_invalid_intermediate_status/5 * ### + +`error_invalid_intermediate_status(Msg1, Msg2, Msg3, RemainingPath, Opts) -> any()` + + + +### error_invalid_message/3 * ### + +`error_invalid_message(Msg1, Msg2, Opts) -> any()` + +Catch all return if the message is invalid. + + + +### find_exported_function/5 ### + +`find_exported_function(Msg, Dev, Key, MaxArity, Opts) -> any()` + +Find the function with the highest arity that has the given name, if it +exists. + +If the device is a module, we look for a function with the given name. + +If the device is a map, we look for a key in the map. First we try to find +the key using its literal value. If that fails, we cast the key to an atom +and try again. + + + +### force_message/2 ### + +`force_message(X1, Opts) -> any()` + + + +### get/2 ### + +`get(Path, Msg) -> any()` + +Shortcut for resolving a key in a message without its status if it is +`ok`. This makes it easier to write complex logic on top of messages while +maintaining a functional style. + +Additionally, this function supports the `{as, Device, Msg}` syntax, which +allows the key to be resolved using another device to resolve the key, +while maintaining the tracability of the `HashPath` of the output message. + +Returns the value of the key if it is found, otherwise returns the default +provided by the user, or `not_found` if no default is provided. + + + +### get/3 ### + +`get(Path, Msg, Opts) -> any()` + + + +### get/4 ### + +`get(Path, Msg, Default, Opts) -> any()` + + + +### get_first/2 ### + +`get_first(Paths, Opts) -> any()` + +take a sequence of base messages and paths, then return the value of the +first message that can be resolved using a path. + + + +### get_first/3 ### + +`get_first(Msgs, Default, Opts) -> any()` + + + +### info/2 ### + +`info(Msg, Opts) -> any()` + +Get the info map for a device, optionally giving it a message if the +device's info function is parameterized by one. + + + +### info/3 * ### + +`info(DevMod, Msg, Opts) -> any()` + + + +### info_handler_to_fun/4 * ### + +`info_handler_to_fun(Handler, Msg, Key, Opts) -> any()` + +Parse a handler key given by a device's `info`. + + + +### internal_opts/1 * ### + +`internal_opts(Opts) -> any()` + +The execution options that are used internally by this module +when calling itself. + + + +### is_exported/2 * ### + +`is_exported(Info, Key) -> any()` + + + +### is_exported/4 ### + +`is_exported(Msg, Dev, Key, Opts) -> any()` + +Check if a device is guarding a key via its `exports` list. Defaults to +true if the device does not specify an `exports` list. The `info` function is +always exported, if it exists. Elements of the `exludes` list are not +exported. Note that we check for info _twice_ -- once when the device is +given but the info result is not, and once when the info result is given. +The reason for this is that `info/3` calls other functions that may need to +check if a key is exported, so we must avoid infinite loops. We must, however, +also return a consistent result in the case that only the info result is +given, so we check for it in both cases. + + + +### keys/1 ### + +`keys(Msg) -> any()` + +Shortcut to get the list of keys from a message. + + + +### keys/2 ### + +`keys(Msg, Opts) -> any()` + + + +### keys/3 ### + +`keys(Msg, Opts, X3) -> any()` + + + +### load_device/2 ### + +`load_device(Map, Opts) -> any()` + +Load a device module from its name or a message ID. +Returns {ok, Executable} where Executable is the device module. On error, +a tuple of the form {error, Reason} is returned. + + + +### maybe_force_message/2 * ### + +`maybe_force_message(X1, Opts) -> any()` + +Force the result of a device call into a message if the result is not +requested by the `Opts`. If the result is a literal, we wrap it in a message +and signal the location of the result inside. We also similarly handle ao-result +when the result is a single value and an explicit status code. + + + +### message_to_device/2 ### + +`message_to_device(Msg, Opts) -> any()` + +Extract the device module from a message. + + + +### message_to_fun/3 ### + +`message_to_fun(Msg, Key, Opts) -> any()` + +Calculate the Erlang function that should be called to get a value for +a given key from a device. + +This comes in 7 forms: +1. The message does not specify a device, so we use the default device. +2. The device has a `handler` key in its `Dev:info()` map, which is a +function that takes a key and returns a function to handle that key. We pass +the key as an additional argument to this function. +3. The device has a function of the name `Key`, which should be called +directly. +4. The device does not implement the key, but does have a default handler +for us to call. We pass it the key as an additional argument. +5. The device does not implement the key, and has no default handler. We use +the default device to handle the key. +Error: If the device is specified, but not loadable, we raise an error. + +Returns {ok | add_key, Fun} where Fun is the function to call, and add_key +indicates that the key should be added to the start of the call's arguments. + + + +### normalize_key/1 ### + +`normalize_key(Key) -> any()` + +Convert a key to a binary in normalized form. + + + +### normalize_key/2 ### + +`normalize_key(Key, Opts) -> any()` + + + +### normalize_keys/1 ### + +`normalize_keys(Msg1) -> any()` + +Ensure that a message is processable by the AO-Core resolver: No lists. + + + +### remove/2 ### + +`remove(Msg, Key) -> any()` + +Remove a key from a message, using its underlying device. + + + +### remove/3 ### + +`remove(Msg, Key, Opts) -> any()` + + + +### resolve/2 ### + +`resolve(SingletonMsg, Opts) -> any()` + +Get the value of a message's key by running its associated device +function. Optionally, takes options that control the runtime environment. +This function returns the raw result of the device function call: +`{ok | error, NewMessage}.` +The resolver is composed of a series of discrete phases: +1: Normalization. +2: Cache lookup. +3: Validation check. +4: Persistent-resolver lookup. +5: Device lookup. +6: Execution. +7: Execution of the `step` hook. +8: Subresolution. +9: Cryptographic linking. +10: Result caching. +11: Notify waiters. +12: Fork worker. +13: Recurse or terminate. + + + +### resolve/3 ### + +`resolve(Msg1, Path, Opts) -> any()` + + + +### resolve_many/2 ### + +`resolve_many(ListMsg, Opts) -> any()` + +Resolve a list of messages in sequence. Take the output of the first +message as the input for the next message. Once the last message is resolved, +return the result. +A `resolve_many` call with only a single ID will attempt to read the message +directly from the store. No execution is performed. + + + +### resolve_stage/4 * ### + +`resolve_stage(X1, Raw, Msg2, Opts) -> any()` + + + +### resolve_stage/5 * ### + +`resolve_stage(X1, Msg1, Msg2, ExecName, Opts) -> any()` + + + +### resolve_stage/6 * ### + +`resolve_stage(X1, Func, Msg1, Msg2, ExecName, Opts) -> any()` + + + +### set/2 ### + +`set(Msg1, Msg2) -> any()` + +Shortcut for setting a key in the message using its underlying device. +Like the `get/3` function, this function honors the `error_strategy` option. +`set` works with maps and recursive paths while maintaining the appropriate +`HashPath` for each step. + + + +### set/3 ### + +`set(RawMsg1, RawMsg2, Opts) -> any()` + + + +### set/4 ### + +`set(Msg1, Key, Value, Opts) -> any()` + + + +### subresolve/4 * ### + +`subresolve(RawMsg1, DevID, ReqPath, Opts) -> any()` + +Execute a sub-resolution. + + + +### truncate_args/2 ### + +`truncate_args(Fun, Args) -> any()` + +Truncate the arguments of a function to the number of arguments it +actually takes. + + + +### verify_device_compatibility/2 * ### + +`verify_device_compatibility(Msg, Opts) -> any()` + +Verify that a device is compatible with the current machine. + diff --git a/docs/resources/source-code/hb_ao_test_vectors.md b/docs/resources/source-code/hb_ao_test_vectors.md new file mode 100644 index 000000000..967e70f79 --- /dev/null +++ b/docs/resources/source-code/hb_ao_test_vectors.md @@ -0,0 +1,279 @@ +# [Module hb_ao_test_vectors.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_ao_test_vectors.erl) + + + + +Uses a series of different `Opts` values to test the resolution engine's +execution under different circumstances. + + + +## Function Index ## + + +
as_path_test/1*
basic_get_test/1*
basic_set_test/1*
continue_as_test/1*
deep_recursive_get_test/1*
deep_set_new_messages_test/0*
deep_set_test/1*
deep_set_with_device_test/1*
denormalized_device_key_test/1*
device_excludes_test/1*
device_exports_test/1*
device_with_default_handler_function_test/1*
device_with_handler_function_test/1*
exec_dummy_device/2*Ensure that we can read a device from the cache then execute it.
gen_default_device/0*Create a simple test device that implements the default handler.
gen_handler_device/0*Create a simple test device that implements the handler key.
generate_device_with_keys_using_args/0*Generates a test device with three keys, each of which uses +progressively more of the arguments that can be passed to a device key.
get_as_with_device_test/1*
get_with_device_test/1*
key_from_id_device_with_args_test/1*Test that arguments are passed to a device key as expected.
key_to_binary_test/1*
list_transform_test/1*
load_as_test/1*
load_device_test/0*
recursive_get_test/1*
resolve_binary_key_test/1*
resolve_from_multiple_keys_test/1*
resolve_id_test/1*
resolve_key_twice_test/1*
resolve_path_element_test/1*
resolve_simple_test/1*
run_all_test_/0*Run each test in the file with each set of options.
run_test/0*
set_with_device_test/1*
start_as_test/1*
start_as_with_parameters_test/1*
step_hook_test/1*
test_opts/0*
test_suite/0*
untrusted_load_device_test/0*
+ + + + +## Function Details ## + + + +### as_path_test/1 * ### + +`as_path_test(Opts) -> any()` + + + +### basic_get_test/1 * ### + +`basic_get_test(Opts) -> any()` + + + +### basic_set_test/1 * ### + +`basic_set_test(Opts) -> any()` + + + +### continue_as_test/1 * ### + +`continue_as_test(Opts) -> any()` + + + +### deep_recursive_get_test/1 * ### + +`deep_recursive_get_test(Opts) -> any()` + + + +### deep_set_new_messages_test/0 * ### + +`deep_set_new_messages_test() -> any()` + + + +### deep_set_test/1 * ### + +`deep_set_test(Opts) -> any()` + + + +### deep_set_with_device_test/1 * ### + +`deep_set_with_device_test(Opts) -> any()` + + + +### denormalized_device_key_test/1 * ### + +`denormalized_device_key_test(Opts) -> any()` + + + +### device_excludes_test/1 * ### + +`device_excludes_test(Opts) -> any()` + + + +### device_exports_test/1 * ### + +`device_exports_test(Opts) -> any()` + + + +### device_with_default_handler_function_test/1 * ### + +`device_with_default_handler_function_test(Opts) -> any()` + + + +### device_with_handler_function_test/1 * ### + +`device_with_handler_function_test(Opts) -> any()` + + + +### exec_dummy_device/2 * ### + +`exec_dummy_device(SigningWallet, Opts) -> any()` + +Ensure that we can read a device from the cache then execute it. By +extension, this will also allow us to load a device from Arweave due to the +remote store implementations. + + + +### gen_default_device/0 * ### + +`gen_default_device() -> any()` + +Create a simple test device that implements the default handler. + + + +### gen_handler_device/0 * ### + +`gen_handler_device() -> any()` + +Create a simple test device that implements the handler key. + + + +### generate_device_with_keys_using_args/0 * ### + +`generate_device_with_keys_using_args() -> any()` + +Generates a test device with three keys, each of which uses +progressively more of the arguments that can be passed to a device key. + + + +### get_as_with_device_test/1 * ### + +`get_as_with_device_test(Opts) -> any()` + + + +### get_with_device_test/1 * ### + +`get_with_device_test(Opts) -> any()` + + + +### key_from_id_device_with_args_test/1 * ### + +`key_from_id_device_with_args_test(Opts) -> any()` + +Test that arguments are passed to a device key as expected. +Particularly, we need to ensure that the key function in the device can +specify any arity (1 through 3) and the call is handled correctly. + + + +### key_to_binary_test/1 * ### + +`key_to_binary_test(Opts) -> any()` + + + +### list_transform_test/1 * ### + +`list_transform_test(Opts) -> any()` + + + +### load_as_test/1 * ### + +`load_as_test(Opts) -> any()` + + + +### load_device_test/0 * ### + +`load_device_test() -> any()` + + + +### recursive_get_test/1 * ### + +`recursive_get_test(Opts) -> any()` + + + +### resolve_binary_key_test/1 * ### + +`resolve_binary_key_test(Opts) -> any()` + + + +### resolve_from_multiple_keys_test/1 * ### + +`resolve_from_multiple_keys_test(Opts) -> any()` + + + +### resolve_id_test/1 * ### + +`resolve_id_test(Opts) -> any()` + + + +### resolve_key_twice_test/1 * ### + +`resolve_key_twice_test(Opts) -> any()` + + + +### resolve_path_element_test/1 * ### + +`resolve_path_element_test(Opts) -> any()` + + + +### resolve_simple_test/1 * ### + +`resolve_simple_test(Opts) -> any()` + + + +### run_all_test_/0 * ### + +`run_all_test_() -> any()` + +Run each test in the file with each set of options. Start and reset +the store for each test. + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### set_with_device_test/1 * ### + +`set_with_device_test(Opts) -> any()` + + + +### start_as_test/1 * ### + +`start_as_test(Opts) -> any()` + + + +### start_as_with_parameters_test/1 * ### + +`start_as_with_parameters_test(Opts) -> any()` + + + +### step_hook_test/1 * ### + +`step_hook_test(InitOpts) -> any()` + + + +### test_opts/0 * ### + +`test_opts() -> any()` + + + +### test_suite/0 * ### + +`test_suite() -> any()` + + + +### untrusted_load_device_test/0 * ### + +`untrusted_load_device_test() -> any()` + diff --git a/docs/resources/source-code/hb_app.md b/docs/resources/source-code/hb_app.md new file mode 100644 index 000000000..d926bb894 --- /dev/null +++ b/docs/resources/source-code/hb_app.md @@ -0,0 +1,33 @@ +# [Module hb_app.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_app.erl) + + + + +The main HyperBEAM application module. + +__Behaviours:__ [`application`](application.md). + + + +## Function Index ## + + +
start/2
stop/1
+ + + + +## Function Details ## + + + +### start/2 ### + +`start(StartType, StartArgs) -> any()` + + + +### stop/1 ### + +`stop(State) -> any()` + diff --git a/docs/resources/source-code/hb_beamr.md b/docs/resources/source-code/hb_beamr.md new file mode 100644 index 000000000..71cbace77 --- /dev/null +++ b/docs/resources/source-code/hb_beamr.md @@ -0,0 +1,243 @@ +# [Module hb_beamr.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_beamr.erl) + + + + +BEAMR: A WAMR wrapper for BEAM. + + + +## Description ## + +Beamr is a library that allows you to run WASM modules in BEAM, using the +Webassembly Micro Runtime (WAMR) as its engine. Each WASM module is +executed using a Linked-In Driver (LID) that is loaded into BEAM. It is +designed with a focus on supporting long-running WASM executions that +interact with Erlang functions and processes easily. + +Because each WASM module runs as an independent async worker, if you plan +to run many instances in parallel, you should be sure to configure the +BEAM to have enough async worker threads enabled (see `erl +A N` in the +Erlang manuals). + +The core API is simple: + +``` + + start(WasmBinary) -> {ok, Port, Imports, Exports} + Where: + WasmBinary is the WASM binary to load. + Port is the port to the LID. + Imports is a list of tuples of the form {Module, Function, + Args, Signature}. + Exports is a list of tuples of the form {Function, Args, + Signature}. + stop(Port) -> ok + call(Port, FunctionName, Args) -> {ok, Result} + Where: + FunctionName is the name of the function to call. + Args is a list of Erlang terms (converted to WASM values by + BEAMR) that match the signature of the function. + Result is a list of Erlang terms (converted from WASM values). + call(Port, FunName, Args[, Import, State, Opts]) -> {ok, Res, NewState} + Where: + ImportFun is a function that will be called upon each import. + ImportFun must have an arity of 2: Taking an arbitrary state + term, and a map containing the port, module, func, args,signature, and the options map of the import. + It must return a tuple of the form {ok, Response, NewState}. + serialize(Port) -> {ok, Mem} + Where: + Port is the port to the LID. + Mem is a binary representing the full WASM state. + deserialize(Port, Mem) -> ok + Where: + Port is the port to the LID. + Mem is a binary output of a previous serialize/1 call. +``` + +BEAMR was designed for use in the HyperBEAM project, but is suitable for +deployment in other Erlang applications that need to run WASM modules. PRs +are welcome. + +## Function Index ## + + +
benchmark_test/0*
call/3Call a function in the WASM executor (see moduledoc for more details).
call/4
call/5
call/6
deserialize/2Deserialize a WASM state from a binary.
dispatch_response/2*Check the type of an import response and dispatch it to a Beamr port.
driver_loads_test/0*
imported_function_test/0*Test that imported functions can be called from the WASM module.
is_valid_arg_list/1*Check that a list of arguments is valid for a WASM function call.
load_driver/0*Load the driver for the WASM executor.
monitor_call/4*Synchonously monitor the WASM executor for a call result and any +imports that need to be handled.
multiclient_test/0*Ensure that processes outside of the initial one can interact with +the WASM executor.
serialize/1Serialize the WASM state to a binary.
simple_wasm_test/0*Test standalone hb_beamr correctly after loading a WASM module.
start/1Start a WASM executor context.
start/2
stop/1Stop a WASM executor context.
stub/3Stub import function for the WASM executor.
wasm64_test/0*Test that WASM Memory64 modules load and execute correctly.
wasm_send/2
worker/2*A worker process that is responsible for handling a WASM instance.
+ + + + +## Function Details ## + + + +### benchmark_test/0 * ### + +`benchmark_test() -> any()` + + + +### call/3 ### + +`call(PID, FuncRef, Args) -> any()` + +Call a function in the WASM executor (see moduledoc for more details). + + + +### call/4 ### + +`call(PID, FuncRef, Args, ImportFun) -> any()` + + + +### call/5 ### + +`call(PID, FuncRef, Args, ImportFun, StateMsg) -> any()` + + + +### call/6 ### + +`call(PID, FuncRef, Args, ImportFun, StateMsg, Opts) -> any()` + + + +### deserialize/2 ### + +`deserialize(WASM, Bin) -> any()` + +Deserialize a WASM state from a binary. + + + +### dispatch_response/2 * ### + +`dispatch_response(WASM, Term) -> any()` + +Check the type of an import response and dispatch it to a Beamr port. + + + +### driver_loads_test/0 * ### + +`driver_loads_test() -> any()` + + + +### imported_function_test/0 * ### + +`imported_function_test() -> any()` + +Test that imported functions can be called from the WASM module. + + + +### is_valid_arg_list/1 * ### + +`is_valid_arg_list(Args) -> any()` + +Check that a list of arguments is valid for a WASM function call. + + + +### load_driver/0 * ### + +`load_driver() -> any()` + +Load the driver for the WASM executor. + + + +### monitor_call/4 * ### + +`monitor_call(WASM, ImportFun, StateMsg, Opts) -> any()` + +Synchonously monitor the WASM executor for a call result and any +imports that need to be handled. + + + +### multiclient_test/0 * ### + +`multiclient_test() -> any()` + +Ensure that processes outside of the initial one can interact with +the WASM executor. + + + +### serialize/1 ### + +`serialize(WASM) -> any()` + +Serialize the WASM state to a binary. + + + +### simple_wasm_test/0 * ### + +`simple_wasm_test() -> any()` + +Test standalone `hb_beamr` correctly after loading a WASM module. + + + +### start/1 ### + +`start(WasmBinary) -> any()` + +Start a WASM executor context. Yields a port to the LID, and the +imports and exports of the WASM module. Optionally, specify a mode +(wasm or aot) to indicate the type of WASM module being loaded. + + + +### start/2 ### + +`start(WasmBinary, Mode) -> any()` + + + +### stop/1 ### + +`stop(WASM) -> any()` + +Stop a WASM executor context. + + + +### stub/3 ### + +`stub(Msg1, Msg2, Opts) -> any()` + +Stub import function for the WASM executor. + + + +### wasm64_test/0 * ### + +`wasm64_test() -> any()` + +Test that WASM Memory64 modules load and execute correctly. + + + +### wasm_send/2 ### + +`wasm_send(WASM, Message) -> any()` + + + +### worker/2 * ### + +`worker(Port, Listener) -> any()` + +A worker process that is responsible for handling a WASM instance. +It wraps the WASM port, handling inputs and outputs from the WASM module. +The last sender to the port is always the recipient of its messages, so +be careful to ensure that there is only one active sender to the port at +any time. + diff --git a/docs/resources/source-code/hb_beamr_io.md b/docs/resources/source-code/hb_beamr_io.md new file mode 100644 index 000000000..6576e44da --- /dev/null +++ b/docs/resources/source-code/hb_beamr_io.md @@ -0,0 +1,156 @@ +# [Module hb_beamr_io.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_beamr_io.erl) + + + + +Simple interface for memory management for Beamr instances. + + + +## Description ## + +It allows for reading and writing to memory, as well as allocating and +freeing memory by calling the WASM module's exported malloc and free +functions. + +Unlike the majority of HyperBEAM modules, this module takes a defensive +approach to type checking, breaking from the conventional Erlang style, +such that failures are caught in the Erlang-side of functions rather than +in the C/WASM-side. + +## Function Index ## + + +
do_read_string/3*
free/2Free space allocated in the Beamr instance's native memory via a +call to the exported free function from the WASM.
malloc/2Allocate space for (via an exported malloc function from the WASM) in +the Beamr instance's native memory.
malloc_test/0*Test allocating and freeing memory.
read/3Read a binary from the Beamr instance's native memory at a given offset +and of a given size.
read_string/2Simple helper function to read a string from the Beamr instance's native +memory at a given offset.
read_string/3*
read_test/0*Test reading memory in and out of bounds.
size/1Get the size (in bytes) of the native memory allocated in the Beamr +instance.
size_test/0*
string_write_and_read_test/0*Write and read strings to memory.
write/3Write a binary to the Beamr instance's native memory at a given offset.
write_string/2Simple helper function to allocate space for (via malloc) and write a +string to the Beamr instance's native memory.
write_test/0*Test writing memory in and out of bounds.
+ + + + +## Function Details ## + + + +### do_read_string/3 * ### + +`do_read_string(WASM, Offset, ChunkSize) -> any()` + + + +### free/2 ### + +`free(WASM, Ptr) -> any()` + +Free space allocated in the Beamr instance's native memory via a +call to the exported free function from the WASM. + + + +### malloc/2 ### + +`malloc(WASM, Size) -> any()` + +Allocate space for (via an exported malloc function from the WASM) in +the Beamr instance's native memory. + + + +### malloc_test/0 * ### + +`malloc_test() -> any()` + +Test allocating and freeing memory. + + + +### read/3 ### + +`read(WASM, Offset, Size) -> any()` + +Read a binary from the Beamr instance's native memory at a given offset +and of a given size. + + + +### read_string/2 ### + +`read_string(Port, Offset) -> any()` + +Simple helper function to read a string from the Beamr instance's native +memory at a given offset. Memory is read by default in chunks of 8 bytes, +but this can be overridden by passing a different chunk size. Strings are +assumed to be null-terminated. + + + +### read_string/3 * ### + +`read_string(WASM, Offset, ChunkSize) -> any()` + + + +### read_test/0 * ### + +`read_test() -> any()` + +Test reading memory in and out of bounds. + + + +### size/1 ### + +`size(WASM) -> any()` + +Get the size (in bytes) of the native memory allocated in the Beamr +instance. Note that WASM memory can never be reduced once granted to an +instance (although it can, of course, be reallocated _inside_ the +environment). + + + +### size_test/0 * ### + +`size_test() -> any()` + + + +### string_write_and_read_test/0 * ### + +`string_write_and_read_test() -> any()` + +Write and read strings to memory. + + + +### write/3 ### + +`write(WASM, Offset, Data) -> any()` + +Write a binary to the Beamr instance's native memory at a given offset. + + + +### write_string/2 ### + +`write_string(WASM, Data) -> any()` + +Simple helper function to allocate space for (via malloc) and write a +string to the Beamr instance's native memory. This can be helpful for easily +pushing a string into the instance, such that the resulting pointer can be +passed to exported functions from the instance. +Assumes that the input is either an iolist or a binary, adding a null byte +to the end of the string. + + + +### write_test/0 * ### + +`write_test() -> any()` + +Test writing memory in and out of bounds. + diff --git a/docs/resources/source-code/hb_cache.md b/docs/resources/source-code/hb_cache.md new file mode 100644 index 000000000..dd5030808 --- /dev/null +++ b/docs/resources/source-code/hb_cache.md @@ -0,0 +1,264 @@ +# [Module hb_cache.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_cache.erl) + + + + +A cache of AO-Core protocol messages and compute results. + + + +## Description ## + +HyperBEAM stores all paths in key value stores, abstracted by the `hb_store` +module. Each store has its own storage backend, but each works with simple +key-value pairs. Each store can write binary keys at paths, and link between +paths. + +There are three layers to HyperBEAMs internal data representation on-disk: + +1. The raw binary data, written to the store at the hash of the content. +Storing binary paths in this way effectively deduplicates the data. +2. The hashpath-graph of all content, stored as a set of links between +hashpaths, their keys, and the data that underlies them. This allows +all messages to share the same hashpath space, such that all requests +from users additively fill-in the hashpath space, minimizing duplicated +compute. +3. Messages, referrable by their IDs (committed or uncommitted). These are +stored as a set of links commitment IDs and the uncommitted message. + +Before writing a message to the store, we convert it to Type-Annotated +Binary Messages (TABMs), such that each of the keys in the message is +either a map or a direct binary. + +## Function Index ## + + +
cache_suite_test_/0*
calculate_all_ids/2*Calculate the IDs for a message.
do_read/4*Read a path from the store.
do_write_message/4*
link/3Make a link from one path to another in the store.
list/2List all items under a given path.
list_numbered/2List all items in a directory, assuming they are numbered.
read/2Read the message at a path.
read_resolved/3Read the output of a prior computation, given Msg1, Msg2, and some +options.
run_test/0*
store_read/3*List all of the subpaths of a given path, read each in turn, returning a +flat map.
store_read/4*
test_deeply_nested_complex_message/1*Test deeply nested item storage and retrieval.
test_device_map_cannot_be_written_test/0*Test that message whose device is #{} cannot be written.
test_message_with_message/1*
test_signed/1
test_signed/2*
test_store_ans104_message/1*
test_store_binary/1*
test_store_simple_signed_message/1*Test storing and retrieving a simple unsigned item.
test_store_simple_unsigned_message/1*Test storing and retrieving a simple unsigned item.
test_store_unsigned_empty_message/1*
test_unsigned/1
to_integer/1*
write/2Write a message to the cache.
write_binary/3Write a raw binary keys into the store and link it at a given hashpath.
write_binary/4*
write_hashpath/2Write a hashpath and its message to the store and link it.
write_hashpath/3*
+ + + + +## Function Details ## + + + +### cache_suite_test_/0 * ### + +`cache_suite_test_() -> any()` + + + +### calculate_all_ids/2 * ### + +`calculate_all_ids(Bin, Opts) -> any()` + +Calculate the IDs for a message. + + + +### do_read/4 * ### + +`do_read(Path, Store, Opts, AlreadyRead) -> any()` + +Read a path from the store. Unsafe: May recurse indefinitely if circular +links are present. + + + +### do_write_message/4 * ### + +`do_write_message(Bin, AllIDs, Store, Opts) -> any()` + + + +### link/3 ### + +`link(Existing, New, Opts) -> any()` + +Make a link from one path to another in the store. +Note: Argument order is `link(Src, Dst, Opts)`. + + + +### list/2 ### + +`list(Path, Opts) -> any()` + +List all items under a given path. + + + +### list_numbered/2 ### + +`list_numbered(Path, Opts) -> any()` + +List all items in a directory, assuming they are numbered. + + + +### read/2 ### + +`read(Path, Opts) -> any()` + +Read the message at a path. Returns in `structured@1.0` format: Either a +richly typed map or a direct binary. + + + +### read_resolved/3 ### + +`read_resolved(MsgID1, MsgID2, Opts) -> any()` + +Read the output of a prior computation, given Msg1, Msg2, and some +options. + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### store_read/3 * ### + +`store_read(Path, Store, Opts) -> any()` + +List all of the subpaths of a given path, read each in turn, returning a +flat map. We track the paths that we have already read to avoid circular +links. + + + +### store_read/4 * ### + +`store_read(Path, Store, Opts, AlreadyRead) -> any()` + + + +### test_deeply_nested_complex_message/1 * ### + +`test_deeply_nested_complex_message(Opts) -> any()` + +Test deeply nested item storage and retrieval + + + +### test_device_map_cannot_be_written_test/0 * ### + +`test_device_map_cannot_be_written_test() -> any()` + +Test that message whose device is `#{}` cannot be written. If it were to +be written, it would cause an infinite loop. + + + +### test_message_with_message/1 * ### + +`test_message_with_message(Opts) -> any()` + + + +### test_signed/1 ### + +`test_signed(Data) -> any()` + + + +### test_signed/2 * ### + +`test_signed(Data, Wallet) -> any()` + + + +### test_store_ans104_message/1 * ### + +`test_store_ans104_message(Opts) -> any()` + + + +### test_store_binary/1 * ### + +`test_store_binary(Opts) -> any()` + + + +### test_store_simple_signed_message/1 * ### + +`test_store_simple_signed_message(Opts) -> any()` + +Test storing and retrieving a simple unsigned item + + + +### test_store_simple_unsigned_message/1 * ### + +`test_store_simple_unsigned_message(Opts) -> any()` + +Test storing and retrieving a simple unsigned item + + + +### test_store_unsigned_empty_message/1 * ### + +`test_store_unsigned_empty_message(Opts) -> any()` + + + +### test_unsigned/1 ### + +`test_unsigned(Data) -> any()` + + + +### to_integer/1 * ### + +`to_integer(Value) -> any()` + + + +### write/2 ### + +`write(RawMsg, Opts) -> any()` + +Write a message to the cache. For raw binaries, we write the data at +the hashpath of the data (by default the SHA2-256 hash of the data). We link +the unattended ID's hashpath for the keys (including `/commitments`) on the +message to the underlying data and recurse. We then link each commitment ID +to the uncommitted message, such that any of the committed or uncommitted IDs +can be read, and once in memory all of the commitments are available. For +deep messages, the commitments will also be read, such that the ID of the +outer message (which does not include its commitments) will be built upon +the commitments of the inner messages. We do not, however, store the IDs from +commitments on signed _inner_ messages. We may wish to revisit this. + + + +### write_binary/3 ### + +`write_binary(Hashpath, Bin, Opts) -> any()` + +Write a raw binary keys into the store and link it at a given hashpath. + + + +### write_binary/4 * ### + +`write_binary(Hashpath, Bin, Store, Opts) -> any()` + + + +### write_hashpath/2 ### + +`write_hashpath(Msg, Opts) -> any()` + +Write a hashpath and its message to the store and link it. + + + +### write_hashpath/3 * ### + +`write_hashpath(HP, Msg, Opts) -> any()` + diff --git a/docs/resources/source-code/hb_cache_control.md b/docs/resources/source-code/hb_cache_control.md new file mode 100644 index 000000000..3a08663a9 --- /dev/null +++ b/docs/resources/source-code/hb_cache_control.md @@ -0,0 +1,243 @@ +# [Module hb_cache_control.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_cache_control.erl) + + + + +Cache control logic for the AO-Core resolver. + + + +## Description ## +It derives cache settings +from request, response, execution-local node Opts, as well as the global +node Opts. It applies these settings when asked to maybe store/lookup in +response to a request. + +## Function Index ## + + +
cache_binary_result_test/0*
cache_message_result_test/0*
cache_source_to_cache_settings/1*Convert a cache source to a cache setting.
derive_cache_settings/2*Derive cache settings from a series of option sources and the opts, +honoring precidence order.
dispatch_cache_write/4*Dispatch the cache write to a worker process if requested.
empty_message_list_test/0*
exec_likely_faster_heuristic/3*Determine whether we are likely to be faster looking up the result in +our cache (hoping we have it), or executing it directly.
hashpath_ignore_prevents_storage_test/0*
is_explicit_lookup/3*
lookup/3*
maybe_lookup/3Handles cache lookup, modulated by the caching options requested by +the user.
maybe_set/2*Takes a key and two maps, returning the first map with the key set to +the value of the second map _if_ the value is not undefined.
maybe_store/4Write a resulting M3 message to the cache if requested.
message_source_cache_control_test/0*
message_without_cache_control_test/0*
msg_precidence_overrides_test/0*
msg_with_cc/1*
multiple_directives_test/0*
necessary_messages_not_found_error/3*Generate a message to return when the necessary messages to execute a +cache lookup are not found in the cache.
no_cache_directive_test/0*
no_store_directive_test/0*
only_if_cached_directive_test/0*
only_if_cached_not_found_error/3*Generate a message to return when only_if_cached was specified, and +we don't have a cached result.
opts_override_message_settings_test/0*
opts_source_cache_control_test/0*
opts_with_cc/1*
specifiers_to_cache_settings/1*Convert a cache control list as received via HTTP headers into a +normalized map of simply whether we should store and/or lookup the result.
+ + + + +## Function Details ## + + + +### cache_binary_result_test/0 * ### + +`cache_binary_result_test() -> any()` + + + +### cache_message_result_test/0 * ### + +`cache_message_result_test() -> any()` + + + +### cache_source_to_cache_settings/1 * ### + +`cache_source_to_cache_settings(Msg) -> any()` + +Convert a cache source to a cache setting. The setting _must_ always be +directly in the source, not an AO-Core-derivable value. The +`to_cache_control_map` function is used as the source of settings in all +cases, except where an `Opts` specifies that hashpaths should not be updated, +which leads to the result not being cached (as it may be stored with an +incorrect hashpath). + + + +### derive_cache_settings/2 * ### + +`derive_cache_settings(SourceList, Opts) -> any()` + +Derive cache settings from a series of option sources and the opts, +honoring precidence order. The Opts is used as the first source. Returns a +map with `store` and `lookup` keys, each of which is a boolean. + +For example, if the last source has a `no_store`, the first expresses no +preference, but the Opts has `cache_control => [always]`, then the result +will contain a `store => true` entry. + + + +### dispatch_cache_write/4 * ### + +`dispatch_cache_write(Msg1, Msg2, Msg3, Opts) -> any()` + +Dispatch the cache write to a worker process if requested. +Invoke the appropriate cache write function based on the type of the message. + + + +### empty_message_list_test/0 * ### + +`empty_message_list_test() -> any()` + + + +### exec_likely_faster_heuristic/3 * ### + +`exec_likely_faster_heuristic(Msg1, Msg2, Opts) -> any()` + +Determine whether we are likely to be faster looking up the result in +our cache (hoping we have it), or executing it directly. + + + +### hashpath_ignore_prevents_storage_test/0 * ### + +`hashpath_ignore_prevents_storage_test() -> any()` + + + +### is_explicit_lookup/3 * ### + +`is_explicit_lookup(Msg1, X2, Opts) -> any()` + + + +### lookup/3 * ### + +`lookup(Msg1, Msg2, Opts) -> any()` + + + +### maybe_lookup/3 ### + +`maybe_lookup(Msg1, Msg2, Opts) -> any()` + +Handles cache lookup, modulated by the caching options requested by +the user. Honors the following `Opts` cache keys: +`only_if_cached`: If set and we do not find a result in the cache, +return an error with a `Cache-Status` of `miss` and +a 504 `Status`. +`no_cache`: If set, the cached values are never used. Returns +`continue` to the caller. + + + +### maybe_set/2 * ### + +`maybe_set(Map1, Map2) -> any()` + +Takes a key and two maps, returning the first map with the key set to +the value of the second map _if_ the value is not undefined. + + + +### maybe_store/4 ### + +`maybe_store(Msg1, Msg2, Msg3, Opts) -> any()` + +Write a resulting M3 message to the cache if requested. The precedence +order of cache control sources is as follows: +1. The `Opts` map (letting the node operator have the final say). +2. The `Msg3` results message (granted by Msg1's device). +3. The `Msg2` message (the user's request). +Msg1 is not used, such that it can specify cache control information about +itself, without affecting its outputs. + + + +### message_source_cache_control_test/0 * ### + +`message_source_cache_control_test() -> any()` + + + +### message_without_cache_control_test/0 * ### + +`message_without_cache_control_test() -> any()` + + + +### msg_precidence_overrides_test/0 * ### + +`msg_precidence_overrides_test() -> any()` + + + +### msg_with_cc/1 * ### + +`msg_with_cc(CC) -> any()` + + + +### multiple_directives_test/0 * ### + +`multiple_directives_test() -> any()` + + + +### necessary_messages_not_found_error/3 * ### + +`necessary_messages_not_found_error(Msg1, Msg2, Opts) -> any()` + +Generate a message to return when the necessary messages to execute a +cache lookup are not found in the cache. + + + +### no_cache_directive_test/0 * ### + +`no_cache_directive_test() -> any()` + + + +### no_store_directive_test/0 * ### + +`no_store_directive_test() -> any()` + + + +### only_if_cached_directive_test/0 * ### + +`only_if_cached_directive_test() -> any()` + + + +### only_if_cached_not_found_error/3 * ### + +`only_if_cached_not_found_error(Msg1, Msg2, Opts) -> any()` + +Generate a message to return when `only_if_cached` was specified, and +we don't have a cached result. + + + +### opts_override_message_settings_test/0 * ### + +`opts_override_message_settings_test() -> any()` + + + +### opts_source_cache_control_test/0 * ### + +`opts_source_cache_control_test() -> any()` + + + +### opts_with_cc/1 * ### + +`opts_with_cc(CC) -> any()` + + + +### specifiers_to_cache_settings/1 * ### + +`specifiers_to_cache_settings(CCSpecifier) -> any()` + +Convert a cache control list as received via HTTP headers into a +normalized map of simply whether we should store and/or lookup the result. + diff --git a/docs/resources/source-code/hb_cache_render.md b/docs/resources/source-code/hb_cache_render.md new file mode 100644 index 000000000..dd12d54f2 --- /dev/null +++ b/docs/resources/source-code/hb_cache_render.md @@ -0,0 +1,181 @@ +# [Module hb_cache_render.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_cache_render.erl) + + + + +A module that helps to render given Key graphs into the .dot files. + + + +## Function Index ## + + +
add_arc/4*Add an arc to the graph.
add_node/3*Add a node to the graph.
cache_path_to_dot/2Generate a dot file from a cache path and options/store.
cache_path_to_dot/3
cache_path_to_graph/3Main function to collect graph elements.
collect_output/2*Helper function to collect output from port.
dot_to_svg/1Convert a dot graph to SVG format.
extract_label/1*Extract a label from a path.
get_graph_data/1Get graph data for the Three.js visualization.
get_label/1*Extract a readable label from a path.
get_node_type/1*Convert node color from hb_cache_render to node type for visualization.
graph_to_dot/1*Generate the DOT file from the graph.
prepare_deeply_nested_complex_message/0
prepare_signed_data/0
prepare_unsigned_data/0
process_composite_node/6*Process a composite (directory) node.
process_simple_node/6*Process a simple (leaf) node.
render/1Render the given Key into svg.
render/2
test_signed/2*
test_unsigned/1*
traverse_store/4*Traverse the store recursively to build the graph.
+ + + + +## Function Details ## + + + +### add_arc/4 * ### + +`add_arc(Graph, From, To, Label) -> any()` + +Add an arc to the graph + + + +### add_node/3 * ### + +`add_node(Graph, ID, Color) -> any()` + +Add a node to the graph + + + +### cache_path_to_dot/2 ### + +`cache_path_to_dot(ToRender, StoreOrOpts) -> any()` + +Generate a dot file from a cache path and options/store + + + +### cache_path_to_dot/3 ### + +`cache_path_to_dot(ToRender, RenderOpts, StoreOrOpts) -> any()` + + + +### cache_path_to_graph/3 ### + +`cache_path_to_graph(ToRender, GraphOpts, StoreOrOpts) -> any()` + +Main function to collect graph elements + + + +### collect_output/2 * ### + +`collect_output(Port, Acc) -> any()` + +Helper function to collect output from port + + + +### dot_to_svg/1 ### + +`dot_to_svg(DotInput) -> any()` + +Convert a dot graph to SVG format + + + +### extract_label/1 * ### + +`extract_label(Path) -> any()` + +Extract a label from a path + + + +### get_graph_data/1 ### + +`get_graph_data(Opts) -> any()` + +Get graph data for the Three.js visualization + + + +### get_label/1 * ### + +`get_label(Path) -> any()` + +Extract a readable label from a path + + + +### get_node_type/1 * ### + +`get_node_type(Color) -> any()` + +Convert node color from hb_cache_render to node type for visualization + + + +### graph_to_dot/1 * ### + +`graph_to_dot(Graph) -> any()` + +Generate the DOT file from the graph + + + +### prepare_deeply_nested_complex_message/0 ### + +`prepare_deeply_nested_complex_message() -> any()` + + + +### prepare_signed_data/0 ### + +`prepare_signed_data() -> any()` + + + +### prepare_unsigned_data/0 ### + +`prepare_unsigned_data() -> any()` + + + +### process_composite_node/6 * ### + +`process_composite_node(Store, Key, Parent, ResolvedPath, JoinedPath, Graph) -> any()` + +Process a composite (directory) node + + + +### process_simple_node/6 * ### + +`process_simple_node(Store, Key, Parent, ResolvedPath, JoinedPath, Graph) -> any()` + +Process a simple (leaf) node + + + +### render/1 ### + +`render(StoreOrOpts) -> any()` + +Render the given Key into svg + + + +### render/2 ### + +`render(ToRender, StoreOrOpts) -> any()` + + + +### test_signed/2 * ### + +`test_signed(Data, Wallet) -> any()` + + + +### test_unsigned/1 * ### + +`test_unsigned(Data) -> any()` + + + +### traverse_store/4 * ### + +`traverse_store(Store, Key, Parent, Graph) -> any()` + +Traverse the store recursively to build the graph + diff --git a/docs/resources/source-code/hb_client.md b/docs/resources/source-code/hb_client.md new file mode 100644 index 000000000..281c3635f --- /dev/null +++ b/docs/resources/source-code/hb_client.md @@ -0,0 +1,98 @@ +# [Module hb_client.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_client.erl) + + + + + + +## Function Index ## + + +
add_route/3
arweave_timestamp/0Grab the latest block information from the Arweave gateway node.
prefix_keys/3*
resolve/4Resolve a message pair on a remote node.
routes/2
upload/2Upload a data item to the bundler node.
upload/3*
upload_empty_message_test/0*
upload_empty_raw_ans104_test/0*
upload_raw_ans104_test/0*
upload_raw_ans104_with_anchor_test/0*
upload_single_layer_message_test/0*
+ + + + +## Function Details ## + + + +### add_route/3 ### + +`add_route(Node, Route, Opts) -> any()` + + + +### arweave_timestamp/0 ### + +`arweave_timestamp() -> any()` + +Grab the latest block information from the Arweave gateway node. + + + +### prefix_keys/3 * ### + +`prefix_keys(Prefix, Message, Opts) -> any()` + + + +### resolve/4 ### + +`resolve(Node, Msg1, Msg2, Opts) -> any()` + +Resolve a message pair on a remote node. +The message pair is first transformed into a singleton request, by +prefixing the keys in both messages for the path segment that they relate to, +and then adjusting the "Path" field from the second message. + + + +### routes/2 ### + +`routes(Node, Opts) -> any()` + + + +### upload/2 ### + +`upload(Msg, Opts) -> any()` + +Upload a data item to the bundler node. + + + +### upload/3 * ### + +`upload(Msg, Opts, X3) -> any()` + + + +### upload_empty_message_test/0 * ### + +`upload_empty_message_test() -> any()` + + + +### upload_empty_raw_ans104_test/0 * ### + +`upload_empty_raw_ans104_test() -> any()` + + + +### upload_raw_ans104_test/0 * ### + +`upload_raw_ans104_test() -> any()` + + + +### upload_raw_ans104_with_anchor_test/0 * ### + +`upload_raw_ans104_with_anchor_test() -> any()` + + + +### upload_single_layer_message_test/0 * ### + +`upload_single_layer_message_test() -> any()` + diff --git a/docs/resources/source-code/hb_crypto.md b/docs/resources/source-code/hb_crypto.md new file mode 100644 index 000000000..110c13a17 --- /dev/null +++ b/docs/resources/source-code/hb_crypto.md @@ -0,0 +1,81 @@ +# [Module hb_crypto.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_crypto.erl) + + + + +Implements the cryptographic functions and wraps the primitives +used in HyperBEAM. + + + +## Description ## + +Abstracted such that this (extremely!) dangerous code +can be carefully managed. + +HyperBEAM currently implements two hashpath algorithms: + +* `sha-256-chain`: A simple chained SHA-256 hash. + +* `accumulate-256`: A SHA-256 hash that chains the given IDs and accumulates +their values into a single commitment. + +The accumulate algorithm is experimental and at this point only exists to +allow us to test multiple HashPath algorithms in HyperBEAM. + +## Function Index ## + + +
accumulate/2Accumulate two IDs into a single commitment.
count_zeroes/1*Count the number of leading zeroes in a bitstring.
sha256/1Wrap Erlang's crypto:hash/2 to provide a standard interface.
sha256_chain/2Add a new ID to the end of a SHA-256 hash chain.
sha256_chain_test/0*Check that sha-256-chain correctly produces a hash matching +the machine's OpenSSL lib's output.
+ + + + +## Function Details ## + + + +### accumulate/2 ### + +`accumulate(ID1, ID2) -> any()` + +Accumulate two IDs into a single commitment. +Experimental! This is not necessarily a cryptographically-secure operation. + + + +### count_zeroes/1 * ### + +`count_zeroes(X1) -> any()` + +Count the number of leading zeroes in a bitstring. + + + +### sha256/1 ### + +`sha256(Data) -> any()` + +Wrap Erlang's `crypto:hash/2` to provide a standard interface. +Under-the-hood, this uses OpenSSL. + + + +### sha256_chain/2 ### + +`sha256_chain(ID1, ID2) -> any()` + +Add a new ID to the end of a SHA-256 hash chain. + + + +### sha256_chain_test/0 * ### + +`sha256_chain_test() -> any()` + +Check that `sha-256-chain` correctly produces a hash matching +the machine's OpenSSL lib's output. Further (in case of a bug in our +or Erlang's usage of OpenSSL), check that the output has at least has +a high level of entropy. + diff --git a/docs/resources/source-code/hb_debugger.md b/docs/resources/source-code/hb_debugger.md new file mode 100644 index 000000000..04b278c31 --- /dev/null +++ b/docs/resources/source-code/hb_debugger.md @@ -0,0 +1,104 @@ +# [Module hb_debugger.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_debugger.erl) + + + + +A module that provides bootstrapping interfaces for external debuggers +to connect to HyperBEAM. + + + +## Description ## + +The simplest way to utilize an external graphical debugger is to use the +`erlang-ls` extension for VS Code, Emacs, or other Language Server Protocol +(LSP) compatible editors. This repository contains a `launch.json` +configuration file for VS Code that can be used to spawn a new HyperBEAM, +attach the debugger to it, and execute the specified `Module:Function(Args)`. +Additionally, the node can be started with `rebar3 debugging` in order to +allow access to the console while also allowing the debugger to attach. + +Boot time is approximately 10 seconds. + +## Function Index ## + + +
await_breakpoint/0Await a new breakpoint being set by the debugger.
await_breakpoint/1*
await_debugger/0*Await a debugger to be attached to the node.
await_debugger/1*
interpret/1*Attempt to interpret a specified module to load it into the debugger.
is_debugging_node_connected/0*Is another Distributed Erlang node connected to us?.
start/0
start_and_break/2A bootstrapping function to wait for an external debugger to be attached, +then add a breakpoint on the specified Module:Function(Args), then call it.
start_and_break/3
+ + + + +## Function Details ## + + + +### await_breakpoint/0 ### + +`await_breakpoint() -> any()` + +Await a new breakpoint being set by the debugger. + + + +### await_breakpoint/1 * ### + +`await_breakpoint(N) -> any()` + + + +### await_debugger/0 * ### + +`await_debugger() -> any()` + +Await a debugger to be attached to the node. + + + +### await_debugger/1 * ### + +`await_debugger(N) -> any()` + + + +### interpret/1 * ### + +`interpret(Module) -> any()` + +Attempt to interpret a specified module to load it into the debugger. +`int:i/1` seems to have an issue that will cause it to fail sporadically +with `error:undef` on some modules. This error appears not to be catchable +through the normal means. Subsequently, we attempt the load in a separate +process and wait for it to complete. If we do not receive a response in a +reasonable amount of time, we assume that the module failed to load and +return `false`. + + + +### is_debugging_node_connected/0 * ### + +`is_debugging_node_connected() -> any()` + +Is another Distributed Erlang node connected to us? + + + +### start/0 ### + +`start() -> any()` + + + +### start_and_break/2 ### + +`start_and_break(Module, Function) -> any()` + +A bootstrapping function to wait for an external debugger to be attached, +then add a breakpoint on the specified `Module:Function(Args)`, then call it. + + + +### start_and_break/3 ### + +`start_and_break(Module, Function, Args) -> any()` + diff --git a/docs/resources/source-code/hb_escape.md b/docs/resources/source-code/hb_escape.md new file mode 100644 index 000000000..f041c669c --- /dev/null +++ b/docs/resources/source-code/hb_escape.md @@ -0,0 +1,118 @@ +# [Module hb_escape.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_escape.erl) + + + + +Escape and unescape mixed case values for use in HTTP headers. + + + +## Description ## +This is necessary for encodings of AO-Core messages for transmission in +HTTP/2 and HTTP/3, because uppercase header keys are explicitly disallowed. +While most map keys in HyperBEAM are normalized to lowercase, IDs are not. +Subsequently, we encode all header keys to lowercase %-encoded URI-style +strings because transmission. + +## Function Index ## + + +
decode/1Decode a URI-encoded string back to a binary.
decode_keys/1Return a message with all of its keys decoded.
encode/1Encode a binary as a URI-encoded string.
encode_keys/1URI encode keys in the base layer of a message.
escape_byte/1*Escape a single byte as a URI-encoded string.
escape_unescape_identity_test/0*
escape_unescape_special_chars_test/0*
hex_digit/1*
hex_value/1*
percent_escape/1*Escape a list of characters as a URI-encoded string.
percent_unescape/1*Unescape a URI-encoded string.
unescape_specific_test/0*
uppercase_test/0*
+ + + + +## Function Details ## + + + +### decode/1 ### + +`decode(Bin) -> any()` + +Decode a URI-encoded string back to a binary. + + + +### decode_keys/1 ### + +`decode_keys(Msg) -> any()` + +Return a message with all of its keys decoded. + + + +### encode/1 ### + +`encode(Bin) -> any()` + +Encode a binary as a URI-encoded string. + + + +### encode_keys/1 ### + +`encode_keys(Msg) -> any()` + +URI encode keys in the base layer of a message. Does not recurse. + + + +### escape_byte/1 * ### + +`escape_byte(C) -> any()` + +Escape a single byte as a URI-encoded string. + + + +### escape_unescape_identity_test/0 * ### + +`escape_unescape_identity_test() -> any()` + + + +### escape_unescape_special_chars_test/0 * ### + +`escape_unescape_special_chars_test() -> any()` + + + +### hex_digit/1 * ### + +`hex_digit(N) -> any()` + + + +### hex_value/1 * ### + +`hex_value(C) -> any()` + + + +### percent_escape/1 * ### + +`percent_escape(Cs) -> any()` + +Escape a list of characters as a URI-encoded string. + + + +### percent_unescape/1 * ### + +`percent_unescape(Cs) -> any()` + +Unescape a URI-encoded string. + + + +### unescape_specific_test/0 * ### + +`unescape_specific_test() -> any()` + + + +### uppercase_test/0 * ### + +`uppercase_test() -> any()` + diff --git a/docs/resources/source-code/hb_event.md b/docs/resources/source-code/hb_event.md new file mode 100644 index 000000000..e00af0725 --- /dev/null +++ b/docs/resources/source-code/hb_event.md @@ -0,0 +1,102 @@ +# [Module hb_event.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_event.erl) + + + + +Wrapper for incrementing prometheus counters. + + + +## Function Index ## + + +
await_prometheus_started/0*Delay the event server until prometheus is started.
handle_events/0*
handle_tracer/3*
increment/3Increment the counter for the given topic and message.
log/1Debugging log logging function.
log/2
log/3
log/4
log/5
log/6
parse_name/1*
server/0*
+ + + + +## Function Details ## + + + +### await_prometheus_started/0 * ### + +`await_prometheus_started() -> any()` + +Delay the event server until prometheus is started. + + + +### handle_events/0 * ### + +`handle_events() -> any()` + + + +### handle_tracer/3 * ### + +`handle_tracer(Topic, X, Opts) -> any()` + + + +### increment/3 ### + +`increment(Topic, Message, Opts) -> any()` + +Increment the counter for the given topic and message. Registers the +counter if it doesn't exist. If the topic is `global`, the message is ignored. +This means that events must specify a topic if they want to be counted, +filtering debug messages. Similarly, events with a topic that begins with +`debug` are ignored. + + + +### log/1 ### + +`log(X) -> any()` + +Debugging log logging function. For now, it just prints to standard +error. + + + +### log/2 ### + +`log(Topic, X) -> any()` + + + +### log/3 ### + +`log(Topic, X, Mod) -> any()` + + + +### log/4 ### + +`log(Topic, X, Mod, Func) -> any()` + + + +### log/5 ### + +`log(Topic, X, Mod, Func, Line) -> any()` + + + +### log/6 ### + +`log(Topic, X, Mod, Func, Line, Opts) -> any()` + + + +### parse_name/1 * ### + +`parse_name(Name) -> any()` + + + +### server/0 * ### + +`server() -> any()` + diff --git a/docs/resources/source-code/hb_examples.md b/docs/resources/source-code/hb_examples.md new file mode 100644 index 000000000..c5fc7b2ca --- /dev/null +++ b/docs/resources/source-code/hb_examples.md @@ -0,0 +1,76 @@ +# [Module hb_examples.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_examples.erl) + + + + +This module contains end-to-end tests for Hyperbeam, accessing through +the HTTP interface. + + + +## Description ## +As well as testing the system, you can use these tests +as examples of how to interact with HyperBEAM nodes. + +## Function Index ## + + +
create_schedule_aos2_test_disabled/0*
paid_wasm_test/0*Gain signed WASM responses from a node and verify them.
relay_with_payments_test/0*Start a node running the simple pay meta device, and use it to relay +a message for a client.
schedule/2*
schedule/3*
schedule/4*
+ + + + +## Function Details ## + + + +### create_schedule_aos2_test_disabled/0 * ### + +`create_schedule_aos2_test_disabled() -> any()` + + + +### paid_wasm_test/0 * ### + +`paid_wasm_test() -> any()` + +Gain signed WASM responses from a node and verify them. +1. Start the client with a small balance. +2. Execute a simple WASM function on the host node. +3. Verify the response is correct and signed by the host node. +4. Get the balance of the client and verify it has been deducted. + + + +### relay_with_payments_test/0 * ### + +`relay_with_payments_test() -> any()` + +Start a node running the simple pay meta device, and use it to relay +a message for a client. We must ensure: +1. When the client has no balance, the relay fails. +2. The operator is able to topup for the client. +3. The client has the correct balance after the topup. +4. The relay succeeds when the client has enough balance. +5. The received message is signed by the host using http-sig and validates +correctly. + + + +### schedule/2 * ### + +`schedule(ProcMsg, Target) -> any()` + + + +### schedule/3 * ### + +`schedule(ProcMsg, Target, Wallet) -> any()` + + + +### schedule/4 * ### + +`schedule(ProcMsg, Target, Wallet, Node) -> any()` + diff --git a/docs/resources/source-code/hb_features.md b/docs/resources/source-code/hb_features.md new file mode 100644 index 000000000..1bb652ad2 --- /dev/null +++ b/docs/resources/source-code/hb_features.md @@ -0,0 +1,64 @@ +# [Module hb_features.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_features.erl) + + + + +A module that exports a list of feature flags that the node supports +using the `-ifdef` macro. + + + +## Description ## +As a consequence, this module acts as a proxy of information between the +build system and the runtime execution environment. + +## Function Index ## + + +
all/0Returns a list of all feature flags that the node supports.
enabled/1Returns true if the feature flag is enabled.
genesis_wasm/0
http3/0
rocksdb/0
test/0
+ + + + +## Function Details ## + + + +### all/0 ### + +`all() -> any()` + +Returns a list of all feature flags that the node supports. + + + +### enabled/1 ### + +`enabled(Feature) -> any()` + +Returns true if the feature flag is enabled. + + + +### genesis_wasm/0 ### + +`genesis_wasm() -> any()` + + + +### http3/0 ### + +`http3() -> any()` + + + +### rocksdb/0 ### + +`rocksdb() -> any()` + + + +### test/0 ### + +`test() -> any()` + diff --git a/docs/resources/source-code/hb_gateway_client.md b/docs/resources/source-code/hb_gateway_client.md new file mode 100644 index 000000000..6a072f0bb --- /dev/null +++ b/docs/resources/source-code/hb_gateway_client.md @@ -0,0 +1,176 @@ +# [Module hb_gateway_client.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_gateway_client.erl) + + + + +Implementation of Arweave's GraphQL API to gain access to specific +items of data stored on the network. + + + +## Description ## +This module must be used to get full HyperBEAM `structured@1.0` form messages +from data items stored on the network, as Arweave gateways do not presently +expose all necessary fields to retrieve this information outside of the +GraphQL API. When gateways integrate serving in `httpsig@1.0` form, this +module will be deprecated. + +## Function Index ## + + +
ans104_no_data_item_test/0*
ao_dataitem_test/0*Test optimistic index.
data/2Get the data associated with a transaction by its ID, using the node's +Arweave gateway peers.
decode_id_or_null/1*
decode_or_null/1*
item_spec/0*Gives the fields of a transaction that are needed to construct an +ANS-104 message.
l1_transaction_test/0*Test l1 message from graphql.
l2_dataitem_test/0*Test l2 message from graphql.
normalize_null/1*
query/2*Run a GraphQL request encoded as a binary.
read/2Get a data item (including data and tags) by its ID, using the node's +GraphQL peers.
result_to_message/2Takes a GraphQL item node, matches it with the appropriate data from a +gateway, then returns {ok, ParsedMsg}.
result_to_message/3*
scheduler_location/2Find the location of the scheduler based on its ID, through GraphQL.
scheduler_location_test/0*Test that we can get the scheduler location.
subindex_to_tags/1*Takes a list of messages with name and value fields, and formats +them as a GraphQL tags argument.
+ + + + +## Function Details ## + + + +### ans104_no_data_item_test/0 * ### + +`ans104_no_data_item_test() -> any()` + + + +### ao_dataitem_test/0 * ### + +`ao_dataitem_test() -> any()` + +Test optimistic index + + + +### data/2 ### + +`data(ID, Opts) -> any()` + +Get the data associated with a transaction by its ID, using the node's +Arweave `gateway` peers. The item is expected to be available in its +unmodified (by caches or other proxies) form at the following location: +https:///raw/ +where `<id>` is the base64-url-encoded transaction ID. + + + +### decode_id_or_null/1 * ### + +`decode_id_or_null(Bin) -> any()` + + + +### decode_or_null/1 * ### + +`decode_or_null(Bin) -> any()` + + + +### item_spec/0 * ### + +`item_spec() -> any()` + +Gives the fields of a transaction that are needed to construct an +ANS-104 message. + + + +### l1_transaction_test/0 * ### + +`l1_transaction_test() -> any()` + +Test l1 message from graphql + + + +### l2_dataitem_test/0 * ### + +`l2_dataitem_test() -> any()` + +Test l2 message from graphql + + + +### normalize_null/1 * ### + +`normalize_null(Bin) -> any()` + + + +### query/2 * ### + +`query(Query, Opts) -> any()` + +Run a GraphQL request encoded as a binary. The node message may contain +a list of URLs to use, optionally as a tuple with an additional map of options +to use for the request. + + + +### read/2 ### + +`read(ID, Opts) -> any()` + +Get a data item (including data and tags) by its ID, using the node's +GraphQL peers. +It uses the following GraphQL schema: +type Transaction { +id: ID! +anchor: String! +signature: String! +recipient: String! +owner: Owner { address: String! key: String! }! +fee: Amount! +quantity: Amount! +data: MetaData! +tags: [Tag { name: String! value: String! }!]! +} +type Amount { +winston: String! +ar: String! +} + + + +### result_to_message/2 ### + +`result_to_message(Item, Opts) -> any()` + +Takes a GraphQL item node, matches it with the appropriate data from a +gateway, then returns `{ok, ParsedMsg}`. + + + +### result_to_message/3 * ### + +`result_to_message(ExpectedID, Item, Opts) -> any()` + + + +### scheduler_location/2 ### + +`scheduler_location(Address, Opts) -> any()` + +Find the location of the scheduler based on its ID, through GraphQL. + + + +### scheduler_location_test/0 * ### + +`scheduler_location_test() -> any()` + +Test that we can get the scheduler location. + + + +### subindex_to_tags/1 * ### + +`subindex_to_tags(Subindex) -> any()` + +Takes a list of messages with `name` and `value` fields, and formats +them as a GraphQL `tags` argument. + diff --git a/docs/resources/source-code/hb_http.md b/docs/resources/source-code/hb_http.md new file mode 100644 index 000000000..e364fdf86 --- /dev/null +++ b/docs/resources/source-code/hb_http.md @@ -0,0 +1,396 @@ +# [Module hb_http.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http.erl) + + + + + + +## Function Index ## + + +
accept_to_codec/2Calculate the codec name to use for a reply given its initiating Cowboy +request, the parsed TABM request, and the response message.
add_cors_headers/2*Add permissive CORS headers to a message, if the message has not already +specified CORS headers.
allowed_status/2*Check if a status is allowed, according to the configuration.
ans104_wasm_test/0*
codec_to_content_type/2*Call the content-type key on a message with the given codec, using +a fast-path for options that are not needed for this one-time lookup.
cors_get_test/0*
default_codec/1*Return the default codec for the given options.
empty_inbox/1*Empty the inbox of the current process for all messages with the given +reference.
encode_reply/3*Generate the headers and body for a HTTP response message.
get/2Gets a URL via HTTP and returns the resulting message in deserialized +form.
get/3
get_deep_signed_wasm_state_test/0*
get_deep_unsigned_wasm_state_test/0*
http_response_to_httpsig/4*Convert a HTTP response to a httpsig message.
httpsig_to_tabm_singleton/3*HTTPSig messages are inherently mixed into the transport layer, so they +require special handling in order to be converted to a normalized message.
maybe_add_unsigned/3*Add the method and path to a message, if they are not already present.
message_to_request/2*Given a message, return the information needed to make the request.
mime_to_codec/2*Find a codec name from a mime-type.
multirequest/5*Dispatch the same HTTP request to many nodes.
multirequest_opt/5*Get a value for a multirequest option from the config or message.
multirequest_opts/3*Get the multirequest options from the config or message.
nested_ao_resolve_test/0*
parallel_multirequest/8*Dispatch the same HTTP request to many nodes in parallel.
parallel_responses/7*Collect the necessary number of responses, and stop workers if +configured to do so.
post/3Posts a message to a URL on a remote peer via HTTP.
post/4
prepare_request/6*Turn a set of request arguments into a request message, formatted in the +preferred format.
remove_unsigned_fields/2*
reply/4Reply to the client's HTTP request with a message.
reply/5*
req_to_tabm_singleton/3Convert a cowboy request to a normalized message.
request/2Posts a binary to a URL on a remote peer via HTTP, returning the raw +binary body.
request/4
request/5
route_to_request/3*Parse a dev_router:route response and return a tuple of request +parameters.
run_wasm_signed_test/0*
run_wasm_unsigned_test/0*
send_encoded_node_message_test/2*
send_flat_encoded_node_message_test/0*
send_json_encoded_node_message_test/0*
send_large_signed_request_test/0*
serial_multirequest/7*Serially request a message, collecting responses until the required +number of responses have been gathered.
simple_ao_resolve_signed_test/0*
simple_ao_resolve_unsigned_test/0*
start/0
wasm_compute_request/3*
wasm_compute_request/4*
+ + + + +## Function Details ## + + + +### accept_to_codec/2 ### + +`accept_to_codec(TABMReq, Opts) -> any()` + +Calculate the codec name to use for a reply given its initiating Cowboy +request, the parsed TABM request, and the response message. The precidence +order for finding the codec is: +1. The `accept-codec` field in the message +2. The `accept` field in the request headers +3. The default codec +Options can be specified in mime-type format (`application/*`) or in +AO device format (`device@1.0`). + + + +### add_cors_headers/2 * ### + +`add_cors_headers(Msg, ReqHdr) -> any()` + +Add permissive CORS headers to a message, if the message has not already +specified CORS headers. + + + +### allowed_status/2 * ### + +`allowed_status(ResponseMsg, Statuses) -> any()` + +Check if a status is allowed, according to the configuration. + + + +### ans104_wasm_test/0 * ### + +`ans104_wasm_test() -> any()` + + + +### codec_to_content_type/2 * ### + +`codec_to_content_type(Codec, Opts) -> any()` + +Call the `content-type` key on a message with the given codec, using +a fast-path for options that are not needed for this one-time lookup. + + + +### cors_get_test/0 * ### + +`cors_get_test() -> any()` + + + +### default_codec/1 * ### + +`default_codec(Opts) -> any()` + +Return the default codec for the given options. + + + +### empty_inbox/1 * ### + +`empty_inbox(Ref) -> any()` + +Empty the inbox of the current process for all messages with the given +reference. + + + +### encode_reply/3 * ### + +`encode_reply(TABMReq, Message, Opts) -> any()` + +Generate the headers and body for a HTTP response message. + + + +### get/2 ### + +`get(Node, Opts) -> any()` + +Gets a URL via HTTP and returns the resulting message in deserialized +form. + + + +### get/3 ### + +`get(Node, PathBin, Opts) -> any()` + + + +### get_deep_signed_wasm_state_test/0 * ### + +`get_deep_signed_wasm_state_test() -> any()` + + + +### get_deep_unsigned_wasm_state_test/0 * ### + +`get_deep_unsigned_wasm_state_test() -> any()` + + + +### http_response_to_httpsig/4 * ### + +`http_response_to_httpsig(Status, HeaderMap, Body, Opts) -> any()` + +Convert a HTTP response to a httpsig message. + + + +### httpsig_to_tabm_singleton/3 * ### + +`httpsig_to_tabm_singleton(Req, Body, Opts) -> any()` + +HTTPSig messages are inherently mixed into the transport layer, so they +require special handling in order to be converted to a normalized message. +In particular, the signatures are verified if present and required by the +node configuration. Additionally, non-committed fields are removed from the +message if it is signed, with the exception of the `path` and `method` fields. + + + +### maybe_add_unsigned/3 * ### + +`maybe_add_unsigned(Req, Msg, Opts) -> any()` + +Add the method and path to a message, if they are not already present. +The precidence order for finding the path is: +1. The path in the message +2. The path in the request URI + + + +### message_to_request/2 * ### + +`message_to_request(M, Opts) -> any()` + +Given a message, return the information needed to make the request. + + + +### mime_to_codec/2 * ### + +`mime_to_codec(X1, Opts) -> any()` + +Find a codec name from a mime-type. + + + +### multirequest/5 * ### + +`multirequest(Config, Method, Path, Message, Opts) -> any()` + +Dispatch the same HTTP request to many nodes. Can be configured to +await responses from all nodes or just one, and to halt all requests after +after it has received the required number of responses, or to leave all +requests running until they have all completed. Default: Race for first +response. + +Expects a config message of the following form: +/Nodes/1..n: Hostname | #{ hostname => Hostname, address => Address } +/Responses: Number of responses to gather +/Stop-After: Should we stop after the required number of responses? +/Parallel: Should we run the requests in parallel? + + + +### multirequest_opt/5 * ### + +`multirequest_opt(Key, Config, Message, Default, Opts) -> any()` + +Get a value for a multirequest option from the config or message. + + + +### multirequest_opts/3 * ### + +`multirequest_opts(Config, Message, Opts) -> any()` + +Get the multirequest options from the config or message. The options in +the message take precidence over the options in the config. + + + +### nested_ao_resolve_test/0 * ### + +`nested_ao_resolve_test() -> any()` + + + +### parallel_multirequest/8 * ### + +`parallel_multirequest(Nodes, Responses, StopAfter, Method, Path, Message, Statuses, Opts) -> any()` + +Dispatch the same HTTP request to many nodes in parallel. + + + +### parallel_responses/7 * ### + +`parallel_responses(Res, Procs, Ref, Awaiting, StopAfter, Statuses, Opts) -> any()` + +Collect the necessary number of responses, and stop workers if +configured to do so. + + + +### post/3 ### + +`post(Node, Message, Opts) -> any()` + +Posts a message to a URL on a remote peer via HTTP. Returns the +resulting message in deserialized form. + + + +### post/4 ### + +`post(Node, Path, Message, Opts) -> any()` + + + +### prepare_request/6 * ### + +`prepare_request(Format, Method, Peer, Path, RawMessage, Opts) -> any()` + +Turn a set of request arguments into a request message, formatted in the +preferred format. + + + +### remove_unsigned_fields/2 * ### + +`remove_unsigned_fields(Msg, Opts) -> any()` + + + +### reply/4 ### + +`reply(Req, TABMReq, Message, Opts) -> any()` + +Reply to the client's HTTP request with a message. + + + +### reply/5 * ### + +`reply(Req, TABMReq, BinStatus, RawMessage, Opts) -> any()` + + + +### req_to_tabm_singleton/3 ### + +`req_to_tabm_singleton(Req, Body, Opts) -> any()` + +Convert a cowboy request to a normalized message. + + + +### request/2 ### + +`request(Message, Opts) -> any()` + +Posts a binary to a URL on a remote peer via HTTP, returning the raw +binary body. + + + +### request/4 ### + +`request(Method, Peer, Path, Opts) -> any()` + + + +### request/5 ### + +`request(Method, Config, Path, Message, Opts) -> any()` + + + +### route_to_request/3 * ### + +`route_to_request(M, X2, Opts) -> any()` + +Parse a `dev_router:route` response and return a tuple of request +parameters. + + + +### run_wasm_signed_test/0 * ### + +`run_wasm_signed_test() -> any()` + + + +### run_wasm_unsigned_test/0 * ### + +`run_wasm_unsigned_test() -> any()` + + + +### send_encoded_node_message_test/2 * ### + +`send_encoded_node_message_test(Config, Codec) -> any()` + + + +### send_flat_encoded_node_message_test/0 * ### + +`send_flat_encoded_node_message_test() -> any()` + + + +### send_json_encoded_node_message_test/0 * ### + +`send_json_encoded_node_message_test() -> any()` + + + +### send_large_signed_request_test/0 * ### + +`send_large_signed_request_test() -> any()` + + + +### serial_multirequest/7 * ### + +`serial_multirequest(Nodes, Remaining, Method, Path, Message, Statuses, Opts) -> any()` + +Serially request a message, collecting responses until the required +number of responses have been gathered. Ensure that the statuses are +allowed, according to the configuration. + + + +### simple_ao_resolve_signed_test/0 * ### + +`simple_ao_resolve_signed_test() -> any()` + + + +### simple_ao_resolve_unsigned_test/0 * ### + +`simple_ao_resolve_unsigned_test() -> any()` + + + +### start/0 ### + +`start() -> any()` + + + +### wasm_compute_request/3 * ### + +`wasm_compute_request(ImageFile, Func, Params) -> any()` + + + +### wasm_compute_request/4 * ### + +`wasm_compute_request(ImageFile, Func, Params, ResultPath) -> any()` + diff --git a/docs/resources/source-code/hb_http_benchmark_tests.md b/docs/resources/source-code/hb_http_benchmark_tests.md new file mode 100644 index 000000000..1bf12fa35 --- /dev/null +++ b/docs/resources/source-code/hb_http_benchmark_tests.md @@ -0,0 +1,5 @@ +# [Module hb_http_benchmark_tests.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_benchmark_tests.erl) + + + + diff --git a/docs/resources/source-code/hb_http_client.md b/docs/resources/source-code/hb_http_client.md new file mode 100644 index 000000000..863031d09 --- /dev/null +++ b/docs/resources/source-code/hb_http_client.md @@ -0,0 +1,210 @@ +# [Module hb_http_client.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_client.erl) + + + + +A wrapper library for gun. + +__Behaviours:__ [`gen_server`](gen_server.md). + + + +## Description ## +This module originates from the Arweave +project, and has been modified for use in HyperBEAM. + +## Function Index ## + + +
await_response/2*
dec_prometheus_gauge/1*Safe wrapper for prometheus_gauge:dec/2.
download_metric/2*
get_status_class/1*Return the HTTP status class label for cowboy_requests_total and +gun_requests_total metrics.
gun_req/3*
handle_call/3
handle_cast/2
handle_info/2
httpc_req/3*
inc_prometheus_counter/3*
inc_prometheus_gauge/1*Safe wrapper for prometheus_gauge:inc/2.
init/1
init_prometheus/1*
log/5*
maybe_invoke_monitor/2*Invoke the HTTP monitor message with AO-Core, if it is set in the +node message key.
method_to_bin/1*
open_connection/2*
parse_peer/2*
record_duration/2*Record the duration of the request in an async process.
record_response_status/3*
reply_error/2*
req/2
req/3*
request/3*
start_link/1
terminate/2
upload_metric/1*
+ + + + +## Function Details ## + + + +### await_response/2 * ### + +`await_response(Args, Opts) -> any()` + + + +### dec_prometheus_gauge/1 * ### + +`dec_prometheus_gauge(Name) -> any()` + +Safe wrapper for prometheus_gauge:dec/2. + + + +### download_metric/2 * ### + +`download_metric(Data, X2) -> any()` + + + +### get_status_class/1 * ### + +`get_status_class(Data) -> any()` + +Return the HTTP status class label for cowboy_requests_total and +gun_requests_total metrics. + + + +### gun_req/3 * ### + +`gun_req(Args, ReestablishedConnection, Opts) -> any()` + + + +### handle_call/3 ### + +`handle_call(Request, From, State) -> any()` + + + +### handle_cast/2 ### + +`handle_cast(Cast, State) -> any()` + + + +### handle_info/2 ### + +`handle_info(Message, State) -> any()` + + + +### httpc_req/3 * ### + +`httpc_req(Args, X2, Opts) -> any()` + + + +### inc_prometheus_counter/3 * ### + +`inc_prometheus_counter(Name, Labels, Value) -> any()` + + + +### inc_prometheus_gauge/1 * ### + +`inc_prometheus_gauge(Name) -> any()` + +Safe wrapper for prometheus_gauge:inc/2. + + + +### init/1 ### + +`init(Opts) -> any()` + + + +### init_prometheus/1 * ### + +`init_prometheus(Opts) -> any()` + + + +### log/5 * ### + +`log(Type, Event, X3, Reason, Opts) -> any()` + + + +### maybe_invoke_monitor/2 * ### + +`maybe_invoke_monitor(Details, Opts) -> any()` + +Invoke the HTTP monitor message with AO-Core, if it is set in the +node message key. We invoke the given message with the `body` set to a signed +version of the details. This allows node operators to configure their machine +to record duration statistics into customized data stores, computations, or +processes etc. Additionally, we include the `http_reference` value, if set in +the given `opts`. + +We use `hb_ao:get` rather than `hb_opts:get`, as settings configured +by the `~router@1.0` route `opts` key are unable to generate atoms. + + + +### method_to_bin/1 * ### + +`method_to_bin(X1) -> any()` + + + +### open_connection/2 * ### + +`open_connection(X1, Opts) -> any()` + + + +### parse_peer/2 * ### + +`parse_peer(Peer, Opts) -> any()` + + + +### record_duration/2 * ### + +`record_duration(Details, Opts) -> any()` + +Record the duration of the request in an async process. We write the +data to prometheus if the application is enabled, as well as invoking the +`http_monitor` if appropriate. + + + +### record_response_status/3 * ### + +`record_response_status(Method, Path, Response) -> any()` + + + +### reply_error/2 * ### + +`reply_error(PendingRequests, Reason) -> any()` + + + +### req/2 ### + +`req(Args, Opts) -> any()` + + + +### req/3 * ### + +`req(Args, ReestablishedConnection, Opts) -> any()` + + + +### request/3 * ### + +`request(PID, Args, Opts) -> any()` + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + + + +### terminate/2 ### + +`terminate(Reason, State) -> any()` + + + +### upload_metric/1 * ### + +`upload_metric(X1) -> any()` + diff --git a/docs/resources/source-code/hb_http_client_sup.md b/docs/resources/source-code/hb_http_client_sup.md new file mode 100644 index 000000000..fb0d04e9a --- /dev/null +++ b/docs/resources/source-code/hb_http_client_sup.md @@ -0,0 +1,33 @@ +# [Module hb_http_client_sup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_client_sup.erl) + + + + +The supervisor for the gun HTTP client wrapper. + +__Behaviours:__ [`supervisor`](supervisor.md). + + + +## Function Index ## + + +
init/1
start_link/1
+ + + + +## Function Details ## + + + +### init/1 ### + +`init(Opts) -> any()` + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + diff --git a/docs/resources/source-code/hb_http_server.md b/docs/resources/source-code/hb_http_server.md new file mode 100644 index 000000000..946292de2 --- /dev/null +++ b/docs/resources/source-code/hb_http_server.md @@ -0,0 +1,185 @@ +# [Module hb_http_server.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_http_server.erl) + + + + +A router that attaches a HTTP server to the AO-Core resolver. + + + +## Description ## + +Because AO-Core is built to speak in HTTP semantics, this module +only has to marshal the HTTP request into a message, and then +pass it to the AO-Core resolver. + +`hb_http:reply/4` is used to respond to the client, handling the +process of converting a message back into an HTTP response. + +The router uses an `Opts` message as its Cowboy initial state, +such that changing it on start of the router server allows for +the execution parameters of all downstream requests to be controlled. + +## Function Index ## + + +
allowed_methods/2Return the list of allowed methods for the HTTP server.
cors_reply/2*Reply to CORS preflight requests.
get_opts/1
handle_request/3*Handle all non-CORS preflight requests as AO-Core requests.
http3_conn_sup_loop/0*
init/2Entrypoint for all HTTP requests.
new_server/1*Trigger the creation of a new HTTP server node.
read_body/1*Helper to grab the full body of a HTTP request, even if it's chunked.
read_body/2*
set_default_opts/1
set_node_opts_test/0*Ensure that the start hook can be used to modify the node options.
set_opts/1Merges the provided Opts with uncommitted values from Request, +preserves the http_server value, and updates node_history by prepending +the Request.
set_opts/2
start/0Starts the HTTP server.
start/1
start_http2/3*
start_http3/3*
start_node/0Test that we can start the server, send a message, and get a response.
start_node/1
+ + + + +## Function Details ## + + + +### allowed_methods/2 ### + +`allowed_methods(Req, State) -> any()` + +Return the list of allowed methods for the HTTP server. + + + +### cors_reply/2 * ### + +`cors_reply(Req, ServerID) -> any()` + +Reply to CORS preflight requests. + + + +### get_opts/1 ### + +`get_opts(NodeMsg) -> any()` + + + +### handle_request/3 * ### + +`handle_request(RawReq, Body, ServerID) -> any()` + +Handle all non-CORS preflight requests as AO-Core requests. Execution +starts by parsing the HTTP request into HyerBEAM's message format, then +passing the message directly to `meta@1.0` which handles calling AO-Core in +the appropriate way. + + + +### http3_conn_sup_loop/0 * ### + +`http3_conn_sup_loop() -> any()` + + + +### init/2 ### + +`init(Req, ServerID) -> any()` + +Entrypoint for all HTTP requests. Receives the Cowboy request option and +the server ID, which can be used to lookup the node message. + + + +### new_server/1 * ### + +`new_server(RawNodeMsg) -> any()` + +Trigger the creation of a new HTTP server node. Accepts a `NodeMsg` +message, which is used to configure the server. This function executed the +`start` hook on the node, giving it the opportunity to modify the `NodeMsg` +before it is used to configure the server. The `start` hook expects gives and +expects the node message to be in the `body` key. + + + +### read_body/1 * ### + +`read_body(Req) -> any()` + +Helper to grab the full body of a HTTP request, even if it's chunked. + + + +### read_body/2 * ### + +`read_body(Req0, Acc) -> any()` + + + +### set_default_opts/1 ### + +`set_default_opts(Opts) -> any()` + + + +### set_node_opts_test/0 * ### + +`set_node_opts_test() -> any()` + +Ensure that the `start` hook can be used to modify the node options. We +do this by creating a message with a device that has a `start` key. This +key takes the message's body (the anticipated node options) and returns a +modified version of that body, which will be used to configure the node. We +then check that the node options were modified as we expected. + + + +### set_opts/1 ### + +`set_opts(Opts) -> any()` + +Merges the provided `Opts` with uncommitted values from `Request`, +preserves the http_server value, and updates node_history by prepending +the `Request`. If a server reference exists, updates the Cowboy environment +variable 'node_msg' with the resulting options map. + + + +### set_opts/2 ### + +`set_opts(Request, Opts) -> any()` + + + +### start/0 ### + +`start() -> any()` + +Starts the HTTP server. Optionally accepts an `Opts` message, which +is used as the source for server configuration settings, as well as the +`Opts` argument to use for all AO-Core resolution requests downstream. + + + +### start/1 ### + +`start(Opts) -> any()` + + + +### start_http2/3 * ### + +`start_http2(ServerID, ProtoOpts, NodeMsg) -> any()` + + + +### start_http3/3 * ### + +`start_http3(ServerID, ProtoOpts, NodeMsg) -> any()` + + + +### start_node/0 ### + +`start_node() -> any()` + +Test that we can start the server, send a message, and get a response. + + + +### start_node/1 ### + +`start_node(Opts) -> any()` + diff --git a/docs/resources/source-code/hb_json.md b/docs/resources/source-code/hb_json.md new file mode 100644 index 000000000..f8dfe8b70 --- /dev/null +++ b/docs/resources/source-code/hb_json.md @@ -0,0 +1,46 @@ +# [Module hb_json.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_json.erl) + + + + +Wrapper for encoding and decoding JSON. + + + +## Description ## +Supports maps and Jiffy's old +`ejson` format. This module abstracts the underlying JSON library, allowing +us to switch between libraries as needed in the future. + +## Function Index ## + + +
decode/1Takes a JSON string and decodes it into an Erlang term.
decode/2
encode/1Takes a term in Erlang's native form and encodes it as a JSON string.
+ + + + +## Function Details ## + + + +### decode/1 ### + +`decode(Bin) -> any()` + +Takes a JSON string and decodes it into an Erlang term. + + + +### decode/2 ### + +`decode(Bin, Opts) -> any()` + + + +### encode/1 ### + +`encode(Term) -> any()` + +Takes a term in Erlang's native form and encodes it as a JSON string. + diff --git a/docs/resources/source-code/hb_keccak.md b/docs/resources/source-code/hb_keccak.md new file mode 100644 index 000000000..ec3d4f0bb --- /dev/null +++ b/docs/resources/source-code/hb_keccak.md @@ -0,0 +1,77 @@ +# [Module hb_keccak.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_keccak.erl) + + + + + + +## Function Index ## + + +
hash_to_checksum_address/2*
init/0*
keccak_256/1
keccak_256_key_test/0*
keccak_256_key_to_address_test/0*
keccak_256_test/0*
key_to_ethereum_address/1
sha3_256/1
sha3_256_test/0*
to_hex/1*
+ + + + +## Function Details ## + + + +### hash_to_checksum_address/2 * ### + +`hash_to_checksum_address(Last40, Hash) -> any()` + + + +### init/0 * ### + +`init() -> any()` + + + +### keccak_256/1 ### + +`keccak_256(Bin) -> any()` + + + +### keccak_256_key_test/0 * ### + +`keccak_256_key_test() -> any()` + + + +### keccak_256_key_to_address_test/0 * ### + +`keccak_256_key_to_address_test() -> any()` + + + +### keccak_256_test/0 * ### + +`keccak_256_test() -> any()` + + + +### key_to_ethereum_address/1 ### + +`key_to_ethereum_address(Key) -> any()` + + + +### sha3_256/1 ### + +`sha3_256(Bin) -> any()` + + + +### sha3_256_test/0 * ### + +`sha3_256_test() -> any()` + + + +### to_hex/1 * ### + +`to_hex(Bin) -> any()` + diff --git a/docs/resources/source-code/hb_logger.md b/docs/resources/source-code/hb_logger.md new file mode 100644 index 000000000..92d077e45 --- /dev/null +++ b/docs/resources/source-code/hb_logger.md @@ -0,0 +1,59 @@ +# [Module hb_logger.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_logger.erl) + + + + + + +## Function Index ## + + +
console/2*
log/2
loop/1*
register/1
report/1
start/0
start/1
+ + + + +## Function Details ## + + + +### console/2 * ### + +`console(State, Act) -> any()` + + + +### log/2 ### + +`log(Monitor, Data) -> any()` + + + +### loop/1 * ### + +`loop(State) -> any()` + + + +### register/1 ### + +`register(Monitor) -> any()` + + + +### report/1 ### + +`report(Monitor) -> any()` + + + +### start/0 ### + +`start() -> any()` + + + +### start/1 ### + +`start(Client) -> any()` + diff --git a/docs/resources/source-code/hb_message.md b/docs/resources/source-code/hb_message.md new file mode 100644 index 000000000..fd6ee87f4 --- /dev/null +++ b/docs/resources/source-code/hb_message.md @@ -0,0 +1,734 @@ +# [Module hb_message.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_message.erl) + + + + +This module acts an adapter between messages, as modeled in the +AO-Core protocol, and their uderlying binary representations and formats. + + + +## Description ## + +Unless you are implementing a new message serialization codec, you should +not need to interact with this module directly. Instead, use the +`hb_ao` interfaces to interact with all messages. The `dev_message` +module implements a device interface for abstracting over the different +message formats. + +`hb_message` and the HyperBEAM caches can interact with multiple different +types of message formats: + +- Richly typed AO-Core structured messages. +- Arweave transations. +- ANS-104 data items. +- HTTP Signed Messages. +- Flat Maps. + +This module is responsible for converting between these formats. It does so +by normalizing messages to a common format: `Type Annotated Binary Messages` +(TABM). TABMs are deep Erlang maps with keys than only contain either other +TABMs or binary values. By marshalling all messages into this format, they +can easily be coerced into other output formats. For example, generating a +`HTTP Signed Message` format output from an Arweave transaction. TABM is +also a simple format from a computational perspective (only binary literals +and O(1) access maps), such that operations upon them are efficient. + +The structure of the conversions is as follows: + +
+Arweave TX/ANS-104 ==> dev_codec_ans104:from/1 ==> TABM
+HTTP Signed Message ==> dev_codec_httpsig_conv:from/1 ==> TABM
+Flat Maps ==> dev_codec_flat:from/1 ==> TABM
+
+TABM ==> dev_codec_structured:to/1 ==> AO-Core Message
+AO-Core Message ==> dev_codec_structured:from/1 ==> TABM
+
+TABM ==> dev_codec_ans104:to/1 ==> Arweave TX/ANS-104
+TABM ==> dev_codec_httpsig_conv:to/1 ==> HTTP Signed Message
+TABM ==> dev_codec_flat:to/1 ==> Flat Maps
+...
+
+ +Additionally, this module provides a number of utility functions for +manipulating messages. For example, `hb_message:sign/2` to sign a message of +arbitrary type, or `hb_message:format/1` to print an AO-Core/TABM message in +a human-readable format. + +The `hb_cache` module is responsible for storing and retrieving messages in +the HyperBEAM stores configured on the node. Each store has its own storage +backend, but each works with simple key-value pairs. Subsequently, the +`hb_cache` module uses TABMs as the internal format for storing and +retrieving messages. + +## Function Index ## + + +
basic_map_codec_test/1*
binary_to_binary_test/1*
commit/2Sign a message with the given wallet.
commit/3
commitment/2Extract a commitment from a message given a committer ID, or a spec +message to match against.
commitment/3
committed/1Return the list of committed keys from a message.
committed/2
committed/3
committed_empty_keys_test/1*
committed_keys_test/1*
complex_signed_message_test/1*
convert/3Convert a message from one format to another.
convert/4
deep_multisignature_test/0*
deeply_nested_committed_keys_test/0*
deeply_nested_message_with_content_test/1*Test that we can convert a 3 layer nested message into a tx record and back.
deeply_nested_message_with_only_content/1*
default_keys_removed_test/0*Test that the filter_default_keys/1 function removes TX fields +that have the default values found in the tx record, but not those that +have been set by the user.
default_tx_list/0Get the ordered list of fields as AO-Core keys and default values of +the tx record.
default_tx_message/0*Get the normalized fields and default values of the tx record.
empty_string_in_tag_test/1*
encode_balance_table/2*
encode_large_balance_table_test/1*
encode_small_balance_table_test/1*
filter_default_keys/1Remove keys from a map that have the default values found in the tx +record.
find_target/3Implements a standard pattern in which the target for an operation is +found by looking for a target key in the request.
format/1Format a message for printing, optionally taking an indentation level +to start from.
format/2
from_tabm/4*
generate_test_suite/1*
get_codec/2*Get a codec from the options.
hashpath_sign_verify_test/1*
id/1Return the ID of a message.
id/2
id/3
large_body_committed_keys_test/1*
match/2Check if two maps match, including recursively checking nested maps.
match/3
match_modes_test/0*
match_test/1*Test that the message matching function works.
matchable_keys/1*
message_suite_test_/0*
message_with_large_keys_test/1*Test that the data field is correctly managed when we have multiple +uses for it (the 'data' key itself, as well as keys that cannot fit in +tags).
message_with_simple_embedded_list_test/1*
minimization_test/0*
minimize/1Remove keys from the map that can be regenerated.
minimize/2*
nested_body_list_test/1*
nested_empty_map_test/1*
nested_message_with_large_content_test/1*Test that the data field is correctly managed when we have multiple +uses for it (the 'data' key itself, as well as keys that cannot fit in +tags).
nested_message_with_large_keys_and_content_test/1*Check that large keys and data fields are correctly handled together.
nested_message_with_large_keys_test/1*
nested_structured_fields_test/1*
normalize/1*Return a map with only the keys that necessary, without those that can +be regenerated.
print/1Pretty-print a message.
print/2*
priv_survives_conversion_test/1*
recursive_nested_list_test/1*
restore_priv/2*Add the existing priv sub-map back to a converted message, honoring +any existing priv sub-map that may already be present.
run_test/0*
set_body_codec_test/1*
sign_node_message_test/1*
signed_deep_message_test/1*
signed_list_test/1*
signed_message_encode_decode_verify_test/1*
signed_message_with_derived_components_test/1*
signed_nested_data_key_test/1*
signed_only_committed_data_field_test/1*
signed_with_inner_signed_message_test/1*
signers/1Return all of the committers on a message that have 'normal', 256 bit, +addresses.
simple_nested_message_test/1*
single_layer_message_to_encoding_test/1*Test that we can convert a message into a tx record and back.
structured_field_atom_parsing_test/1*Structured field parsing tests.
structured_field_decimal_parsing_test/1*
tabm_ao_ids_equal_test/1*
test_codecs/0*
to_tabm/3*
type/1Return the type of an encoded message.
uncommitted/1Return the unsigned version of a message in AO-Core format.
unsigned_id_test/1*
verify/1wrapper function to verify a message.
verify/2
with_commitments/2Filter messages that do not match the 'spec' given.
with_commitments/3*
with_only_committed/1Return a message with only the committed keys.
with_only_committed/2
with_only_committers/2Return the message with only the specified committers attached.
without_commitments/2Filter messages that match the 'spec' given.
without_commitments/3*
+ + + + +## Function Details ## + + + +### basic_map_codec_test/1 * ### + +`basic_map_codec_test(Codec) -> any()` + + + +### binary_to_binary_test/1 * ### + +`binary_to_binary_test(Codec) -> any()` + + + +### commit/2 ### + +`commit(Msg, WalletOrOpts) -> any()` + +Sign a message with the given wallet. + + + +### commit/3 ### + +`commit(Msg, Wallet, Format) -> any()` + + + +### commitment/2 ### + +`commitment(Committer, Msg) -> any()` + +Extract a commitment from a message given a `committer` ID, or a spec +message to match against. Returns only the first matching commitment, or +`not_found`. + + + +### commitment/3 ### + +`commitment(CommitterID, Msg, Opts) -> any()` + + + +### committed/1 ### + +`committed(Msg) -> any()` + +Return the list of committed keys from a message. + + + +### committed/2 ### + +`committed(Msg, Committers) -> any()` + + + +### committed/3 ### + +`committed(Msg, List, Opts) -> any()` + + + +### committed_empty_keys_test/1 * ### + +`committed_empty_keys_test(Codec) -> any()` + + + +### committed_keys_test/1 * ### + +`committed_keys_test(Codec) -> any()` + + + +### complex_signed_message_test/1 * ### + +`complex_signed_message_test(Codec) -> any()` + + + +### convert/3 ### + +`convert(Msg, TargetFormat, Opts) -> any()` + +Convert a message from one format to another. Taking a message in the +source format, a target format, and a set of opts. If not given, the source +is assumed to be `structured@1.0`. Additional codecs can be added by ensuring they +are part of the `Opts` map -- either globally, or locally for a computation. + +The encoding happens in two phases: +1. Convert the message to a TABM. +2. Convert the TABM to the target format. + +The conversion to a TABM is done by the `structured@1.0` codec, which is always +available. The conversion from a TABM is done by the target codec. + + + +### convert/4 ### + +`convert(Msg, TargetFormat, SourceFormat, Opts) -> any()` + + + +### deep_multisignature_test/0 * ### + +`deep_multisignature_test() -> any()` + + + +### deeply_nested_committed_keys_test/0 * ### + +`deeply_nested_committed_keys_test() -> any()` + + + +### deeply_nested_message_with_content_test/1 * ### + +`deeply_nested_message_with_content_test(Codec) -> any()` + +Test that we can convert a 3 layer nested message into a tx record and back. + + + +### deeply_nested_message_with_only_content/1 * ### + +`deeply_nested_message_with_only_content(Codec) -> any()` + + + +### default_keys_removed_test/0 * ### + +`default_keys_removed_test() -> any()` + +Test that the filter_default_keys/1 function removes TX fields +that have the default values found in the tx record, but not those that +have been set by the user. + + + +### default_tx_list/0 ### + +`default_tx_list() -> any()` + +Get the ordered list of fields as AO-Core keys and default values of +the tx record. + + + +### default_tx_message/0 * ### + +`default_tx_message() -> any()` + +Get the normalized fields and default values of the tx record. + + + +### empty_string_in_tag_test/1 * ### + +`empty_string_in_tag_test(Codec) -> any()` + + + +### encode_balance_table/2 * ### + +`encode_balance_table(Size, Codec) -> any()` + + + +### encode_large_balance_table_test/1 * ### + +`encode_large_balance_table_test(Codec) -> any()` + + + +### encode_small_balance_table_test/1 * ### + +`encode_small_balance_table_test(Codec) -> any()` + + + +### filter_default_keys/1 ### + +`filter_default_keys(Map) -> any()` + +Remove keys from a map that have the default values found in the tx +record. + + + +### find_target/3 ### + +`find_target(Self, Req, Opts) -> any()` + +Implements a standard pattern in which the target for an operation is +found by looking for a `target` key in the request. If the target is `self`, +or not present, the operation is performed on the original message. Otherwise, +the target is expected to be a key in the message, and the operation is +performed on the value of that key. + + + +### format/1 ### + +`format(Item) -> any()` + +Format a message for printing, optionally taking an indentation level +to start from. + + + +### format/2 ### + +`format(Bin, Indent) -> any()` + + + +### from_tabm/4 * ### + +`from_tabm(Msg, TargetFormat, OldPriv, Opts) -> any()` + + + +### generate_test_suite/1 * ### + +`generate_test_suite(Suite) -> any()` + + + +### get_codec/2 * ### + +`get_codec(TargetFormat, Opts) -> any()` + +Get a codec from the options. + + + +### hashpath_sign_verify_test/1 * ### + +`hashpath_sign_verify_test(Codec) -> any()` + + + +### id/1 ### + +`id(Msg) -> any()` + +Return the ID of a message. + + + +### id/2 ### + +`id(Msg, Committers) -> any()` + + + +### id/3 ### + +`id(Msg, RawCommitters, Opts) -> any()` + + + +### large_body_committed_keys_test/1 * ### + +`large_body_committed_keys_test(Codec) -> any()` + + + +### match/2 ### + +`match(Map1, Map2) -> any()` + +Check if two maps match, including recursively checking nested maps. +Takes an optional mode argument to control the matching behavior: +`strict`: All keys in both maps be present and match. +`only_present`: Only present keys in both maps must match. +`primary`: Only the primary map's keys must be present. + + + +### match/3 ### + +`match(Map1, Map2, Mode) -> any()` + + + +### match_modes_test/0 * ### + +`match_modes_test() -> any()` + + + +### match_test/1 * ### + +`match_test(Codec) -> any()` + +Test that the message matching function works. + + + +### matchable_keys/1 * ### + +`matchable_keys(Map) -> any()` + + + +### message_suite_test_/0 * ### + +`message_suite_test_() -> any()` + + + +### message_with_large_keys_test/1 * ### + +`message_with_large_keys_test(Codec) -> any()` + +Test that the data field is correctly managed when we have multiple +uses for it (the 'data' key itself, as well as keys that cannot fit in +tags). + + + +### message_with_simple_embedded_list_test/1 * ### + +`message_with_simple_embedded_list_test(Codec) -> any()` + + + +### minimization_test/0 * ### + +`minimization_test() -> any()` + + + +### minimize/1 ### + +`minimize(Msg) -> any()` + +Remove keys from the map that can be regenerated. Optionally takes an +additional list of keys to include in the minimization. + + + +### minimize/2 * ### + +`minimize(RawVal, ExtraKeys) -> any()` + + + +### nested_body_list_test/1 * ### + +`nested_body_list_test(Codec) -> any()` + + + +### nested_empty_map_test/1 * ### + +`nested_empty_map_test(Codec) -> any()` + + + +### nested_message_with_large_content_test/1 * ### + +`nested_message_with_large_content_test(Codec) -> any()` + +Test that the data field is correctly managed when we have multiple +uses for it (the 'data' key itself, as well as keys that cannot fit in +tags). + + + +### nested_message_with_large_keys_and_content_test/1 * ### + +`nested_message_with_large_keys_and_content_test(Codec) -> any()` + +Check that large keys and data fields are correctly handled together. + + + +### nested_message_with_large_keys_test/1 * ### + +`nested_message_with_large_keys_test(Codec) -> any()` + + + +### nested_structured_fields_test/1 * ### + +`nested_structured_fields_test(Codec) -> any()` + + + +### normalize/1 * ### + +`normalize(Map) -> any()` + +Return a map with only the keys that necessary, without those that can +be regenerated. + + + +### print/1 ### + +`print(Msg) -> any()` + +Pretty-print a message. + + + +### print/2 * ### + +`print(Msg, Indent) -> any()` + + + +### priv_survives_conversion_test/1 * ### + +`priv_survives_conversion_test(Codec) -> any()` + + + +### recursive_nested_list_test/1 * ### + +`recursive_nested_list_test(Codec) -> any()` + + + +### restore_priv/2 * ### + +`restore_priv(Msg, EmptyPriv) -> any()` + +Add the existing `priv` sub-map back to a converted message, honoring +any existing `priv` sub-map that may already be present. + + + +### run_test/0 * ### + +`run_test() -> any()` + + + +### set_body_codec_test/1 * ### + +`set_body_codec_test(Codec) -> any()` + + + +### sign_node_message_test/1 * ### + +`sign_node_message_test(Codec) -> any()` + + + +### signed_deep_message_test/1 * ### + +`signed_deep_message_test(Codec) -> any()` + + + +### signed_list_test/1 * ### + +`signed_list_test(Codec) -> any()` + + + +### signed_message_encode_decode_verify_test/1 * ### + +`signed_message_encode_decode_verify_test(Codec) -> any()` + + + +### signed_message_with_derived_components_test/1 * ### + +`signed_message_with_derived_components_test(Codec) -> any()` + + + +### signed_nested_data_key_test/1 * ### + +`signed_nested_data_key_test(Codec) -> any()` + + + +### signed_only_committed_data_field_test/1 * ### + +`signed_only_committed_data_field_test(Codec) -> any()` + + + +### signed_with_inner_signed_message_test/1 * ### + +`signed_with_inner_signed_message_test(Codec) -> any()` + + + +### signers/1 ### + +`signers(Msg) -> any()` + +Return all of the committers on a message that have 'normal', 256 bit, +addresses. + + + +### simple_nested_message_test/1 * ### + +`simple_nested_message_test(Codec) -> any()` + + + +### single_layer_message_to_encoding_test/1 * ### + +`single_layer_message_to_encoding_test(Codec) -> any()` + +Test that we can convert a message into a tx record and back. + + + +### structured_field_atom_parsing_test/1 * ### + +`structured_field_atom_parsing_test(Codec) -> any()` + +Structured field parsing tests. + + + +### structured_field_decimal_parsing_test/1 * ### + +`structured_field_decimal_parsing_test(Codec) -> any()` + + + +### tabm_ao_ids_equal_test/1 * ### + +`tabm_ao_ids_equal_test(Codec) -> any()` + + + +### test_codecs/0 * ### + +`test_codecs() -> any()` + + + +### to_tabm/3 * ### + +`to_tabm(Msg, SourceFormat, Opts) -> any()` + + + +### type/1 ### + +`type(TX) -> any()` + +Return the type of an encoded message. + + + +### uncommitted/1 ### + +`uncommitted(Bin) -> any()` + +Return the unsigned version of a message in AO-Core format. + + + +### unsigned_id_test/1 * ### + +`unsigned_id_test(Codec) -> any()` + + + +### verify/1 ### + +`verify(Msg) -> any()` + +wrapper function to verify a message. + + + +### verify/2 ### + +`verify(Msg, Committers) -> any()` + + + +### with_commitments/2 ### + +`with_commitments(Spec, Msg) -> any()` + +Filter messages that do not match the 'spec' given. The underlying match +is performed in the `only_present` mode, such that match specifications only +need to specify the keys that must be present. + + + +### with_commitments/3 * ### + +`with_commitments(Spec, Msg, Opts) -> any()` + + + +### with_only_committed/1 ### + +`with_only_committed(Msg) -> any()` + +Return a message with only the committed keys. If no commitments are +present, the message is returned unchanged. This means that you need to +check if the message is: +- Committed +- Verifies +...before using the output of this function as the 'canonical' message. This +is such that expensive operations like signature verification are not +performed unless necessary. + + + +### with_only_committed/2 ### + +`with_only_committed(Msg, Opts) -> any()` + + + +### with_only_committers/2 ### + +`with_only_committers(Msg, Committers) -> any()` + +Return the message with only the specified committers attached. + + + +### without_commitments/2 ### + +`without_commitments(Spec, Msg) -> any()` + +Filter messages that match the 'spec' given. Inverts the `with_commitments/2` +function, such that only messages that do _not_ match the spec are returned. + + + +### without_commitments/3 * ### + +`without_commitments(Spec, Msg, Opts) -> any()` + diff --git a/docs/resources/source-code/hb_metrics_collector.md b/docs/resources/source-code/hb_metrics_collector.md new file mode 100644 index 000000000..f9ba6fdea --- /dev/null +++ b/docs/resources/source-code/hb_metrics_collector.md @@ -0,0 +1,43 @@ +# [Module hb_metrics_collector.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_metrics_collector.erl) + + + + +__Behaviours:__ [`prometheus_collector`](prometheus_collector.md). + + + +## Function Index ## + + +
collect_metrics/2
collect_mf/2
create_gauge/3*
deregister_cleanup/1
+ + + + +## Function Details ## + + + +### collect_metrics/2 ### + +`collect_metrics(X1, SystemLoad) -> any()` + + + +### collect_mf/2 ### + +`collect_mf(Registry, Callback) -> any()` + + + +### create_gauge/3 * ### + +`create_gauge(Name, Help, Data) -> any()` + + + +### deregister_cleanup/1 ### + +`deregister_cleanup(X1) -> any()` + diff --git a/docs/resources/source-code/hb_name.md b/docs/resources/source-code/hb_name.md new file mode 100644 index 000000000..6adfbadaf --- /dev/null +++ b/docs/resources/source-code/hb_name.md @@ -0,0 +1,136 @@ +# [Module hb_name.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_name.erl) + + + + +An abstraction for name registration/deregistration in Hyperbeam. + + + +## Description ## +Its motivation is to provide a way to register names that are not necessarily +atoms, but can be any term (for example: hashpaths or `process@1.0` IDs). +An important characteristic of these functions is that they are atomic: +There can only ever be one registrant for a given name at a time. + +## Function Index ## + + +
all/0List the names in the registry.
all_test/0*
atom_test/0*
basic_test/1*
cleanup_test/0*
concurrency_test/0*
dead_process_test/0*
ets_lookup/1*
lookup/1Lookup a name -> PID.
register/1Register a name.
register/2
spawn_test_workers/1*
start/0
start_ets/0*
term_test/0*
unregister/1Unregister a name.
wait_for_cleanup/2*
+ + + + +## Function Details ## + + + +### all/0 ### + +`all() -> any()` + +List the names in the registry. + + + +### all_test/0 * ### + +`all_test() -> any()` + + + +### atom_test/0 * ### + +`atom_test() -> any()` + + + +### basic_test/1 * ### + +`basic_test(Term) -> any()` + + + +### cleanup_test/0 * ### + +`cleanup_test() -> any()` + + + +### concurrency_test/0 * ### + +`concurrency_test() -> any()` + + + +### dead_process_test/0 * ### + +`dead_process_test() -> any()` + + + +### ets_lookup/1 * ### + +`ets_lookup(Name) -> any()` + + + +### lookup/1 ### + +`lookup(Name) -> any()` + +Lookup a name -> PID. + + + +### register/1 ### + +`register(Name) -> any()` + +Register a name. If the name is already registered, the registration +will fail. The name can be any Erlang term. + + + +### register/2 ### + +`register(Name, Pid) -> any()` + + + +### spawn_test_workers/1 * ### + +`spawn_test_workers(Name) -> any()` + + + +### start/0 ### + +`start() -> any()` + + + +### start_ets/0 * ### + +`start_ets() -> any()` + + + +### term_test/0 * ### + +`term_test() -> any()` + + + +### unregister/1 ### + +`unregister(Name) -> any()` + +Unregister a name. + + + +### wait_for_cleanup/2 * ### + +`wait_for_cleanup(Name, Retries) -> any()` + diff --git a/docs/resources/source-code/hb_opts.md b/docs/resources/source-code/hb_opts.md new file mode 100644 index 000000000..fcb0bea22 --- /dev/null +++ b/docs/resources/source-code/hb_opts.md @@ -0,0 +1,167 @@ +# [Module hb_opts.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_opts.erl) + + + + +A module for interacting with local and global options inside +HyperBEAM. + + + +## Description ## + +Options are set globally, but can also be overridden using an +an optional local `Opts` map argument. Many functions across the HyperBEAM +environment accept an `Opts` argument, which can be used to customize +behavior. + +Options set in an `Opts` map must _never_ change the behavior of a function +that should otherwise be deterministic. Doing so may lead to loss of funds +by the HyperBEAM node operator, as the results of their executions will be +different than those of other node operators. If they are economically +staked on the correctness of these results, they may experience punishments +for non-verifiable behavior. Instead, if a local node setting makes +deterministic behavior impossible, the caller should fail the execution +with a refusal to execute. + +## Function Index ## + + +
cached_os_env/2*Cache the result of os:getenv/1 in the process dictionary, as it never +changes during the lifetime of a node.
check_required_opts/2Utility function to check for required options in a list.
config_lookup/2*An abstraction for looking up configuration variables.
default_message/0The default configuration options of the hyperbeam node.
get/1Get an option from the global options, optionally overriding with a +local Opts map if prefer or only is set to local.
get/2
get/3
global_get/2*Get an environment variable or configuration key.
load/1Parse a flat@1.0 encoded file into a map, matching the types of the +keys to those in the default message.
load_bin/1
mimic_default_types/2Mimic the types of the default message for a given map.
normalize_default/1*Get an option from environment variables, optionally consulting the +hb_features of the node if a conditional default tuple is provided.
validate_node_history/1Validate that the node_history length is within an acceptable range.
validate_node_history/3
+ + + + +## Function Details ## + + + +### cached_os_env/2 * ### + +`cached_os_env(Key, DefaultValue) -> any()` + +Cache the result of os:getenv/1 in the process dictionary, as it never +changes during the lifetime of a node. + + + +### check_required_opts/2 ### + +

+check_required_opts(KeyValuePairs::[{binary(), term()}], Opts::map()) -> {ok, map()} | {error, binary()}
+
+
+ +`KeyValuePairs`: A list of {Name, Value} pairs to check.
`Opts`: The original options map to return if validation succeeds.
+ +returns: `{ok, Opts}` if all required options are present, or +`{error, <<"Missing required parameters: ", MissingOptsStr/binary>>}` +where `MissingOptsStr` is a comma-separated list of missing option names. + +Utility function to check for required options in a list. +Takes a list of {Name, Value} pairs and returns: +- {ok, Opts} when all required options are present (Value =/= not_found) +- {error, ErrorMsg} with a message listing all missing options when any are not_found + + + +### config_lookup/2 * ### + +`config_lookup(Key, Default) -> any()` + +An abstraction for looking up configuration variables. In the future, +this is the function that we will want to change to support a more dynamic +configuration system. + + + +### default_message/0 ### + +`default_message() -> any()` + +The default configuration options of the hyperbeam node. + + + +### get/1 ### + +`get(Key) -> any()` + +Get an option from the global options, optionally overriding with a +local `Opts` map if `prefer` or `only` is set to `local`. If the `only` +option is provided in the `local` map, only keys found in the corresponding +(`local` or `global`) map will be returned. This function also offers users +a way to specify a default value to return if the option is not set. + +`prefer` defaults to `local`. + + + +### get/2 ### + +`get(Key, Default) -> any()` + + + +### get/3 ### + +`get(Key, Default, Opts) -> any()` + + + +### global_get/2 * ### + +`global_get(Key, Default) -> any()` + +Get an environment variable or configuration key. + + + +### load/1 ### + +`load(Path) -> any()` + +Parse a `flat@1.0` encoded file into a map, matching the types of the +keys to those in the default message. + + + +### load_bin/1 ### + +`load_bin(Bin) -> any()` + + + +### mimic_default_types/2 ### + +`mimic_default_types(Map, Mode) -> any()` + +Mimic the types of the default message for a given map. + + + +### normalize_default/1 * ### + +`normalize_default(Default) -> any()` + +Get an option from environment variables, optionally consulting the +`hb_features` of the node if a conditional default tuple is provided. + + + +### validate_node_history/1 ### + +`validate_node_history(Opts) -> any()` + +Validate that the node_history length is within an acceptable range. + + + +### validate_node_history/3 ### + +`validate_node_history(Opts, MinLength, MaxLength) -> any()` + diff --git a/docs/resources/source-code/hb_path.md b/docs/resources/source-code/hb_path.md new file mode 100644 index 000000000..d3811c713 --- /dev/null +++ b/docs/resources/source-code/hb_path.md @@ -0,0 +1,297 @@ +# [Module hb_path.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_path.erl) + + + + +This module provides utilities for manipulating the paths of a +message: Its request path (referred to in messages as just the `Path`), and +its HashPath. + + + +## Description ## + +A HashPath is a rolling Merkle list of the messages that have been applied +in order to generate a given message. Because applied messages can +themselves be the result of message applications with the AO-Core protocol, +the HashPath can be thought of as the tree of messages that represent the +history of a given message. The initial message on a HashPath is referred to +by its ID and serves as its user-generated 'root'. + +Specifically, the HashPath can be generated by hashing the previous HashPath +and the current message. This means that each message in the HashPath is +dependent on all previous messages. + +``` + + Msg1.HashPath = Msg1.ID + Msg3.HashPath = Msg1.Hash(Msg1.HashPath, Msg2.ID) + Msg3.{...} = AO-Core.apply(Msg1, Msg2) + ... +``` + +A message's ID itself includes its HashPath, leading to the mixing of +a Msg2's merkle list into the resulting Msg3's HashPath. This allows a single +message to represent a history _tree_ of all of the messages that were +applied to generate it -- rather than just a linear history. + +A message may also specify its own algorithm for generating its HashPath, +which allows for custom logic to be used for representing the history of a +message. When Msg2's are applied to a Msg1, the resulting Msg3's HashPath +will be generated according to Msg1's algorithm choice. + +## Function Index ## + + +
do_to_binary/1*
from_message/2Extract the request path or hashpath from a message.
hashpath/2Add an ID of a Msg2 to the HashPath of another message.
hashpath/3
hashpath/4
hashpath_alg/1Get the hashpath function for a message from its HashPath-Alg.
hashpath_direct_msg2_test/0*
hashpath_test/0*
hd/2Extract the first key from a Message2's Path field.
hd_test/0*
matches/2Check if two keys match.
multiple_hashpaths_test/0*
normalize/1Normalize a path to a binary, removing the leading slash if present.
pop_from_message_test/0*
pop_from_path_list_test/0*
pop_request/2Pop the next element from a request path or path list.
priv_remaining/2Return the Remaining-Path of a message, from its hidden AO-Core +key.
priv_store_remaining/2Store the remaining path of a message in its hidden AO-Core key.
push_request/2Add a message to the head (next to execute) of a request path.
queue_request/2Queue a message at the back of a request path.
regex_matches/2Check if two keys match using regex.
regex_matches_test/0*
term_to_path_parts/1Convert a term into an executable path.
term_to_path_parts/2
term_to_path_parts_test/0*
tl/2Return the message without its first path element.
tl_test/0*
to_binary/1Convert a path of any form to a binary.
to_binary_test/0*
validate_path_transitions/2*
verify_hashpath/2Verify the HashPath of a message, given a list of messages that +represent its history.
verify_hashpath_test/0*
+ + + + +## Function Details ## + + + +### do_to_binary/1 * ### + +`do_to_binary(Path) -> any()` + + + +### from_message/2 ### + +`from_message(X1, Msg) -> any()` + +Extract the request path or hashpath from a message. We do not use +AO-Core for this resolution because this function is called from inside AO-Core +itself. This imparts a requirement: the message's device must store a +viable hashpath and path in its Erlang map at all times, unless the message +is directly from a user (in which case paths and hashpaths will not have +been assigned yet). + + + +### hashpath/2 ### + +`hashpath(Bin, Opts) -> any()` + +Add an ID of a Msg2 to the HashPath of another message. + + + +### hashpath/3 ### + +`hashpath(Msg1, Msg2, Opts) -> any()` + + + +### hashpath/4 ### + +`hashpath(Msg1, Msg2, HashpathAlg, Opts) -> any()` + + + +### hashpath_alg/1 ### + +`hashpath_alg(Msg) -> any()` + +Get the hashpath function for a message from its HashPath-Alg. +If no hashpath algorithm is specified, the protocol defaults to +`sha-256-chain`. + + + +### hashpath_direct_msg2_test/0 * ### + +`hashpath_direct_msg2_test() -> any()` + + + +### hashpath_test/0 * ### + +`hashpath_test() -> any()` + + + +### hd/2 ### + +`hd(Msg2, Opts) -> any()` + +Extract the first key from a `Message2`'s `Path` field. +Note: This function uses the `dev_message:get/2` function, rather than +a generic call as the path should always be an explicit key in the message. + + + +### hd_test/0 * ### + +`hd_test() -> any()` + + + +### matches/2 ### + +`matches(Key1, Key2) -> any()` + +Check if two keys match. + + + +### multiple_hashpaths_test/0 * ### + +`multiple_hashpaths_test() -> any()` + + + +### normalize/1 ### + +`normalize(Path) -> any()` + +Normalize a path to a binary, removing the leading slash if present. + + + +### pop_from_message_test/0 * ### + +`pop_from_message_test() -> any()` + + + +### pop_from_path_list_test/0 * ### + +`pop_from_path_list_test() -> any()` + + + +### pop_request/2 ### + +`pop_request(Msg, Opts) -> any()` + +Pop the next element from a request path or path list. + + + +### priv_remaining/2 ### + +`priv_remaining(Msg, Opts) -> any()` + +Return the `Remaining-Path` of a message, from its hidden `AO-Core` +key. Does not use the `get` or set `hb_private` functions, such that it +can be safely used inside the main AO-Core resolve function. + + + +### priv_store_remaining/2 ### + +`priv_store_remaining(Msg, RemainingPath) -> any()` + +Store the remaining path of a message in its hidden `AO-Core` key. + + + +### push_request/2 ### + +`push_request(Msg, Path) -> any()` + +Add a message to the head (next to execute) of a request path. + + + +### queue_request/2 ### + +`queue_request(Msg, Path) -> any()` + +Queue a message at the back of a request path. `path` is the only +key that we cannot use dev_message's `set/3` function for (as it expects +the compute path to be there), so we use `maps:put/3` instead. + + + +### regex_matches/2 ### + +`regex_matches(Path1, Path2) -> any()` + +Check if two keys match using regex. + + + +### regex_matches_test/0 * ### + +`regex_matches_test() -> any()` + + + +### term_to_path_parts/1 ### + +`term_to_path_parts(Path) -> any()` + +Convert a term into an executable path. Supports binaries, lists, and +atoms. Notably, it does not support strings as lists of characters. + + + +### term_to_path_parts/2 ### + +`term_to_path_parts(Binary, Opts) -> any()` + + + +### term_to_path_parts_test/0 * ### + +`term_to_path_parts_test() -> any()` + + + +### tl/2 ### + +`tl(Msg2, Opts) -> any()` + +Return the message without its first path element. Note that this +is the only transformation in AO-Core that does _not_ make a log of its +transformation. Subsequently, the message's IDs will not be verifiable +after executing this transformation. +This may or may not be the mainnet behavior we want. + + + +### tl_test/0 * ### + +`tl_test() -> any()` + + + +### to_binary/1 ### + +`to_binary(Path) -> any()` + +Convert a path of any form to a binary. + + + +### to_binary_test/0 * ### + +`to_binary_test() -> any()` + + + +### validate_path_transitions/2 * ### + +`validate_path_transitions(X, Opts) -> any()` + + + +### verify_hashpath/2 ### + +`verify_hashpath(Rest, Opts) -> any()` + +Verify the HashPath of a message, given a list of messages that +represent its history. + + + +### verify_hashpath_test/0 * ### + +`verify_hashpath_test() -> any()` + diff --git a/docs/resources/source-code/hb_persistent.md b/docs/resources/source-code/hb_persistent.md new file mode 100644 index 000000000..bfb5bdc31 --- /dev/null +++ b/docs/resources/source-code/hb_persistent.md @@ -0,0 +1,278 @@ +# [Module hb_persistent.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_persistent.erl) + + + + +Creates and manages long-lived AO-Core resolution processes. + + + +## Description ## + +These can be useful for situations where a message is large and expensive +to serialize and deserialize, or when executions should be deliberately +serialized to avoid parallel executions of the same computation. This +module is called during the core `hb_ao` execution process, so care +must be taken to avoid recursive spawns/loops. + +Built using the `pg` module, which is a distributed Erlang process group +manager. + +## Function Index ## + + +
await/4If there was already an Erlang process handling this execution, +we should register with them and wait for them to notify us of +completion.
deduplicated_execution_test/0*Test merging and returning a value with a persistent worker.
default_await/5Default await function that waits for a resolution from a worker.
default_grouper/3Create a group name from a Msg1 and Msg2 pair as a tuple.
default_worker/3A server function for handling persistent executions.
do_monitor/1*
do_monitor/2*
find_execution/2*Find a group with the given name.
find_or_register/3Register the process to lead an execution if none is found, otherwise +signal that we should await resolution.
find_or_register/4*
forward_work/2Forward requests to a newly delegated execution process.
group/3Calculate the group name for a Msg1 and Msg2 pair.
notify/4Check our inbox for processes that are waiting for the resolution +of this execution.
persistent_worker_test/0*Test spawning a default persistent worker.
register_groupname/2*Register for performing an AO-Core resolution.
send_response/4*Helper function that wraps responding with a new Msg3.
spawn_after_execution_test/0*
spawn_test_client/2*
spawn_test_client/3*
start/0*Ensure that the pg module is started.
start_monitor/0Start a monitor that prints the current members of the group every +n seconds.
start_monitor/1
start_worker/2Start a worker process that will hold a message in memory for +future executions.
start_worker/3
stop_monitor/1
test_device/0*
test_device/1*
unregister/3*Unregister for being the leader on an AO-Core resolution.
unregister_groupname/2*
unregister_notify/4Unregister as the leader for an execution and notify waiting processes.
wait_for_test_result/1*
worker_event/5*Log an event with the worker process.
+ + + + +## Function Details ## + + + +### await/4 ### + +`await(Worker, Msg1, Msg2, Opts) -> any()` + +If there was already an Erlang process handling this execution, +we should register with them and wait for them to notify us of +completion. + + + +### deduplicated_execution_test/0 * ### + +`deduplicated_execution_test() -> any()` + +Test merging and returning a value with a persistent worker. + + + +### default_await/5 ### + +`default_await(Worker, GroupName, Msg1, Msg2, Opts) -> any()` + +Default await function that waits for a resolution from a worker. + + + +### default_grouper/3 ### + +`default_grouper(Msg1, Msg2, Opts) -> any()` + +Create a group name from a Msg1 and Msg2 pair as a tuple. + + + +### default_worker/3 ### + +`default_worker(GroupName, Msg1, Opts) -> any()` + +A server function for handling persistent executions. + + + +### do_monitor/1 * ### + +`do_monitor(Group) -> any()` + + + +### do_monitor/2 * ### + +`do_monitor(Group, Last) -> any()` + + + +### find_execution/2 * ### + +`find_execution(Groupname, Opts) -> any()` + +Find a group with the given name. + + + +### find_or_register/3 ### + +`find_or_register(Msg1, Msg2, Opts) -> any()` + +Register the process to lead an execution if none is found, otherwise +signal that we should await resolution. + + + +### find_or_register/4 * ### + +`find_or_register(GroupName, Msg1, Msg2, Opts) -> any()` + + + +### forward_work/2 ### + +`forward_work(NewPID, Opts) -> any()` + +Forward requests to a newly delegated execution process. + + + +### group/3 ### + +`group(Msg1, Msg2, Opts) -> any()` + +Calculate the group name for a Msg1 and Msg2 pair. Uses the Msg1's +`group` function if it is found in the `info`, otherwise uses the default. + + + +### notify/4 ### + +`notify(GroupName, Msg2, Msg3, Opts) -> any()` + +Check our inbox for processes that are waiting for the resolution +of this execution. Comes in two forms: +1. Notify on group name alone. +2. Notify on group name and Msg2. + + + +### persistent_worker_test/0 * ### + +`persistent_worker_test() -> any()` + +Test spawning a default persistent worker. + + + +### register_groupname/2 * ### + +`register_groupname(Groupname, Opts) -> any()` + +Register for performing an AO-Core resolution. + + + +### send_response/4 * ### + +`send_response(Listener, GroupName, Msg2, Msg3) -> any()` + +Helper function that wraps responding with a new Msg3. + + + +### spawn_after_execution_test/0 * ### + +`spawn_after_execution_test() -> any()` + + + +### spawn_test_client/2 * ### + +`spawn_test_client(Msg1, Msg2) -> any()` + + + +### spawn_test_client/3 * ### + +`spawn_test_client(Msg1, Msg2, Opts) -> any()` + + + +### start/0 * ### + +`start() -> any()` + +Ensure that the `pg` module is started. + + + +### start_monitor/0 ### + +`start_monitor() -> any()` + +Start a monitor that prints the current members of the group every +n seconds. + + + +### start_monitor/1 ### + +`start_monitor(Group) -> any()` + + + +### start_worker/2 ### + +`start_worker(Msg, Opts) -> any()` + +Start a worker process that will hold a message in memory for +future executions. + + + +### start_worker/3 ### + +`start_worker(GroupName, NotMsg, Opts) -> any()` + + + +### stop_monitor/1 ### + +`stop_monitor(PID) -> any()` + + + +### test_device/0 * ### + +`test_device() -> any()` + + + +### test_device/1 * ### + +`test_device(Base) -> any()` + + + +### unregister/3 * ### + +`unregister(Msg1, Msg2, Opts) -> any()` + +Unregister for being the leader on an AO-Core resolution. + + + +### unregister_groupname/2 * ### + +`unregister_groupname(Groupname, Opts) -> any()` + + + +### unregister_notify/4 ### + +`unregister_notify(GroupName, Msg2, Msg3, Opts) -> any()` + +Unregister as the leader for an execution and notify waiting processes. + + + +### wait_for_test_result/1 * ### + +`wait_for_test_result(Ref) -> any()` + + + +### worker_event/5 * ### + +`worker_event(Group, Data, Msg1, Msg2, Opts) -> any()` + +Log an event with the worker process. If we used the default grouper +function, we should also include the Msg1 and Msg2 in the event. If we did not, +we assume that the group name expresses enough information to identify the +request. + diff --git a/docs/resources/source-code/hb_private.md b/docs/resources/source-code/hb_private.md new file mode 100644 index 000000000..4da4ce7d0 --- /dev/null +++ b/docs/resources/source-code/hb_private.md @@ -0,0 +1,130 @@ +# [Module hb_private.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_private.erl) + + + + +This module provides basic helper utilities for managing the +private element of a message, which can be used to store state that is +not included in serialized messages, or those granted to users via the +APIs. + + + +## Description ## + +Private elements of a message can be useful for storing state that +is only relevant temporarily. For example, a device might use the private +element to store a cache of values that are expensive to recompute. They +should _not_ be used for encoding state that makes the execution of a +device non-deterministic (unless you are sure you know what you are doing). + +The `set` and `get` functions of this module allow you to run those keys +as AO-Core paths if you would like to have private `devices` in the +messages non-public zone. + +See `hb_ao` for more information about the AO-Core protocol +and private elements of messages. + +## Function Index ## + + +
from_message/1Return the private key from a message.
get/3Helper for getting a value from the private element of a message.
get/4
get_private_key_test/0*
is_private/1Check if a key is private.
priv_ao_opts/1*The opts map that should be used when resolving paths against the +private element of a message.
remove_private_specifier/1*Remove the first key from the path if it is a private specifier.
reset/1Unset all of the private keys in a message.
set/3
set/4Helper function for setting a key in the private element of a message.
set_priv/2Helper function for setting the complete private element of a message.
set_private_test/0*
+ + + + +## Function Details ## + + + +### from_message/1 ### + +`from_message(Msg) -> any()` + +Return the `private` key from a message. If the key does not exist, an +empty map is returned. + + + +### get/3 ### + +`get(Key, Msg, Opts) -> any()` + +Helper for getting a value from the private element of a message. Uses +AO-Core resolve under-the-hood, removing the private specifier from the +path if it exists. + + + +### get/4 ### + +`get(InputPath, Msg, Default, Opts) -> any()` + + + +### get_private_key_test/0 * ### + +`get_private_key_test() -> any()` + + + +### is_private/1 ### + +`is_private(Key) -> any()` + +Check if a key is private. + + + +### priv_ao_opts/1 * ### + +`priv_ao_opts(Opts) -> any()` + +The opts map that should be used when resolving paths against the +private element of a message. + + + +### remove_private_specifier/1 * ### + +`remove_private_specifier(InputPath) -> any()` + +Remove the first key from the path if it is a private specifier. + + + +### reset/1 ### + +`reset(Msg) -> any()` + +Unset all of the private keys in a message. + + + +### set/3 ### + +`set(Msg, PrivMap, Opts) -> any()` + + + +### set/4 ### + +`set(Msg, InputPath, Value, Opts) -> any()` + +Helper function for setting a key in the private element of a message. + + + +### set_priv/2 ### + +`set_priv(Msg, PrivMap) -> any()` + +Helper function for setting the complete private element of a message. + + + +### set_private_test/0 * ### + +`set_private_test() -> any()` + diff --git a/docs/resources/source-code/hb_process_monitor.md b/docs/resources/source-code/hb_process_monitor.md new file mode 100644 index 000000000..b154476be --- /dev/null +++ b/docs/resources/source-code/hb_process_monitor.md @@ -0,0 +1,59 @@ +# [Module hb_process_monitor.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_process_monitor.erl) + + + + + + +## Function Index ## + + +
handle_crons/1*
server/1*
start/1
start/2
start/3
stop/1
ticker/2*
+ + + + +## Function Details ## + + + +### handle_crons/1 * ### + +`handle_crons(State) -> any()` + + + +### server/1 * ### + +`server(State) -> any()` + + + +### start/1 ### + +`start(ProcID) -> any()` + + + +### start/2 ### + +`start(ProcID, Rate) -> any()` + + + +### start/3 ### + +`start(ProcID, Rate, Cursor) -> any()` + + + +### stop/1 ### + +`stop(PID) -> any()` + + + +### ticker/2 * ### + +`ticker(Monitor, Rate) -> any()` + diff --git a/docs/resources/source-code/hb_router.md b/docs/resources/source-code/hb_router.md new file mode 100644 index 000000000..94038cc12 --- /dev/null +++ b/docs/resources/source-code/hb_router.md @@ -0,0 +1,29 @@ +# [Module hb_router.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_router.erl) + + + + + + +## Function Index ## + + +
find/2
find/3
+ + + + +## Function Details ## + + + +### find/2 ### + +`find(Type, ID) -> any()` + + + +### find/3 ### + +`find(Type, ID, Address) -> any()` + diff --git a/docs/resources/source-code/hb_singleton.md b/docs/resources/source-code/hb_singleton.md new file mode 100644 index 000000000..652f2b669 --- /dev/null +++ b/docs/resources/source-code/hb_singleton.md @@ -0,0 +1,431 @@ +# [Module hb_singleton.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_singleton.erl) + + + + +A parser that translates AO-Core HTTP API requests in TABM format +into an ordered list of messages to evaluate. + + + +## Description ## + +The details of this format +are described in `docs/ao-core-http-api.md`. + +Syntax overview: + +``` + + Singleton: Message containing keys and a path field, + which may also contain a query string of key-value pairs. + Path: + - /Part1/Part2/.../PartN/ => [Part1, Part2, ..., PartN] + - /ID/Part2/.../PartN => [ID, Part2, ..., PartN] + Part: (Key + Resolution), Device?, #{ K => V}? + - Part => #{ path => Part } + - Part&Key=Value => #{ path => Part, Key => Value } + - Part&Key => #{ path => Part, Key => true } + - Part&k1=v1&k2=v2 => #{ path => Part, k1 => `<<"v1">>, k2 => <<"v2">> }' + - Part~Device => {as, Device, #{ path => Part }} + - Part~D&K1=V1 => {as, D, #{ path => Part, K1 => `<<"v1">> }}' + - pt&k1+int=1 => #{ path => pt, k1 => 1 } + - pt~d&k1+int=1 => {as, d, #{ path => pt, k1 => 1 }} + - (/nested/path) => Resolution of the path /nested/path + - (/nested/path&k1=v1) => (resolve /nested/path)#{k1 => v1} + - (/nested/path~D&K1=V1) => (resolve /nested/path)#{K1 => V1} + - pt&k1+res=(/a/b/c) => #{ path => pt, k1 => (resolve /a/b/c) } + Key: + - key: <<"value">> => #{ key => <<"value">>, ... } for all messages + - n.key: <<"value">> => #{ key => <<"value">>, ... } for Nth message + - key+Int: 1 => #{ key => 1, ... } + - key+Res: /nested/path => #{ key => (resolve /nested/path), ... } + - N.Key+Res=(/a/b/c) => #{ Key => (resolve /a/b/c), ... } +``` + + + +## Data Types ## + + + + +### ao_message() ### + + +

+ao_message() = map() | binary()
+
+ + + + +### tabm_message() ### + + +

+tabm_message() = map()
+
+ + + +## Function Index ## + + +
all_path_parts/2*Extract all of the parts from the binary, given (a list of) separators.
append_path/2*
apply_types/1*Step 3: Apply types to values and remove specifiers.
basic_hashpath_test/0*
basic_hashpath_to_test/0*
build/3*
build_messages/2*Step 5: Merge the base message with the scoped messages.
decode_string/1*Attempt Cowboy URL decode, then sanitize the result.
from/1Normalize a singleton TABM message into a list of executable AO-Core +messages.
group_scoped/2*Step 4: Group headers/query by N-scope.
inlined_keys_test/0*
inlined_keys_to_test/0*
maybe_join/2*Join a list of items with a separator, or return the first item if there +is only one item.
maybe_subpath/1*Check if the string is a subpath, returning it in parsed form, +or the original string with a specifier.
maybe_typed/2*Parse a key's type (applying it to the value) and device name if present.
multiple_inlined_keys_test/0*
multiple_inlined_keys_to_test/0*
multiple_messages_test/0*
multiple_messages_to_test/0*
normalize_base/1*Normalize the base path.
parse_explicit_message_test/0*
parse_full_path/1*Parse the relative reference into path, query, and fragment.
parse_inlined_key_val/1*Extrapolate the inlined key-value pair from a path segment.
parse_inlined_keys/2*Parse inlined key-value pairs from a path segment.
parse_part/1*Parse a path part into a message or an ID.
parse_part_mods/2*Parse part modifiers: +1.
parse_scope/1*Get the scope of a key.
part/2*Extract the characters from the binary until a separator is found.
part/4*
path_messages/1*Step 2: Decode, split and sanitize the path.
path_parts/2*Split the path into segments, filtering out empty segments and +segments that are too long.
path_parts_test/0*
scoped_key_test/0*
scoped_key_to_test/0*
simple_to_test/0*
single_message_test/0*
subpath_in_inlined_test/0*
subpath_in_inlined_to_test/0*
subpath_in_key_test/0*
subpath_in_key_to_test/0*
subpath_in_path_test/0*
subpath_in_path_to_test/0*
to/1Convert a list of AO-Core message into TABM message.
to_suite_test_/0*
type/1*
typed_key_test/0*
typed_key_to_test/0*
+ + + + +## Function Details ## + + + +### all_path_parts/2 * ### + +`all_path_parts(Sep, Bin) -> any()` + +Extract all of the parts from the binary, given (a list of) separators. + + + +### append_path/2 * ### + +`append_path(PathPart, Message) -> any()` + + + +### apply_types/1 * ### + +`apply_types(Msg) -> any()` + +Step 3: Apply types to values and remove specifiers. + + + +### basic_hashpath_test/0 * ### + +`basic_hashpath_test() -> any()` + + + +### basic_hashpath_to_test/0 * ### + +`basic_hashpath_to_test() -> any()` + + + +### build/3 * ### + +`build(I, Rest, ScopedKeys) -> any()` + + + +### build_messages/2 * ### + +`build_messages(Msgs, ScopedModifications) -> any()` + +Step 5: Merge the base message with the scoped messages. + + + +### decode_string/1 * ### + +`decode_string(B) -> any()` + +Attempt Cowboy URL decode, then sanitize the result. + + + +### from/1 ### + +`from(Path) -> any()` + +Normalize a singleton TABM message into a list of executable AO-Core +messages. + + + +### group_scoped/2 * ### + +`group_scoped(Map, Msgs) -> any()` + +Step 4: Group headers/query by N-scope. +`N.Key` => applies to Nth step. Otherwise => `global` + + + +### inlined_keys_test/0 * ### + +`inlined_keys_test() -> any()` + + + +### inlined_keys_to_test/0 * ### + +`inlined_keys_to_test() -> any()` + + + +### maybe_join/2 * ### + +`maybe_join(Items, Sep) -> any()` + +Join a list of items with a separator, or return the first item if there +is only one item. If there are no items, return an empty binary. + + + +### maybe_subpath/1 * ### + +`maybe_subpath(Str) -> any()` + +Check if the string is a subpath, returning it in parsed form, +or the original string with a specifier. + + + +### maybe_typed/2 * ### + +`maybe_typed(Key, Value) -> any()` + +Parse a key's type (applying it to the value) and device name if present. + + + +### multiple_inlined_keys_test/0 * ### + +`multiple_inlined_keys_test() -> any()` + + + +### multiple_inlined_keys_to_test/0 * ### + +`multiple_inlined_keys_to_test() -> any()` + + + +### multiple_messages_test/0 * ### + +`multiple_messages_test() -> any()` + + + +### multiple_messages_to_test/0 * ### + +`multiple_messages_to_test() -> any()` + + + +### normalize_base/1 * ### + +`normalize_base(Rest) -> any()` + +Normalize the base path. + + + +### parse_explicit_message_test/0 * ### + +`parse_explicit_message_test() -> any()` + + + +### parse_full_path/1 * ### + +`parse_full_path(RelativeRef) -> any()` + +Parse the relative reference into path, query, and fragment. + + + +### parse_inlined_key_val/1 * ### + +`parse_inlined_key_val(Bin) -> any()` + +Extrapolate the inlined key-value pair from a path segment. If the +key has a value, it may provide a type (as with typical keys), but if a +value is not provided, it is assumed to be a boolean `true`. + + + +### parse_inlined_keys/2 * ### + +`parse_inlined_keys(InlinedMsgBin, Msg) -> any()` + +Parse inlined key-value pairs from a path segment. Each key-value pair +is separated by `&` and is of the form `K=V`. + + + +### parse_part/1 * ### + +`parse_part(ID) -> any()` + +Parse a path part into a message or an ID. +Applies the syntax rules outlined in the module doc, in the following order: +1. ID +2. Part subpath resolutions +3. Inlined key-value pairs +4. Device specifier + + + +### parse_part_mods/2 * ### + +`parse_part_mods(X1, Msg) -> any()` + +Parse part modifiers: +1. `~Device` => `{as, Device, Msg}` +2. `&K=V` => `Msg#{ K => V }` + + + +### parse_scope/1 * ### + +`parse_scope(KeyBin) -> any()` + +Get the scope of a key. Adds 1 to account for the base message. + + + +### part/2 * ### + +`part(Sep, Bin) -> any()` + +Extract the characters from the binary until a separator is found. +The first argument of the function is an explicit separator character, or +a list of separator characters. Returns a tuple with the separator, the +accumulated characters, and the rest of the binary. + + + +### part/4 * ### + +`part(Seps, X2, Depth, CurrAcc) -> any()` + + + +### path_messages/1 * ### + +`path_messages(RawBin) -> any()` + +Step 2: Decode, split and sanitize the path. Split by `/` but avoid +subpath components, such that their own path parts are not dissociated from +their parent path. + + + +### path_parts/2 * ### + +`path_parts(Sep, PathBin) -> any()` + +Split the path into segments, filtering out empty segments and +segments that are too long. + + + +### path_parts_test/0 * ### + +`path_parts_test() -> any()` + + + +### scoped_key_test/0 * ### + +`scoped_key_test() -> any()` + + + +### scoped_key_to_test/0 * ### + +`scoped_key_to_test() -> any()` + + + +### simple_to_test/0 * ### + +`simple_to_test() -> any()` + + + +### single_message_test/0 * ### + +`single_message_test() -> any()` + + + +### subpath_in_inlined_test/0 * ### + +`subpath_in_inlined_test() -> any()` + + + +### subpath_in_inlined_to_test/0 * ### + +`subpath_in_inlined_to_test() -> any()` + + + +### subpath_in_key_test/0 * ### + +`subpath_in_key_test() -> any()` + + + +### subpath_in_key_to_test/0 * ### + +`subpath_in_key_to_test() -> any()` + + + +### subpath_in_path_test/0 * ### + +`subpath_in_path_test() -> any()` + + + +### subpath_in_path_to_test/0 * ### + +`subpath_in_path_to_test() -> any()` + + + +### to/1 ### + +

+to(Messages::[ao_message()]) -> tabm_message()
+
+
+ +Convert a list of AO-Core message into TABM message. + + + +### to_suite_test_/0 * ### + +`to_suite_test_() -> any()` + + + +### type/1 * ### + +`type(Value) -> any()` + + + +### typed_key_test/0 * ### + +`typed_key_test() -> any()` + + + +### typed_key_to_test/0 * ### + +`typed_key_to_test() -> any()` + diff --git a/docs/resources/source-code/hb_store.md b/docs/resources/source-code/hb_store.md new file mode 100644 index 000000000..0e8ad542f --- /dev/null +++ b/docs/resources/source-code/hb_store.md @@ -0,0 +1,250 @@ +# [Module hb_store.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store.erl) + + + + + + +## Function Index ## + + +
add_path/2Add two path components together.
add_path/3
behavior_info/1
call_all/3*Call a function on all modules in the store.
call_function/3*Call a function on the first store module that succeeds.
filter/2Takes a store object and a filter function or match spec, returning a +new store object with only the modules that match the filter.
generate_test_suite/1
generate_test_suite/2
get_store_scope/1*Ask a store for its own scope.
hierarchical_path_resolution_test/1*Ensure that we can resolve links through a directory.
join/1Join a list of path components together.
list/2List the keys in a group in the store.
make_group/2Make a group in the store.
make_link/3Make a link from one path to another in the store.
path/1Create a path from a list of path components.
path/2
read/2Read a key from the store.
reset/1Delete all of the keys in a store.
resolve/2Follow links through the store to resolve a path to its ultimate target.
resursive_path_resolution_test/1*Ensure that we can resolve links recursively.
scope/2Limit the store scope to only a specific (set of) option(s).
simple_path_resolution_test/1*Test path resolution dynamics.
sort/2Order a store by a preference of its scopes.
start/1
stop/1
store_suite_test_/0*
test_stores/0
type/2Get the type of element of a given path in the store.
write/3Write a key with a value to the store.
+ + + + +## Function Details ## + + + +### add_path/2 ### + +`add_path(Path1, Path2) -> any()` + +Add two path components together. If no store implements the add_path +function, we concatenate the paths. + + + +### add_path/3 ### + +`add_path(Store, Path1, Path2) -> any()` + + + +### behavior_info/1 ### + +`behavior_info(X1) -> any()` + + + +### call_all/3 * ### + +`call_all(X, Function, Args) -> any()` + +Call a function on all modules in the store. + + + +### call_function/3 * ### + +`call_function(X, Function, Args) -> any()` + +Call a function on the first store module that succeeds. Returns its +result, or no_viable_store if none of the stores succeed. + + + +### filter/2 ### + +`filter(Module, Filter) -> any()` + +Takes a store object and a filter function or match spec, returning a +new store object with only the modules that match the filter. The filter +function takes 2 arguments: the scope and the options. It calls the store's +scope function to get the scope of the module. + + + +### generate_test_suite/1 ### + +`generate_test_suite(Suite) -> any()` + + + +### generate_test_suite/2 ### + +`generate_test_suite(Suite, Stores) -> any()` + + + +### get_store_scope/1 * ### + +`get_store_scope(Store) -> any()` + +Ask a store for its own scope. If it doesn't have one, return the +default scope (local). + + + +### hierarchical_path_resolution_test/1 * ### + +`hierarchical_path_resolution_test(Opts) -> any()` + +Ensure that we can resolve links through a directory. + + + +### join/1 ### + +`join(Path) -> any()` + +Join a list of path components together. + + + +### list/2 ### + +`list(Modules, Path) -> any()` + +List the keys in a group in the store. Use only in debugging. +The hyperbeam model assumes that stores are built as efficient hash-based +structures, so this is likely to be very slow for most stores. + + + +### make_group/2 ### + +`make_group(Modules, Path) -> any()` + +Make a group in the store. A group can be seen as a namespace or +'directory' in a filesystem. + + + +### make_link/3 ### + +`make_link(Modules, Existing, New) -> any()` + +Make a link from one path to another in the store. + + + +### path/1 ### + +`path(Path) -> any()` + +Create a path from a list of path components. If no store implements +the path function, we return the path with the 'default' transformation (id). + + + +### path/2 ### + +`path(X1, Path) -> any()` + + + +### read/2 ### + +`read(Modules, Key) -> any()` + +Read a key from the store. + + + +### reset/1 ### + +`reset(Modules) -> any()` + +Delete all of the keys in a store. Should be used with extreme +caution. Lost data can lose money in many/most of hyperbeam's use cases. + + + +### resolve/2 ### + +`resolve(Modules, Path) -> any()` + +Follow links through the store to resolve a path to its ultimate target. + + + +### resursive_path_resolution_test/1 * ### + +`resursive_path_resolution_test(Opts) -> any()` + +Ensure that we can resolve links recursively. + + + +### scope/2 ### + +`scope(Scope, Opts) -> any()` + +Limit the store scope to only a specific (set of) option(s). +Takes either an Opts message or store, and either a single scope or a list +of scopes. + + + +### simple_path_resolution_test/1 * ### + +`simple_path_resolution_test(Opts) -> any()` + +Test path resolution dynamics. + + + +### sort/2 ### + +`sort(Stores, PreferenceOrder) -> any()` + +Order a store by a preference of its scopes. This is useful for making +sure that faster (or perhaps cheaper) stores are used first. If a list is +provided, it will be used as a preference order. If a map is provided, +scopes will be ordered by the scores in the map. Any unknown scopes will +default to a score of 0. + + + +### start/1 ### + +`start(Modules) -> any()` + + + +### stop/1 ### + +`stop(Modules) -> any()` + + + +### store_suite_test_/0 * ### + +`store_suite_test_() -> any()` + + + +### test_stores/0 ### + +`test_stores() -> any()` + + + +### type/2 ### + +`type(Modules, Path) -> any()` + +Get the type of element of a given path in the store. This can be +a performance killer if the store is remote etc. Use only when necessary. + + + +### write/3 ### + +`write(Modules, Key, Value) -> any()` + +Write a key with a value to the store. + diff --git a/docs/resources/source-code/hb_store_fs.md b/docs/resources/source-code/hb_store_fs.md new file mode 100644 index 000000000..9e55fef1b --- /dev/null +++ b/docs/resources/source-code/hb_store_fs.md @@ -0,0 +1,149 @@ +# [Module hb_store_fs.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_fs.erl) + + + + + + +## Function Index ## + + +
add_prefix/2*Add the directory prefix to a path.
list/2List contents of a directory in the store.
make_group/2Create a directory (group) in the store.
make_link/3Create a symlink, handling the case where the link would point to itself.
read/1*
read/2Read a key from the store, following symlinks as needed.
remove_prefix/2*Remove the directory prefix from a path.
reset/1Reset the store by completely removing its directory and recreating it.
resolve/2Replace links in a path successively, returning the final path.
resolve/3*
scope/1The file-based store is always local, for now.
start/1Initialize the file system store with the given data directory.
stop/1Stop the file system store.
type/1*
type/2Determine the type of a key in the store.
write/3Write a value to the specified path in the store.
+ + + + +## Function Details ## + + + +### add_prefix/2 * ### + +`add_prefix(X1, Path) -> any()` + +Add the directory prefix to a path. + + + +### list/2 ### + +`list(Opts, Path) -> any()` + +List contents of a directory in the store. + + + +### make_group/2 ### + +`make_group(Opts, Path) -> any()` + +Create a directory (group) in the store. + + + +### make_link/3 ### + +`make_link(Opts, Link, New) -> any()` + +Create a symlink, handling the case where the link would point to itself. + + + +### read/1 * ### + +`read(Path) -> any()` + + + +### read/2 ### + +`read(Opts, Key) -> any()` + +Read a key from the store, following symlinks as needed. + + + +### remove_prefix/2 * ### + +`remove_prefix(X1, Path) -> any()` + +Remove the directory prefix from a path. + + + +### reset/1 ### + +`reset(X1) -> any()` + +Reset the store by completely removing its directory and recreating it. + + + +### resolve/2 ### + +`resolve(Opts, RawPath) -> any()` + +Replace links in a path successively, returning the final path. +Each element of the path is resolved in turn, with the result of each +resolution becoming the prefix for the next resolution. This allows +paths to resolve across many links. For example, a structure as follows: + +/a/b/c: "Not the right data" +/a/b -> /a/alt-b +/a/alt-b/c: "Correct data" + +will resolve "a/b/c" to "Correct data". + + + +### resolve/3 * ### + +`resolve(Opts, CurrPath, Rest) -> any()` + + + +### scope/1 ### + +`scope(X1) -> any()` + +The file-based store is always local, for now. In the future, we may +want to allow that an FS store is shared across a cluster and thus remote. + + + +### start/1 ### + +`start(X1) -> any()` + +Initialize the file system store with the given data directory. + + + +### stop/1 ### + +`stop(X1) -> any()` + +Stop the file system store. Currently a no-op. + + + +### type/1 * ### + +`type(Path) -> any()` + + + +### type/2 ### + +`type(Opts, Key) -> any()` + +Determine the type of a key in the store. + + + +### write/3 ### + +`write(Opts, PathComponents, Value) -> any()` + +Write a value to the specified path in the store. + diff --git a/docs/resources/source-code/hb_store_gateway.md b/docs/resources/source-code/hb_store_gateway.md new file mode 100644 index 000000000..2411d5d80 --- /dev/null +++ b/docs/resources/source-code/hb_store_gateway.md @@ -0,0 +1,133 @@ +# [Module hb_store_gateway.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_gateway.erl) + + + + +A store module that reads data from the nodes Arweave gateway and +GraphQL routes, additionally including additional store-specific routes. + + + +## Function Index ## + + +
cache_read_message_test/0*Ensure that saving to the gateway store works.
external_http_access_test/0*Test that the default node config allows for data to be accessed.
graphql_as_store_test_/0*Store is accessible via the default options.
graphql_from_cache_test/0*Stored messages are accessible via hb_cache accesses.
list/2
manual_local_cache_test/0*
maybe_cache/2*Cache the data if the cache is enabled.
read/2Read the data at the given key from the GraphQL route.
resolve/2
resolve_on_gateway_test_/0*
scope/1The scope of a GraphQL store is always remote, due to performance.
specific_route_test/0*Routes can be specified in the options, overriding the default routes.
store_opts_test/0*Test to verify store opts is being set for Data-Protocol ao.
type/2Get the type of the data at the given key.
+ + + + +## Function Details ## + + + +### cache_read_message_test/0 * ### + +`cache_read_message_test() -> any()` + +Ensure that saving to the gateway store works. + + + +### external_http_access_test/0 * ### + +`external_http_access_test() -> any()` + +Test that the default node config allows for data to be accessed. + + + +### graphql_as_store_test_/0 * ### + +`graphql_as_store_test_() -> any()` + +Store is accessible via the default options. + + + +### graphql_from_cache_test/0 * ### + +`graphql_from_cache_test() -> any()` + +Stored messages are accessible via `hb_cache` accesses. + + + +### list/2 ### + +`list(StoreOpts, Key) -> any()` + + + +### manual_local_cache_test/0 * ### + +`manual_local_cache_test() -> any()` + + + +### maybe_cache/2 * ### + +`maybe_cache(StoreOpts, Data) -> any()` + +Cache the data if the cache is enabled. The `store` option may either +be `false` to disable local caching, or a store definition to use as the +cache. + + + +### read/2 ### + +`read(StoreOpts, Key) -> any()` + +Read the data at the given key from the GraphQL route. Will only attempt +to read the data if the key is an ID. + + + +### resolve/2 ### + +`resolve(X1, Key) -> any()` + + + +### resolve_on_gateway_test_/0 * ### + +`resolve_on_gateway_test_() -> any()` + + + +### scope/1 ### + +`scope(X1) -> any()` + +The scope of a GraphQL store is always remote, due to performance. + + + +### specific_route_test/0 * ### + +`specific_route_test() -> any()` + +Routes can be specified in the options, overriding the default routes. +We test this by inversion: If the above cache read test works, then we know +that the default routes allow access to the item. If the test below were to +produce the same result, despite an empty 'only' route list, then we would +know that the module is not respecting the route list. + + + +### store_opts_test/0 * ### + +`store_opts_test() -> any()` + +Test to verify store opts is being set for Data-Protocol ao + + + +### type/2 ### + +`type(StoreOpts, Key) -> any()` + +Get the type of the data at the given key. We potentially cache the +result, so that we don't have to read the data from the GraphQL route +multiple times. + diff --git a/docs/resources/source-code/hb_store_remote_node.md b/docs/resources/source-code/hb_store_remote_node.md new file mode 100644 index 000000000..34a886174 --- /dev/null +++ b/docs/resources/source-code/hb_store_remote_node.md @@ -0,0 +1,100 @@ +# [Module hb_store_remote_node.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_remote_node.erl) + + + + +A store module that reads data from another AO node. + + + +## Description ## +Notably, this store only provides the _read_ side of the store interface. +The write side could be added, returning an commitment that the data has +been written to the remote node. In that case, the node would probably want +to upload it to an Arweave bundler to ensure persistence, too. + +## Function Index ## + + +
make_link/3Link a source to a destination in the remote node.
read/2Read a key from the remote node.
read_test/0*Test that we can create a store, write a random message to it, then +start a remote node with that store, and read the message from it.
resolve/2Resolve a key path in the remote store.
scope/1Return the scope of this store.
type/2Determine the type of value at a given key.
write/3Write a key to the remote node.
+ + + + +## Function Details ## + + + +### make_link/3 ### + +`make_link(Opts, Source, Destination) -> any()` + +Link a source to a destination in the remote node. + +Constructs an HTTP POST link request. If a wallet is provided, +the message is signed. Returns {ok, Path} on HTTP 200, or +{error, Reason} on failure. + + + +### read/2 ### + +`read(Opts, Key) -> any()` + +Read a key from the remote node. + +Makes an HTTP GET request to the remote node and returns the +committed message. + + + +### read_test/0 * ### + +`read_test() -> any()` + +Test that we can create a store, write a random message to it, then +start a remote node with that store, and read the message from it. + + + +### resolve/2 ### + +`resolve(X1, Key) -> any()` + +Resolve a key path in the remote store. + +For the remote node store, the key is returned as-is. + + + +### scope/1 ### + +`scope(Arg) -> any()` + +Return the scope of this store. + +For the remote store, the scope is always `remote`. + + + +### type/2 ### + +`type(Opts, Key) -> any()` + +Determine the type of value at a given key. + +Remote nodes support only the `simple` type or `not_found`. + + + +### write/3 ### + +`write(Opts, Key, Value) -> any()` + +Write a key to the remote node. + +Constructs an HTTP POST write request. If a wallet is provided, +the message is signed. Returns {ok, Path} on HTTP 200, or +{error, Reason} on failure. + diff --git a/docs/resources/source-code/hb_store_rocksdb.md b/docs/resources/source-code/hb_store_rocksdb.md new file mode 100644 index 000000000..1da975be5 --- /dev/null +++ b/docs/resources/source-code/hb_store_rocksdb.md @@ -0,0 +1,367 @@ +# [Module hb_store_rocksdb.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_store_rocksdb.erl) + + + + +A process wrapper over rocksdb storage. + +__Behaviours:__ [`gen_server`](gen_server.md), [`hb_store`](hb_store.md). + + + +## Description ## + +Replicates functionality of the +hb_fs_store module. + +Encodes the item types with the help of prefixes, see `encode_value/2` +and `decode_value/1` + + +## Data Types ## + + + + +### key() ### + + +

+key() = binary() | list()
+
+ + + + +### value() ### + + +

+value() = binary() | list()
+
+ + + + +### value_type() ### + + +

+value_type() = link | raw | group
+
+ + + +## Function Index ## + + +
add_path/3Add two path components together.
code_change/3
collect/1*
collect/2*
convert_if_list/1*
decode_value/1*
do_read/2*
do_resolve/3*
do_write/3*Write given Key and Value to the database.
enabled/0Returns whether the RocksDB store is enabled.
encode_value/2*
ensure_dir/2*
ensure_dir/3*
ensure_list/1*Ensure that the given filename is a list, not a binary.
handle_call/3
handle_cast/2
handle_info/2
init/1
join/1*
list/0List all items registered in rocksdb store.
list/2Returns the full list of items stored under the given path.
make_group/2Creates group under the given path.
make_link/3
maybe_append_key_to_group/2*
maybe_convert_to_binary/1*
maybe_create_dir/3*
open_rockdb/1*
path/2Return path.
read/2Read data by the key.
reset/1
resolve/2Replace links in a path with the target of the link.
scope/1Return scope (local).
start/1
start_link/1Start the RocksDB store.
stop/1
terminate/2
type/2Get type of the current item.
write/3Write given Key and Value to the database.
+ + + + +## Function Details ## + + + +### add_path/3 ### + +`add_path(Opts, Path1, Path2) -> any()` + +Add two path components together. // is not used + + + +### code_change/3 ### + +`code_change(OldVsn, State, Extra) -> any()` + + + +### collect/1 * ### + +`collect(Iterator) -> any()` + + + +### collect/2 * ### + +`collect(Iterator, Acc) -> any()` + + + +### convert_if_list/1 * ### + +`convert_if_list(Value) -> any()` + + + +### decode_value/1 * ### + +

+decode_value(X1::binary()) -> {value_type(), binary()}
+
+
+ + + +### do_read/2 * ### + +`do_read(Opts, Key) -> any()` + + + +### do_resolve/3 * ### + +`do_resolve(Opts, FinalPath, Rest) -> any()` + + + +### do_write/3 * ### + +

+do_write(Opts, Key, Value) -> Result
+
+ +
  • Opts = map()
  • Key = key()
  • Value = value()
  • Result = ok | {error, any()}
+ +Write given Key and Value to the database + + + +### enabled/0 ### + +`enabled() -> any()` + +Returns whether the RocksDB store is enabled. + + + +### encode_value/2 * ### + +

+encode_value(X1::value_type(), Value::binary()) -> binary()
+
+
+ + + +### ensure_dir/2 * ### + +`ensure_dir(DBHandle, BaseDir) -> any()` + + + +### ensure_dir/3 * ### + +`ensure_dir(DBHandle, CurrentPath, Rest) -> any()` + + + +### ensure_list/1 * ### + +`ensure_list(Value) -> any()` + +Ensure that the given filename is a list, not a binary. + + + +### handle_call/3 ### + +`handle_call(Request, From, State) -> any()` + + + +### handle_cast/2 ### + +`handle_cast(Request, State) -> any()` + + + +### handle_info/2 ### + +`handle_info(Info, State) -> any()` + + + +### init/1 ### + +`init(Dir) -> any()` + + + +### join/1 * ### + +`join(Key) -> any()` + + + +### list/0 ### + +`list() -> any()` + +List all items registered in rocksdb store. Should be used only +for testing/debugging, as the underlying operation is doing full traversal +on the KV storage, and is slow. + + + +### list/2 ### + +

+list(Opts, Path) -> Result
+
+ +
  • Opts = any()
  • Path = any()
  • Result = {ok, [string()]} | {error, term()}
+ +Returns the full list of items stored under the given path. Where the path +child items is relevant to the path of parentItem. (Same as in `hb_store_fs`). + + + +### make_group/2 ### + +

+make_group(Opts, Key) -> Result
+
+ +
  • Opts = any()
  • Key = binary()
  • Result = ok | {error, already_added}
+ +Creates group under the given path. + + + +### make_link/3 ### + +

+make_link(Opts::any(), Key1::key(), New::key()) -> ok
+
+
+ + + +### maybe_append_key_to_group/2 * ### + +`maybe_append_key_to_group(Key, CurrentDirContents) -> any()` + + + +### maybe_convert_to_binary/1 * ### + +`maybe_convert_to_binary(Value) -> any()` + + + +### maybe_create_dir/3 * ### + +`maybe_create_dir(DBHandle, DirPath, Value) -> any()` + + + +### open_rockdb/1 * ### + +`open_rockdb(RawDir) -> any()` + + + +### path/2 ### + +`path(Opts, Path) -> any()` + +Return path + + + +### read/2 ### + +

+read(Opts, Key) -> Result
+
+ +
  • Opts = map()
  • Key = key() | list()
  • Result = {ok, value()} | not_found | {error, {corruption, string()}} | {error, any()}
+ +Read data by the key. +Recursively follows link messages + + + +### reset/1 ### + +

+reset(Opts::[]) -> ok | no_return()
+
+
+ + + +### resolve/2 ### + +

+resolve(Opts, Path) -> Result
+
+ +
  • Opts = any()
  • Path = binary() | list()
  • Result = not_found | string()
+ +Replace links in a path with the target of the link. + + + +### scope/1 ### + +`scope(X1) -> any()` + +Return scope (local) + + + +### start/1 ### + +`start(Opts) -> any()` + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + +Start the RocksDB store. + + + +### stop/1 ### + +

+stop(Opts::any()) -> ok
+
+
+ + + +### terminate/2 ### + +`terminate(Reason, State) -> any()` + + + +### type/2 ### + +

+type(Opts, Key) -> Result
+
+ +
  • Opts = map()
  • Key = binary()
  • Result = composite | simple | not_found
+ +Get type of the current item + + + +### write/3 ### + +

+write(Opts, Key, Value) -> Result
+
+ +
  • Opts = map()
  • Key = key()
  • Value = value()
  • Result = ok | {error, any()}
+ +Write given Key and Value to the database + diff --git a/docs/resources/source-code/hb_structured_fields.md b/docs/resources/source-code/hb_structured_fields.md new file mode 100644 index 000000000..5aa7d74fc --- /dev/null +++ b/docs/resources/source-code/hb_structured_fields.md @@ -0,0 +1,521 @@ +# [Module hb_structured_fields.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_structured_fields.erl) + + + + +A module for parsing and converting between Erlang and HTTP Structured +Fields, as described in RFC-9651. + + + +## Description ## + +The mapping between Erlang and structured headers types is as follow: + +List: list() +Inner list: {list, [item()], params()} +Dictionary: [{binary(), item()}] +There is no distinction between empty list and empty dictionary. +Item with parameters: {item, bare_item(), params()} +Parameters: [{binary(), bare_item()}] +Bare item: one bare_item() that can be of type: +Integer: integer() +Decimal: {decimal, {integer(), integer()}} +String: {string, binary()} +Token: {token, binary()} +Byte sequence: {binary, binary()} +Boolean: boolean() + + +## Data Types ## + + + + +### sh_bare_item() ### + + +

+sh_bare_item() = integer() | sh_decimal() | boolean() | {string | token | binary, binary()}
+
+ + + + +### sh_decimal() ### + + +

+sh_decimal() = {decimal, {integer(), integer()}}
+
+ + + + +### sh_dictionary() ### + + +

+sh_dictionary() = [{binary(), sh_item() | sh_inner_list()}]
+
+ + + + +### sh_inner_list() ### + + +

+sh_inner_list() = {list, [sh_item()], sh_params()}
+
+ + + + +### sh_item() ### + + +

+sh_item() = {item, sh_bare_item(), sh_params()}
+
+ + + + +### sh_list() ### + + +

+sh_list() = [sh_item() | sh_inner_list()]
+
+ + + + +### sh_params() ### + + +

+sh_params() = [{binary(), sh_bare_item()}]
+
+ + + +## Function Index ## + + +
bare_item/1
dictionary/1
e2t/1*
e2tb/1*
e2tp/1*
escape_string/2*
exp_div/1*
expected_to_term/1*
from_bare_item/1Convert an SF bare_item to an Erlang term.
inner_list/1*
item/1
item_or_inner_list/1*
key_to_binary/1*Convert an Erlang term to a binary key.
list/1
params/1*
parse_bare_item/1Parse an integer or decimal.
parse_before_param/2*
parse_binary/2*Parse a byte sequence binary.
parse_decimal/5*Parse a decimal binary.
parse_dict_before_member/2*Parse a binary SF dictionary before a member.
parse_dict_before_sep/2*Parse a binary SF dictionary before a separator.
parse_dict_key/3*
parse_dictionary/1Parse a binary SF dictionary.
parse_inner_list/2*
parse_item/1Parse a binary SF item to an SF item.
parse_item1/1*
parse_list/1Parse a binary SF list.
parse_list_before_member/2*Parse a binary SF list before a member.
parse_list_before_sep/2*Parse a binary SF list before a separator.
parse_list_member/2*Parse a binary SF list before a member.
parse_number/3*Parse an integer or decimal binary.
parse_param/3*
parse_string/2*Parse a string binary.
parse_struct_hd_test_/0*
parse_token/2*Parse a token binary.
raw_to_binary/1*
struct_hd_identity_test_/0*
to_bare_item/1*Convert an Erlang term to an SF bare_item.
to_dictionary/1Convert a map to a dictionary.
to_dictionary/2*
to_dictionary_depth_test/0*
to_dictionary_test/0*
to_inner_item/1*Convert an Erlang term to an SF item.
to_inner_list/1*Convert an inner list to an SF term.
to_inner_list/2*
to_inner_list/3*
to_item/1Convert an item to a dictionary.
to_item/2
to_item_or_inner_list/1*Convert an Erlang term to an SF item or inner_list.
to_item_test/0*
to_list/1Convert a list to an SF term.
to_list/2*
to_list_depth_test/0*
to_list_test/0*
to_param/1*Convert an Erlang term to an SF parameter.
trim_ws/1*
trim_ws_end/2*
+ + + + +## Function Details ## + + + +### bare_item/1 ### + +`bare_item(Integer) -> any()` + + + +### dictionary/1 ### + +

+dictionary(Map::#{binary() => sh_item() | sh_inner_list()} | sh_dictionary()) -> iolist()
+
+
+ + + +### e2t/1 * ### + +`e2t(Dict) -> any()` + + + +### e2tb/1 * ### + +`e2tb(V) -> any()` + + + +### e2tp/1 * ### + +`e2tp(Params) -> any()` + + + +### escape_string/2 * ### + +`escape_string(X1, Acc) -> any()` + + + +### exp_div/1 * ### + +`exp_div(N) -> any()` + + + +### expected_to_term/1 * ### + +`expected_to_term(Dict) -> any()` + + + +### from_bare_item/1 ### + +`from_bare_item(BareItem) -> any()` + +Convert an SF `bare_item` to an Erlang term. + + + +### inner_list/1 * ### + +`inner_list(X1) -> any()` + + + +### item/1 ### + +

+item(X1::sh_item()) -> iolist()
+
+
+ + + +### item_or_inner_list/1 * ### + +`item_or_inner_list(Value) -> any()` + + + +### key_to_binary/1 * ### + +`key_to_binary(Key) -> any()` + +Convert an Erlang term to a binary key. + + + +### list/1 ### + +

+list(List::sh_list()) -> iolist()
+
+
+ + + +### params/1 * ### + +`params(Params) -> any()` + + + +### parse_bare_item/1 ### + +`parse_bare_item(X1) -> any()` + +Parse an integer or decimal. + + + +### parse_before_param/2 * ### + +`parse_before_param(X1, Acc) -> any()` + + + +### parse_binary/2 * ### + +`parse_binary(X1, Acc) -> any()` + +Parse a byte sequence binary. + + + +### parse_decimal/5 * ### + +`parse_decimal(R, L1, L2, IntAcc, FracAcc) -> any()` + +Parse a decimal binary. + + + +### parse_dict_before_member/2 * ### + +`parse_dict_before_member(X1, Acc) -> any()` + +Parse a binary SF dictionary before a member. + + + +### parse_dict_before_sep/2 * ### + +`parse_dict_before_sep(X1, Acc) -> any()` + +Parse a binary SF dictionary before a separator. + + + +### parse_dict_key/3 * ### + +`parse_dict_key(R, Acc, K) -> any()` + + + +### parse_dictionary/1 ### + +

+parse_dictionary(X1::binary()) -> sh_dictionary()
+
+
+ +Parse a binary SF dictionary. + + + +### parse_inner_list/2 * ### + +`parse_inner_list(R0, Acc) -> any()` + + + +### parse_item/1 ### + +

+parse_item(Bin::binary()) -> sh_item()
+
+
+ +Parse a binary SF item to an SF `item`. + + + +### parse_item1/1 * ### + +`parse_item1(Bin) -> any()` + + + +### parse_list/1 ### + +

+parse_list(Bin::binary()) -> sh_list()
+
+
+ +Parse a binary SF list. + + + +### parse_list_before_member/2 * ### + +`parse_list_before_member(R, Acc) -> any()` + +Parse a binary SF list before a member. + + + +### parse_list_before_sep/2 * ### + +`parse_list_before_sep(X1, Acc) -> any()` + +Parse a binary SF list before a separator. + + + +### parse_list_member/2 * ### + +`parse_list_member(R0, Acc) -> any()` + +Parse a binary SF list before a member. + + + +### parse_number/3 * ### + +`parse_number(R, L, Acc) -> any()` + +Parse an integer or decimal binary. + + + +### parse_param/3 * ### + +`parse_param(R, Acc, K) -> any()` + + + +### parse_string/2 * ### + +`parse_string(X1, Acc) -> any()` + +Parse a string binary. + + + +### parse_struct_hd_test_/0 * ### + +`parse_struct_hd_test_() -> any()` + + + +### parse_token/2 * ### + +`parse_token(R, Acc) -> any()` + +Parse a token binary. + + + +### raw_to_binary/1 * ### + +`raw_to_binary(RawList) -> any()` + + + +### struct_hd_identity_test_/0 * ### + +`struct_hd_identity_test_() -> any()` + + + +### to_bare_item/1 * ### + +`to_bare_item(BareItem) -> any()` + +Convert an Erlang term to an SF `bare_item`. + + + +### to_dictionary/1 ### + +`to_dictionary(Map) -> any()` + +Convert a map to a dictionary. + + + +### to_dictionary/2 * ### + +`to_dictionary(Dict, Rest) -> any()` + + + +### to_dictionary_depth_test/0 * ### + +`to_dictionary_depth_test() -> any()` + + + +### to_dictionary_test/0 * ### + +`to_dictionary_test() -> any()` + + + +### to_inner_item/1 * ### + +`to_inner_item(Item) -> any()` + +Convert an Erlang term to an SF `item`. + + + +### to_inner_list/1 * ### + +`to_inner_list(Inner) -> any()` + +Convert an inner list to an SF term. + + + +### to_inner_list/2 * ### + +`to_inner_list(Inner, Params) -> any()` + + + +### to_inner_list/3 * ### + +`to_inner_list(Inner, Rest, Params) -> any()` + + + +### to_item/1 ### + +`to_item(Item) -> any()` + +Convert an item to a dictionary. + + + +### to_item/2 ### + +`to_item(Item, Params) -> any()` + + + +### to_item_or_inner_list/1 * ### + +`to_item_or_inner_list(ItemOrInner) -> any()` + +Convert an Erlang term to an SF `item` or `inner_list`. + + + +### to_item_test/0 * ### + +`to_item_test() -> any()` + + + +### to_list/1 ### + +`to_list(List) -> any()` + +Convert a list to an SF term. + + + +### to_list/2 * ### + +`to_list(Acc, Rest) -> any()` + + + +### to_list_depth_test/0 * ### + +`to_list_depth_test() -> any()` + + + +### to_list_test/0 * ### + +`to_list_test() -> any()` + + + +### to_param/1 * ### + +`to_param(X1) -> any()` + +Convert an Erlang term to an SF `parameter`. + + + +### trim_ws/1 * ### + +`trim_ws(R) -> any()` + + + +### trim_ws_end/2 * ### + +`trim_ws_end(Value, N) -> any()` + diff --git a/docs/resources/source-code/hb_sup.md b/docs/resources/source-code/hb_sup.md new file mode 100644 index 000000000..3b29e9f87 --- /dev/null +++ b/docs/resources/source-code/hb_sup.md @@ -0,0 +1,45 @@ +# [Module hb_sup.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_sup.erl) + + + + +__Behaviours:__ [`supervisor`](supervisor.md). + + + +## Function Index ## + + +
init/1
start_link/0
start_link/1
store_children/1*Generate a child spec for stores in the given Opts.
+ + + + +## Function Details ## + + + +### init/1 ### + +`init(Opts) -> any()` + + + +### start_link/0 ### + +`start_link() -> any()` + + + +### start_link/1 ### + +`start_link(Opts) -> any()` + + + +### store_children/1 * ### + +`store_children(Store) -> any()` + +Generate a child spec for stores in the given Opts. + diff --git a/docs/resources/source-code/hb_test_utils.md b/docs/resources/source-code/hb_test_utils.md new file mode 100644 index 000000000..46ba4d961 --- /dev/null +++ b/docs/resources/source-code/hb_test_utils.md @@ -0,0 +1,48 @@ +# [Module hb_test_utils.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_test_utils.erl) + + + + +Simple utilities for testing HyperBEAM. + + + +## Function Index ## + + +
run/4
satisfies_requirements/1*Determine if the environment satisfies the given test requirements.
suite_with_opts/2Run each test in a suite with each set of options.
+ + + + +## Function Details ## + + + +### run/4 ### + +`run(Name, OptsName, Suite, OptsList) -> any()` + + + +### satisfies_requirements/1 * ### + +`satisfies_requirements(Requirements) -> any()` + +Determine if the environment satisfies the given test requirements. +Requirements is a list of atoms, each corresponding to a module that must +return true if it exposes an `enabled/0` function. + + + +### suite_with_opts/2 ### + +`suite_with_opts(Suite, OptsList) -> any()` + +Run each test in a suite with each set of options. Start and reset +the store(s) for each test. Expects suites to be a list of tuples with +the test name, description, and test function. +The list of `Opts` should contain maps with the `name` and `opts` keys. +Each element may also contain a `skip` key with a list of test names to skip. +They can also contain a `desc` key with a description of the options. + diff --git a/docs/resources/source-code/hb_tracer.md b/docs/resources/source-code/hb_tracer.md new file mode 100644 index 000000000..a880ee8be --- /dev/null +++ b/docs/resources/source-code/hb_tracer.md @@ -0,0 +1,78 @@ +# [Module hb_tracer.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_tracer.erl) + + + + +A module for tracing the flow of requests through the system. + + + +## Description ## +This allows for tracking the lifecycle of a request from HTTP receipt through processing and response. + +## Function Index ## + + +
checkmark_emoji/0*
failure_emoji/0*
format_error_trace/1Format a trace for error in a user-friendly emoji oriented output.
get_trace/1Exports the complete queue of events.
record_step/2Register a new step into a tracer.
stage_to_emoji/1*
start_trace/0Start a new tracer acting as queue of events registered.
trace_loop/1*
+ + + + +## Function Details ## + + + +### checkmark_emoji/0 * ### + +`checkmark_emoji() -> any()` + + + +### failure_emoji/0 * ### + +`failure_emoji() -> any()` + + + +### format_error_trace/1 ### + +`format_error_trace(Trace) -> any()` + +Format a trace for error in a user-friendly emoji oriented output + + + +### get_trace/1 ### + +`get_trace(TracePID) -> any()` + +Exports the complete queue of events + + + +### record_step/2 ### + +`record_step(TracePID, Step) -> any()` + +Register a new step into a tracer + + + +### stage_to_emoji/1 * ### + +`stage_to_emoji(Stage) -> any()` + + + +### start_trace/0 ### + +`start_trace() -> any()` + +Start a new tracer acting as queue of events registered. + + + +### trace_loop/1 * ### + +`trace_loop(Trace) -> any()` + diff --git a/docs/resources/source-code/hb_util.md b/docs/resources/source-code/hb_util.md new file mode 100644 index 000000000..711e9acc5 --- /dev/null +++ b/docs/resources/source-code/hb_util.md @@ -0,0 +1,645 @@ +# [Module hb_util.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_util.erl) + + + + +A collection of utility functions for building with HyperBEAM. + + + +## Function Index ## + + +
add_commas/1*
all_hb_modules/0Get all loaded modules that are loaded and are part of HyperBEAM.
atom/1Coerce a string to an atom.
bin/1Coerce a value to a binary.
count/2
debug_fmt/1Convert a term to a string for debugging print purposes.
debug_fmt/2
debug_print/4Print a message to the standard error stream, prefixed by the amount +of time that has elapsed since the last call to this function.
decode/1Try to decode a URL safe base64 into a binary or throw an error when +invalid.
deep_merge/2Deep merge two maps, recursively merging nested maps.
do_debug_fmt/2*
do_to_lines/1*
encode/1Encode a binary to URL safe base64 binary string.
eunit_print/2Format and print an indented string to standard error.
find_value/2Find the value associated with a key in parsed a JSON structure list.
find_value/3
float/1Coerce a string to a float.
format_address/2*If the user attempts to print a wallet, format it as an address.
format_binary/1Format a binary as a short string suitable for printing.
format_debug_trace/3*Generate the appropriate level of trace for a given call.
format_indented/2Format a string with an indentation level.
format_indented/3
format_maybe_multiline/2Format a map as either a single line or a multi-line string depending +on the value of the debug_print_map_line_threshold runtime option.
format_trace/1Format a stack trace as a list of strings, one for each stack frame.
format_trace/2*
format_trace_short/1Format a trace to a short string.
format_trace_short/4*
format_tuple/2*Helper function to format tuples with arity greater than 2.
get_trace/0*Get the trace of the current process.
hd/1Get the first element (the lowest integer key >= 1) of a numbered map.
hd/2
hd/3
hd/5*
human_id/1Convert a native binary ID to a human readable ID.
human_int/1Add , characters to a number every 3 digits to make it human readable.
id/1Return the human-readable form of an ID of a message when given either +a message explicitly, raw encoded ID, or an Erlang Arweave tx record.
id/2
int/1Coerce a string to an integer.
is_hb_module/1Is the given module part of HyperBEAM?.
is_hb_module/2
is_human_binary/1*Determine whether a binary is human-readable.
is_ordered_list/1Determine if the message given is an ordered list, starting from 1.
is_ordered_list/2*
is_string_list/1Is the given term a string list?.
key_to_atom/2Convert keys in a map to atoms, lowering - to _.
list/1Coerce a value to a list.
list_to_numbered_map/1Convert a list of elements to a map with numbered keys.
maybe_throw/2Throw an exception if the Opts map has an error_strategy key with the +value throw.
mean/1
message_to_ordered_list/1Take a message with numbered keys and convert it to a list of tuples +with the associated key as an integer and a value.
message_to_ordered_list/2
message_to_ordered_list/4*
native_id/1Convert a human readable ID to a native binary ID.
normalize_trace/1*Remove all calls from this module from the top of a trace.
number/1Label a list of elements with a number.
ok/1Unwrap a tuple of the form {ok, Value}, or throw/return, depending on +the value of the error_strategy option.
ok/2
pick_weighted/2*
print_trace/3*
print_trace/4Print the trace of the current stack, up to the first non-hyperbeam +module.
print_trace_short/4Print a trace to the standard error stream.
remove_common/2Remove the common prefix from two strings, returning the remainder of the +first string.
remove_trailing_noise/1*
remove_trailing_noise/2
safe_decode/1Safely decode a URL safe base64 into a binary returning an ok or error +tuple.
safe_encode/1Safely encode a binary to URL safe base64.
short_id/1Return a short ID for the different types of IDs used in AO-Core.
shuffle/1*Shuffle a list.
stddev/1
to_hex/1Convert a binary to a hex string.
to_lines/1*
to_lower/1Convert a binary to a lowercase.
to_sorted_keys/1Given a map or KVList, return a deterministically ordered list of its keys.
to_sorted_list/1Given a map or KVList, return a deterministically sorted list of its +key-value pairs.
trace_macro_helper/5Utility function to help macro ?trace/0 remove the first frame of the +stack trace.
until/1Utility function to wait for a condition to be true.
until/2
until/3
variance/1
weighted_random/1Return a random element from a list, weighted by the values in the list.
+ + + + +## Function Details ## + + + +### add_commas/1 * ### + +`add_commas(Rest) -> any()` + + + +### all_hb_modules/0 ### + +`all_hb_modules() -> any()` + +Get all loaded modules that are loaded and are part of HyperBEAM. + + + +### atom/1 ### + +`atom(Str) -> any()` + +Coerce a string to an atom. + + + +### bin/1 ### + +`bin(Value) -> any()` + +Coerce a value to a binary. + + + +### count/2 ### + +`count(Item, List) -> any()` + + + +### debug_fmt/1 ### + +`debug_fmt(X) -> any()` + +Convert a term to a string for debugging print purposes. + + + +### debug_fmt/2 ### + +`debug_fmt(X, Indent) -> any()` + + + +### debug_print/4 ### + +`debug_print(X, Mod, Func, LineNum) -> any()` + +Print a message to the standard error stream, prefixed by the amount +of time that has elapsed since the last call to this function. + + + +### decode/1 ### + +`decode(Input) -> any()` + +Try to decode a URL safe base64 into a binary or throw an error when +invalid. + + + +### deep_merge/2 ### + +`deep_merge(Map1, Map2) -> any()` + +Deep merge two maps, recursively merging nested maps. + + + +### do_debug_fmt/2 * ### + +`do_debug_fmt(Wallet, Indent) -> any()` + + + +### do_to_lines/1 * ### + +`do_to_lines(In) -> any()` + + + +### encode/1 ### + +`encode(Bin) -> any()` + +Encode a binary to URL safe base64 binary string. + + + +### eunit_print/2 ### + +`eunit_print(FmtStr, FmtArgs) -> any()` + +Format and print an indented string to standard error. + + + +### find_value/2 ### + +`find_value(Key, List) -> any()` + +Find the value associated with a key in parsed a JSON structure list. + + + +### find_value/3 ### + +`find_value(Key, Map, Default) -> any()` + + + +### float/1 ### + +`float(Str) -> any()` + +Coerce a string to a float. + + + +### format_address/2 * ### + +`format_address(Wallet, Indent) -> any()` + +If the user attempts to print a wallet, format it as an address. + + + +### format_binary/1 ### + +`format_binary(Bin) -> any()` + +Format a binary as a short string suitable for printing. + + + +### format_debug_trace/3 * ### + +`format_debug_trace(Mod, Func, Line) -> any()` + +Generate the appropriate level of trace for a given call. + + + +### format_indented/2 ### + +`format_indented(Str, Indent) -> any()` + +Format a string with an indentation level. + + + +### format_indented/3 ### + +`format_indented(RawStr, Fmt, Ind) -> any()` + + + +### format_maybe_multiline/2 ### + +`format_maybe_multiline(X, Indent) -> any()` + +Format a map as either a single line or a multi-line string depending +on the value of the `debug_print_map_line_threshold` runtime option. + + + +### format_trace/1 ### + +`format_trace(Stack) -> any()` + +Format a stack trace as a list of strings, one for each stack frame. +Each stack frame is formatted if it matches the `stack_print_prefixes` +option. At the first frame that does not match a prefix in the +`stack_print_prefixes` option, the rest of the stack is not formatted. + + + +### format_trace/2 * ### + +`format_trace(Rest, Prefixes) -> any()` + + + +### format_trace_short/1 ### + +`format_trace_short(Trace) -> any()` + +Format a trace to a short string. + + + +### format_trace_short/4 * ### + +`format_trace_short(Max, Latch, Trace, Prefixes) -> any()` + + + +### format_tuple/2 * ### + +`format_tuple(Tuple, Indent) -> any()` + +Helper function to format tuples with arity greater than 2. + + + +### get_trace/0 * ### + +`get_trace() -> any()` + +Get the trace of the current process. + + + +### hd/1 ### + +`hd(Message) -> any()` + +Get the first element (the lowest integer key >= 1) of a numbered map. +Optionally, it takes a specifier of whether to return the key or the value, +as well as a standard map of HyperBEAM runtime options. + + + +### hd/2 ### + +`hd(Message, ReturnType) -> any()` + + + +### hd/3 ### + +`hd(Message, ReturnType, Opts) -> any()` + + + +### hd/5 * ### + +`hd(Map, Rest, Index, ReturnType, Opts) -> any()` + + + +### human_id/1 ### + +`human_id(Bin) -> any()` + +Convert a native binary ID to a human readable ID. If the ID is already +a human readable ID, it is returned as is. If it is an ethereum address, it +is returned as is. + + + +### human_int/1 ### + +`human_int(Int) -> any()` + +Add `,` characters to a number every 3 digits to make it human readable. + + + +### id/1 ### + +`id(Item) -> any()` + +Return the human-readable form of an ID of a message when given either +a message explicitly, raw encoded ID, or an Erlang Arweave `tx` record. + + + +### id/2 ### + +`id(TX, Type) -> any()` + + + +### int/1 ### + +`int(Str) -> any()` + +Coerce a string to an integer. + + + +### is_hb_module/1 ### + +`is_hb_module(Atom) -> any()` + +Is the given module part of HyperBEAM? + + + +### is_hb_module/2 ### + +`is_hb_module(Atom, Prefixes) -> any()` + + + +### is_human_binary/1 * ### + +`is_human_binary(Bin) -> any()` + +Determine whether a binary is human-readable. + + + +### is_ordered_list/1 ### + +`is_ordered_list(Msg) -> any()` + +Determine if the message given is an ordered list, starting from 1. + + + +### is_ordered_list/2 * ### + +`is_ordered_list(N, Msg) -> any()` + + + +### is_string_list/1 ### + +`is_string_list(MaybeString) -> any()` + +Is the given term a string list? + + + +### key_to_atom/2 ### + +`key_to_atom(Key, Mode) -> any()` + +Convert keys in a map to atoms, lowering `-` to `_`. + + + +### list/1 ### + +`list(Value) -> any()` + +Coerce a value to a list. + + + +### list_to_numbered_map/1 ### + +`list_to_numbered_map(List) -> any()` + +Convert a list of elements to a map with numbered keys. + + + +### maybe_throw/2 ### + +`maybe_throw(Val, Opts) -> any()` + +Throw an exception if the Opts map has an `error_strategy` key with the +value `throw`. Otherwise, return the value. + + + +### mean/1 ### + +`mean(List) -> any()` + + + +### message_to_ordered_list/1 ### + +`message_to_ordered_list(Message) -> any()` + +Take a message with numbered keys and convert it to a list of tuples +with the associated key as an integer and a value. Optionally, it takes a +standard map of HyperBEAM runtime options. + + + +### message_to_ordered_list/2 ### + +`message_to_ordered_list(Message, Opts) -> any()` + + + +### message_to_ordered_list/4 * ### + +`message_to_ordered_list(Message, Keys, Key, Opts) -> any()` + + + +### native_id/1 ### + +`native_id(Bin) -> any()` + +Convert a human readable ID to a native binary ID. If the ID is already +a native binary ID, it is returned as is. + + + +### normalize_trace/1 * ### + +`normalize_trace(Rest) -> any()` + +Remove all calls from this module from the top of a trace. + + + +### number/1 ### + +`number(List) -> any()` + +Label a list of elements with a number. + + + +### ok/1 ### + +`ok(Value) -> any()` + +Unwrap a tuple of the form `{ok, Value}`, or throw/return, depending on +the value of the `error_strategy` option. + + + +### ok/2 ### + +`ok(Other, Opts) -> any()` + + + +### pick_weighted/2 * ### + +`pick_weighted(Rest, Remaining) -> any()` + + + +### print_trace/3 * ### + +`print_trace(Stack, Label, CallerInfo) -> any()` + + + +### print_trace/4 ### + +`print_trace(Stack, CallMod, CallFunc, CallLine) -> any()` + +Print the trace of the current stack, up to the first non-hyperbeam +module. Prints each stack frame on a new line, until it finds a frame that +does not start with a prefix in the `stack_print_prefixes` hb_opts. +Optionally, you may call this function with a custom label and caller info, +which will be used instead of the default. + + + +### print_trace_short/4 ### + +`print_trace_short(Trace, Mod, Func, Line) -> any()` + +Print a trace to the standard error stream. + + + +### remove_common/2 ### + +`remove_common(MainStr, SubStr) -> any()` + +Remove the common prefix from two strings, returning the remainder of the +first string. This function also coerces lists to binaries where appropriate, +returning the type of the first argument. + + + +### remove_trailing_noise/1 * ### + +`remove_trailing_noise(Str) -> any()` + + + +### remove_trailing_noise/2 ### + +`remove_trailing_noise(Str, Noise) -> any()` + + + +### safe_decode/1 ### + +`safe_decode(E) -> any()` + +Safely decode a URL safe base64 into a binary returning an ok or error +tuple. + + + +### safe_encode/1 ### + +`safe_encode(Bin) -> any()` + +Safely encode a binary to URL safe base64. + + + +### short_id/1 ### + +`short_id(Bin) -> any()` + +Return a short ID for the different types of IDs used in AO-Core. + + + +### shuffle/1 * ### + +`shuffle(List) -> any()` + +Shuffle a list. + + + +### stddev/1 ### + +`stddev(List) -> any()` + + + +### to_hex/1 ### + +`to_hex(Bin) -> any()` + +Convert a binary to a hex string. Do not use this for anything other than +generating a lower-case, non-special character id. It should not become part of +the core protocol. We use b64u for efficient encoding. + + + +### to_lines/1 * ### + +`to_lines(Elems) -> any()` + + + +### to_lower/1 ### + +`to_lower(Str) -> any()` + +Convert a binary to a lowercase. + + + +### to_sorted_keys/1 ### + +`to_sorted_keys(Msg) -> any()` + +Given a map or KVList, return a deterministically ordered list of its keys. + + + +### to_sorted_list/1 ### + +`to_sorted_list(Msg) -> any()` + +Given a map or KVList, return a deterministically sorted list of its +key-value pairs. + + + +### trace_macro_helper/5 ### + +`trace_macro_helper(Fun, X2, Mod, Func, Line) -> any()` + +Utility function to help macro `?trace/0` remove the first frame of the +stack trace. + + + +### until/1 ### + +`until(Condition) -> any()` + +Utility function to wait for a condition to be true. Optionally, +you can pass a function that will be called with the current count of +iterations, returning an integer that will be added to the count. Once the +condition is true, the function will return the count. + + + +### until/2 ### + +`until(Condition, Count) -> any()` + + + +### until/3 ### + +`until(Condition, Fun, Count) -> any()` + + + +### variance/1 ### + +`variance(List) -> any()` + + + +### weighted_random/1 ### + +`weighted_random(List) -> any()` + +Return a random element from a list, weighted by the values in the list. + diff --git a/docs/resources/source-code/hb_volume.md b/docs/resources/source-code/hb_volume.md new file mode 100644 index 000000000..8df22b076 --- /dev/null +++ b/docs/resources/source-code/hb_volume.md @@ -0,0 +1,146 @@ +# [Module hb_volume.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/hb_volume.erl) + + + + + + +## Function Index ## + + +
change_node_store/2
check_for_device/1
create_actual_partition/2*
create_mount_info/3*
create_partition/2
format_disk/2
get_partition_info/1*
list_partitions/0
mount_disk/4
mount_opened_volume/3*
parse_disk_info/2*
parse_disk_line/2*
parse_disk_model_line/2*
parse_disk_units_line/2*
parse_io_size_line/2*
parse_sector_size_line/2*
process_disk_line/2*
update_store_config/2*
+ + + + +## Function Details ## + + + +### change_node_store/2 ### + +

+change_node_store(StorePath::binary(), CurrentStore::list()) -> {ok, map()} | {error, binary()}
+
+
+ + + +### check_for_device/1 ### + +

+check_for_device(Device::binary()) -> boolean()
+
+
+ + + +### create_actual_partition/2 * ### + +`create_actual_partition(Device, PartType) -> any()` + + + +### create_mount_info/3 * ### + +`create_mount_info(Partition, MountPoint, VolumeName) -> any()` + + + +### create_partition/2 ### + +

+create_partition(Device::binary(), PartType::binary()) -> {ok, map()} | {error, binary()}
+
+
+ + + +### format_disk/2 ### + +

+format_disk(Partition::binary(), EncKey::binary()) -> {ok, map()} | {error, binary()}
+
+
+ + + +### get_partition_info/1 * ### + +`get_partition_info(Device) -> any()` + + + +### list_partitions/0 ### + +

+list_partitions() -> {ok, map()} | {error, binary()}
+
+
+ + + +### mount_disk/4 ### + +

+mount_disk(Partition::binary(), EncKey::binary(), MountPoint::binary(), VolumeName::binary()) -> {ok, map()} | {error, binary()}
+
+
+ + + +### mount_opened_volume/3 * ### + +`mount_opened_volume(Partition, MountPoint, VolumeName) -> any()` + + + +### parse_disk_info/2 * ### + +`parse_disk_info(Device, Lines) -> any()` + + + +### parse_disk_line/2 * ### + +`parse_disk_line(Line, Info) -> any()` + + + +### parse_disk_model_line/2 * ### + +`parse_disk_model_line(Line, Info) -> any()` + + + +### parse_disk_units_line/2 * ### + +`parse_disk_units_line(Line, Info) -> any()` + + + +### parse_io_size_line/2 * ### + +`parse_io_size_line(Line, Info) -> any()` + + + +### parse_sector_size_line/2 * ### + +`parse_sector_size_line(Line, Info) -> any()` + + + +### process_disk_line/2 * ### + +`process_disk_line(Line, X2) -> any()` + + + +### update_store_config/2 * ### + +

+update_store_config(StoreConfig::term(), NewPath::binary()) -> term()
+
+
+ diff --git a/docs/resources/source-code/index.md b/docs/resources/source-code/index.md new file mode 100644 index 000000000..e0bb2d593 --- /dev/null +++ b/docs/resources/source-code/index.md @@ -0,0 +1,23 @@ +# Source Code Documentation + +Welcome to the source code documentation for HyperBEAM. This section provides detailed insights into the codebase, helping developers understand the structure, functionality, and implementation details of HyperBEAM and its components. + +## Overview + +HyperBEAM is built with a modular architecture to ensure scalability, maintainability, and extensibility. The source code is organized into distinct components, each serving a specific purpose within the ecosystem. + +## Sections + +- **HyperBEAM Core**: The main framework that orchestrates data processing, storage, and routing. +- **Compute Unit**: Handles computational tasks and integrates with the HyperBEAM core for distributed processing. +- **Trusted Execution Environment (TEE)**: Ensures secure execution of sensitive operations. +- **Client Libraries**: Tools and SDKs for interacting with HyperBEAM, including the JavaScript client. + +## Getting Started + +To explore the source code, you can clone the repository from [GitHub](https://github.com/permaweb/HyperBEAM). + +## Navigation + +Use the navigation menu to dive into specific parts of the codebase. Each module includes detailed documentation, code comments, and examples to assist in understanding and contributing to the project. + diff --git a/docs/resources/source-code/rsa_pss.md b/docs/resources/source-code/rsa_pss.md new file mode 100644 index 000000000..afa0543e3 --- /dev/null +++ b/docs/resources/source-code/rsa_pss.md @@ -0,0 +1,158 @@ +# [Module rsa_pss.erl](https://github.com/permaweb/HyperBEAM/blob/main/src/rsa_pss.erl) + + + + +Distributed under the Mozilla Public License v2.0. + +Copyright (c) 2014-2015, Andrew Bennett + +__Authors:__ Andrew Bennett ([`andrew@pixid.com`](mailto:andrew@pixid.com)). + + + +## Description ## +Original available at: +https://github.com/potatosalad/erlang-crypto_rsassa_pss + + +## Data Types ## + + + + +### rsa_digest_type() ### + + +

+rsa_digest_type() = md5 | sha | sha224 | sha256 | sha384 | sha512
+
+ + + + +### rsa_private_key() ### + + +

+rsa_private_key() = #RSAPrivateKey{}
+
+ + + + +### rsa_public_key() ### + + +

+rsa_public_key() = #RSAPublicKey{}
+
+ + + +## Function Index ## + + +
dp/2*
ep/2*
int_to_bit_size/1*
int_to_bit_size/2*
int_to_byte_size/1*
int_to_byte_size/2*
mgf1/3*
mgf1/5*
normalize_to_key_size/2*
pad_to_key_size/2*
sign/3
sign/4
verify/4
verify_legacy/4
+ + + + +## Function Details ## + + + +### dp/2 * ### + +`dp(B, X2) -> any()` + + + +### ep/2 * ### + +`ep(B, X2) -> any()` + + + +### int_to_bit_size/1 * ### + +`int_to_bit_size(I) -> any()` + + + +### int_to_bit_size/2 * ### + +`int_to_bit_size(I, B) -> any()` + + + +### int_to_byte_size/1 * ### + +`int_to_byte_size(I) -> any()` + + + +### int_to_byte_size/2 * ### + +`int_to_byte_size(I, B) -> any()` + + + +### mgf1/3 * ### + +`mgf1(DigestType, Seed, Len) -> any()` + + + +### mgf1/5 * ### + +`mgf1(DigestType, Seed, Len, T, Counter) -> any()` + + + +### normalize_to_key_size/2 * ### + +`normalize_to_key_size(Bits, A) -> any()` + + + +### pad_to_key_size/2 * ### + +`pad_to_key_size(Bytes, Data) -> any()` + + + +### sign/3 ### + +

+sign(Message, DigestType, PrivateKey) -> Signature
+
+ + + + + +### sign/4 ### + +

+sign(Message, DigestType, Salt, PrivateKey) -> Signature
+
+ + + + + +### verify/4 ### + +

+verify(Message, DigestType, Signature, PublicKey) -> boolean()
+
+ + + + + +### verify_legacy/4 ### + +`verify_legacy(Message, DigestType, Signature, PublicKey) -> any()` + diff --git a/docs/resources/source-code/stylesheet.css b/docs/resources/source-code/stylesheet.css new file mode 100644 index 000000000..ab170c091 --- /dev/null +++ b/docs/resources/source-code/stylesheet.css @@ -0,0 +1,55 @@ +/* standard EDoc style sheet */ +body { + font-family: Verdana, Arial, Helvetica, sans-serif; + margin-left: .25in; + margin-right: .2in; + margin-top: 0.2in; + margin-bottom: 0.2in; + color: #000000; + background-color: #ffffff; +} +h1,h2 { + margin-left: -0.2in; +} +div.navbar { + background-color: #add8e6; + padding: 0.2em; +} +h2.indextitle { + padding: 0.4em; + background-color: #add8e6; +} +h3.function,h3.typedecl { + background-color: #add8e6; + padding-left: 1em; +} +div.spec { + margin-left: 2em; + background-color: #eeeeee; +} +a.module { + text-decoration:none +} +a.module:hover { + background-color: #eeeeee; +} +ul.definitions { + list-style-type: none; +} +ul.index { + list-style-type: none; + background-color: #eeeeee; +} + +/* + * Minor style tweaks + */ +ul { + list-style-type: square; +} +table { + border-collapse: collapse; +} +td { + padding: 3 +} diff --git a/docs/run/configuring-your-machine.md b/docs/run/configuring-your-machine.md new file mode 100644 index 000000000..1154297b4 --- /dev/null +++ b/docs/run/configuring-your-machine.md @@ -0,0 +1,157 @@ +# Configuring Your HyperBEAM Node + +This guide details the various ways to configure your HyperBEAM node's behavior, including ports, storage, keys, and logging. + +## Configuration (`config.flat`) + +The primary way to configure your HyperBEAM node is through a `config.flat` file located in the node's working directory or specified by the `HB_CONFIG_LOCATION` environment variable. + +This file uses a simple `Key = Value.` format (note the period at the end of each line). + +**Example `config.flat`:** + +```erlang +% Set the HTTP port +port = 8080. + +% Specify the Arweave key file +priv_key_location = "/path/to/your/wallet.json". + +% Set the data store directory +% Note: Storage configuration can be complex. See below. +% store = [{local, [{root, <<"./node_data_mainnet">>}]}]. % Example of complex config, not for config.flat + +% Enable verbose logging for specific modules +% debug_print = [hb_http, dev_router]. % Example of complex config, not for config.flat +``` + +Below is a reference of commonly used configuration keys. Remember that `config.flat` only supports simple key-value pairs (Atoms, Strings, Integers, Booleans). For complex configurations (Lists, Maps), you must use environment variables or `hb:start_mainnet/1`. + +### Core Configuration + +These options control fundamental HyperBEAM behavior. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `port` | Integer | 8734 | HTTP API port | +| `hb_config_location` | String | "config.flat" | Path to configuration file | +| `priv_key_location` | String | "hyperbeam-key.json" | Path to operator wallet key file | +| `mode` | Atom | debug | Execution mode (debug, prod) | + +### Server & Network Configuration + +These options control networking behavior and HTTP settings. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `host` | String | "localhost" | Choice of remote node for non-local tasks | +| `gateway` | String | "https://arweave.net" | Default gateway | +| `bundler_ans104` | String | "https://up.arweave.net:443" | Location of ANS-104 bundler | +| `protocol` | Atom | http2 | Protocol for HTTP requests (http1, http2, http3) | +| `http_client` | Atom | gun | HTTP client to use (gun, httpc) | +| `http_connect_timeout` | Integer | 5000 | HTTP connection timeout in milliseconds | +| `http_keepalive` | Integer | 120000 | HTTP keepalive time in milliseconds | +| `http_request_send_timeout` | Integer | 60000 | HTTP request send timeout in milliseconds | +| `relay_http_client` | Atom | httpc | HTTP client for the relay device | + + +### Security & Identity + +These options control identity and security settings. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `scheduler_location_ttl` | Integer | 604800000 | TTL for scheduler registration (7 days in ms) | + + +### Caching & Storage + +These options control caching behavior. **Note:** Detailed storage configuration (`store` option) involves complex data structures and cannot be set via `config.flat`. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `cache_lookup_hueristics` | Boolean | false | Whether to use caching heuristics or always consult the local data store | +| `access_remote_cache_for_client` | Boolean | false | Whether to access data from remote caches for client requests | +| `store_all_signed` | Boolean | true | Whether the node should store all signed messages | +| `await_inprogress` | Atom/Boolean | named | Whether to await in-progress executions (false, named, true) | + + +### Execution & Processing + +These options control how HyperBEAM executes messages and processes. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `scheduling_mode` | Atom | local_confirmation | When to inform recipients about scheduled assignments (aggressive, local_confirmation, remote_confirmation) | +| `compute_mode` | Atom | lazy | Whether to execute more messages after returning a result (aggressive, lazy) | +| `process_workers` | Boolean | true | Whether the node should use persistent processes | +| `client_error_strategy` | Atom | throw | What to do if a client error occurs | +| `wasm_allow_aot` | Boolean | false | Allow ahead-of-time compilation for WASM | + +### Device Management + +These options control how HyperBEAM manages devices. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `load_remote_devices` | Boolean | false | Whether to load devices from remote signers | + + +### Debug & Development + +These options control debugging and development features. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `debug_stack_depth` | Integer | 40 | Maximum stack depth for debug printing | +| `debug_print_map_line_threshold` | Integer | 30 | Maximum lines for map printing | +| `debug_print_binary_max` | Integer | 60 | Maximum binary size for debug printing | +| `debug_print_indent` | Integer | 2 | Indentation for debug printing | +| `debug_print_trace` | Atom | short | Trace mode (short, false) | +| `short_trace_len` | Integer | 5 | Length of short traces | +| `debug_hide_metadata` | Boolean | true | Whether to hide metadata in debug output | +| `debug_ids` | Boolean | false | Whether to print IDs in debug output | +| `debug_hide_priv` | Boolean | true | Whether to hide private data in debug output | + + +**Note:** For the *absolute complete* and most up-to-date list, including complex options not suitable for `config.flat`, refer to the `default_message/0` function in the `hb_opts` module source code. + +## Overrides (Environment Variables & Args) + +You can override settings from `config.flat` or provide values if the file is missing using environment variables or command-line arguments. + +**Using Environment Variables:** + +Environment variables typically use an `HB_` prefix followed by the configuration key in uppercase. + +* **`HB_PORT=`:** Overrides `hb_port`. + * Example: `HB_PORT=8080 rebar3 shell` +* **`HB_KEY=`:** Overrides `hb_key`. + * Example: `HB_KEY=~/.keys/arweave_key.json rebar3 shell` +* **`HB_STORE=`:** Overrides `hb_store`. + * Example: `HB_STORE=./node_data_1 rebar3 shell` +* **`HB_PRINT=`:** Overrides `hb_print`. `` can be `true` (or `1`), or a comma-separated list of modules/topics (e.g., `hb_path,hb_ao,ao_result`). + * Example: `HB_PRINT=hb_http,dev_router rebar3 shell` +* **`HB_CONFIG_LOCATION=`:** Specifies a custom location for the configuration file. + +**Using `erl_opts` (Direct Erlang VM Arguments):** + +You can also pass arguments directly to the Erlang VM using the `- ` format within `erl_opts`. This is generally less common for application configuration than `config.flat` or environment variables. + +```bash +rebar3 shell --erl_opts "-hb_port 8080 -hb_key path/to/key.json" +``` + +**Order of Precedence:** + +1. Command-line arguments (`erl_opts`). +2. Settings in `config.flat`. +3. Environment variables (`HB_*`). +4. Default values from `hb_opts.erl`. + +## Configuration in Releases + +When running a release build (see [Running a HyperBEAM Node](./running-a-hyperbeam-node.md)), configuration works similarly: + +1. A `config.flat` file will be present in the release directory (e.g., `_build/default/rel/hb/config.flat`). Edit this file to set your desired parameters for the release environment. +2. Environment variables (`HB_*`) can still be used to override the settings in the release's `config.flat` when starting the node using the `bin/hb` script. diff --git a/docs/run/joining-running-a-router.md b/docs/run/joining-running-a-router.md new file mode 100644 index 000000000..268990cf2 --- /dev/null +++ b/docs/run/joining-running-a-router.md @@ -0,0 +1,68 @@ +# Joining or Running a Router Node + +Router nodes play a crucial role in the HyperBEAM network by directing incoming HTTP requests to appropriate worker nodes capable of handling the requested computation or data retrieval. They act as intelligent load balancers and entry points into the AO ecosystem. + +!!! info "Advanced Topic" + Configuring and running a production-grade router involves considerations beyond the scope of this introductory guide, including network topology, security, high availability, and performance tuning. + +## What is a Router? + +In HyperBEAM, the `dev_router` module (and associated logic) implements routing functionality. A node configured as a router typically: + +1. Receives external HTTP requests (HyperPATH calls). +2. Parses the request path to determine the target process, device, and desired operation. +3. Consults its routing table or logic to select an appropriate downstream worker node (which might be itself or another node). +4. Forwards the request to the selected worker. +5. Receives the response from the worker. +6. Returns the response to the original client. + +Routers often maintain information about the capabilities and load of worker nodes they know about. + +## Configuring Routing Behavior + +Routing logic is primarily configured through node options, often managed via `hb_opts` or environment variables when starting the node. Key aspects include: + +* **Route Definitions:** Defining patterns (templates) and corresponding downstream targets (worker node URLs or internal handlers). Routes are typically ordered by precedence. +* **Load Balancing Strategy:** How the router chooses among multiple potential workers for a given route (e.g., round-robin, least connections, latency-based). +* **Worker Discovery/Management:** How the router learns about available worker nodes and their status. + +**Example Configuration Snippet (Conceptual - from `hb_opts` or config file):** + +```erlang +{ + routes, + [ + #{ template => "/~meta@1.0/.*", target => self }, % Handle meta locally + #{ template => "/PROCESS_ID1~process@1.0/.*", target => "http://worker1.example.com" }, + #{ template => "/PROCESS_ID2~process@1.0/.*", target => "http://worker2.example.com" }, + #{ template => "/.*~wasm64@1.0/.*", target => ["http://wasm_worker1", "http://wasm_worker2"], strategy => round_robin }, % Route WASM requests + #{ template => "/.*", target => "http://default_worker.example.com" } % Default fallback + ] +}, +{ router_load_balancing_strategy, latency_aware } +``` + +*(Note: The actual configuration format and options should be verified in the `hb_opts.erl` and `dev_router.erl` source code.)* + +## Running a Simple Router + +While a dedicated router setup is complex, any HyperBEAM node implicitly performs some level of routing, especially if it needs to interact with other nodes (e.g., via the `~relay@1.0` device). The default configuration might route certain requests internally or have basic forwarding capabilities. + +To run a node that explicitly acts *more* like a router, you would typically configure it with specific `routes` pointing to other worker nodes, potentially disabling local execution for certain devices it intends to forward. + +## Joining an Existing Router Network + +As a user or developer, you typically don't *run* the main public routers (like `router-1.forward.computer`). Instead, you configure your client applications (or your own local node if it needs to relay requests) to *use* these public routers as entry points. + +When making HyperPATH calls, you simply target the public router's URL: + +``` +https:///~/... +``` +The router handles directing your request to an appropriate compute node. + +## Further Exploration + +* Examine the `dev_router.erl` source code for detailed implementation. +* Review the available configuration options in `hb_opts.erl` related to routing (`routes`, strategies, etc.). +* Consult community channels or advanced documentation for best practices on deploying production routers. diff --git a/docs/run/running-a-hyperbeam-node.md b/docs/run/running-a-hyperbeam-node.md new file mode 100644 index 000000000..30f9df072 --- /dev/null +++ b/docs/run/running-a-hyperbeam-node.md @@ -0,0 +1,250 @@ +# Running a HyperBEAM Node + +This guide provides the basics for running your own HyperBEAM node, installing dependencies, and connecting to the AO network. + +## System Dependencies + +To successfully build and run a HyperBEAM node, your system needs several software dependencies installed. + +=== "macOS" + Install core dependencies using [Homebrew](https://brew.sh/): + + ```bash + brew install cmake git pkg-config openssl ncurses + ``` + +=== "Linux (Debian/Ubuntu)" + Install core dependencies using `apt`: + ```bash + sudo apt-get update && sudo apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + git \ + pkg-config \ + ncurses-dev \ + libssl-dev \ + sudo \ + curl + ca-certificates + ``` + +=== "Windows (WSL)" + Using the Windows Subsystem for Linux (WSL) with a distribution like Ubuntu is recommended. Follow the Linux (Debian/Ubuntu) instructions within your WSL environment. + + + +### Erlang/OTP + +HyperBEAM is built on Erlang/OTP. You need a compatible version installed (check the `rebar.config` or project documentation for specific version requirements, **typically OTP 27**). + +Installation methods: + +=== "macOS (brew)" + ```bash + brew install erlang + ``` + +=== "Linux (apt)" + ```bash + sudo apt install erlang + ``` + + +=== "Source Build" + Download from [erlang.org](https://www.erlang.org/downloads) and follow the build instructions for your platform. + +### Rebar3 + +Rebar3 is the build tool for Erlang projects. + +Installation methods: + +=== "macOS (brew)" + ```bash + brew install rebar3 + ``` + +=== "Linux / macOS (Direct Download)" + Get the `rebar3` binary from the [official website](https://rebar3.org/). Place the downloaded `rebar3` file in your system's `PATH` (e.g., `/usr/local/bin`) and make it executable (`chmod +x rebar3`). + + + +### Node.js + +Node.js might be required for certain JavaScript-related tools or dependencies. + +Installation methods: + +=== "macOS (brew)" + ```bash + brew install node + ``` + +=== "Linux (apt)" + ```bash + # Check your distribution's recommended method, might need nodesource repo + sudo apt install nodejs npm + ``` + +=== "asdf (Recommended)" + `asdf-vm` with the `asdf-nodejs` plugin is recommended. + ```bash + asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git + asdf install nodejs # e.g., lts + asdf global nodejs + ``` + +### Rust + +Rust is needed if you intend to work with or build components involving WebAssembly (WASM) or certain Native Implemented Functions (NIFs) used by some devices (like `~snp@1.0`). + +The recommended way to install Rust on **all platforms** is via `rustup`: + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +source "$HOME/.cargo/env" # Or follow the instructions provided by rustup +``` + +## Prerequisites for Running + +Before starting a node, ensure you have: + +* Installed the [system dependencies](#system-dependencies) mentioned above. +* Cloned the [HyperBEAM repository](https://github.com/permaweb/HyperBEAM) (`git clone ...`). +* Compiled the source code (`rebar3 compile` in the repo directory). +* An Arweave **wallet keyfile** (e.g., generated via [Wander](https://www.wander.app)). The path to this file is typically set via the `hb_key` configuration option (see [Configuring Your HyperBEAM Node](./configuring-your-machine.md)). + +## Starting a Basic Node + +The simplest way to start a HyperBEAM node for development or testing is using `rebar3` from the repository's root directory: + +```bash +rebar3 shell +``` + +This command: + +1. Starts the Erlang Virtual Machine (BEAM) with all HyperBEAM modules loaded. +2. Initializes the node with default settings (from `hb_opts.erl`). +3. Starts the default HTTP server (typically on **port 10000**), making the node accessible via HyperPATHs. +4. Drops you into an interactive Erlang shell where you can interact with the running node. + +This basic setup is suitable for local development and exploring HyperBEAM's functionalities. + +## Optional Build Profiles + +HyperBEAM uses build profiles to enable optional features, often requiring extra dependencies. To run a node with specific profiles enabled, use `rebar3 as ... shell`: + +**Available Profiles (Examples):** + +* `genesis_wasm`: Enables Genesis WebAssembly support. +* `rocksdb`: Enables the RocksDB storage backend. +* `http3`: Enables HTTP/3 support. + +**Example Usage:** + +```bash +# Start with RocksDB profile +rebar3 as rocksdb shell + +# Start with RocksDB and Genesis WASM profiles +rebar3 as rocksdb, genesis_wasm shell +``` + +*Note: Choose profiles **before** starting the shell, as they affect compile-time options.* + +## Node Configuration + +HyperBEAM offers various configuration options (port, key file, data storage, logging, etc.). These are primarily set using a `config.flat` file and can be overridden by environment variables or command-line arguments. + +See the dedicated **[Configuring Your HyperBEAM Node](./configuring-your-machine.md)** guide for detailed information on all configuration methods and options. + +## Verify Installation + +To quickly check if your node is running and accessible, you can send a request to its `~meta@1.0` device (assuming default port 10000): + +```bash +curl http://localhost:10000/~meta@1.0/info +``` + +A JSON response containing node information indicates success. + +## Running for Production (Mainnet) + +While you can connect to the main AO network using the `rebar3 shell` for testing purposes (potentially using specific configurations or helper functions like `hb:start_mainnet/1` if available and applicable), the standard and recommended method for a stable production deployment (like running on the mainnet) is to build and run a **release**. + +**1. Build the Release:** + +From the root of the HyperBEAM repository, build the release package. You might include specific profiles needed for your mainnet setup (e.g., `rocksdb` if you intend to use it): + +```bash +# Build release with default profile +rebar3 release + +# Or, build with specific profiles (example) +# rebar3 as rocksdb release +``` + +This command compiles the project and packages it along with the Erlang Runtime System (ERTS) and all dependencies into a directory, typically `_build/default/rel/hb`. + +**2. Configure the Release:** + +Navigate into the release directory (e.g., `cd _build/default/rel/hb`). Ensure you have a correctly configured `config.flat` file here. See the [configuration guide](./configuring-your-machine.md) for details on setting mainnet parameters (port, key file location, store path, specific peers, etc.). Environment variables can also be used to override settings in the release's `config.flat` when starting the node. + +**3. Start the Node:** + +Use the generated start script (`bin/hb`) to run the node: + +```bash +# Start the node in the foreground (logs to console) +./bin/hb console + +# Start the node as a background daemon +./bin/hb start + +# Check the status +./bin/hb ping +./bin/hb status + +# Stop the node +./bin/hb stop +``` + +Consult the generated `bin/hb` script or Erlang/OTP documentation for more advanced start-up options (e.g., attaching a remote shell). + +Running as a release provides a more robust, isolated, and manageable way to operate a node compared to running directly from the `rebar3 shell`. + +## Stopping the Node (rebar3 shell) + +To stop the node running *within the `rebar3 shell`*, press `Ctrl+C` twice or use the Erlang command `q().`. + +## Next Steps + +* **Configure Your Node:** Deep dive into [configuration options](./configuring-your-machine.md). +* **TEE Nodes:** Learn about running nodes in [Trusted Execution Environments](./tee-nodes.md) for enhanced security. +* **Routers:** Understand how to configure and run a [router node](./joining-running-a-router.md). diff --git a/docs/run/tee-nodes.md b/docs/run/tee-nodes.md new file mode 100644 index 000000000..7da306ab7 --- /dev/null +++ b/docs/run/tee-nodes.md @@ -0,0 +1,31 @@ +# Trusted Execution Environment (TEE) + +!!! info "Documentation Coming Soon" + Detailed documentation about Trusted Execution Environment support in HyperBEAM is currently being developed and will be available soon. + +## Overview + +HyperBEAM supports Trusted Execution Environments (TEEs) through the `~snp@1.0` device, which enables secure, trust-minimized computation on remote machines. TEEs provide hardware-level isolation and attestation capabilities that allow users to verify that their code is running in a protected environment, exactly as intended, even on untrusted hardware. + +The `~snp@1.0` device in HyperBEAM is used to generate and validate proofs that a node is executing inside a Trusted Execution Environment. Nodes executing inside these environments use an ephemeral key pair that provably only exists inside the TEE, and can sign attestations of AO-Core executions in a trust-minimized way. + +## Key Features + +- Hardware-level isolation for secure computation +- Remote attestation capabilities +- Protected execution state +- Confidential computing support +- Compatibility with AMD SEV-SNP technology + +## Coming Soon + +Detailed documentation on the following topics will be added: + +- TEE setup and configuration +- Using the `~snp@1.0` device +- Verifying TEE attestations +- Developing for TEEs +- Security considerations +- Performance characteristics + +If you intend to offer TEE-based computation of AO-Core devices, please see the [HyperBEAM OS repository](https://github.com/permaweb/hb-os) for preliminary details on configuration and deployment. \ No newline at end of file diff --git a/docs/theme/templates/base.html b/docs/theme/templates/base.html new file mode 100644 index 000000000..22c8a5ff4 --- /dev/null +++ b/docs/theme/templates/base.html @@ -0,0 +1,605 @@ +{#- + This file was automatically generated - do not edit + -#} + {% import "partials/language.html" as lang with context %} + + + + {% block site_meta %} + + + {% if page.meta and page.meta.description %} + + {% elif config.site_description %} + + {% endif %} + {% if page.meta and page.meta.author %} + + {% elif config.site_author %} + + {% endif %} + {% if page.canonical_url %} + + {% endif %} + {% if page.previous_page %} + + {% endif %} + {% if page.next_page %} + + {% endif %} + {% if "rss" in config.plugins %} + + + {% endif %} + + + {% endblock %} + {% block htmltitle %} + {% if page.meta and page.meta.title %} + {{ page.meta.title }} - {{ config.site_name }} + {% elif page.title and not page.is_homepage %} + {{ page.title | striptags }} - {{ config.site_name }} + {% else %} + {{ config.site_name }} + {% endif %} + {% endblock %} + {% block styles %} + + {% if config.theme.palette %} + {% set palette = config.theme.palette %} + + {% endif %} + {% include "partials/icons.html" %} + {% endblock %} + {% block libs %} + {% for script in config.extra.polyfills %} + {{ script | script_tag }} + {% endfor %} + {% endblock %} + {% block fonts %} + {% if config.theme.font != false %} + {% set text = config.theme.font.text | d("Roboto", true) %} + {% set code = config.theme.font.code | d("Roboto Mono", true) %} + + + + {% endif %} + {% endblock %} + {% for path in config.extra_css %} + + {% endfor %} + {% include "partials/javascripts/base.html" %} + {% block analytics %} + {% include "partials/integrations/analytics.html" %} + {% endblock %} + {% if page.meta and page.meta.meta %} + {% for tag in page.meta.meta %} + + {% endfor %} + {% endif %} + {% block extrahead %}{% endblock %} + + {% set direction = config.theme.direction or lang.t("direction") %} + {% if config.theme.palette %} + {% set palette = config.theme.palette %} + {% if not palette is mapping %} + {% set palette = palette | first %} + {% endif %} + {% set scheme = palette.scheme | d("default", true) %} + {% set primary = palette.primary | d("indigo", true) %} + {% set accent = palette.accent | d("indigo", true) %} + + {% else %} + + {% endif %} + {% set features = config.theme.features or [] %} + + + +
+ {% if page.toc | first is defined %} + {% set skip = page.toc | first %} + + {{ lang.t("action.skip") }} + + {% endif %} +
+
+ {% if self.announce() %} + + {% endif %} +
+ {% if config.extra.version %} + + {% endif %} + {% block header %} + {% include "partials/header.html" %} + {% endblock %} +
+ {% block hero %}{% endblock %} + {% block tabs %} + {% if "navigation.tabs.sticky" not in features %} + {% if "navigation.tabs" in features %} + {% include "partials/tabs.html" %} + {% endif %} + {% endif %} + {% endblock %} +
+ {% if page.is_homepage %} + +
+ + + front-rocks + + + + front-boulder + + + + + mid-rocks + + + + back-rocks + + + + + background-rocks + + + + + background-chroma-space + + +
+
+
+
+
+
+

Hyperbeam

+

+ A Decentralized Operating System. + Powered By AO. +

+
+
+ +
+
+
+ + + + + + + + + +
+
+
+
+
+ +
+
+
+
+

What is hyperBEAM?

+
+
+
+ +
+
+
+
+

+ It is an implementation of AO-Core, + letting you offer and procure usage + of your machine in its network. +

+

+ HyperBEAM is a client implementation + of the AO-Core protocol, written in + Erlang. +

+
+
+ Parallel +

+ Programs run as independent + processes. +

+
+
+ Async +

+ Communicate via asynchronous + message passing. +

+
+
+ Distributed +

+ Operate across a distributed + network of nodes. +

+
+
+
+
+ + + +
+
+
+
+
+
+
+
+
+
+

What Can I Build?

+
+
+
+
+
+
+ 01 +

+ Create New Devices for AO + processes. +

+
+

+ Bringing further capabilities to its + network of 7m+ smart contracts. +

+
+
+ what-is-hyperbeam-fig +
+
+
+
+
+
+
+ 02 +

+ Rock solid Decentralized + infrastructure apps at any + scale. +

+
+

+ Core fundamentals supported by + Arweave as the permanent data ledger + and AO as the decentralized + supercomputer. +

+
+
+ what-is-hyperbeam-fig +
+
+
+
+
+
+ 03 +

+ Bringing the power of + decentralization to your web2 and + web3 applications. +

+

+ Permissionless in combination with + reduced costs. +

+
+
+ + + +
+
+ +
+
+
+
+
+
+
+
+
+

How Do I Get Involved?

+
+

Monetize Your Harware.

+

+ Join AO-Core's peer-to-peer network and + build towards a decentralized future. +

+
+
+
+ +
+
+

+ Bringing further capabilities to its network + of 7m+ smart contracts. +

+
+
+

+ Core fundamentals supported by Arweave as + the permanent data ledger and AO as the + decentralized supercomputer. +

+
+
+

+ Permissionless in combination with reduced + costs. +

+
+
+

+ Permissionless in combination with reduced + costs. +

+
+
+ +
+
+ {% else %} +
+ {% block site_nav %} + {% if nav %} + {% if page.meta and page.meta.hide %} + {% set hidden = "hidden" if "navigation" in page.meta.hide %} + {% endif %} + + {% endif %} + {% if "toc.integrate" not in features %} + {% if page.meta and page.meta.hide %} + {% set hidden = "hidden" if "toc" in page.meta.hide %} + {% endif %} + + {% endif %} + {% endblock %} + {% block container %} +
+
+ {% block content %} + {% include "partials/content.html" %} + {% endblock %} +
+
+ {% endblock %} + {% include "partials/javascripts/content.html" %} +
+ {% if "navigation.top" in features %} + {% include "partials/top.html" %} + {% endif %} + {% endif %} +
+ + {% block footer %} + {% include "partials/footer.html" %} + {% endblock %} +
+
+
+
+ {% if "navigation.instant.progress" in features %} + {% include "partials/progress.html" %} + {% endif %} + {% if config.extra.consent %} + + {% include "partials/javascripts/consent.html" %} + {% endif %} + {% block config %} + {% set _ = namespace() %} + {% set _.tags = config.extra.tags %} + {%- if config.extra.version -%} + {%- set mike = config.plugins.mike -%} + {%- if not mike or mike.config.version_selector -%} + {%- set _.version = config.extra.version -%} + {%- endif -%} + {%- endif -%} + + {% endblock %} + {% block scripts %} + + {% for script in config.extra_javascript %} + {{ script | script_tag }} + {% endfor %} + {% endblock %} + + \ No newline at end of file diff --git a/docs/theme/templates/partials/header.html b/docs/theme/templates/partials/header.html new file mode 100644 index 000000000..80de1259e --- /dev/null +++ b/docs/theme/templates/partials/header.html @@ -0,0 +1,56 @@ +{#- This file was automatically generated - do not edit -#} {% set class = +"md-header" %} {% if "navigation.tabs.sticky" in features %} {% set class = +class ~ " md-header--shadow md-header--lifted" %} {% elif "navigation.tabs" not +in features %} {% set class = class ~ " md-header--shadow" %} {% endif %} + +
+ + {% if "navigation.tabs.sticky" in features %} {% if "navigation.tabs" in + features %} {% include "partials/tabs.html" %} {% endif %} {% endif %} +
diff --git a/erlang_ls.config b/erlang_ls.config index 483ff9687..097464093 100644 --- a/erlang_ls.config +++ b/erlang_ls.config @@ -6,18 +6,11 @@ diagnostics: apps_dirs: - "src" - "src/*" -deps_dirs: - - "_build/default/lib/b64fast" - - "_build/default/lib/cowboy" - - "_build/default/lib/gun" - - "_build/default/lib/jiffy" include_dirs: - "src/include" include_dirs: - "src" - "src/include" - - "_build/default/lib/" - - "_build/default/lib/*/include" lenses: enabled: - ct-run-test diff --git a/metrics/grafana/provisioning/dashboards/beam.json b/metrics/grafana/provisioning/dashboards/beam.json new file mode 100644 index 000000000..71bd99e3d --- /dev/null +++ b/metrics/grafana/provisioning/dashboards/beam.json @@ -0,0 +1,1350 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "hidden", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 16, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "process_uptime_seconds{instance=\"$node\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "intervalFactor": 2, + "legendFormat": "", + "range": true, + "refId": "A", + "step": 4, + "useBackend": false + } + ], + "title": "Uptime", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Reductions" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 6, + "y": 0 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(erlang_vm_statistics_context_switches{instance=\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Context Switches", + "refId": "B", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(erlang_vm_statistics_reductions_total{instance=\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reductions", + "refId": "C", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "irate(erlang_vm_statistics_runtime_milliseconds{instance=\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Runtime", + "range": true, + "refId": "D", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "system_load{instance=\"$node\"}/256", + "hide": false, + "instant": false, + "legendFormat": "Load average (5)", + "range": true, + "refId": "E" + } + ], + "title": "Load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 14, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "erlang_vm_process_limit{instance=\"$node\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "Process Limit", + "refId": "A", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "erlang_vm_process_count{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Processes", + "refId": "B", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "erlang_vm_statistics_run_queues_length_total{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Run Queues Length", + "refId": "C", + "step": 2 + } + ], + "title": "Processes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "erlang_vm_memory_bytes_total{instance=\"$node\", kind=\"system\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "System memory", + "range": true, + "refId": "B", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "erlang_vm_memory_system_bytes_total{instance=\"$node\", usage=\"atom\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Atoms", + "refId": "C", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "erlang_vm_memory_system_bytes_total{instance=\"$node\", usage=\"binary\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Binary", + "refId": "D", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "erlang_vm_memory_system_bytes_total{instance=\"$node\", usage=\"code\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Code", + "refId": "E", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "erlang_vm_memory_system_bytes_total{instance=\"$node\", usage=\"ets\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ETS", + "range": true, + "refId": "F", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "G" + } + ], + "title": "VM Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 12, + "y": 7 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "editorMode": "code", + "expr": "sum(erlang_vm_allocators{usage=\"carriers_size\", instance=~\"$node\"}) by (alloc)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory breakdown", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "editorMode": "code", + "expr": "avg by (method) (rate(cowboy_request_duration_seconds_sum[5m]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Average request duration (seconds)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 12, + "y": 14 + }, + "id": 17, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (method, status_class) (rate(cowboy_requests_total[5m])) * 60\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests rates (per minute)", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(erlang_vm_statistics_bytes_output_total{instance=\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Output Bytes", + "metric": "erlang_vm_statistics_bytes_output_total", + "refId": "A", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(erlang_vm_statistics_bytes_received_total{instance=\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Received Bytes", + "metric": "erlang_vm_statistics_bytes_received_total", + "refId": "B", + "step": 2 + } + ], + "title": "VM IO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Words Reclaimed" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Bytes Reclaimed" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 11, + "x": 12, + "y": 22 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(erlang_vm_statistics_garbage_collection_number_of_gcs{instance=\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Number of GCs", + "metric": "erlang_vm_statistics_garbage_collection_number_of_gcs", + "refId": "A", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(erlang_vm_statistics_garbage_collection_bytes_reclaimed{instance=\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bytes Reclaimed", + "metric": "erlang_vm_statistics_garbage_collection_words_reclaimed", + "refId": "B", + "step": 2 + } + ], + "title": "VM GC", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max Ports" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ports" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 28 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_open_fds{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Open FDs", + "metric": "", + "refId": "A", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_max_fds{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Max FDs", + "refId": "B", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "erlang_vm_port_limit{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Max Ports", + "refId": "C", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "erlang_vm_port_count{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Ports", + "refId": "D", + "step": 2 + } + ], + "title": "File Descriptors & Ports", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "172.17.0.1:4000", + "value": "172.17.0.1:4000" + }, + "datasource": "Prometheus", + "includeAll": false, + "label": "Node", + "name": "node", + "options": [], + "query": "label_values(erlang_vm_process_count, instance)", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "BEAM", + "uid": "ce50mecnfs4cga", + "version": 7, + "weekStart": "" +} diff --git a/metrics/grafana/provisioning/dashboards/events.json b/metrics/grafana/provisioning/dashboards/events.json new file mode 100644 index 000000000..4a97c65f4 --- /dev/null +++ b/metrics/grafana/provisioning/dashboards/events.json @@ -0,0 +1,131 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.1", + "repeat": "event", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "$event", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$event", + "type": "stat" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "metrics(event_)", + "includeAll": true, + "label": "Event", + "multi": true, + "name": "event", + "options": [], + "query": { + "qryType": 2, + "query": "metrics(event_)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Events", + "uid": "bedg03eysra4ga", + "version": 11, + "weekStart": "" +} \ No newline at end of file diff --git a/metrics/grafana/provisioning/dashboards/topics.json b/metrics/grafana/provisioning/dashboards/topics.json new file mode 100644 index 000000000..6dd2e7ffb --- /dev/null +++ b/metrics/grafana/provisioning/dashboards/topics.json @@ -0,0 +1,126 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.1", + "repeat": "topic", + "repeatDirection": "h", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (topic) ({__name__=~\"event_.*\", topic=\"$topic\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$topic", + "type": "stat" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values({__name__=~\"event_.*\"}, topic)", + "includeAll": true, + "multi": true, + "name": "topic", + "options": [], + "query": { + "qryType": 5, + "query": "label_values({__name__=~\"event_.*\"}, topic)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Topics", + "uid": "aedgmwhhbg8w0b", + "version": 14, + "weekStart": "" +} \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..02649fbca --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,252 @@ +site_name: 'HyperBEAM - Documentation' +repo_url: https://github.com/permaweb/HyperBEAM +repo_name: 'permaweb/HyperBEAM' +site_url: http://[::]:8000/ + +docs_dir: docs +site_dir: mkdocs-site + +use_directory_urls: false + +nav: + - Introduction: + - What is AO-Core?: 'introduction/what-is-ao-core.md' + - What is HyperBEAM?: 'introduction/what-is-hyperbeam.md' + - AO Devices: 'introduction/ao-devices.md' + - Pathing in AO-Core: 'introduction/pathing-in-ao-core.md' + - Run a Node: + - Running a HyperBEAM node: 'run/running-a-hyperbeam-node.md' + - Configuring your machine: 'run/configuring-your-machine.md' + - TEE nodes: 'run/tee-nodes.md' + - Joining/running a router: 'run/joining-running-a-router.md' + - Build on HyperBEAM: + - Get started building on AO-Core: 'build/get-started-building-on-ao-core.md' + - Exposing process state: 'build/exposing-process-state.md' + - Serverless decentralized compute: 'build/serverless-decentralized-compute.md' + - Extending HyperBEAM: 'build/extending-hyperbeam.md' + - Devices: + - Overview: 'devices/overview.md' + - '~meta@1.0': 'devices/meta-at-1-0.md' + - '~process@1.0': 'devices/process-at-1-0.md' + - '~message@1.0': 'devices/message-at-1-0.md' + - '~wasm64@1.0': 'devices/wasm64-at-1-0.md' + - '~lua@5.3a': 'devices/lua-at-5-3a.md' + - '~json@1.0': 'devices/json-at-1-0.md' + - '~scheduler@1.0': 'devices/scheduler-at-1-0.md' + - '~relay@1.0': 'devices/relay-at-1-0.md' + - Resources: + # - Overview: 'resources/source-code/index.md' + - FAQ: 'resources/reference/faq.md' + - LLMs.txt: 'resources/llms.md' + - Glossary: 'resources/reference/glossary.md' + - Source Code Modules: + - Modules: + - ar_bundles: 'resources/source-code/ar_bundles.md' + - ar_deep_hash: 'resources/source-code/ar_deep_hash.md' + - ar_rate_limiter: 'resources/source-code/ar_rate_limiter.md' + - ar_timestamp: 'resources/source-code/ar_timestamp.md' + - ar_tx: 'resources/source-code/ar_tx.md' + - ar_wallet: 'resources/source-code/ar_wallet.md' + - dev_cache: 'resources/source-code/dev_cache.md' + - dev_cacheviz: 'resources/source-code/dev_cacheviz.md' + - dev_codec_ans104: 'resources/source-code/dev_codec_ans104.md' + - dev_codec_flat: 'resources/source-code/dev_codec_flat.md' + - dev_codec_httpsig_conv: 'resources/source-code/dev_codec_httpsig_conv.md' + - dev_codec_httpsig: 'resources/source-code/dev_codec_httpsig.md' + - dev_codec_json: 'resources/source-code/dev_codec_json.md' + - dev_codec_structured: 'resources/source-code/dev_codec_structured.md' + - dev_cron: 'resources/source-code/dev_cron.md' + - dev_cu: 'resources/source-code/dev_cu.md' + - dev_dedup: 'resources/source-code/dev_dedup.md' + - dev_delegated_compute: 'resources/source-code/dev_delegated_compute.md' + - dev_faff: 'resources/source-code/dev_faff.md' + - dev_genesis_wasm: 'resources/source-code/dev_genesis_wasm.md' + - dev_green_zone: 'resources/source-code/dev_green_zone.md' + - dev_hyperbuddy: 'resources/source-code/dev_hyperbuddy.md' + - dev_json_iface: 'resources/source-code/dev_json_iface.md' + - dev_local_name: 'resources/source-code/dev_local_name.md' + - dev_lookup: 'resources/source-code/dev_lookup.md' + - dev_lua_lib: 'resources/source-code/dev_lua_lib.md' + - dev_lua_test: 'resources/source-code/dev_lua_test.md' + - dev_lua: 'resources/source-code/dev_lua.md' + - dev_manifest: 'resources/source-code/dev_manifest.md' + - dev_message: 'resources/source-code/dev_message.md' + - dev_meta: 'resources/source-code/dev_meta.md' + - dev_monitor: 'resources/source-code/dev_monitor.md' + - dev_multipass: 'resources/source-code/dev_multipass.md' + - dev_name: 'resources/source-code/dev_name.md' + - dev_node_process: 'resources/source-code/dev_node_process.md' + - dev_p4: 'resources/source-code/dev_p4.md' + - dev_patch: 'resources/source-code/dev_patch.md' + - dev_poda: 'resources/source-code/dev_poda.md' + - dev_process_cache: 'resources/source-code/dev_process_cache.md' + - dev_process_worker: 'resources/source-code/dev_process_worker.md' + - dev_process: 'resources/source-code/dev_process.md' + - dev_push: 'resources/source-code/dev_push.md' + - dev_relay: 'resources/source-code/dev_relay.md' + - dev_router: 'resources/source-code/dev_router.md' + - dev_scheduler_cache: 'resources/source-code/dev_scheduler_cache.md' + - dev_scheduler_formats: 'resources/source-code/dev_scheduler_formats.md' + - dev_scheduler_registry: 'resources/source-code/dev_scheduler_registry.md' + - dev_scheduler_server: 'resources/source-code/dev_scheduler_server.md' + - dev_scheduler: 'resources/source-code/dev_scheduler.md' + - dev_simple_pay: 'resources/source-code/dev_simple_pay.md' + - dev_snp_nif: 'resources/source-code/dev_snp_nif.md' + - dev_snp: 'resources/source-code/dev_snp.md' + - dev_stack: 'resources/source-code/dev_stack.md' + - dev_test: 'resources/source-code/dev_test.md' + - dev_wasi: 'resources/source-code/dev_wasi.md' + - dev_wasm: 'resources/source-code/dev_wasm.md' + - hb_ao_test_vectors: 'resources/source-code/hb_ao_test_vectors.md' + - hb_ao: 'resources/source-code/hb_ao.md' + - hb_app: 'resources/source-code/hb_app.md' + - hb_beamr_io: 'resources/source-code/hb_beamr_io.md' + - hb_beamr: 'resources/source-code/hb_beamr.md' + - hb_cache_control: 'resources/source-code/hb_cache_control.md' + - hb_cache_render: 'resources/source-code/hb_cache_render.md' + - hb_cache: 'resources/source-code/hb_cache.md' + - hb_client: 'resources/source-code/hb_client.md' + - hb_crypto: 'resources/source-code/hb_crypto.md' + - hb_debugger: 'resources/source-code/hb_debugger.md' + - hb_escape: 'resources/source-code/hb_escape.md' + - hb_event: 'resources/source-code/hb_event.md' + - hb_examples: 'resources/source-code/hb_examples.md' + - hb_features: 'resources/source-code/hb_features.md' + - hb_gateway_client: 'resources/source-code/hb_gateway_client.md' + - hb_http_benchmark_tests: 'resources/source-code/hb_http_benchmark_tests.md' + - hb_http_client_sup: 'resources/source-code/hb_http_client_sup.md' + - hb_http_client: 'resources/source-code/hb_http_client.md' + - hb_http_server: 'resources/source-code/hb_http_server.md' + - hb_http: 'resources/source-code/hb_http.md' + - hb_json: 'resources/source-code/hb_json.md' + - hb_logger: 'resources/source-code/hb_logger.md' + - hb_message: 'resources/source-code/hb_message.md' + - hb_metrics_collector: 'resources/source-code/hb_metrics_collector.md' + - hb_name: 'resources/source-code/hb_name.md' + - hb_opts: 'resources/source-code/hb_opts.md' + - hb_path: 'resources/source-code/hb_path.md' + - hb_persistent: 'resources/source-code/hb_persistent.md' + - hb_private: 'resources/source-code/hb_private.md' + - hb_process_monitor: 'resources/source-code/hb_process_monitor.md' + - hb_router: 'resources/source-code/hb_router.md' + - hb_singleton: 'resources/source-code/hb_singleton.md' + - hb_store_fs: 'resources/source-code/hb_store_fs.md' + - hb_store_gateway: 'resources/source-code/hb_store_gateway.md' + - hb_store_remote_node: 'resources/source-code/hb_store_remote_node.md' + - hb_store_rocksdb: 'resources/source-code/hb_store_rocksdb.md' + - hb_store: 'resources/source-code/hb_store.md' + - hb_structured_fields: 'resources/source-code/hb_structured_fields.md' + - hb_sup: 'resources/source-code/hb_sup.md' + - hb_test_utils: 'resources/source-code/hb_test_utils.md' + - hb_tracer: 'resources/source-code/hb_tracer.md' + - hb_util: 'resources/source-code/hb_util.md' + - hb_volume: 'resources/source-code/hb_volume.md' + - hb: 'resources/source-code/hb.md' + - rsa_pss: 'resources/source-code/rsa_pss.md' + # - Troubleshooting: 'resources/reference/troubleshooting.md' + + # - Community: + # - Contribute Overview: 'community/guidelines.md' + # - Development Setup: 'community/setup.md' + # - Contributing Documentation: 'community/contributing-docs.md' + +markdown_extensions: + - attr_list + - md_in_html + - admonition + - pymdownx.superfences + - pymdownx.betterem + - pymdownx.caret + - pymdownx.mark + - pymdownx.tilde + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.critic + - pymdownx.keys + - def_list + - pymdownx.tasklist: + custom_checkbox: true + - toc: + toc_depth: 2 + permalink: true + +theme: + name: material + custom_dir: docs/theme/templates + language: en + logo: https://arweave.net/e8SdCkAlqpMqvBSUuHu7sYpfZWoJsRKG7XuK0EXon_0 + favicon: https://arweave.net/zMT0qotUQUmPUYhGcgLr80XhG7GRmYXeLWWGitok6Ao + icon: + repo: fontawesome/brands/github + features: + # - navigation.instant + - navigation.instant.progress + - navigation.instant.prefetch + - navigation.tracking + - navigation.sections + - navigation.path + - navigation.expand + - navigation.tabs + - navigation.tabs.sticky + - navigation.indexes + - navigation.prune + - toc.integrate + - content.tooltips + - content.code.copy + - content.code.select + - content.code.annotate + - navigation.footer + palette: + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: white + accent: blue + toggle: + icon: material/brightness-7 + name: Switch to dark mode + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + scheme: default + primary: white + accent: blue + toggle: + icon: material/brightness-7 + name: Switch to light mode + font: + text: DM Sans + nav_style: default + highlightjs: true + + +plugins: + - search + - git-revision-date-localized + +extra_css: + - assets/style.css + +extra_javascript: + - js/utc-time.js + - js/custom-header.js + - js/parallax.js + - js/navigation.js + - js/toc-highlight.js + - js/disable-preload-transition.js + - js/header-scroll.js + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/permaweb/hyperBEAM + name: GitHub + generator: false + diff --git a/native/dev_snp_nif/.gitignore b/native/dev_snp_nif/.gitignore new file mode 100644 index 000000000..be2bbcfd0 --- /dev/null +++ b/native/dev_snp_nif/.gitignore @@ -0,0 +1,3 @@ +files +target +Cargo.lock \ No newline at end of file diff --git a/native/dev_snp_nif/Cargo.lock b/native/dev_snp_nif/Cargo.lock new file mode 100644 index 000000000..01d2cb41c --- /dev/null +++ b/native/dev_snp_nif/Cargo.lock @@ -0,0 +1,1711 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitfield" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c821a6e124197eb56d907ccc2188eab1038fb919c914f47976e64dd8dbc855d1" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" + +[[package]] +name = "cc" +version = "1.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "codicon" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12170080f3533d6f09a19f81596f836854d0fa4867dc32c8172b8474b4e9de61" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "dev_snp_nif" +version = "0.1.0" +dependencies = [ + "bincode", + "hex", + "openssl", + "reqwest", + "rustler", + "serde", + "serde_json", + "sev", + "snafu", + "tokio", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inventory" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b31349d02fe60f80bbbab1a9402364cad7460626d6030494b08ac4a2075bf81" +dependencies = [ + "rustversion", +] + +[[package]] +name = "iocuddle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8972d5be69940353d5347a1344cb375d9b457d6809b428b05bb1ca2fb9ce007" + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.8.0", + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + +[[package]] +name = "log" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags 2.8.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.8.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustler" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7b219d7473cf473409665a4898d66688b34736e51bb5791098b0d3390e4c98" +dependencies = [ + "inventory", + "libloading", + "regex-lite", + "rustler_codegen", +] + +[[package]] +name = "rustler_codegen" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743ec5267bd5f18fd88d89f7e729c0f43b97d9c2539959915fa1f234300bb621" +dependencies = [ + "heck", + "inventory", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustversion" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.8.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sev" +version = "5.0.0" +source = "git+https://github.com/PeterFarber/sev.git#436e0faec7fa4010e36a44b59508b00571fb1b5a" +dependencies = [ + "base64 0.22.1", + "bincode", + "bitfield", + "bitflags 1.3.2", + "byteorder", + "codicon", + "dirs", + "hex", + "iocuddle", + "lazy_static", + "libc", + "openssl", + "rand", + "serde", + "serde-big-array", + "serde_bytes", + "static_assertions", + "uuid", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "snafu" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "2.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +dependencies = [ + "cfg-if", + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" +dependencies = [ + "serde", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/native/dev_snp_nif/Cargo.toml b/native/dev_snp_nif/Cargo.toml new file mode 100644 index 000000000..1179031e9 --- /dev/null +++ b/native/dev_snp_nif/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "dev_snp_nif" +version = "0.1.0" +edition = "2021" + +[lib] +name = "dev_snp_nif" +path = "src/lib.rs" +crate-type = ["dylib"] + +[dependencies] +rustler = "0.36.0" +sev = { git = "https://github.com/PeterFarber/sev.git", features = ["openssl"] } +openssl = "0.10.66" +bincode = "1.3" +snafu = "0.8.2" +hex = "0.4.3" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +reqwest = { version="0.11.10", features = ["blocking"]} +tokio = {version = "1.29.1", features =["rt-multi-thread"] } \ No newline at end of file diff --git a/native/dev_snp_nif/src/attestation.rs b/native/dev_snp_nif/src/attestation.rs new file mode 100644 index 000000000..eb5507fcc --- /dev/null +++ b/native/dev_snp_nif/src/attestation.rs @@ -0,0 +1,89 @@ +use rustler::{Binary, Encoder, Env, NifResult, Term}; +use rustler::types::atom::{self, ok}; +use sev::firmware::guest::{Firmware, AttestationReport}; +use serde_json::to_string; +use crate::logging::log_message; + +/// Generates an attestation report using the provided unique data and VMPL value. +/// +/// # Arguments +/// * `env` - The Rustler environment, used to encode the return value. +/// * `unique_data` - A 64-byte binary input containing unique data for the attestation report. +/// * `vmpl` - The Virtual Machine Privilege Level (VMPL) to be used in the report. +/// +/// # Returns +/// A tuple containing an `ok` atom and the serialized attestation report in JSON format. +/// If an error occurs during the generation or serialization process, an error is returned. +/// +/// # Example +/// ```erlang +/// {ok, JsonReport} = dev_snp_nif:generate_attestation_report(UniqueDataBinary, VMPL). +/// ``` +#[rustler::nif] +pub fn generate_attestation_report<'a>( + env: Env<'a>, + unique_data: Binary, + vmpl: u32, +) -> NifResult> { + log_message("INFO", file!(), line!(), "Starting attestation report generation..."); + + // Step 1: Convert the binary input to a fixed-size array. + let unique_data_array: [u8; 64] = match unique_data.as_slice().try_into() { + Ok(data) => data, + Err(_) => { + let msg = "Input binary must be exactly 64 bytes long."; + log_message("ERROR", file!(), line!(), msg); + return Err(rustler::Error::BadArg); + } + }; + + // Step 2: Open the firmware interface. + let mut firmware = match Firmware::open() { + Ok(fw) => { + log_message("INFO", file!(), line!(), "Firmware opened successfully."); + fw + } + Err(err) => { + let msg = format!("Failed to open firmware: {:?}", err); + log_message("ERROR", file!(), line!(), &msg); + return Ok((atom::error(), msg).encode(env)); + } + }; + + // Step 3: Generate the attestation report. + let report: AttestationReport = match firmware.get_report(None, Some(unique_data_array), Some(vmpl)) { + Ok(report) => { + log_message("INFO", file!(), line!(), "Attestation report generated successfully."); + report + } + Err(err) => { + let msg = format!("Failed to generate attestation report: {:?}", err); + log_message("ERROR", file!(), line!(), &msg); + return Ok((atom::error(), msg).encode(env)); + } + }; + + // Step 4: Serialize the report into a JSON string for output. + let report_json = match to_string(&report) { + Ok(json) => { + log_message("INFO", file!(), line!(), "Attestation report serialized to JSON format."); + json + } + Err(err) => { + let msg = format!("Failed to serialize attestation report: {:?}", err); + log_message("ERROR", file!(), line!(), &msg); + return Ok((atom::error(), msg).encode(env)); + } + }; + + // Step 5: Log the serialized JSON for debugging purposes. + // log_message( + // "INFO", + // file!(), + // line!(), + // &format!("Generated report JSON: {:?}", report_json), + // ); + + // Step 6: Return the result as a tuple with the `ok` atom. + Ok((ok(), report_json).encode(env)) +} diff --git a/native/dev_snp_nif/src/digest.rs b/native/dev_snp_nif/src/digest.rs new file mode 100644 index 000000000..483c026f6 --- /dev/null +++ b/native/dev_snp_nif/src/digest.rs @@ -0,0 +1,145 @@ +use rustler::{Encoder, Env, MapIterator, NifResult, Term}; +use rustler::types::atom::{self, ok}; +use sev::measurement::snp::{snp_calc_launch_digest, SnpMeasurementArgs}; +use sev::measurement::vcpu_types::CpuType; +use sev::measurement::vmsa::{GuestFeatures, VMMType}; +use crate::logging::log_message; +use std::path::PathBuf; +use bincode; + +/// Struct to hold launch digest arguments passed from Erlang +#[derive(Debug)] +struct LaunchDigestArgs { + vcpus: u32, + vcpu_type: u8, + vmm_type: u8, + guest_features: u64, + ovmf_hash_str: String, + kernel_hash: String, + initrd_hash: String, + append_hash: String, +} + +/// Computes the launch digest using the input arguments provided as an Erlang map. +/// +/// # Arguments +/// * `env` - The Rustler environment, used to encode the return value. +/// * `input_map` - An Erlang map containing the input parameters required for the calculation. +/// +/// # Returns +/// A tuple containing an `ok` atom and the calculated and serialized launch digest. +/// If the input is invalid or an error occurs during calculation, an error is returned. +/// +/// # Expected Input Map Keys: +/// - `"vcpus"`: Number of virtual CPUs (u32). +/// - `"vcpu_type"`: Type of the virtual CPU (u8). +/// - `"vmm_type"`: Type of the Virtual Machine Monitor (u8). +/// - `"guest_features"`: Features of the guest (u64). +/// - `"ovmf_hash_str"`: Hash of the OVMF firmware (String). +/// - `"kernel_hash"`: Hash of the kernel (String). +/// - `"initrd_hash"`: Hash of the initrd (String). +/// - `"append_hash"`: Hash of the kernel command line arguments (String). +/// +/// # Example +/// ```erlang +/// {ok, LaunchDigest} = dev_snp_nif:compute_launch_digest(InputMap). +/// ``` +#[rustler::nif] +pub fn compute_launch_digest<'a>(env: Env<'a>, input_map: Term<'a>) -> NifResult> { + //log_message("INFO", file!(), line!(), "Starting launch digest calculation..."); + + // Step 1: Validate that the input is a map. + if !input_map.is_map() { + log_message("ERROR", file!(), line!(), "Provided input is not a map."); + return Err(rustler::Error::BadArg); + } + + // Step 2: Helper function to decode string values from the map. + fn decode_string(value: Term) -> NifResult { + match value.get_type() { + rustler::TermType::List => { + let list: Vec = value.decode()?; + String::from_utf8(list).map_err(|_| rustler::Error::BadArg) + } + _ => value.decode(), + } + } + + // Step 3: Parse input map into LaunchDigestArgs. + let mut args = LaunchDigestArgs { + vcpus: 0, + vcpu_type: 0, + vmm_type: 0, + guest_features: 0, + ovmf_hash_str: String::new(), + kernel_hash: String::new(), + initrd_hash: String::new(), + append_hash: String::new(), + }; + + let map_iter = MapIterator::new(input_map).unwrap(); + for (key, value) in map_iter { + let key_str = key.atom_to_string()?.to_string(); + match key_str.as_str() { + "vcpus" => args.vcpus = value.decode()?, + "vcpu_type" => args.vcpu_type = value.decode()?, + "vmm_type" => args.vmm_type = value.decode()?, + "guest_features" => args.guest_features = value.decode()?, + "firmware" => args.ovmf_hash_str = decode_string(value)?, + "kernel" => args.kernel_hash = decode_string(value)?, + "initrd" => args.initrd_hash = decode_string(value)?, + "append" => args.append_hash = decode_string(value)?, + _ => log_message("WARN", file!(), line!(), &format!("Unexpected key: {}", key_str)), + } + } + + //log_message("INFO", file!(), line!(), &format!("Parsed arguments: {:?}", args)); + + // Step 4: Prepare SnpMeasurementArgs for digest calculation. + let ovmf_file = "test/OVMF-1.55.fd".to_owned(); + let measurement_args = SnpMeasurementArgs { + ovmf_file: Some(PathBuf::from(ovmf_file)), + kernel_file: None, + initrd_file: None, + append: None, + + vcpus: args.vcpus, + vcpu_type: CpuType::try_from(args.vcpu_type).unwrap(), + vmm_type: Some(VMMType::try_from(args.vmm_type).unwrap()), + guest_features: GuestFeatures(args.guest_features), + ovmf_hash_str: Some(args.ovmf_hash_str.as_str()), + kernel_hash: Some(hex::decode(args.kernel_hash).unwrap().try_into().unwrap()), + initrd_hash: Some(hex::decode(args.initrd_hash).unwrap().try_into().unwrap()), + append_hash: Some(hex::decode(args.append_hash).unwrap().try_into().unwrap()), + }; + + // Step 5: Compute the launch digest. + let digest = match snp_calc_launch_digest(measurement_args) { + Ok(digest) => digest, + Err(err) => { + let msg = format!("Failed to compute launch digest: {:?}", err); + log_message("ERROR", file!(), line!(), &msg); + return Ok((atom::error(), msg).encode(env)); + } + }; + + // Step 6: Serialize the digest. + let serialized_digest = match bincode::serialize(&digest) { + Ok(serialized) => serialized, + Err(err) => { + let msg = format!("Failed to serialize launch digest: {:?}", err); + log_message("ERROR", file!(), line!(), &msg); + return Ok((atom::error(), msg).encode(env)); + } + }; + + //log_message( + // "INFO", + // file!(), + // line!(), + // "Launch digest successfully computed and serialized.", + //); + + // Step 7: Return the calculated and serialized digest. + Ok((ok(), serialized_digest).encode(env)) +} diff --git a/native/dev_snp_nif/src/helpers.rs b/native/dev_snp_nif/src/helpers.rs new file mode 100644 index 000000000..b74482264 --- /dev/null +++ b/native/dev_snp_nif/src/helpers.rs @@ -0,0 +1,110 @@ +use sev::certs::snp::{ca, Certificate}; +use sev::firmware::host::TcbVersion; +use crate::logging::log_message; +use reqwest::blocking::get; + +/// Base URL for AMD's Key Distribution Service (KDS). +const KDS_CERT_SITE: &str = "https://kdsintf.amd.com"; +/// Endpoint for the VCEK API. +const KDS_VCEK: &str = "/vcek/v1"; +/// Endpoint for the Certificate Chain API. +const KDS_CERT_CHAIN: &str = "cert_chain"; + +/// Requests the AMD certificate chain (ASK + ARK) for the given SEV product name. +/// +/// # Arguments +/// * `sev_prod_name` - The SEV product name (e.g., "Milan"). +/// +/// # Returns +/// A `ca::Chain` containing the ASK and ARK certificates. +/// +/// # Errors +/// Returns an error if the request fails, the response is invalid, or the certificate parsing fails. +/// +/// # Example +/// ```erlang +/// {ok, CertChain} = dev_snp_nif:request_cert_chain("Milan"). +pub fn request_cert_chain(sev_prod_name: &str) -> Result> { +// Blocking version of reqwest + let url = format!("{KDS_CERT_SITE}{KDS_VCEK}/{sev_prod_name}/{KDS_CERT_CHAIN}"); + // log_message( + // "INFO", + // file!(), + // line!(), + // &format!("Requesting AMD certificate chain from: {url}"), + // ); + + // Perform the blocking GET request + let response = get(&url)?; + let body = response.bytes()?; + + // Parse the response as a PEM-encoded certificate chain + let chain = openssl::x509::X509::stack_from_pem(&body)?; + if chain.len() < 2 { + return Err("Expected at least two certificates (ARK and ASK) in the chain".into()); + } + + // Convert ARK and ASK into the `ca::Chain` structure required by the SEV crate + let ark = chain[1].to_pem()?; + let ask = chain[0].to_pem()?; + let ca_chain = ca::Chain::from_pem(&ark, &ask)?; + + //log_message( + // "INFO", + // file!(), + // line!(), + // "Successfully fetched AMD certificate chain.", + //); + + Ok(ca_chain) +} + +/// Requests the VCEK for the given chip ID and reported TCB. +/// +/// # Arguments +/// * `chip_id` - The unique 64-byte chip ID. +/// * `reported_tcb` - The TCB version of the platform. +/// +/// # Returns +/// A `Certificate` representing the VCEK. +/// +/// # Errors +/// Returns an error if the request fails, the response is invalid, or the certificate parsing fails. +/// +/// # Example +/// ```erlang +/// {ok, VcekCert} = dev_snp_nif:request_vcek(ChipIdBinary, ReportedTcbMap). +/// ``` +pub fn request_vcek( + chip_id: [u8; 64], + reported_tcb: TcbVersion, +) -> Result> { + use reqwest::blocking::get; // Blocking version of reqwest + + let hw_id = chip_id + .iter() + .map(|byte| format!("{:02x}", byte)) + .collect::(); + + let url = format!( + "{KDS_CERT_SITE}{KDS_VCEK}/Milan/{hw_id}?blSPL={:02}&teeSPL={:02}&snpSPL={:02}&ucodeSPL={:02}", + reported_tcb.bootloader, reported_tcb.tee, reported_tcb.snp, reported_tcb.microcode + ); + + // log_message( + // "INFO", + // file!(), + // line!(), + // &format!("Requesting VCEK from: {url}"), + // ); + + // Perform the blocking GET request + let response = get(&url)?; + let rsp_bytes = response.bytes()?; + + // Parse the VCEK response as a DER-encoded certificate + let vcek_cert = Certificate::from_der(&rsp_bytes)?; + + // log_message("INFO", file!(), line!(), "Successfully fetched VCEK."); + Ok(vcek_cert) +} diff --git a/native/dev_snp_nif/src/lib.rs b/native/dev_snp_nif/src/lib.rs new file mode 100644 index 000000000..abfc92abc --- /dev/null +++ b/native/dev_snp_nif/src/lib.rs @@ -0,0 +1,13 @@ +/// Entry point for the Rustler NIF module. +/// This file defines the available NIF functions and organizes them into modules. + +mod logging; +mod snp_support; +mod attestation; +mod digest; +mod verification; +mod helpers; + +rustler::init!( + "dev_snp_nif"// Module name as used in Erlang. +); diff --git a/native/dev_snp_nif/src/logging.rs b/native/dev_snp_nif/src/logging.rs new file mode 100644 index 000000000..31be106fa --- /dev/null +++ b/native/dev_snp_nif/src/logging.rs @@ -0,0 +1,28 @@ +use std::thread; +use std::time::SystemTime; + +/// Logs messages with details including thread ID, timestamp, file, and line number. +/// +/// # Arguments +/// - `log_level`: The log level (e.g., "INFO", "ERROR"). +/// - `file`: The file where the log is being generated. +/// - `line`: The line number of the log statement. +/// - `message`: The log message. +/// +/// # Example +/// ```rust +/// log_message("INFO", file!(), line!(), "This is a log message."); +/// ``` +pub fn log_message(log_level: &str, file: &str, line: u32, message: &str) { + let thread_id = thread::current().id(); + let now = SystemTime::now(); + let timestamp = now + .duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + println!( + "[{}#{:?} @ {}:{}] [{}] {}", + log_level, thread_id, file, line, timestamp, message + ); +} diff --git a/native/dev_snp_nif/src/snp_support.rs b/native/dev_snp_nif/src/snp_support.rs new file mode 100644 index 000000000..0ca9da69c --- /dev/null +++ b/native/dev_snp_nif/src/snp_support.rs @@ -0,0 +1,44 @@ +use rustler::{Encoder, Env, NifResult, Term}; +use rustler::types::atom::ok; +use sev::firmware::guest::Firmware; +use crate::logging::log_message; + +/// Checks if Secure Nested Paging (SNP) is supported by the system. +/// +/// # Arguments +/// * `env` - The Rustler environment, used to encode the return value. +/// +/// # Returns +/// A tuple containing an `ok` atom and a boolean value: +/// - `true` if the firmware indicates that SNP is supported. +/// - `false` if SNP is not supported or if the firmware cannot be accessed. +/// +/// # Example +/// ```erlang +/// {ok, Supported} = dev_snp_nif:check_snp_support(). +/// ``` +#[rustler::nif] +pub fn check_snp_support<'a>(env: Env<'a>) -> NifResult> { + //log_message("INFO", file!(), line!(), "Checking SNP support..."); + + // Step 1: Attempt to open the firmware interface. + // If the firmware is accessible, SNP is supported; otherwise, it is not. + let is_supported = match Firmware::open() { + Ok(_) => { + //log_message("INFO", file!(), line!(), "SNP is supported."); + true // SNP is supported. + } + Err(_) => { + // log_message( + // "ERROR", + // file!(), + // line!(), + // "Failed to open firmware. SNP is not supported.", + // ); + false // SNP is not supported. + } + }; + + // Step 2: Return the result as a tuple with the `ok` atom and the boolean value. + Ok((ok(), is_supported).encode(env)) +} diff --git a/native/dev_snp_nif/src/verification.rs b/native/dev_snp_nif/src/verification.rs new file mode 100644 index 000000000..e8636e851 --- /dev/null +++ b/native/dev_snp_nif/src/verification.rs @@ -0,0 +1,310 @@ +use rustler::{Binary, Encoder, Env, NifResult, Term}; +use rustler::types::atom::{self, ok}; +use serde_json::Value; +use serde::Deserialize; +use sev::certs::snp::{ecdsa::Signature, Chain, Verifiable}; +use sev::firmware::host::TcbVersion; +use sev::firmware::guest::{AttestationReport, GuestPolicy, PlatformInfo}; +use crate::helpers::{request_cert_chain, request_vcek}; +use crate::logging::log_message; + +/// Verifies whether the measurement in the attestation report matches the expected measurement. +/// +/// # Arguments +/// * `env` - The Rustler environment, used to encode the return value. +/// * `_report` - A binary containing the serialized attestation report (JSON format). +/// * `_expected_measurement` - A binary containing the expected measurement (as a byte array). +/// +/// # Returns +/// A tuple with: +/// - `ok` atom and a success message if the measurements match. +/// - `error` atom and an error message if the measurements do not match. +#[rustler::nif] +fn verify_measurement<'a>( + env: Env<'a>, + _report: Binary, + _expected_measurement: Binary, +) -> NifResult> { + //log_message("INFO", file!(), line!(), "Starting measurement verification..."); + + // Define a struct for deserializing the attestation report. + #[derive(Debug, Deserialize)] + struct AttestationReport { + measurement: Vec, + // Additional fields can be added here if needed. + } + + // Step 1: Deserialize the JSON report. + let report: AttestationReport = match serde_json::from_slice(_report.as_slice()) { + Ok(parsed_report) => { + //log_message( + // "INFO", + // file!(), + // line!(), + // &format!("Successfully parsed report: {:?}", parsed_report), + //); + parsed_report + } + Err(err) => { + log_message( + "ERROR", + file!(), + line!(), + &format!("Failed to deserialize report: {:?}", err), + ); + return Ok((atom::error(), "Invalid report format").encode(env)); + } + }; + + // Step 2: Extract the actual measurement from the report. + let actual_measurement = &report.measurement; + // log_message( + // "INFO", + // file!(), + // line!(), + // &format!("Extracted actual measurement: {:?}", actual_measurement), + // ); + + // Step 3: Decode the expected measurement from the input binary. + let expected_measurement: Vec = _expected_measurement.as_slice().to_vec(); + // log_message( + // "INFO", + // file!(), + // line!(), + // &format!("Decoded expected measurement: {:?}", expected_measurement), + // ); + + // Step 4: Compare the actual and expected measurements. + if actual_measurement == &expected_measurement { + //log_message("INFO", file!(), line!(), "Measurements match."); + Ok((atom::ok(), true).encode(env)) + } else { + //log_message("ERROR", file!(), line!(), "Measurements do not match."); + Ok((atom::error(), false).encode(env)) + } +} + + +/// Verifies the signature of an attestation report. +/// +/// # Arguments +/// * `env` - The Rustler environment, used to encode the return value. +/// * `report` - A binary containing the serialized attestation report. +/// +/// # Returns +/// A tuple with: +/// - `ok` atom and a success message if the signature is valid. +/// - `error` atom and an error message if the signature verification fails. +#[rustler::nif] +fn verify_signature<'a>( + env: Env<'a>, + report: Binary<'a>, +) -> NifResult> { + // log_message("INFO", file!(), line!(), "Verifying signature..."); + + // Step 1: Parse the report JSON into a serde Value object. + let json_data = match serde_json::from_slice::(report.as_slice()) { + Ok(data) => data, + Err(err) => { + return Ok(( + rustler::types::atom::error(), + format!("Failed to parse JSON: {}", err), + ) + .encode(env)); + } + }; + + // Step 2: Map JSON fields to the AttestationReport struct. + // Each field is individually parsed to ensure type safety. + let attestation_report = AttestationReport { + version: json_data["version"].as_u64().unwrap_or(0) as u32, + guest_svn: json_data["guest_svn"].as_u64().unwrap_or(0) as u32, + policy: GuestPolicy(json_data["policy"].as_u64().unwrap_or(0)), + family_id: json_data["family_id"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 16]), + image_id: json_data["image_id"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 16]), + vmpl: json_data["vmpl"].as_u64().unwrap_or(0) as u32, + sig_algo: json_data["sig_algo"].as_u64().unwrap_or(0) as u32, + current_tcb: TcbVersion { + bootloader: json_data["current_tcb"]["bootloader"].as_u64().unwrap_or(0) as u8, + tee: json_data["current_tcb"]["tee"].as_u64().unwrap_or(0) as u8, + snp: json_data["current_tcb"]["snp"].as_u64().unwrap_or(0) as u8, + microcode: json_data["current_tcb"]["microcode"].as_u64().unwrap_or(0) as u8, + _reserved: [0; 4], + }, + plat_info: PlatformInfo(json_data["plat_info"].as_u64().unwrap_or(0)), + _author_key_en: json_data["_author_key_en"].as_u64().unwrap_or(0) as u32, + _reserved_0: json_data["_reserved_0"].as_u64().unwrap_or(0) as u32, + report_data: json_data["report_data"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 64]), + measurement: json_data["measurement"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 48]), + host_data: json_data["host_data"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 32]), + id_key_digest: json_data["id_key_digest"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 48]), + author_key_digest: json_data["author_key_digest"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 48]), + report_id: json_data["report_id"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 32]), + report_id_ma: json_data["report_id_ma"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 32]), + reported_tcb: TcbVersion { + bootloader: json_data["reported_tcb"]["bootloader"] + .as_u64() + .unwrap_or(0) as u8, + tee: json_data["reported_tcb"]["tee"].as_u64().unwrap_or(0) as u8, + snp: json_data["reported_tcb"]["snp"].as_u64().unwrap_or(0) as u8, + microcode: json_data["reported_tcb"]["microcode"].as_u64().unwrap_or(0) as u8, + _reserved: [0; 4], + }, + _reserved_1: [0; 24], + chip_id: json_data["chip_id"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 64]), + committed_tcb: TcbVersion { + bootloader: json_data["committed_tcb"]["bootloader"] + .as_u64() + .unwrap_or(0) as u8, + tee: json_data["committed_tcb"]["tee"].as_u64().unwrap_or(0) as u8, + snp: json_data["committed_tcb"]["snp"].as_u64().unwrap_or(0) as u8, + microcode: json_data["committed_tcb"]["microcode"] + .as_u64() + .unwrap_or(0) as u8, + _reserved: [0; 4], + }, + current_build: json_data["current_build"].as_u64().unwrap_or(0) as u8, + current_minor: json_data["current_minor"].as_u64().unwrap_or(0) as u8, + current_major: json_data["current_major"].as_u64().unwrap_or(0) as u8, + _reserved_2: json_data["_reserved_2"].as_u64().unwrap_or(0) as u8, + committed_build: json_data["committed_build"].as_u64().unwrap_or(0) as u8, + committed_minor: json_data["committed_minor"].as_u64().unwrap_or(0) as u8, + committed_major: json_data["committed_major"].as_u64().unwrap_or(0) as u8, + _reserved_3: json_data["_reserved_3"].as_u64().unwrap_or(0) as u8, + launch_tcb: TcbVersion { + bootloader: json_data["launch_tcb"]["bootloader"].as_u64().unwrap_or(0) as u8, + tee: json_data["launch_tcb"]["tee"].as_u64().unwrap_or(0) as u8, + snp: json_data["launch_tcb"]["snp"].as_u64().unwrap_or(0) as u8, + microcode: json_data["launch_tcb"]["microcode"].as_u64().unwrap_or(0) as u8, + _reserved: [0; 4], + }, + _reserved_4: [0; 168], + signature: Signature { + r: json_data["signature"]["r"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 72]), + s: json_data["signature"]["s"] + .as_array() + .unwrap_or(&vec![]) + .iter() + .map(|v| v.as_u64().unwrap_or(0) as u8) + .collect::>() + .try_into() + .unwrap_or([0; 72]), + _reserved: [0; 368], + }, + }; + + // Step 3: Extract the chip ID and TCB version. + let chip_id_array: [u8; 64] = attestation_report + .chip_id + .try_into() + .expect("chip_id must be 64 bytes"); + let tcb_version = attestation_report.current_tcb; + + // Step 4: Request the certificate chain and VCEK. + let ca = request_cert_chain("Milan").unwrap(); + let vcek = request_vcek(chip_id_array, tcb_version).unwrap(); + + // Step 5: Verify the certificate chain. + if let Err(e) = ca.verify() { + log_message( + "ERROR", + file!(), + line!(), + &format!("CA chain verification failed: {:?}", e), + ); + return Ok((atom::error(), format!("CA verification failed: {:?}", e)).encode(env)); + } + //log_message("INFO", file!(), line!(), "CA chain verification successful."); + + // Step 6: Verify the attestation report. + let cert_chain = Chain { ca, vek: vcek }; + if let Err(e) = (&cert_chain, &attestation_report).verify() { + log_message( + "ERROR", + file!(), + line!(), + &format!("Attestation report verification failed: {:?}", e), + ); + return Ok((atom::error(), format!("Report verification failed: {:?}", e)).encode(env)); + } + + //log_message("INFO", file!(), line!(), "Signature verification successful."); + Ok((ok(), true).encode(env)) +} diff --git a/native/digest_calc/.gitignore b/native/digest_calc/.gitignore new file mode 100644 index 000000000..be2bbcfd0 --- /dev/null +++ b/native/digest_calc/.gitignore @@ -0,0 +1,3 @@ +files +target +Cargo.lock \ No newline at end of file diff --git a/native/digest_calc/Cargo.toml b/native/digest_calc/Cargo.toml new file mode 100644 index 000000000..f58f90b5b --- /dev/null +++ b/native/digest_calc/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "digest_calc" +version = "0.1.0" +edition = "2021" + +[dependencies] +sev = { git = "https://github.com/PeterFarber/sev.git", features = ["openssl"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_yaml = "0.9.34" +openssl = "0.10.66" +bincode = "1.3" +clap = "3.0" \ No newline at end of file diff --git a/native/digest_calc/config.yml b/native/digest_calc/config.yml new file mode 100644 index 000000000..8ba31931f --- /dev/null +++ b/native/digest_calc/config.yml @@ -0,0 +1,64 @@ +# Kernel configuration file path +kernel_file: "/home/peterfarber/_Current/HyperBEAM/native/digest_calc/files/kernel" # Path to the kernel binary + +# Initrd configuration file path +initrd_file: "/home/peterfarber/_Current/HyperBEAM/native/digest_calc/files/initrd" # Path to the initrd (initial RAM disk) + +# OVMF (Open Virtual Machine Firmware) file path +ovmf_file: "/home/peterfarber/_Current/HyperBEAM/native/digest_calc/files/ovmf" # Path to the OVMF file used for virtual machine boot + +# Kernel command line arguments +cmdline: "console=ttyS0 earlyprintk=serial root=/dev/sda boot=verity verity_disk=/dev/sdb verity_roothash=7270de8ae229d0e8c219170b2c8b34d20d544d74f77c9469b81d22b1697ad3aa" +# Kernel boot arguments including console settings, verity disk and root hash for secure boot + +# Number of virtual CPUs to be allocated for the virtual machine +vcpus: 1 # Set to 1 for a single virtual CPU (adjust as necessary) + +# VCPU Types: List of available virtual CPU models. These are typically based on physical CPU models. +# Each of these types corresponds to a specific configuration of CPU features that the virtual machine will use. +# Examples include specific features for hardware virtualization, optimizations, or security configurations. + +vcpu_type: "EpycV4" # Choose the CPU model for the virtual machine (EpycV4 is a modern AMD CPU type) + +# Virtual Machine Monitor (VMM) Types: Specifies the hypervisor used for the VM. +# - QEMU: A generic and widely used open-source hypervisor. +# - EC2: Amazon's EC2 instance type for cloud-based VMs. +# - KRUN: A specialized hypervisor used in specific security and research contexts. +vmm_type: "QEMU" # Choose the type of hypervisor. QEMU is commonly used for local virtual machines. + +# Guest Features: +# The guest features are represented by individual bits in a 64-bit integer. +# Each bit in the 64-bit integer corresponds to an enabled or disabled feature for the virtual machine. +# The bits can be set to '1' to enable a specific feature, or '0' to disable it. + +# Here's the breakdown of the available bits: + +# | Bit | Feature | +# |-----|----------------------| +# | 0 | SNPActive | # Enables Secure Nested Paging (SNP), enhancing security for the VM +# | 1 | vTOM | # Enables virtual Trusted Opaque Memory, a security feature +# | 2 | ReflectVC | # Enables Reflection for VC (Virtualization Context) +# | 3 | RestrictedInjection | # Restricts certain types of injection into the VM +# | 4 | AlternateInjection | # Enables alternate forms of injection into the VM +# | 5 | DebugSwap | # Allows for debugging VM swap operations +# | 6 | PreventHostIBS | # Prevents Instruction-Based Sampling on the host system +# | 7 | BTBIsolation | # Isolates Branch Target Buffer for security +# | 8 | VmplSSS | # Enables Virtual Memory for Secure State (SSS) support +# | 9 | SecureTSC | # Enables Secure Time Stamp Counter, preventing time-based attacks +# | 10 | VmgexitParameter | # Enables parameters related to VM exit for performance tuning +# | 11 | Reserved, SBZ | # Reserved bit, should not be used, always zero +# | 12 | IbsVirtualization | # Allows Ibs (Interrupt-based Sampling) virtualization +# | 13 | Reserved, SBZ | # Reserved bit, should not be used, always zero +# | 14 | VmsaRegProt | # Enables VM-Sensitive Register Protection +# | 15 | SmtProtection | # Protects against Simultaneous Multithreading (SMT) attacks + +# The value is represented as a hexadecimal string where each bit corresponds to a feature. +# For example, a value of "0000000000000001" means: +# - Bit 0 (SNPActive) is enabled. +# All other bits are set to 0. + +guest_features: "0000000000000001" # Bit 0: SNPActive Enabled (secure virtualization enabled) + +# Notes: +# - You can adjust this value to enable or disable features by modifying the appropriate bits. +# - To turn on additional features, set the corresponding bit to '1' (e.g., "0000000000000011" to enable SNPActive and vTOM). diff --git a/native/digest_calc/src/main.rs b/native/digest_calc/src/main.rs new file mode 100644 index 000000000..aba6bd5be --- /dev/null +++ b/native/digest_calc/src/main.rs @@ -0,0 +1,358 @@ +// This program calculates the SEV-SNP launch digest, which is used for +// verifying the integrity of a virtual machine at launch. It computes +// cryptographic hashes of the kernel, initrd, cmdline, and OVMF files, +// and generates the corresponding launch digest required for secure attestation +// in SEV-SNP environments. + +use bincode; +use clap::{App, Arg}; +use serde::{Deserialize, Serialize}; +use sev::error::MeasurementError; +use sev::measurement::sev_hashes::SevHashes; +use sev::measurement::snp::{ + calc_snp_ovmf_hash, snp_calc_launch_digest, SnpLaunchDigest, SnpMeasurementArgs, +}; +use sev::measurement::vcpu_types::CpuType; +use sev::measurement::vmsa::{GuestFeatures, VMMType}; +use std::fs; +use std::path::PathBuf; + +/// Struct to hold the arguments received from the command line. +#[derive(Serialize)] +struct Arguments { + config: Option, // Path to the configuration file + kernel_file: Option, // Path to the kernel file + initrd_file: Option, // Path to the initrd file + ovmf_file: Option, // Path to the OVMF file + cmdline: Option, // Kernel command line + vcpus: Option, // Number of virtual CPUs + vcpu_type: Option, // Type of virtual CPU + vmm_type: Option, // Virtual Machine Monitor type + guest_features: Option, // Guest features as a hex value +} + +/// Struct to hold the configuration loaded from a YAML file. +#[derive(Debug, Deserialize, Serialize)] +struct Config { + kernel_file: String, + initrd_file: String, + ovmf_file: String, + cmdline: String, + vcpus: Option, + vcpu_type: Option, + vmm_type: Option, + guest_features: Option, +} + +/// Converts a byte slice to a hexadecimal string representation. +fn bytes_to_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{:02x}", byte)).collect() +} + +/// Calculates the launch measurement digest using the SEV-SNP arguments. +fn calculate_launch_measurment( + snp_measure_args: SnpMeasurementArgs, +) -> Result<[u8; 384 / 8], String> { + // Calculate the launch digest + let ld = snp_calc_launch_digest(snp_measure_args) + .map_err(|e| format!("Failed to compute launch digest: {:?}", e))?; + + // Serialize the launch digest + let ld_vec = bincode::serialize(&ld).map_err(|e| { + format!( + "Failed to bincode serialize SnpLaunchDigest to Vec: {:?}", + e + ) + })?; + + // Convert the serialized data into a fixed-length byte array + let ld_arr: [u8; 384 / 8] = ld_vec + .try_into() + .map_err(|_| "SnpLaunchDigest has unexpected length".to_string())?; + + Ok(ld_arr) +} + +/// Calculates the OVMF file hash. +pub fn get_ovmf_hash_from_file(ovmf_file: PathBuf) -> Result { + calc_snp_ovmf_hash(ovmf_file) +} + +/// Retrieves the hashes for kernel, initrd, and cmdline files. +pub fn get_hashes_from_files( + kernel_file: PathBuf, + initrd_file: Option, + append: Option<&str>, +) -> Result { + SevHashes::new(kernel_file, initrd_file, append) +} + +fn main() { + // Starting message + println!("=== Digest Calculator Starting ==="); + + println!("\n=== Getting Command Line Arguments ==="); + // Parse command line arguments using the clap library + let matches = App::new("SEV SNP Measurement") + .version("1.0") + .author("Peter Farber ") + .about( + "SEV-SNP Launch Digest Calculation\n\n\ + Example commands:\n\ + 1. Basic example with default values:\n\ + ./sev_snp_measurement --kernel_file /path/to/kernel --ovmf_file /path/to/ovmf --cmdline \"root=/dev/sda console=ttyS0\"\n\ + 2. Specify all arguments, including optional ones:\n\ + ./sev_snp_measurement --kernel_file /path/to/kernel --initrd_file /path/to/initrd --ovmf_file /path/to/ovmf --cmdline \"root=/dev/sda console=ttyS0\" --vcpus 2 --vcpu_type EpycV4 --vmm_type QEMU --guest_features 0x1\n\ + 3. Use a different VMM type and guest features:\n\ + ./sev_snp_measurement --kernel_file /path/to/kernel --ovmf_file /path/to/ovmf --cmdline \"root=/dev/sda console=ttyS0\" --vcpus 4 --vcpu_type EpycMilan --vmm_type EC2 --guest_features 0x2\n" + ) + .arg(Arg::new("config") + .help("Path to the YAML configuration file") + .takes_value(true)) + .arg(Arg::new("kernel_file") + .help("The path to the kernel file (required)") + .required_unless_present("config") + .takes_value(true)) + .arg(Arg::new("initrd_file") + .help("The path to the initrd file (required)") + .required_unless_present("config") + .takes_value(true)) + .arg(Arg::new("ovmf_file") + .help("The path to the OVMF file (required)") + .required_unless_present("config") + .takes_value(true)) + .arg(Arg::new("cmdline") + .help("The kernel command line (required)") + .required_unless_present("config") + .takes_value(true)) + .arg(Arg::new("vcpus") + .help("Number of virtual CPUs (default: 1)") + .takes_value(true) + .default_value("1")) + .arg(Arg::new("vcpu_type") + .help("The type of virtual CPU (default: EpycV4)\n\ + Available options:\n\ + \"Epyc\" => CpuType::Epyc,\n\ + \"EpycV1\" => CpuType::EpycV1,\n\ + \"EpycV2\" => CpuType::EpycV2,\n\ + \"EpycIBPB\" => CpuType::EpycIBPB,\n\ + \"EpycV3\" => CpuType::EpycV3,\n\ + \"EpycV4\" => CpuType::EpycV4,\n\ + \"EpycRome\" => CpuType::EpycRome,\n\ + \"EpycRomeV1\" => CpuType::EpycRomeV1,\n\ + \"EpycRomeV2\" => CpuType::EpycRomeV2,\n\ + \"EpycRomeV3\" => CpuType::EpycRomeV3,\n\ + \"EpycMilan\" => CpuType::EpycMilan,\n\ + \"EpycMilanV1\" => CpuType::EpycMilanV1,\n\ + \"EpycMilanV2\" => CpuType::EpycMilanV2,\n\ + \"EpycGenoa\" => CpuType::EpycGenoa,\n\ + \"EpycGenoaV1\" => CpuType::EpycGenoaV1") + .takes_value(true) + .default_value("EpycV4")) + .arg(Arg::new("vmm_type") + .help("The VMM type (default: QEMU)\n\ + Available options:\n\ + \"QEMU\" => Some(VMMType::QEMU),\n\ + \"EC2\" => Some(VMMType::EC2),\n\ + \"KRUN\" => Some(VMMType::KRUN),") + .takes_value(true) + .default_value("QEMU")) + .arg(Arg::new("guest_features") + .help("Guest features as a hex value (default: 0x1)\n\ + Available features:\n\ + | 0 | SNPActive |\n\ + | 1 | vTOM |\n\ + | 2 | ReflectVC |\n\ + | 3 | RestrictedInjection |\n\ + | 4 | AlternateInjection |\n\ + | 5 | DebugSwap |\n\ + | 6 | PreventHostIBS |\n\ + | 7 | BTBIsolation |\n\ + | 8 | VmplSSS |\n\ + | 9 | SecureTSC |\n\ + | 10 | VmgexitParameter |\n\ + | 11 | Reserved, SBZ |\n\ + | 12 | IbsVirtualization |\n\ + | 13 | Reserved, SBZ |\n\ + | 14 | VmsaRegProt |\n\ + | 15 | SmtProtection |\n\ + | 63:16 | Reserved, SBZ |\n\n\ + Example Usage:\n\ + 1. Enable SNPActive (bit 0):\n\ + guest_features 0000000000000001\n") + .takes_value(true) + .default_value("0x1")) + .get_matches(); + + // Store the parsed command line arguments + let args = Arguments { + config: matches.value_of("config").map(String::from), + kernel_file: matches.value_of("kernel_file").map(String::from), + initrd_file: matches.value_of("initrd_file").map(String::from), + ovmf_file: matches.value_of("ovmf_file").map(String::from), + cmdline: matches.value_of("cmdline").map(String::from), + vcpus: matches.value_of("vcpus").map(|v| v.parse().unwrap()), + vcpu_type: matches.value_of("vcpu_type").map(String::from), + vmm_type: matches.value_of("vmm_type").map(String::from), + guest_features: matches.value_of("guest_features").map(String::from), + }; + + // Output arguments in a nicely formatted JSON style + let formatted_json = serde_json::to_string_pretty(&args).unwrap(); + println!("{}", formatted_json); + + println!("\n=== Parsing Command Line Arguments ==="); + + // Check if a config file path is provided, and load the configuration + let config: Option = if let Some(config_path) = matches.value_of("config") { + let config_content = fs::read_to_string(config_path) + .map_err(|e| format!("Failed to read config file: {:?}", e)) + .unwrap(); + let config: Config = serde_yaml::from_str(&config_content) + .map_err(|e| format!("Failed to parse config file: {:?}", e)) + .unwrap(); + Some(config) + } else { + None + }; + + // If a config file is loaded, print it as formatted JSON + if let Some(config) = config.as_ref() { + let formatted_json = serde_json::to_string_pretty(&config).unwrap(); + println!("{}", formatted_json); + } else { + println!("No config loaded."); + } + + // Retrieve the kernel file from either the config or the command line arguments + let kernel_file = config + .as_ref() + .and_then(|c| Some(c.kernel_file.clone())) + .unwrap_or_else(|| matches.value_of("kernel_file").unwrap().to_owned()); + + // Process other command line arguments or config values similarly... + let initrd_file = config + .as_ref() + .and_then(|c| Some(c.initrd_file.clone())) + .or_else(|| matches.value_of("initrd_file").map(|s| s.to_owned())); + + let ovmf_file = config + .as_ref() + .and_then(|c| Some(c.ovmf_file.clone())) + .unwrap_or_else(|| matches.value_of("ovmf_file").unwrap().to_owned()); + + let cmdline = config + .as_ref() + .and_then(|c| Some(c.cmdline.clone())) + .unwrap_or_else(|| matches.value_of("cmdline").unwrap().to_owned()); + + let vcpus: u32 = config + .as_ref() + .and_then(|c| c.vcpus) + .unwrap_or_else(|| matches.value_of("vcpus").unwrap().parse().unwrap()); + + let vcpu_type = config + .as_ref() + .and_then(|c| c.vcpu_type.clone()) + .unwrap_or_else(|| matches.value_of("vcpu_type").unwrap().to_owned()); + + // Process virtual CPU type + let vcpu_type = match vcpu_type.as_str() { + "Epyc" => CpuType::Epyc, + "EpycV1" => CpuType::EpycV1, + "EpycV2" => CpuType::EpycV2, + "EpycIBPB" => CpuType::EpycIBPB, + "EpycV3" => CpuType::EpycV3, + "EpycV4" => CpuType::EpycV4, + "EpycRome" => CpuType::EpycRome, + "EpycRomeV1" => CpuType::EpycRomeV1, + "EpycRomeV2" => CpuType::EpycRomeV2, + "EpycRomeV3" => CpuType::EpycRomeV3, + "EpycMilan" => CpuType::EpycMilan, + "EpycMilanV1" => CpuType::EpycMilanV1, + "EpycMilanV2" => CpuType::EpycMilanV2, + "EpycGenoa" => CpuType::EpycGenoa, + "EpycGenoaV1" => CpuType::EpycGenoaV1, + _ => CpuType::EpycV4, // Default to EpycV4 + }; + + let vmm_type = config + .as_ref() + .and_then(|c| c.vmm_type.clone()) + .unwrap_or_else(|| matches.value_of("vmm_type").unwrap().to_owned()); + + // Resolve the VMM type + let vmm_type = match vmm_type.as_str() { + "QEMU" => Some(VMMType::QEMU), + "EC2" => Some(VMMType::EC2), + "KRUN" => Some(VMMType::KRUN), + _ => Some(VMMType::QEMU), // Default to QEMU + }; + + let guest_features_string = config + .as_ref() + .and_then(|c| c.guest_features.clone()) + .unwrap_or_else(|| matches.value_of("guest_features").unwrap().to_owned()); + + let guest_features: u64 = + u64::from_str_radix(&guest_features_string, 2).unwrap(); + + + // Step 1: Get the hash of the OVMF file + let ovmf_hash = get_ovmf_hash_from_file(ovmf_file.clone().into()).unwrap(); + let ovmf_bytes: Vec = bincode::serialize(&ovmf_hash).unwrap(); + let ovmf_binding = ovmf_hash.get_hex_ld(); + + println!("\n===== OVFM ====="); + println!("Bytes: {:x?}", ovmf_bytes); + println!("Hash: {:x?}", ovmf_binding); + + // Step 2: Get the hash of the kernel, initrd, and cmdline + let SevHashes { + kernel_hash, + initrd_hash, + cmdline_hash, + } = get_hashes_from_files( + kernel_file.clone().into(), + initrd_file.clone().map(|file| file.into()), + Some(cmdline.as_str()), + ) + .unwrap(); + + println!("\n===== Kernel ====="); + println!("Bytes: {:x?}", kernel_hash); + println!("Hash: {}", bytes_to_hex(&kernel_hash)); + + println!("\n===== Initrd ====="); + println!("Bytes {:x?}", initrd_hash); + println!("Hash: {}", bytes_to_hex(&initrd_hash)); + + + println!("\n===== Cmdline ====="); + println!("Bytes: {:x?}", cmdline_hash); + println!("Hash: {}", bytes_to_hex(&cmdline_hash)); + + // Step 3: Calculate the launch digest + let arguments = SnpMeasurementArgs { + ovmf_file: Some(PathBuf::from(ovmf_file)), + kernel_file: None, + initrd_file: None, + append: None, + + vcpus, + vcpu_type, + vmm_type, + guest_features: GuestFeatures(guest_features), + + ovmf_hash_str: Some(ovmf_binding.as_str()), + kernel_hash: Some(kernel_hash), + initrd_hash: Some(initrd_hash), + append_hash: Some(cmdline_hash), + }; + + let expected_hash = calculate_launch_measurment(arguments).unwrap(); + + println!("\n===== Expected ====="); + println!("Bytes: {:x?}", expected_hash); + println!("Hash: {}", bytes_to_hex(&expected_hash)); +} diff --git a/native/genesis-wasm/launch-monitored.sh b/native/genesis-wasm/launch-monitored.sh new file mode 100755 index 000000000..e223a4703 --- /dev/null +++ b/native/genesis-wasm/launch-monitored.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +"$@" & +CHILD_PID=$! + +while kill -0 "$PPID" 2>/dev/null; do + sleep 1 +done + +kill -TERM "$CHILD_PID" 2>/dev/null \ No newline at end of file diff --git a/native/hb_beamr/hb_beamr.c b/native/hb_beamr/hb_beamr.c new file mode 100644 index 000000000..f754bf47e --- /dev/null +++ b/native/hb_beamr/hb_beamr.c @@ -0,0 +1,288 @@ +#include "include/hb_helpers.h" +#include "include/hb_logging.h" +#include "include/hb_driver.h" +#include "include/hb_wasm.h" + +// Declare the atoms used in Erlang driver communication +ErlDrvTermData atom_ok; +ErlDrvTermData atom_error; +ErlDrvTermData atom_import; +ErlDrvTermData atom_execution_result; + +static ErlDrvData wasm_driver_start(ErlDrvPort port, char *buff) { + ErlDrvSysInfo info; + driver_system_info(&info, sizeof(info)); + DRV_DEBUG("Starting WASM driver"); + DRV_DEBUG("Port: %p", port); + DRV_DEBUG("Buff: %s", buff); + DRV_DEBUG("Caller PID: %d", driver_caller(port)); + DRV_DEBUG("ERL_DRV_EXTENDED_MAJOR_VERSION: %d", ERL_DRV_EXTENDED_MAJOR_VERSION); + DRV_DEBUG("ERL_DRV_EXTENDED_MINOR_VERSION: %d", ERL_DRV_EXTENDED_MINOR_VERSION); + DRV_DEBUG("ERL_DRV_FLAG_USE_PORT_LOCKING: %d", ERL_DRV_FLAG_USE_PORT_LOCKING); + DRV_DEBUG("info.major_version: %d", info.driver_major_version); + DRV_DEBUG("info.minor_version: %d", info.driver_major_version); + DRV_DEBUG("info.thread_support: %d", info.thread_support); + DRV_DEBUG("info.smp_support: %d", info.smp_support); + DRV_DEBUG("info.async_threads: %d", info.async_threads); + DRV_DEBUG("info.scheduler_threads: %d", info.scheduler_threads); + DRV_DEBUG("info.nif_major_version: %d", info.nif_major_version); + DRV_DEBUG("info.nif_minor_version: %d", info.nif_minor_version); + DRV_DEBUG("info.dirty_scheduler_support: %d", info.dirty_scheduler_support); + DRV_DEBUG("info.erts_version: %s", info.erts_version); + DRV_DEBUG("info.otp_release: %s", info.otp_release); + Proc* proc = driver_alloc(sizeof(Proc)); + proc->port = port; + DRV_DEBUG("Port: %p", proc->port); + proc->port_term = driver_mk_port(proc->port); + DRV_DEBUG("Port term: %p", proc->port_term); + proc->is_running = erl_drv_mutex_create("wasm_instance_mutex"); + proc->is_initialized = 0; + proc->current_import = NULL; + proc->start_time = time(NULL); + return (ErlDrvData)proc; +} + +static void wasm_driver_stop(ErlDrvData raw) { + Proc* proc = (Proc*)raw; + DRV_DEBUG("Stopping WASM driver"); + + if(proc->current_import) { + DRV_DEBUG("Shutting down during import response..."); + proc->current_import->error_message = "WASM driver unloaded during import response"; + proc->current_import->ready = 1; + DRV_DEBUG("Signalling import_response with error"); + drv_signal(proc->current_import->response_ready, proc->current_import->cond, &proc->current_import->ready); + DRV_DEBUG("Signalled worker to fail. Locking is_running mutex to shutdown"); + } + + // We need to first grab the lock, then unlock it and destroy it. Must be a better way... + DRV_DEBUG("Grabbing is_running mutex to shutdown..."); + drv_lock(proc->is_running); + drv_unlock(proc->is_running); + DRV_DEBUG("Destroying is_running mutex"); + erl_drv_mutex_destroy(proc->is_running); + // Cleanup WASM resources + DRV_DEBUG("Cleaning up WASM resources"); + if (proc->is_initialized) { + DRV_DEBUG("Deleting WASM instance"); + wasm_instance_delete(proc->instance); + DRV_DEBUG("Deleted WASM instance"); + wasm_module_delete(proc->module); + DRV_DEBUG("Deleted WASM module"); + wasm_store_delete(proc->store); + DRV_DEBUG("Deleted WASM store"); + } + DRV_DEBUG("Freeing proc"); + driver_free(proc); + DRV_DEBUG("Freed proc"); +} + +static void wasm_driver_output(ErlDrvData raw, char *buff, ErlDrvSizeT bufflen) { + DRV_DEBUG("WASM driver output received"); + Proc* proc = (Proc*)raw; + //DRV_DEBUG("Port: %p", proc->port); + //DRV_DEBUG("Port term: %p", proc->port_term); + + int index = 0; + int version; + if(ei_decode_version(buff, &index, &version) != 0) { + send_error(proc, "Failed to decode message header (version)."); + return; + } + //DRV_DEBUG("Received term has version: %d", version); + //DRV_DEBUG("Index: %d. buff_len: %d. buff: %p", index, bufflen, buff); + int arity; + ei_decode_tuple_header(buff, &index, &arity); + //DRV_DEBUG("Term arity: %d", arity); + + char command[MAXATOMLEN]; + ei_decode_atom(buff, &index, command); + DRV_DEBUG("Port %p received command: %s, arity: %d", proc->port, command, arity); + + if (strcmp(command, "init") == 0) { + // Start async initialization + proc->pid = driver_caller(proc->port); + //DRV_DEBUG("Caller PID: %d", proc->pid); + int size, type, mode_size; + char* mode; + ei_get_type(buff, &index, &type, &size); + //DRV_DEBUG("WASM binary size: %d bytes. Type: %c", size, type); + void* wasm_binary = driver_alloc(size); + long size_l = (long)size; + ei_decode_binary(buff, &index, wasm_binary, &size_l); + ei_get_type(buff, &index, &type, &mode_size); + // the init message size + '\0' character + mode = driver_alloc(mode_size + 1); + ei_decode_atom(buff, &index, mode); + LoadWasmReq* mod_bin = driver_alloc(sizeof(LoadWasmReq)); + mod_bin->proc = proc; + mod_bin->binary = wasm_binary; + mod_bin->size = size; + mod_bin->mode = mode; + //DRV_DEBUG("Calling for async thread to init"); + driver_async(proc->port, NULL, wasm_initialize_runtime, mod_bin, NULL); + } else if (strcmp(command, "call") == 0) { + if (!proc->is_initialized) { + send_error(proc, "Cannot run WASM function as module not initialized."); + return; + } + // Extract the function name and the args from the Erlang term and generate the wasm_val_vec_t + char* function_name = driver_alloc(MAXATOMLEN); + ei_decode_string(buff, &index, function_name); + DRV_DEBUG("Function name: %s", function_name); + proc->current_function = function_name; + + DRV_DEBUG("Decoding args. Buff: %p. Index: %d", buff, index); + proc->current_args = decode_list(buff, &index); + + driver_async(proc->port, NULL, wasm_execute_function, proc, NULL); + } + // else if (strcmp(command, "indirect_call") == 0) { + // if (!proc->is_initialized) { + // send_error(proc, "Cannot run WASM indirect function as module not initialized."); + // return; + // } + // DRV_DEBUG("Decoding indirect call"); + // ei_decode_long(buff, &index, &proc->current_function_ix); + // proc->current_args = decode_list(buff, &index); + // DRV_DEBUG("Calling indirect call invoker"); + // driver_async(proc->port, NULL, async_indirect_call, proc, NULL); + // } + else if (strcmp(command, "import_response") == 0) { + // Handle import response + // TODO: We should probably start a mutex on the current_import object here. + // At the moment current_import->response_ready must not be locked so that signalling can happen. + DRV_DEBUG("Import response received. Providing..."); + if (proc->current_import) { + DRV_DEBUG("Decoding import response from Erlang..."); + proc->current_import->result_terms = decode_list(buff, &index); + proc->current_import->error_message = NULL; + + // Signal that the response is ready + drv_signal( + proc->current_import->response_ready, + proc->current_import->cond, + &proc->current_import->ready); + } else { + DRV_DEBUG("[error] No pending import response waiting"); + send_error(proc, "No pending import response waiting"); + } + } else if (strcmp(command, "write") == 0) { + DRV_DEBUG("Write received"); + long ptr, size; + int type; + ei_decode_tuple_header(buff, &index, &arity); + ei_decode_long(buff, &index, &ptr); + ei_get_type(buff, &index, &type, &size); + long size_l = (long)size; + char* wasm_binary; + int res = ei_decode_bitstring(buff, &index, &wasm_binary, NULL, &size_l); + DRV_DEBUG("Decoded binary. Res: %d. Size (bits): %ld", res, size_l); + long size_bytes = size_l / 8; + DRV_DEBUG("Write received. Ptr: %ld. Bytes: %ld", ptr, size_bytes); + long memory_size = get_memory_size(proc); + if(ptr + size_bytes > memory_size) { + DRV_DEBUG("Write request out of bounds."); + send_error(proc, "Write request out of bounds"); + return; + } + byte_t* memory_data = wasm_memory_data(get_memory(proc)); + DRV_DEBUG("Memory location to write to: %p", ptr+memory_data); + + memcpy(memory_data + ptr, wasm_binary, size_bytes); + DRV_DEBUG("Write complete"); + + ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * 2); + msg[0] = ERL_DRV_ATOM; + msg[1] = atom_ok; + erl_drv_output_term(proc->port_term, msg, 2); + } + else if (strcmp(command, "read") == 0) { + DRV_DEBUG("Read received"); + long ptr, size; + ei_decode_tuple_header(buff, &index, &arity); + ei_decode_long(buff, &index, &ptr); + ei_decode_long(buff, &index, &size); + long size_l = (long)size; + long memory_size = get_memory_size(proc); + DRV_DEBUG("Read received. Ptr: %ld. Size: %ld. Memory size: %ld", ptr, size_l, memory_size); + if(ptr + size_l > memory_size) { + DRV_DEBUG("Read request out of bounds."); + send_error(proc, "Read request out of bounds"); + return; + } + byte_t* memory_data = wasm_memory_data(get_memory(proc)); + DRV_DEBUG("Memory location to read from: %p", memory_data + ptr); + + char* out_binary = driver_alloc(size_l); + memcpy(out_binary, memory_data + ptr, size_l); + + DRV_DEBUG("Read complete. Binary: %p", out_binary); + + ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * 7); + int msg_index = 0; + msg[msg_index++] = ERL_DRV_ATOM; + msg[msg_index++] = atom_execution_result; + msg[msg_index++] = ERL_DRV_BUF2BINARY; + msg[msg_index++] = (ErlDrvTermData)out_binary; + msg[msg_index++] = size_l; + msg[msg_index++] = ERL_DRV_TUPLE; + msg[msg_index++] = 2; + + int msg_res = erl_drv_output_term(proc->port_term, msg, msg_index); + DRV_DEBUG("Read response sent: %d", msg_res); + } + else if (strcmp(command, "size") == 0) { + DRV_DEBUG("Size received"); + long size = get_memory_size(proc); + DRV_DEBUG("Size: %ld", size); + + ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * 6); + int msg_index = 0; + msg[msg_index++] = ERL_DRV_ATOM; + msg[msg_index++] = atom_execution_result; + msg[msg_index++] = ERL_DRV_INT; + msg[msg_index++] = size; + msg[msg_index++] = ERL_DRV_TUPLE; + msg[msg_index++] = 2; + erl_drv_output_term(proc->port_term, msg, msg_index); + } + else { + DRV_DEBUG("Unknown command: %s", command); + send_error(proc, "Unknown command"); + } +} + +static ErlDrvEntry wasm_driver_entry = { + NULL, + wasm_driver_start, + wasm_driver_stop, + wasm_driver_output, + NULL, + NULL, + "hb_beamr", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + ERL_DRV_EXTENDED_MARKER, + ERL_DRV_EXTENDED_MAJOR_VERSION, + ERL_DRV_EXTENDED_MINOR_VERSION, + ERL_DRV_FLAG_USE_PORT_LOCKING, + NULL, + NULL, + NULL +}; + +DRIVER_INIT(wasm_driver) { + atom_ok = driver_mk_atom("ok"); + atom_error = driver_mk_atom("error"); + atom_import = driver_mk_atom("import"); + atom_execution_result = driver_mk_atom("execution_result"); + return &wasm_driver_entry; +} \ No newline at end of file diff --git a/native/hb_beamr/hb_driver.c b/native/hb_beamr/hb_driver.c new file mode 100644 index 000000000..3a930c307 --- /dev/null +++ b/native/hb_beamr/hb_driver.c @@ -0,0 +1,39 @@ +#include "include/hb_driver.h" +#include "include/hb_logging.h" +#include "include/hb_helpers.h" + + +void drv_lock(ErlDrvMutex* mutex) { + DRV_DEBUG("Locking: %s", erl_drv_mutex_name(mutex)); + erl_drv_mutex_lock(mutex); + DRV_DEBUG("Locked: %s", erl_drv_mutex_name(mutex)); +} + + +void drv_unlock(ErlDrvMutex* mutex) { + DRV_DEBUG("Unlocking: %s", erl_drv_mutex_name(mutex)); + erl_drv_mutex_unlock(mutex); + DRV_DEBUG("Unlocked: %s", erl_drv_mutex_name(mutex)); +} + +void drv_signal(ErlDrvMutex* mut, ErlDrvCond* cond, int* ready) { + DRV_DEBUG("Signaling: %s. Pre-signal ready state: %d", erl_drv_cond_name(cond), *ready); + drv_lock(mut); + *ready = 1; + erl_drv_cond_signal(cond); + drv_unlock(mut); + DRV_DEBUG("Signaled: %s. Post-signal ready state: %d", erl_drv_cond_name(cond), *ready); +} + +void drv_wait(ErlDrvMutex* mut, ErlDrvCond* cond, int* ready) { + DRV_DEBUG("Started to wait: %s. Ready: %d", erl_drv_cond_name(cond), *ready); + DRV_DEBUG("Mutex: %s", erl_drv_mutex_name(mut)); + drv_lock(mut); + while (!*ready) { + DRV_DEBUG("Waiting: %s", erl_drv_cond_name(cond)); + erl_drv_cond_wait(cond, mut); + DRV_DEBUG("Woke up: Ready: %d", *ready); + } + drv_unlock(mut); + DRV_DEBUG("Finish waiting: %s", erl_drv_cond_name(cond)); +} diff --git a/native/hb_beamr/hb_helpers.c b/native/hb_beamr/hb_helpers.c new file mode 100644 index 000000000..6c32a0c64 --- /dev/null +++ b/native/hb_beamr/hb_helpers.c @@ -0,0 +1,205 @@ +#include "include/hb_helpers.h" +#include "include/hb_logging.h" + +// Returns the string name corresponding to the wasm type +const char* get_wasm_type_name(wasm_valkind_t kind) { + switch (kind) { + case WASM_I32: return "i32"; + case WASM_I64: return "i64"; + case WASM_F32: return "f32"; + case WASM_F64: return "f64"; + default: return "unknown"; + } +} + +const char* wasm_externtype_to_kind_string(const wasm_externtype_t* type) { + switch (wasm_externtype_kind(type)) { + case WASM_EXTERN_FUNC: return "func"; + case WASM_EXTERN_GLOBAL: return "global"; + case WASM_EXTERN_TABLE: return "table"; + case WASM_EXTERN_MEMORY: return "memory"; + default: return "unknown"; + } +} + +// Helper function to convert wasm_valtype_t to char +char wasm_valtype_kind_to_char(const wasm_valtype_t* valtype) { + switch (wasm_valtype_kind(valtype)) { + case WASM_I32: return 'i'; + case WASM_I64: return 'I'; + case WASM_F32: return 'f'; + case WASM_F64: return 'F'; + case WASM_EXTERNREF: return 'e'; + case WASM_V128: return 'v'; + case WASM_FUNCREF: return 'f'; + default: return 'u'; + } +} + +int wasm_val_to_erl_term(ErlDrvTermData* term, const wasm_val_t* val) { + DRV_DEBUG("Adding wasm val to erl term"); + DRV_DEBUG("Val of: %d", val->of.i32); + switch (val->kind) { + case WASM_I32: + term[0] = ERL_DRV_INT; + term[1] = val->of.i32; + return 2; + case WASM_I64: + term[0] = ERL_DRV_INT64; + term[1] = (ErlDrvTermData) &val->of.i64; + return 2; + case WASM_F32: + term[0] = ERL_DRV_FLOAT; + term[1] = (ErlDrvTermData) &val->of.f32; + return 2; + case WASM_F64: + term[0] = ERL_DRV_FLOAT; + term[1] = (ErlDrvTermData) &val->of.f64; + return 2; + default: + DRV_DEBUG("Unsupported result type: %d", val->kind); + return 0; + } +} + +int erl_term_to_wasm_val(wasm_val_t* val, ei_term* term) { + DRV_DEBUG("Converting erl term to wasm val. Term: %d. Size: %d", term->value.i_val, term->size); + switch (val->kind) { + case WASM_I32: + val->of.i32 = (int) term->value.i_val; + break; + case WASM_I64: + val->of.i64 = (long) term->value.i_val; + break; + case WASM_F32: + val->of.f32 = (float) term->value.d_val; + break; + case WASM_F64: + val->of.f64 = term->value.d_val; + break; + default: + DRV_DEBUG("Unsupported parameter type: %d", val->kind); + return -1; + } + return 0; +} + +int erl_terms_to_wasm_vals(wasm_val_vec_t* vals, ei_term* terms) { + DRV_DEBUG("Converting erl terms to wasm vals"); + DRV_DEBUG("Vals: %d", vals->size); + for(int i = 0; i < vals->size; i++) { + DRV_DEBUG("Converting term %d: %p", i, &vals->data[i]); + int res = erl_term_to_wasm_val(&vals->data[i], &terms[i]); + if(res == -1) { + DRV_DEBUG("Failed to convert term to wasm val"); + return -1; + } + } + return 0; +} + +ei_term* decode_list(char* buff, int* index) { + int arity, type; + + if(ei_get_type(buff, index, &type, &arity) == -1) { + DRV_DEBUG("Failed to get type"); + return NULL; + } + DRV_DEBUG("Decoded header. Arity: %d", arity); + + ei_term* res = driver_alloc(sizeof(ei_term) * arity); + + if(type == ERL_LIST_EXT) { + //DRV_DEBUG("Decoding list"); + ei_decode_list_header(buff, index, &arity); + //DRV_DEBUG("Decoded list header. Arity: %d", arity); + for(int i = 0; i < arity; i++) { + ei_decode_ei_term(buff, index, &res[i]); + DRV_DEBUG("Decoded term (assuming int) %d: %d", i, res[i].value.i_val); + } + } + else if(type == ERL_STRING_EXT) { + //DRV_DEBUG("Decoding list encoded as string"); + unsigned char* str = driver_alloc(arity * sizeof(char) + 1); + ei_decode_string(buff, index, str); + for(int i = 0; i < arity; i++) { + res[i].ei_type = ERL_INTEGER_EXT; + res[i].value.i_val = (long) str[i]; + DRV_DEBUG("Decoded term %d: %d", i, res[i].value.i_val); + } + driver_free(str); + } + else { + DRV_DEBUG("Unknown type: %d", type); + return NULL; + } + + return res; +} + +int get_function_sig(const wasm_externtype_t* type, char* type_str) { + if (wasm_externtype_kind(type) == WASM_EXTERN_FUNC) { + const wasm_functype_t* functype = wasm_externtype_as_functype_const(type); + const wasm_valtype_vec_t* params = wasm_functype_params(functype); + const wasm_valtype_vec_t* results = wasm_functype_results(functype); + + if(!params || !results) { + DRV_DEBUG("Export function params/results are NULL"); + return 0; + } + + type_str[0] = '('; + size_t offset = 1; + + for (size_t i = 0; i < params->size; i++) { + type_str[offset++] = wasm_valtype_kind_to_char(params->data[i]); + } + type_str[offset++] = ')'; + + for (size_t i = 0; i < results->size; i++) { + type_str[offset++] = wasm_valtype_kind_to_char(results->data[i]); + } + type_str[offset] = '\0'; + + return 1; + } + return 0; +} + +wasm_func_t* get_exported_function(Proc* proc, const char* target_name) { + wasm_extern_vec_t exports; + wasm_instance_exports(proc->instance, &exports); + wasm_exporttype_vec_t export_types; + wasm_module_exports(proc->module, &export_types); + wasm_func_t* func = NULL; + + for (size_t i = 0; i < exports.size; ++i) { + wasm_extern_t* ext = exports.data[i]; + if (wasm_extern_kind(ext) == WASM_EXTERN_FUNC) { + const wasm_name_t* exp_name = wasm_exporttype_name(export_types.data[i]); + if (exp_name && exp_name->size == strlen(target_name) + 1 && + strncmp(exp_name->data, target_name, exp_name->size - 1) == 0) { + func = wasm_extern_as_func(ext); + break; + } + } + } + + return func; +} + +wasm_memory_t* get_memory(Proc* proc) { + wasm_extern_vec_t exports; + wasm_instance_exports(proc->instance, &exports); + for (size_t i = 0; i < exports.size; i++) { + if (wasm_extern_kind(exports.data[i]) == WASM_EXTERN_MEMORY) { + return wasm_extern_as_memory(exports.data[i]); + } + } + return NULL; +} + +long get_memory_size(Proc* proc) { + wasm_memory_t* memory = get_memory(proc); + return wasm_memory_size(memory) * 65536; +} diff --git a/native/hb_beamr/hb_logging.c b/native/hb_beamr/hb_logging.c new file mode 100644 index 000000000..7e91fd477 --- /dev/null +++ b/native/hb_beamr/hb_logging.c @@ -0,0 +1,36 @@ +#include "include/hb_logging.h" + +extern ErlDrvTermData atom_error; + +void beamr_print(int print, const char* file, int line, const char* format, ...) { + va_list args; + va_start(args, format); + if(print) { + pthread_t thread_id = pthread_self(); + printf("[DBG#%p @ %s:%d] ", thread_id, file, line); + vprintf(format, args); + printf("\r\n"); + } + va_end(args); +} + +void send_error(Proc* proc, const char* message_fmt, ...) { + va_list args; + va_start(args, message_fmt); + char* message = driver_alloc(256); + vsnprintf(message, 256, message_fmt, args); + DRV_DEBUG("Sending error message: %s", message); + ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * 7); + int msg_index = 0; + msg[msg_index++] = ERL_DRV_ATOM; + msg[msg_index++] = atom_error; + msg[msg_index++] = ERL_DRV_STRING; + msg[msg_index++] = (ErlDrvTermData)message; + msg[msg_index++] = strlen(message); + msg[msg_index++] = ERL_DRV_TUPLE; + msg[msg_index++] = 2; + + int msg_res = erl_drv_output_term(proc->port_term, msg, msg_index); + DRV_DEBUG("Sent error message. Res: %d", msg_res); + va_end(args); +} \ No newline at end of file diff --git a/native/hb_beamr/hb_wasm.c b/native/hb_beamr/hb_wasm.c new file mode 100644 index 000000000..e119c1c3f --- /dev/null +++ b/native/hb_beamr/hb_wasm.c @@ -0,0 +1,582 @@ +#include "include/hb_wasm.h" +#include "include/hb_logging.h" +#include "include/hb_helpers.h" +#include "include/hb_driver.h" + +extern ErlDrvTermData atom_ok; +extern ErlDrvTermData atom_import; +extern ErlDrvTermData atom_execution_result; + +wasm_trap_t* wasm_handle_import(void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results) { + DRV_DEBUG("generic_import_handler called"); + ImportHook* import_hook = (ImportHook*)env; + Proc* proc = import_hook->proc; + + // Check if the field name is "invoke"; if not, exit early + if (strncmp(import_hook->field_name, "invoke", 6) == 0) { + wasm_execute_indirect_function(proc, import_hook->field_name, args, results); + return NULL; + } + + DRV_DEBUG("Proc: %p. Args size: %d", proc, args->size); + DRV_DEBUG("Import name: %s.%s [%s]", import_hook->module_name, import_hook->field_name, import_hook->signature); + + // Initialize the message object + ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * ((2+(2*3)) + ((args->size + 1) * 2) + ((results->size + 1) * 2) + 2)); + int msg_index = 0; + msg[msg_index++] = ERL_DRV_ATOM; + msg[msg_index++] = atom_import; + msg[msg_index++] = ERL_DRV_STRING; + msg[msg_index++] = (ErlDrvTermData) import_hook->module_name; + msg[msg_index++] = strlen(import_hook->module_name); + msg[msg_index++] = ERL_DRV_STRING; + msg[msg_index++] = (ErlDrvTermData) import_hook->field_name; + msg[msg_index++] = strlen(import_hook->field_name); + + // Encode args + for (size_t i = 0; i < args->size; i++) { + msg_index += wasm_val_to_erl_term(&msg[msg_index], &args->data[i]); + } + msg[msg_index++] = ERL_DRV_NIL; + msg[msg_index++] = ERL_DRV_LIST; + msg[msg_index++] = args->size + 1; + + // Encode function signature + msg[msg_index++] = ERL_DRV_STRING; + msg[msg_index++] = (ErlDrvTermData) import_hook->signature; + msg[msg_index++] = strlen(import_hook->signature) - 1; + + // Prepare the message to send to the Erlang side + msg[msg_index++] = ERL_DRV_TUPLE; + msg[msg_index++] = 5; + + // Initialize the result vector and set the required result types + proc->current_import = driver_alloc(sizeof(ImportResponse)); + + // Create and initialize a is_running and condition variable for the response + proc->current_import->response_ready = erl_drv_mutex_create("response_mutex"); + proc->current_import->cond = erl_drv_cond_create("response_cond"); + proc->current_import->ready = 0; + + DRV_DEBUG("Sending %d terms...", msg_index); + // Send the message to the caller process + int msg_res = erl_drv_output_term(proc->port_term, msg, msg_index); + // Wait for the response (we set this directly after the message was sent + // so we have the lock, before Erlang sends us data back) + drv_wait(proc->current_import->response_ready, proc->current_import->cond, &proc->current_import->ready); + + DRV_DEBUG("Response ready"); + + // Handle error in the response + if (proc->current_import->error_message) { + DRV_DEBUG("Import execution failed. Error message: %s", proc->current_import->error_message); + wasm_name_t message; + wasm_name_new_from_string_nt(&message, proc->current_import->error_message); + wasm_trap_t* trap = wasm_trap_new(proc->store, &message); + driver_free(proc->current_import); + proc->current_import = NULL; + return trap; + } + + // Convert the response back to WASM values + const wasm_valtype_vec_t* result_types = wasm_functype_results(wasm_func_type(import_hook->stub_func)); + for(int i = 0; i < proc->current_import->result_length; i++) { + results->data[i].kind = wasm_valtype_kind(result_types->data[i]); + } + int res = erl_terms_to_wasm_vals(results, proc->current_import->result_terms); + if(res == -1) { + DRV_DEBUG("Failed to convert terms to wasm vals"); + return NULL; + } + + results->num_elems = result_types->num_elems; + + // Clean up + DRV_DEBUG("Cleaning up import response"); + erl_drv_cond_destroy(proc->current_import->cond); + erl_drv_mutex_destroy(proc->current_import->response_ready); + driver_free(proc->current_import); + + proc->current_import = NULL; + return NULL; +} + +void wasm_initialize_runtime(void* raw) { + DRV_DEBUG("Initializing WASM module"); + LoadWasmReq* mod_bin = (LoadWasmReq*)raw; + Proc* proc = mod_bin->proc; + drv_lock(proc->is_running); + // Initialize WASM engine, store, etc. + +#if HB_DEBUG==1 + wasm_runtime_set_log_level(WASM_LOG_LEVEL_VERBOSE); +#else + wasm_runtime_set_log_level(WASM_LOG_LEVEL_ERROR); +#endif + + DRV_DEBUG("Mode: %s", mod_bin->mode); + + // if(strcmp(mod_bin->mode, "wasm") == 0) { + // DRV_DEBUG("Using WASM mode."); + // wasm_runtime_set_default_running_mode(Mode_Interp); + // } else { + // DRV_DEBUG("Using AOT mode."); + // } + + proc->engine = wasm_engine_new(); + DRV_DEBUG("Created engine"); + proc->store = wasm_store_new(proc->engine); + DRV_DEBUG("Created store"); + + + // Load WASM module + wasm_byte_vec_t binary; + wasm_byte_vec_new(&binary, mod_bin->size, (const wasm_byte_t*)mod_bin->binary); + + proc->module = wasm_module_new(proc->store, &binary); + DRV_DEBUG("Module created: %p", proc->module); + if (!proc->module) { + DRV_DEBUG("Failed to create module"); + send_error(proc, "Failed to create module."); + wasm_byte_vec_delete(&binary); + wasm_store_delete(proc->store); + wasm_engine_delete(proc->engine); + drv_unlock(proc->is_running); + return; + } + //wasm_byte_vec_delete(&binary); + DRV_DEBUG("Created module"); + + // Get imports + wasm_importtype_vec_t imports; + wasm_module_imports(proc->module, &imports); + DRV_DEBUG("Imports size: %d", imports.size); + wasm_extern_t *stubs[imports.size]; + + // Get exports + wasm_exporttype_vec_t exports; + wasm_module_exports(proc->module, &exports); + + // Create Erlang lists for imports + int init_msg_size = sizeof(ErlDrvTermData) * (2 + 3 + 5 + (13 * imports.size) + (11 * exports.size)); + ErlDrvTermData* init_msg = driver_alloc(init_msg_size); + int msg_i = 0; + + // 2 in the init_msg_size + init_msg[msg_i++] = ERL_DRV_ATOM; + init_msg[msg_i++] = atom_execution_result; + + // Process imports + for (int i = 0; i < imports.size; ++i) { + //DRV_DEBUG("Processing import %d", i); + const wasm_importtype_t* import = imports.data[i]; + const wasm_name_t* module_name = wasm_importtype_module(import); + const wasm_name_t* name = wasm_importtype_name(import); + const wasm_externtype_t* type = wasm_importtype_type(import); + + //DRV_DEBUG("Import: %s.%s", module_name->data, name->data); + + char* type_str = driver_alloc(256); + // TODO: What happpens here? + if(!get_function_sig(type, type_str)) { + // TODO: Handle other types of imports? + continue; + } + // 13 items in the each import message + init_msg[msg_i++] = ERL_DRV_ATOM; + init_msg[msg_i++] = driver_mk_atom((char*)wasm_externtype_to_kind_string(type)); + init_msg[msg_i++] = ERL_DRV_STRING; + init_msg[msg_i++] = (ErlDrvTermData)module_name->data; + init_msg[msg_i++] = module_name->size - 1; + init_msg[msg_i++] = ERL_DRV_STRING; + init_msg[msg_i++] = (ErlDrvTermData)name->data; + init_msg[msg_i++] = name->size - 1; + init_msg[msg_i++] = ERL_DRV_STRING; + init_msg[msg_i++] = (ErlDrvTermData)type_str; + init_msg[msg_i++] = strlen(type_str); + init_msg[msg_i++] = ERL_DRV_TUPLE; + init_msg[msg_i++] = 4; + + DRV_DEBUG("Creating callback for %s.%s [%s]", module_name->data, name->data, type_str); + ImportHook* hook = driver_alloc(sizeof(ImportHook)); + hook->module_name = module_name->data; + hook->field_name = name->data; + hook->proc = proc; + hook->signature = type_str; + + hook->stub_func = + wasm_func_new_with_env( + proc->store, + wasm_externtype_as_functype_const(type), + wasm_handle_import, + hook, + NULL + ); + stubs[i] = wasm_func_as_extern(hook->stub_func); + } + + init_msg[msg_i++] = ERL_DRV_NIL; + init_msg[msg_i++] = ERL_DRV_LIST; + init_msg[msg_i++] = imports.size + 1; + + // Create proc! + wasm_extern_vec_t externs; + wasm_extern_vec_new(&externs, imports.size, stubs); + wasm_trap_t* trap = NULL; + proc->instance = wasm_instance_new_with_args(proc->store, proc->module, &externs, &trap, 0x10000, 0x10000); + if (!proc->instance) { + DRV_DEBUG("Failed to create WASM instance"); + send_error(proc, "Failed to create WASM instance (although module was created)."); + drv_unlock(proc->is_running); + return; + } + + wasm_extern_vec_t exported_externs; + wasm_instance_exports(proc->instance, &exported_externs); + + // Refresh the exports now that we have an instance + wasm_module_exports(proc->module, &exports); + for (size_t i = 0; i < exports.size; i++) { + //DRV_DEBUG("Processing export %d", i); + const wasm_exporttype_t* export = exports.data[i]; + const wasm_name_t* name = wasm_exporttype_name(export); + const wasm_externtype_t* type = wasm_exporttype_type(export); + char* kind_str = (char*) wasm_externtype_to_kind_string(type); + + // Check if the export is the indirect function table + if (strcmp(name->data, "__indirect_function_table") == 0) { + DRV_DEBUG("Found indirect function table: %s. Index: %d", name->data, i); + proc->indirect_func_table_ix = i; + const wasm_tabletype_t* table_type = wasm_externtype_as_tabletype_const(type); + const wasm_limits_t* table_limits = wasm_tabletype_limits(table_type); + // Retrieve the indirect function table + proc->indirect_func_table = wasm_extern_as_table(exported_externs.data[i]); + + } + + char* type_str = driver_alloc(256); + get_function_sig(type, type_str); + DRV_DEBUG("Export: %s [%s] -> %s", name->data, kind_str, type_str); + + // 10 elements for each exported function + init_msg[msg_i++] = ERL_DRV_ATOM; + init_msg[msg_i++] = driver_mk_atom(kind_str); + init_msg[msg_i++] = ERL_DRV_STRING; + init_msg[msg_i++] = (ErlDrvTermData)name->data; + init_msg[msg_i++] = name->size - 1; + init_msg[msg_i++] = ERL_DRV_STRING; + init_msg[msg_i++] = (ErlDrvTermData)type_str; + init_msg[msg_i++] = strlen(type_str); + init_msg[msg_i++] = ERL_DRV_TUPLE; + init_msg[msg_i++] = 3; + } + + // 5 closing elements + init_msg[msg_i++] = ERL_DRV_NIL; + init_msg[msg_i++] = ERL_DRV_LIST; + init_msg[msg_i++] = (exports.size) + 1; + init_msg[msg_i++] = ERL_DRV_TUPLE; + init_msg[msg_i++] = 3; + + DRV_DEBUG("Sending init message to Erlang. Elements: %d", msg_i); + + int send_res = erl_drv_output_term(proc->port_term, init_msg, msg_i); + DRV_DEBUG("Send result: %d", send_res); + + proc->current_import = NULL; + proc->is_initialized = 1; + drv_unlock(proc->is_running); +} + +void wasm_execute_function(void* raw) { + Proc* proc = (Proc*)raw; + DRV_DEBUG("Calling function: %s", proc->current_function); + drv_lock(proc->is_running); + char* function_name = proc->current_function; + + // Find the function in the exports + wasm_func_t* func = get_exported_function(proc, function_name); + if (!func) { + send_error(proc, "Function not found: %s", function_name); + drv_unlock(proc->is_running); + return; + } + DRV_DEBUG("Func: %p", func); + + const wasm_functype_t* func_type = wasm_func_type(func); + const wasm_valtype_vec_t* param_types = wasm_functype_params(func_type); + const wasm_valtype_vec_t* result_types = wasm_functype_results(func_type); + + wasm_val_vec_t args, results; + wasm_val_vec_new_uninitialized(&args, param_types->size); + args.num_elems = param_types->num_elems; + // CONV: ei_term* -> wasm_val_vec_t + for(int i = 0; i < param_types->size; i++) { + args.data[i].kind = wasm_valtype_kind(param_types->data[i]); + } + int res = erl_terms_to_wasm_vals(&args, proc->current_args); + + for(int i = 0; i < args.size; i++) { + DRV_DEBUG("Arg %d: %d", i, args.data[i].of.i64); + DRV_DEBUG("Source term: %d", proc->current_args[i].value.i_val); + } + + if(res == -1) { + send_error(proc, "Failed to convert terms to wasm vals"); + drv_unlock(proc->is_running); + return; + } + + wasm_val_vec_new_uninitialized(&results, result_types->size); + results.num_elems = result_types->num_elems; + for (size_t i = 0; i < result_types->size; i++) { + results.data[i].kind = wasm_valtype_kind(result_types->data[i]); + } + + proc->exec_env = wasm_runtime_get_exec_env_singleton(func->inst_comm_rt); + + // Call the function + DRV_DEBUG("Calling function: %s", function_name); + wasm_trap_t* trap = wasm_func_call(func, &args, &results); + + + if (trap) { + wasm_message_t trap_msg; + wasm_trap_message(trap, &trap_msg); + // wasm_frame_t* origin = wasm_trap_origin(trap); + // int32_t func_index = wasm_frame_func_index(origin); + // int32_t func_offset = wasm_frame_func_offset(origin); + // char* func_name; + + // DRV_DEBUG("WASM Exception: [func_index: %d, func_offset: %d] %.*s", func_index, func_offset, trap_msg.size, trap_msg.data); + send_error(proc, "%.*s", trap_msg.size, trap_msg.data); + drv_unlock(proc->is_running); + return; + } + + // Send the results back to Erlang + DRV_DEBUG("Results size: %d", results.size); + ErlDrvTermData* msg = driver_alloc(sizeof(ErlDrvTermData) * (7 + (results.size * 2))); + DRV_DEBUG("Allocated msg"); + int msg_index = 0; + msg[msg_index++] = ERL_DRV_ATOM; + msg[msg_index++] = atom_execution_result; + for (size_t i = 0; i < results.size; i++) { + DRV_DEBUG("Processing result %d", i); + DRV_DEBUG("Result type: %d", results.data[i].kind); + switch(results.data[i].kind) { + case WASM_I32: + DRV_DEBUG("Value: %d", results.data[i].of.i32); + break; + case WASM_I64: + DRV_DEBUG("Value: %ld", results.data[i].of.i64); + break; + case WASM_F32: + DRV_DEBUG("Value: %f", results.data[i].of.f32); + break; + case WASM_F64: + DRV_DEBUG("Value: %f", results.data[i].of.f64); + break; + default: + DRV_DEBUG("Unknown result type.", results.data[i].kind); + break; + } + + int res_size = wasm_val_to_erl_term(&msg[msg_index], &results.data[i]); + msg_index += res_size; + } + msg[msg_index++] = ERL_DRV_NIL; + msg[msg_index++] = ERL_DRV_LIST; + msg[msg_index++] = results.size + 1; + msg[msg_index++] = ERL_DRV_TUPLE; + msg[msg_index++] = 2; + DRV_DEBUG("Sending %d terms", msg_index); + int response_msg_res = erl_drv_output_term(proc->port_term, msg, msg_index); + driver_free(msg); + DRV_DEBUG("Msg: %d", response_msg_res); + + wasm_val_vec_delete(&results); + proc->current_import = NULL; + + DRV_DEBUG("Unlocking is_running mutex: %p", proc->is_running); + drv_unlock(proc->is_running); +} + +int wasm_execute_indirect_function(Proc* proc, const char *field_name, const wasm_val_vec_t* input_args, wasm_val_vec_t* output_results) { + + + DRV_DEBUG("================================================="); + DRV_DEBUG("Starting function invocation"); + DRV_DEBUG("================================================="); + + wasm_table_t* indirect_function_table = proc->indirect_func_table; + + + int result = 0; + DRV_DEBUG("Function name: %s", field_name); + +// Extract the function index from the input arguments + int function_index = input_args->data[0].of.i32; + DRV_DEBUG("Function index retrieved from input_args: %d", function_index); + + // Get the function reference from the table and cast it to a function + wasm_ref_t* function_ref = wasm_table_get(indirect_function_table, function_index); + const wasm_func_t* func = wasm_ref_as_func(function_ref); + DRV_DEBUG("Function pointer: %p", func); + + // Retrieve the function type and log its parameters and results + const wasm_functype_t* function_type = wasm_func_type(func); + if (!function_type) { + DRV_DEBUG("Failed to retrieve function type for function at index %d", function_index); + } + + // Log the function's parameter types + const wasm_valtype_vec_t* param_types = wasm_functype_params(function_type); + DRV_DEBUG("Function at index %d has %zu parameters", function_index, param_types->size); + for (size_t j = 0; j < param_types->size; ++j) { + const wasm_valtype_t* param_type = param_types->data[j]; + wasm_valkind_t param_kind = wasm_valtype_kind(param_type); + DRV_DEBUG("Param %zu: %s", j, get_wasm_type_name(param_kind)); + } + + + // Log the function's result types + const wasm_valtype_vec_t* result_types = wasm_functype_results(function_type); + DRV_DEBUG("Function at index %d has %zu results", function_index, result_types->size); + for (size_t k = 0; k < result_types->size; ++k) { + const wasm_valtype_t* result_type = result_types->data[k]; + wasm_valkind_t result_kind = wasm_valtype_kind(result_type); + DRV_DEBUG("Result %zu: %s", k, get_wasm_type_name(result_kind)); + } + + // Prepare the arguments for the function call + wasm_val_vec_t prepared_args; + // If there are no arguments or only one argument (function index), no preparation is needed + if (input_args->size <= 1) { + DRV_DEBUG("Not enough arguments to create new wasm_val_vec_t"); + return 0; + } + + // Allocate memory for the prepared arguments + wasm_val_t* prepared_data = malloc(sizeof(wasm_val_t) * (input_args->size - 1)); + + // Copy the arguments starting from the second element (skip function index) + for (size_t i = 1; i < input_args->size; ++i) { + prepared_data[i - 1] = input_args->data[i]; + } + + // Create a new wasm_val_vec_t with the prepared arguments + wasm_val_vec_new(&prepared_args, input_args->size - 1, prepared_data); + DRV_DEBUG("Prepared %zu arguments for function call", prepared_args.size); + + uint64_t argc = prepared_args.size; + uint64_t* argv = malloc(sizeof(uint64_t) * argc); + + // Convert prepared arguments to an array of 64-bit integers + for (uint64_t i = 0; i < argc; ++i) { + argv[i] = prepared_args.data[i].of.i64; + } + + + /* ---------------- STACK SAVE -----------------*/ + + // const char* stack_save_name = "emscripten_stack_get_current"; + // wasm_val_t *stack_save_params = NULL; + // wasm_val_t stack_save_results[1]; + // if (call_exported_function_runtime(proc, stack_save_name, stack_save_params, stack_save_results) != 0) { + // DRV_DEBUG("Failed to call stack save function"); + // } + + /* ---------------- STACK SAVE -----------------*/ + + // Attempt to call the function and check for any exceptions + if (!wasm_runtime_call_indirect(proc->exec_env, function_index, argc, argv)) { + if (wasm_runtime_get_exception(proc->exec_env)) { + DRV_DEBUG("%s", wasm_runtime_get_exception(proc->exec_env)); + } + DRV_DEBUG("WASM function call failed"); + result = -1; + } + + if(result != 0) { + + + + } + + + // Free allocated memory + free(argv); + free(prepared_args.data); + DRV_DEBUG("Function call completed successfully"); + return result; +} + +int wasm_execute_exported_function(Proc* proc, const char *function_name, wasm_val_t* params, wasm_val_t * results) { + DRV_DEBUG("=== Calling Runtime Export Function ==="); + DRV_DEBUG("= Function name: %s", function_name); + + + // Get exported wasm_func_t pointer by function name + wasm_func_t* func = get_exported_function(proc, function_name); + if(!func) { + DRV_DEBUG("= Failed to get exported function"); + return -1; + } + + // Get the function type + const wasm_functype_t* function_type = wasm_func_type(func); + if (!function_type) { + DRV_DEBUG("= Failed to get function type"); + return -1; + } + + // Get the function's parameter types and set the argument types for args + const wasm_valtype_vec_t* param_types = wasm_functype_params(function_type); + if(!param_types) { + DRV_DEBUG("= Failed to get function parameters"); + return -1; + } + + DRV_DEBUG("= Function has %zu parameters", param_types->size); + for (size_t j = 0; j < param_types->size; ++j) { + const wasm_valtype_t* param_type = param_types->data[j]; + wasm_valkind_t param_kind = wasm_valtype_kind(param_type); + params[j].kind = param_kind; + DRV_DEBUG("= Param %zu: %s, %i", j, get_wasm_type_name(param_kind), params[j].of.i64); + } + + + // Get the function's result types and set the result types for results + const wasm_valtype_vec_t* result_types = wasm_functype_results(function_type); + if (!result_types) { + DRV_DEBUG("= Failed to get function results"); + return -1; + } + DRV_DEBUG("= Function has %zu results", result_types->size); + + for (size_t k = 0; k < result_types->size; ++k) { + const wasm_valtype_t* result_type = result_types->data[k]; + wasm_valkind_t result_kind = wasm_valtype_kind(result_type); + results[k].kind = result_kind; + results[k].of.i64 = 0; // Initialize result value + DRV_DEBUG("= Result %zu: %s, %i", k, get_wasm_type_name(result_kind), results[k].of.i64); + } + + + // Call the exported function + if (wasm_runtime_call_wasm_a(proc->exec_env, func->func_comm_rt, result_types->size, results, param_types->size, params)) { + DRV_DEBUG("= Function call successful"); + } else { + const char* exception = wasm_runtime_get_exception(proc->exec_env); + DRV_DEBUG("= Function call failed: %s", exception); + return -1; + } + + // Retrieve the stack pointer result (as i64) + int64_t stack_pointer = results[0].of.i64; // Assuming the result is i64 + DRV_DEBUG("Stack pointer: %lld", stack_pointer); + + return 0; +} + diff --git a/native/hb_beamr/include/hb_core.h b/native/hb_beamr/include/hb_core.h new file mode 100644 index 000000000..a63e9ec7d --- /dev/null +++ b/native/hb_beamr/include/hb_core.h @@ -0,0 +1,102 @@ +#ifndef HB_CORE_H +#define HB_CORE_H + +#include +#include +#include +#include +#include +#include +#include +#include + +// Structure to represent the response for an import operation +typedef struct { + ErlDrvMutex* response_ready; // Mutex to synchronize response readiness + ErlDrvCond* cond; // Condition variable to signal readiness + int ready; // Flag indicating if the response is ready + char* error_message; // Error message (if any) + ei_term* result_terms; // List of result terms from the import + int result_length; // Length of the result_terms +} ImportResponse; + +// Structure to represent a WASM process instance +typedef struct { + wasm_engine_t* engine; // WASM engine instance + wasm_instance_t* instance; // WASM instance + wasm_module_t* module; // WASM module + wasm_store_t* store; // WASM store + ErlDrvPort port; // Erlang port associated with this process + ErlDrvTermData port_term; // Erlang term representation of the port + ErlDrvMutex* is_running; // Mutex to track if the process is running + char* current_function; // Current function being executed + long current_function_ix; // Index of the current function + int indirect_func_table_ix; // Index of the indirect function table + wasm_table_t* indirect_func_table; // Indirect function table + wasm_exec_env_t exec_env; // Execution environment for the WASM instance + ei_term* current_args; // Arguments for the current function + int current_args_length; // Length of the current arguments + ImportResponse* current_import; // Import response structure + ErlDrvTermData pid; // PID of the Erlang process + int is_initialized; // Flag to check if the process is initialized + time_t start_time; // Start time of the process +} Proc; + +// Structure to represent an import hook +typedef struct { + char* module_name; // Name of the module + char* field_name; // Name of the field (function) + char* signature; // Function signature + Proc* proc; // The associated process + wasm_func_t* stub_func; // WASM function pointer for the import +} ImportHook; + +// Structure to represent the request for loading a WASM binary +typedef struct { + void* binary; // Binary data for the WASM module + long size; // Size of the binary + Proc* proc; // The associated process + char* mode; // Mode of the WASM module +} LoadWasmReq; + +// NO_PROD: Import these from headers instead + +// Structure for a common WASM module instance +typedef struct WASMModuleInstanceCommon { + uint32_t module_type; // Type of the module + uint8_t module_inst_data[1]; // Module instance data +} WASMModuleInstanceCommon; + +// Structure to store host information about the WASM instance +struct wasm_host_info { + void *info; // Pointer to host info + void (*finalizer)(void *); // Finalizer function for the host info +}; + +// Structure representing a WASM function (extended with host-specific details) +struct wasm_func_t { + wasm_store_t *store; // WASM store + wasm_name_t *module_name; // Module name for the function + wasm_name_t *name; // Function name + uint16_t kind; // Function kind (e.g., export) + struct wasm_host_info host_info; // Host-specific information + wasm_functype_t *type; // Function type (parameters and results) + uint16_t param_count; // Number of parameters + uint16_t result_count; // Number of results + bool with_env; // Whether the function has an environment + union { + wasm_func_callback_t cb; // Callback function + struct callback_ext { + void *env; // Environment for the callback + wasm_func_callback_with_env_t cb; // Callback function with environment + void (*finalizer)(void *); // Finalizer for the callback + } cb_env; + } u; + uint16_t func_idx_rt; // Function index in the runtime + WASMModuleInstanceCommon *inst_comm_rt; // Module instance data + WASMFunctionInstanceCommon *func_comm_rt; // Function instance data +}; + + + +#endif // HB_CORE_H \ No newline at end of file diff --git a/native/hb_beamr/include/hb_driver.h b/native/hb_beamr/include/hb_driver.h new file mode 100644 index 000000000..5d782f160 --- /dev/null +++ b/native/hb_beamr/include/hb_driver.h @@ -0,0 +1,47 @@ +#ifndef HB_DRIVER_H +#define HB_DRIVER_H + +#include "hb_core.h" + +/* + * Function: drv_lock + * -------------------- + * Locks the specified mutex to ensure exclusive access to shared resources. + * + * mutex: The mutex to be locked. + */ +void drv_lock(ErlDrvMutex* mutex); + +/* + * Function: drv_unlock + * -------------------- + * Unlocks the specified mutex, allowing other threads to access the shared resource. + * + * mutex: The mutex to be unlocked. + */ +void drv_unlock(ErlDrvMutex* mutex); + +/* + * Function: drv_signal + * -------------------- + * Signals the condition variable, notifying one or more threads waiting for the condition. + * + * mut: The mutex used to synchronize access to shared resources. + * cond: The condition variable to signal. + * ready: A flag indicating the state of the condition, typically set to 1 to signal that the condition is met. + */ +void drv_signal(ErlDrvMutex* mut, ErlDrvCond* cond, int* ready); + +/* + * Function: drv_wait + * -------------------- + * Causes the current thread to wait for a signal on the condition variable, holding the provided mutex. + * The thread will be blocked until the condition variable is signaled. + * + * mut: The mutex used to synchronize access to shared resources. + * cond: The condition variable to wait on. + * ready: A flag indicating the state of the condition. The thread will wait until this is set to 1. + */ +void drv_wait(ErlDrvMutex* mut, ErlDrvCond* cond, int* ready); + +#endif diff --git a/native/hb_beamr/include/hb_helpers.h b/native/hb_beamr/include/hb_helpers.h new file mode 100644 index 000000000..2d2aa35ed --- /dev/null +++ b/native/hb_beamr/include/hb_helpers.h @@ -0,0 +1,133 @@ +#ifndef HB_HELPERS_H +#define HB_HELPERS_H + +#include "hb_core.h" + +/* + * Function: get_wasm_type_name + * -------------------- + * Returns the string name corresponding to the given WASM value type. + * + * kind: The WASM value type kind (e.g., WASM_I32, WASM_I64, WASM_F32, WASM_F64). + * + * returns: A string representing the value type (e.g., "i32", "i64", etc.). + */ +const char* get_wasm_type_name(wasm_valkind_t kind); + +/* + * Function: wasm_externtype_to_kind_string + * -------------------- + * Converts a WASM external type to its corresponding kind string. + * + * type: A pointer to the WASM external type to convert. + * + * returns: A string representing the kind of the external type (e.g., "func", "global", "table", "memory"). + */ +const char* wasm_externtype_to_kind_string(const wasm_externtype_t* type); + +/* + * Function: wasm_valtype_kind_to_char + * -------------------- + * Converts a WASM value type to its corresponding character representation. + * + * valtype: The WASM value type to convert. + * + * returns: A character representing the value type (e.g., 'i' for i32, 'f' for f32). + */ +char wasm_valtype_kind_to_char(const wasm_valtype_t* valtype); + +/* + * Function: wasm_val_to_erl_term + * -------------------- + * Converts a WASM value to an Erlang term. + * + * term: The Erlang term data to be filled with the converted value. + * val: The WASM value to convert. + * + * returns: 2 on success (size of the term), 0 if conversion fails. + */ +int wasm_val_to_erl_term(ErlDrvTermData* term, const wasm_val_t* val); + +/* + * Function: erl_term_to_wasm_val + * -------------------- + * Converts an Erlang term to a WASM value. + * + * val: The WASM value to be populated. + * term: The Erlang term to convert. + * + * returns: 0 on success, -1 if conversion fails. + */ +int erl_term_to_wasm_val(wasm_val_t* val, ei_term* term); + +/* + * Function: erl_terms_to_wasm_vals + * -------------------- + * Converts a list of Erlang terms to a vector of WASM values. + * + * vals: The WASM values vector to be filled. + * terms: The list of Erlang terms to convert. + * + * returns: 0 on success, -1 on failure. + */ +int erl_terms_to_wasm_vals(wasm_val_vec_t* vals, ei_term* terms); + +/* + * Function: decode_list + * -------------------- + * Decodes a list of Erlang terms from a provided binary buffer. + * + * buff: The binary buffer containing the Erlang encoded terms. + * index: The index in the buffer to start decoding from. + * + * returns: A pointer to the decoded list of Erlang terms, or NULL if an error occurs. + */ +ei_term* decode_list(char* buff, int* index); + +/* + * Function: get_function_sig + * -------------------- + * Retrieves the function signature as a string from a WASM external type. + * + * type: The WASM external type representing a function. + * type_str: The string that will hold the function signature. + * + * returns: 1 on success, 0 if the external type is not a function or there is an error. + */ +int get_function_sig(const wasm_externtype_t* type, char* type_str); + +/* + * Function: get_exported_function + * -------------------- + * Retrieves an exported function from the WASM instance by its name. + * + * proc: The process structure containing the WASM instance. + * target_name: The name of the exported function to retrieve. + * + * returns: A pointer to the exported WASM function, or NULL if the function is not found. + */ +wasm_func_t* get_exported_function(Proc* proc, const char* target_name); + +/* + * Function: get_memory + * -------------------- + * Retrieves the WASM memory associated with the given process. + * + * proc: The process structure containing the WASM instance. + * + * returns: A pointer to the WASM memory object, or NULL if not found. + */ +wasm_memory_t* get_memory(Proc* proc); + +/* + * Function: get_memory_size + * -------------------- + * Retrieves the size of the WASM memory in bytes. + * + * proc: The process structure containing the WASM instance. + * + * returns: The size of the WASM memory in bytes. + */ +long get_memory_size(Proc* proc); + +#endif // HB_HELPERS_H diff --git a/native/hb_beamr/include/hb_logging.h b/native/hb_beamr/include/hb_logging.h new file mode 100644 index 000000000..a5d9adee4 --- /dev/null +++ b/native/hb_beamr/include/hb_logging.h @@ -0,0 +1,40 @@ +#ifndef HB_LOGGING_H +#define HB_LOGGING_H + +#include "hb_core.h" +// Enable debug logging by default if not defined +#define HB_DEBUG 0 +#ifndef HB_DEBUG +#endif + + +#define DRV_DEBUG(format, ...) beamr_print(HB_DEBUG, __FILE__, __LINE__, format, ##__VA_ARGS__) +#define DRV_PRINT(format, ...) beamr_print(1, __FILE__, __LINE__, format, ##__VA_ARGS__) + +/* + * Function: beamr_print + * -------------------- + * This function prints a formatted message to the standard output, prefixed with the thread + * ID, file name, and line number where the log was generated. + * + * print: A flag that controls whether the message is printed (1 to print, 0 to skip). + * file: The source file name where the log was generated. + * line: The line number where the log was generated. + * format: The format string for the message. + * ...: The variables to be printed in the format. + */ +void beamr_print(int print, const char* file, int line, const char* format, ...); + +/* + * Function: send_error + * -------------------- + * This function sends an error message to the Erlang process, formatted according to the provided + * message format and arguments. The message is also logged using the DRV_DEBUG macro. + * + * proc: The process to send the error message to. + * message_fmt: The format string for the error message. + * ...: The variables to be printed in the error message. + */ +void send_error(Proc* proc, const char* message_fmt, ...); + +#endif // HB_LOGGING_H diff --git a/native/hb_beamr/include/hb_wasm.h b/native/hb_beamr/include/hb_wasm.h new file mode 100644 index 000000000..e5046fc19 --- /dev/null +++ b/native/hb_beamr/include/hb_wasm.h @@ -0,0 +1,66 @@ +#ifndef HB_WASM_H +#define HB_WASM_H + +#include "hb_core.h" + +/* + * Function: wasm_handle_import + * -------------------- + * Handles the import for processing WASM imports. + * + * env: The environment (import hook) associated with the import. + * args: The arguments for the import function. + * results: The results of the import will be stored here. + * + * returns: A WASM trap in case of an error, or NULL on success. + */ +wasm_trap_t* wasm_handle_import(void* env, const wasm_val_vec_t* args, wasm_val_vec_t* results); + +/* + * Function: wasm_initialize_runtime + * -------------------- + * Initializes the WASM module asynchronously. + * + * raw: A pointer to the raw data for the initialization. + */ +void wasm_initialize_runtime(void* raw); + +/* + * Function: wasm_execute_function_async + * -------------------- + * Asynchronously executes a WASM function. + * + * raw: A pointer to the process structure containing the function call details. + */ +void wasm_execute_function(void* raw); + +/* + * Function: wasm_execute_indirect_function + * -------------------- + * Executes an indirect WASM function asynchronously. + * + * proc: The current process structure. + * function_name: The name of the indirect function to call. + * input_args: The input arguments for the function call. + * output_results: The results of the function call will be stored here. + * + * returns: 0 on success or -1 on failure. + */ +int wasm_execute_indirect_function(Proc* proc, const char *function_name, const wasm_val_vec_t* input_args, wasm_val_vec_t* output_results); + +/* + * Function: wasm_execute_exported_function + * -------------------- + * Invokes a specific exported WASM function from the runtime environment. + * + * proc: The current process structure. + * function_name: The name of the function to call. + * params: The parameters to pass to the function. + * results: The results of the function call will be stored here. + * + * returns: A 64-bit status code indicating success (0) or failure (-1). + */ +int wasm_execute_exported_function(Proc* proc, const char *function_name, wasm_val_t* params, wasm_val_t * results); + + +#endif \ No newline at end of file diff --git a/native/hb_keccak/hb_keccak.c b/native/hb_keccak/hb_keccak.c new file mode 100644 index 000000000..f65e39b2f --- /dev/null +++ b/native/hb_keccak/hb_keccak.c @@ -0,0 +1,174 @@ +/** libkeccak-tiny + * + * A single-file implementation of SHA-3 and SHAKE. + * + * Implementor: David Leon Gil + * License: CC0, attribution kindly requested. Blame taken too, + * but not liability. + */ +#include "include/hb_keccak.h" + +#include +#include +#include +#include + +/******** The Keccak-f[1600] permutation ********/ + +/*** Constants. ***/ +static const uint8_t rho[24] = \ + { 1, 3, 6, 10, 15, 21, + 28, 36, 45, 55, 2, 14, + 27, 41, 56, 8, 25, 43, + 62, 18, 39, 61, 20, 44}; +static const uint8_t pi[24] = \ + {10, 7, 11, 17, 18, 3, + 5, 16, 8, 21, 24, 4, + 15, 23, 19, 13, 12, 2, + 20, 14, 22, 9, 6, 1}; +static const uint64_t RC[24] = \ + {1ULL, 0x8082ULL, 0x800000000000808aULL, 0x8000000080008000ULL, + 0x808bULL, 0x80000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x8aULL, 0x88ULL, 0x80008009ULL, 0x8000000aULL, + 0x8000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, 0x800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, 0x80000001ULL, 0x8000000080008008ULL}; + +/*** Helper macros to unroll the permutation. ***/ +#define rol(x, s) (((x) << s) | ((x) >> (64 - s))) +#define REPEAT6(e) e e e e e e +#define REPEAT24(e) REPEAT6(e e e e) +#define REPEAT5(e) e e e e e +#define FOR5(v, s, e) \ + v = 0; \ + REPEAT5(e; v += s;) + +/*** Keccak-f[1600] ***/ +static inline void keccakf(void* state) { + uint64_t* a = (uint64_t*)state; + uint64_t b[5] = {0}; + uint64_t t = 0; + uint8_t x, y; + + for (int i = 0; i < 24; i++) { + // Theta + FOR5(x, 1, + b[x] = 0; + FOR5(y, 5, + b[x] ^= a[x + y]; )) + FOR5(x, 1, + FOR5(y, 5, + a[y + x] ^= b[(x + 4) % 5] ^ rol(b[(x + 1) % 5], 1); )) + // Rho and pi + t = a[1]; + x = 0; + REPEAT24(b[0] = a[pi[x]]; + a[pi[x]] = rol(t, rho[x]); + t = b[0]; + x++; ) + // Chi + FOR5(y, + 5, + FOR5(x, 1, + b[x] = a[y + x];) + FOR5(x, 1, + a[y + x] = b[x] ^ ((~b[(x + 1) % 5]) & b[(x + 2) % 5]); )) + // Iota + a[0] ^= RC[i]; + } +} + +/******** The FIPS202-defined functions. ********/ + +/*** Some helper macros. ***/ + +#define _(S) do { S } while (0) +#define FOR(i, ST, L, S) \ + _(for (size_t i = 0; i < L; i += ST) { S; }) +#define mkapply_ds(NAME, S) \ + static inline void NAME(uint8_t* dst, \ + const uint8_t* src, \ + size_t len) { \ + FOR(i, 1, len, S); \ + } +#define mkapply_sd(NAME, S) \ + static inline void NAME(const uint8_t* src, \ + uint8_t* dst, \ + size_t len) { \ + FOR(i, 1, len, S); \ + } + +mkapply_ds(xorin, dst[i] ^= src[i]) // xorin +mkapply_sd(setout, dst[i] = src[i]) // setout + +#define P keccakf +#define Plen 200 + +// Fold P*F over the full blocks of an input. +#define foldP(I, L, F) \ + while (L >= rate) { \ + F(a, I, rate); \ + P(a); \ + I += rate; \ + L -= rate; \ + } + +/** The sponge-based hash construction. **/ +static inline int hash(uint8_t* out, size_t outlen, + const uint8_t* in, size_t inlen, + size_t rate, uint8_t delim) { + if ((out == NULL) || ((in == NULL) && inlen != 0) || (rate >= Plen)) { + return -1; + } + uint8_t a[Plen] = {0}; + // Absorb input. + foldP(in, inlen, xorin); + // Xor in the DS and pad frame. + a[inlen] ^= delim; + a[rate - 1] ^= 0x80; + // Xor in the last block. + xorin(a, in, inlen); + // Apply P + P(a); + // Squeeze output. + foldP(out, outlen, setout); + setout(a, out, outlen); + memset(a, 0, 200); + return 0; +} + +/*** Helper macros to define SHA3 and SHAKE instances. ***/ +#define defshake(bits) \ + int shake##bits(uint8_t* out, size_t outlen, \ + const uint8_t* in, size_t inlen) { \ + return hash(out, outlen, in, inlen, 200 - (bits / 4), 0x1f); \ + } + + +#define defalgo(algoname, bits, pad) \ + int algoname ## bits(uint8_t* out, size_t outlen, \ + const uint8_t* in, size_t inlen) { \ + if (outlen > (bits/8)) { \ + return -1; \ + } \ + return hash(out, outlen, in, inlen, 200 - (bits / 4), pad); \ + } + +#define defsha3(bits) defalgo(sha3_, bits, 0x06) +#define defkeccak(bits) defalgo(keccak_, bits, 0x01) + +/*** FIPS202 SHAKE VOFs ***/ +defshake(128) +defshake(256) + +/*** FIPS202 SHA3 FOFs ***/ +defsha3(224) +defsha3(256) +defsha3(384) +defsha3(512) + +/*** ORIGINAL KECCAK SUBMISSION ***/ +defkeccak(224) +defkeccak(256) +defkeccak(384) +defkeccak(512) \ No newline at end of file diff --git a/native/hb_keccak/hb_keccak_nif.c b/native/hb_keccak/hb_keccak_nif.c new file mode 100644 index 000000000..4f6a26237 --- /dev/null +++ b/native/hb_keccak/hb_keccak_nif.c @@ -0,0 +1,40 @@ +#include "erl_nif.h" +#include "include/hb_keccak.h" +#include +#include + +static ERL_NIF_TERM nif_sha3_256(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ErlNifBinary input; + if (!enif_inspect_binary(env, argv[0], &input)) { + return enif_make_badarg(env); + } + + uint8_t output[32]; + sha3_256(output, 32, input.data, input.size); // this is the actual C implementation + + ERL_NIF_TERM result; + uint8_t* bin = enif_make_new_binary(env, 32, &result); + memcpy(bin, output, 32); + return result; +} + +static ERL_NIF_TERM nif_keccak_256(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ErlNifBinary input; + if (!enif_inspect_binary(env, argv[0], &input)) { + return enif_make_badarg(env); + } + uint8_t output[32]; + keccak_256(output, 32, input.data, input.size); + + ERL_NIF_TERM result; + uint8_t* bin = enif_make_new_binary(env, 32, &result); + memcpy(bin, output, 32); + return result; +} + +static ErlNifFunc nif_funcs[] = { + {"sha3_256", 1, nif_sha3_256}, + {"keccak_256", 1, nif_keccak_256} +}; + +ERL_NIF_INIT(hb_keccak, nif_funcs, NULL, NULL, NULL, NULL) diff --git a/native/hb_keccak/include/hb_keccak.h b/native/hb_keccak/include/hb_keccak.h new file mode 100644 index 000000000..96aa3143b --- /dev/null +++ b/native/hb_keccak/include/hb_keccak.h @@ -0,0 +1,23 @@ +#ifndef KECCAK_FIPS202_H +#define KECCAK_FIPS202_H +#define __STDC_WANT_LIB_EXT1__ 1 +#include +#include + +#define decshake(bits) \ + int shake##bits(uint8_t*, size_t, const uint8_t*, size_t); + +#define decsha3(bits) \ + int sha3_##bits(uint8_t*, size_t, const uint8_t*, size_t); + +#define deckeccak(bits) \ + int keccak_##bits(uint8_t*, size_t, const uint8_t*, size_t); + +decshake(128) +decshake(256) +decsha3(224) +decsha3(256) +decsha3(384) +decsha3(512) +deckeccak(256) +#endif \ No newline at end of file diff --git a/packer.pkr.hcl b/packer.pkr.hcl deleted file mode 100644 index b0369a818..000000000 --- a/packer.pkr.hcl +++ /dev/null @@ -1,105 +0,0 @@ -packer { - required_plugins { - googlecompute = { - version = ">= 1.1.6" - source = "github.com/hashicorp/googlecompute" - } - } -} - -# Define required variables -variable "project_id" { - type = string - default = "hyperbeam-cd" -} - -variable "region" { - type = string - default = "us-east1" -} - -variable "zone" { - type = string - default = "us-east1-c" -} - -variable "image_name" { - type = string - default = "hyperbeam-image" -} - -# Source block to define GCP builder -source "googlecompute" "ubuntu" { - project_id = var.project_id - source_image_family = "ubuntu-2204-lts" - image_name = var.image_name - zone = var.zone - machine_type = "n1-standard-1" - ssh_username = "packer" -} - -# Define the build stage -build { - sources = ["source.googlecompute.ubuntu"] - - # Add a provisioner to download and install go-tpm-tools - provisioner "shell" { - inline = [ - "sudo apt-get update -y", - "sudo apt-get install -y wget tar", - - # Download the go-tpm-tools binary archive - "wget https://github.com/google/go-tpm-tools/releases/download/v0.4.4/go-tpm-tools_Linux_x86_64.tar.gz -O /tmp/go-tpm-tools.tar.gz", - - # Extract the binary - "tar -xzf /tmp/go-tpm-tools.tar.gz -C /tmp", - - # Move the gotpm binary to /usr/local/bin - "sudo mv /tmp/gotpm /usr/local/bin/", - - # Clean up - "rm -f /tmp/go-tpm-tools.tar.gz /tmp/LICENSE /tmp/README.md" - ] - } - - - # Upload the pre-built release (with ERTS included) to the instance - provisioner "file" { - source = "./_build/default/rel/ao" - destination = "/tmp/hyperbeam" - } - - provisioner "shell" { - inline = [ - # Move the release to /opt with sudo - "sudo mv /tmp/hyperbeam /opt/hyperbeam", - "sudo chmod -R 755 /opt/hyperbeam", - - # Create a symlink to make it easier to run the app - "sudo ln -s /opt/hyperbeam/bin/hyperbeam /usr/local/bin/hyperbeam", - - # (Optional) If you want to create a systemd service to manage the app - "echo '[Unit]' | sudo tee /etc/systemd/system/hyperbeam.service", - "echo 'Description=Permaweb Node' | sudo tee -a /etc/systemd/system/hyperbeam.service", - "echo '[Service]' | sudo tee -a /etc/systemd/system/hyperbeam.service", - "echo 'Type=simple' | sudo tee -a /etc/systemd/system/hyperbeam.service", - "echo 'ExecStart=/opt/hyperbeam/bin/hyperbeam foreground' | sudo tee -a /etc/systemd/system/hyperbeam.service", - "echo 'Restart=on-failure' | sudo tee -a /etc/systemd/system/hyperbeam.service", - "echo '[Install]' | sudo tee -a /etc/systemd/system/hyperbeam.service", - "echo 'WantedBy=multi-user.target' | sudo tee -a /etc/systemd/system/hyperbeam.service", - - # Enable and start the service - "sudo systemctl enable hyperbeam", - "sudo systemctl start hyperbeam" - ] - } - - # Disable ssh - # provisioner "shell" { - # inline = [ - # "sudo systemctl stop ssh", - # "sudo systemctl disable ssh" - # ] - # } -} - diff --git a/rebar.config b/rebar.config index 719f3ea30..76a625337 100644 --- a/rebar.config +++ b/rebar.config @@ -1,11 +1,74 @@ {erl_opts, [debug_info, {d, 'COWBOY_QUICER', 1}, {d, 'GUN_QUICER', 1}]}. -{plugins, [pc]}. +{plugins, [pc, rebar3_rustler, rebar_edown_plugin]}. -{overrides, [ - {add, cowboy, [{erl_opts, [{d, 'COWBOY_QUICER', 1}]}]}, - {add, gun, [{erl_opts, [{d, 'GUN_QUICER', 1}]}]} +{profiles, [ + {no_events, [{erl_opts, [{d, 'NO_EVENTS', true}]}]}, + {store_events, [{erl_opts, [{d, 'STORE_EVENTS', true}]}]}, + {ao_profiling, [{erl_opts, [{d, 'AO_PROFILING', true}]}]}, + {eflame, + [ + {deps, + [ + {eflame, + {git, + "https://github.com/samcamwilliams/eflame.git", + {ref, "d81a6e174956b4b0aca13363d51e4f51a5fabbd2"} + } + } + ] + }, + {erl_opts, [{d, 'ENABLE_EFLAME', true}]} + ] + }, + {genesis_wasm, [ + {erl_opts, [{d, 'ENABLE_GENESIS_WASM', true}]}, + {pre_hooks, [ + {compile, "make -C \"${REBAR_ROOT_DIR}\" setup-genesis-wasm"} + ]}, + {relx, [ + {overlay, [ + {copy, + "_build/genesis_wasm/genesis-wasm-server", + "genesis-wasm-server" + } + ]} + ]} + ]}, + {rocksdb, [ + {deps, [{rocksdb, "1.8.0"}]}, + {erl_opts, [ + {d, 'ENABLE_ROCKSDB', true} + ]} + ]}, + {http3, [ + {deps, [ + {quicer, {git, "https://github.com/emqx/quic.git", + {ref, "e2e9ab3e1ec53e20d5f9401359cc6ee8f971d112"}} + } + ]}, + {erl_opts, [ + {d, 'ENABLE_HTTP3', true}, + {d, 'COWBOY_QUICER', 1}, + {d, 'GUN_QUICER', 1} + ]}, + {overrides, [ + {add, cowboy, [{erl_opts, [{d, 'COWBOY_QUICER', 1}]}]}, + {add, gun, [{erl_opts, [{d, 'GUN_QUICER', 1}]}]} + ]} + ]} ]}. + +{cargo_opts, [ + {src_dir, "native/dev_snp_nif"}, + {src_dir, "deps/elmdb/native/elmdb_nif"} +]}. + +{overrides, []}. + {pre_hooks, [ + {compile, "bash -c \"echo '-define(HB_BUILD_SOURCE, <<\\\"$(git rev-parse HEAD)\\\">>).\n' > ${REBAR_ROOT_DIR}/_build/hb_buildinfo.hrl\""}, + {compile, "bash -c \"echo '-define(HB_BUILD_SOURCE_SHORT, <<\\\"$(git rev-parse --short HEAD)\\\">>).\n' >> ${REBAR_ROOT_DIR}/_build/hb_buildinfo.hrl\""}, + {compile, "bash -c \"echo '-define(HB_BUILD_TIME, $(date +%s)).\n' >> ${REBAR_ROOT_DIR}/_build/hb_buildinfo.hrl\""}, {compile, "make -C \"${REBAR_ROOT_DIR}\" wamr"} ]}. @@ -18,29 +81,49 @@ {post_hooks, [ {"(linux|darwin|solaris)", clean, "rm -rf \"${REBAR_ROOT_DIR}/_build\" \"${REBAR_ROOT_DIR}/priv\""}, - {"(linux|darwin|solaris)", compile, "echo 'Post-compile hooks executed'"} + {"(linux|darwin|solaris)", compile, "echo 'Post-compile hooks executed'"}, + { compile, "rm -f native/hb_beamr/*.o native/hb_beamr/*.d"}, + { compile, "rm -f native/hb_keccak/*.o native/hb_keccak/*.d"}, + { compile, "mkdir -p priv/html"}, + { compile, "cp -R src/html/* priv/html"}, + { compile, "cp _build/default/lib/elmdb/priv/crates/elmdb_nif/elmdb_nif.so _build/default/lib/elmdb/priv/elmdb_nif.so 2>/dev/null || true" } ]}. {provider_hooks, [ - {post, [ - {compile, {pc, compile}}, - {clean, {pc, clean}} - ]} + {pre, [ + {compile, {cargo, build}} + ]}, + {post, [ + {compile, {pc, compile}}, + {clean, {pc, clean}}, + {clean, {cargo, clean}} + ]} ]}. {port_specs, [ - {"./priv/hb_beamr.so", ["./c_src/hb_beamr.c"]} + {"./priv/hb_beamr.so", [ + "./native/hb_beamr/hb_beamr.c", + "./native/hb_beamr/hb_wasm.c", + "./native/hb_beamr/hb_driver.c", + "./native/hb_beamr/hb_helpers.c", + "./native/hb_beamr/hb_logging.c" + ]}, + {"./priv/hb_keccak.so", [ + "./native/hb_keccak/hb_keccak.c", + "./native/hb_keccak/hb_keccak_nif.c" + ]} ]}. {deps, [ + {elmdb, { git, "https://github.com/twilson63/elmdb-rs.git", {branch, "feat/match" }}}, {b64fast, {git, "https://github.com/ArweaveTeam/b64fast.git", {ref, "58f0502e49bf73b29d95c6d02460d1fb8d2a5273"}}}, - {jiffy, {git, "https://github.com/ArweaveTeam/jiffy.git", {ref, "74c956defa9116c85d76f77c3e9b5bd6de7bd39a"}}}, {cowboy, {git, "https://github.com/ninenines/cowboy", {ref, "022013b6c4e967957c7e0e7e7cdefa107fc48741"}}}, {gun, {git, "https://github.com/ninenines/gun", {ref, "8efcedd3a089e6ab5317e4310fed424a4ee130f8"}}}, - {quicer, {git, "https://github.com/qzhuyan/quic.git", {ref, "97d8be9fb8017f4578248f96f5b35f8e357df792"}}}, + {graphql, "0.17.1", {pkg, graphql_erl}}, {prometheus, "4.11.0"}, {prometheus_cowboy, "0.1.8"}, - {rocksdb, "1.8.0"} + {gun, "0.10.0"}, + {luerl, "1.3.0"} ]}. {shell, [ @@ -51,15 +134,49 @@ {apps, [hb]} ]}. -{eunit_opts, [verbose]}. +% Increase `scale_timeouts` when running on a slower machine. +{eunit_opts, [verbose, {scale_timeouts, 10}]}. {relx, [ - {release, {'hb', "0.0.1"}, [hb, jiffy, cowboy, gun, b64fast]}, + {release, {'hb', "0.0.1"}, [hb, b64fast, cowboy, gun, luerl, prometheus, prometheus_cowboy, elmdb]}, {include_erts, true}, - {extended_start_script, true} + {extended_start_script, true}, + {overlay, [ + {mkdir, "bin/priv"}, + {copy, "priv", "bin/priv"}, + {copy, "config.flat", "config.flat"} + ]} +]}. + +{dialyzer, [ + {plt_extra_apps, [public_key, ranch, cowboy, prometheus, prometheus_cowboy, b64fast, eunit, gun]}, + incremental, + {warnings, [no_improper_lists, no_unused]} +]}. + +{alias, [ + {debugger, + [ + {shell, "--sname hb --setcookie hb-debug --eval hb_debugger:start()."} + ] + }, + {'lua-test', + [ + {eunit, "--module dev_lua_test"} + ] + }, + {'deploy-scripts', + [ + {shell, "--eval hb:deploy_scripts()."} + ] + } ]}. -% {dist_node, [ -% {setcookie, 'hb'}, -% {name, 'hb@hb-node'} -% ]}. +{edoc_opts, [ + {doclet, edown_doclet}, + {dir, "docs/resources/source-code"}, + {preprocess, true}, + {preprocess, true}, + {private, true}, + {hidden, true} +]}. \ No newline at end of file diff --git a/rebar.lock b/rebar.lock index d9d89b6b4..07dc97c23 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,5 +1,5 @@ {"1.2.0", -[{<<"accept">>,{pkg,<<"accept">>,<<"0.3.5">>},2}, +[{<<"accept">>,{pkg,<<"accept">>,<<"0.3.7">>},2}, {<<"b64fast">>, {git,"https://github.com/ArweaveTeam/b64fast.git", {ref,"58f0502e49bf73b29d95c6d02460d1fb8d2a5273"}}, @@ -10,46 +10,41 @@ 0}, {<<"cowlib">>, {git,"https://github.com/ninenines/cowlib", - {ref,"1c3d5defba28e92a88ce45c440d57e178ab1c514"}}, + {ref,"e2d7749f61b89cc6f8779ba66a5a8ab0fe85c827"}}, 1}, + {<<"elmdb">>, + {git,"https://github.com/twilson63/elmdb-rs.git", + {ref,"90c8857cd4ccff341fbe415b96bc5703d17ff7f0"}}, + 0}, + {<<"graphql">>,{pkg,<<"graphql_erl">>,<<"0.17.1">>},0}, {<<"gun">>, {git,"https://github.com/ninenines/gun", {ref,"8efcedd3a089e6ab5317e4310fed424a4ee130f8"}}, 0}, - {<<"jiffy">>, - {git,"https://github.com/ArweaveTeam/jiffy.git", - {ref,"74c956defa9116c85d76f77c3e9b5bd6de7bd39a"}}, - 0}, + {<<"luerl">>,{pkg,<<"luerl">>,<<"1.3.0">>},0}, {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.11.0">>},0}, {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.8">>},0}, - {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.11">>},1}, + {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.15">>},1}, {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},1}, - {<<"quicer">>, - {git,"https://github.com/qzhuyan/quic.git", - {ref,"97d8be9fb8017f4578248f96f5b35f8e357df792"}}, - 0}, {<<"ranch">>, {git,"https://github.com/ninenines/ranch", {ref,"a692f44567034dacf5efcaa24a24183788594eb7"}}, - 1}, - {<<"rocksdb">>,{pkg,<<"rocksdb">>,<<"1.8.0">>},0}, - {<<"snabbkaffe">>, - {git,"https://github.com/kafka4beam/snabbkaffe.git", - {ref,"b59298334ed349556f63405d1353184c63c66534"}}, 1}]}. [ {pkg_hash,[ - {<<"accept">>, <<"B33B127ABCA7CC948BBE6CAA4C263369ABF1347CFA9D8E699C6D214660F10CD1">>}, + {<<"accept">>, <<"CD6E34A2D7E28CA38B2D3CB233734CA0C221EFBC1F171F91FEC5F162CC2D18DA">>}, + {<<"graphql">>, <<"EB59FCBB39F667DC1C78C950426278015C3423F7A6ED2A121D3DB8B1D2C5F8B4">>}, + {<<"luerl">>, <<"B56423DDB721432AB980B818FEECB84ADBAB115E2E11522CF94BCD0729CAA501">>}, {<<"prometheus">>, <<"B95F8DE8530F541BD95951E18E355A840003672E5EDA4788C5FA6183406BA29A">>}, {<<"prometheus_cowboy">>, <<"CFCE0BC7B668C5096639084FCD873826E6220EA714BF60A716F5BD080EF2A99C">>}, - {<<"prometheus_httpd">>, <<"F616ED9B85B536B195D94104063025A91F904A4CFC20255363F49A197D96C896">>}, - {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>}, - {<<"rocksdb">>, <<"0AE072F9818DAC03E18BA0E4B436450D24040DFB1A526E2198B451FD9FA0284F">>}]}, + {<<"prometheus_httpd">>, <<"8F767D819A5D36275EAB9264AFF40D87279151646776069BF69FBDBBD562BD75">>}, + {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>}]}, {pkg_hash_ext,[ - {<<"accept">>, <<"11B18C220BCC2EAB63B5470C038EF10EB6783BCB1FCDB11AA4137DEFA5AC1BB8">>}, + {<<"accept">>, <<"CA69388943F5DAD2E7232A5478F16086E3C872F48E32B88B378E1885A59F5649">>}, + {<<"graphql">>, <<"4D0F08EC57EF0983E2596763900872B1AB7E94F8EE3817B9F67EEC911FF7C386">>}, + {<<"luerl">>, <<"6B3138AA829F0FBC4CD0F083F273B4030A2B6CE99155194A6DB8C67B2C3480A4">>}, {<<"prometheus">>, <<"719862351AABF4DF7079B05DC085D2BBCBE3AC0AC3009E956671B1D5AB88247D">>}, {<<"prometheus_cowboy">>, <<"BA286BECA9302618418892D37BCD5DC669A6CC001F4EB6D6AF85FF81F3F4F34C">>}, - {<<"prometheus_httpd">>, <<"0BBE831452CFDF9588538EB2F570B26F30C348ADAE5E95A7D87F35A5910BCF92">>}, - {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>}, - {<<"rocksdb">>, <<"185E645EA480E9325D5EFE362BF3D2A38EDFC31B5145031B0CBEED978E89523C">>}]} + {<<"prometheus_httpd">>, <<"67736D000745184D5013C58A63E947821AB90CB9320BC2E6AE5D3061C6FFE039">>}, + {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>}]} ]. diff --git a/release/snpguest b/release/snpguest new file mode 100755 index 000000000..cd52f894e Binary files /dev/null and b/release/snpguest differ diff --git a/scripts/dynamic-router.lua b/scripts/dynamic-router.lua new file mode 100644 index 000000000..3f37e118e --- /dev/null +++ b/scripts/dynamic-router.lua @@ -0,0 +1,448 @@ +--- A dynamic route generator in an AO `~process@1.0'. +--- This generator grants a routing table, found at `/now/routes', that is +--- compatible with the `~router@1.0' interface. Subsequently, it can be +--- used for routing by HyperBEAM nodes' via setting the `router@1.0/provider' +--- node message key. +--- +--- The configuration options are as follows: +--- /is-admissible = A message to call with the registration request's body. Should +--- return a boolean indicating whether the peer is admissible. +--- /sampling-rate = The frequency at which random sampling of registered nodes +--- should be performed, rather than scored routing. Default = 0.1. +--- /pricing-weight = The level to which pricing should be preferred relative to +--- performance in the scoring algorithm. Default = 1. +--- /performance-weight = The level to which performance should be preferred +--- relative to pricing in the scoring algorithm. +--- Default = 1. +--- /score-preference = The level to which the scoring algorithm influence routing +--- decisions amongst scored route generations. Default = 1, +--- yielding an exponential decay in preference for better +--- performing nodes. Default = 1. +--- /performance-period = Alters the rate at which performance scores are modified +--- by new performance ratings. A lower period implies faster +--- changes to the score. +--- /recalculate-every = The number of messages to process between recalculating +--- the routing table. Default = 1000. +local function ensure_defaults(state) + state.routes = state.routes or {} + state["is-admissible"] = + state["is-admissible"] or { + path = "/default", + default = "true" + } + state["sampling-rate"] = state["sampling-rate"] or 0.1 + state["pricing-weight"] = state["pricing-weight"] or 1 + state["performance-weight"] = state["performance-weight"] or 1 + state["score-preference"] = state["score-preference"] or 1 + state["recalculate-every"] = state["recalculate-every"] or 1000 + state["performance-period"] = state["performance-period"] or 1000 + state["initial-performance"] = state["initial-performance"] or 30000 + return state +end + +-- Find the current route message for a template. +local function current_route(routes, template, opts) + -- Find the existing route that matches the template, if it exists. + local status, res = + ao.resolve({ + path = "/~router@1.0/match", + ["route-path"] = template, -- Only supports binary templates for now. + routes = routes + }) + if status == "ok" then + -- We found an existing route for this template. Return it as-is. + return res + else + -- We haven't found a route for this template, so we need to create a new + -- one. We set the reference to the next available index in the routes + -- table. + return { + strategy = "By-Weight", + template = template, + nodes = {}, + reference = "routes/" .. tostring(#routes + 1) + } + end +end + +-- Compute the decay for a given score, modulated by the score preference. +local function decay(state, score) + return math.exp(-state["score-preference"] * score) +end + +-- Calculate statistics for a given key across all nodes in a route. +local function calculate_stats(nodes, key) + local stats = { + count = 0, + total = 0, + max = 0, + mean = 0, + values = {} + } + + for _, n in ipairs(nodes) do + stats.count = stats.count + 1 + stats.total = stats.total + n[key] + if n[key] > stats.max then + stats.max = n[key] + end + if stats.min == nil or n[key] < stats.min then + stats.min = n[key] + end + table.insert(stats.values, n[key]) + end + + stats.mean = stats.total / stats.count + + -- Add a function that returns the percentile of a node for the given key. + table.sort(stats.values) + stats.percentile = + function(n) + local n_key = n[key] + for ix, v in ipairs(stats.values) do + if n_key <= v then + return (ix-1) / stats.count + end + end + end + + return stats +end + +-- Compute the scores for all routes. Outputs a single weight value per node, +-- where a higher value indicates that the node should be picked more frequently. +-- Each of the 'scoring' factors, in their natural state, are worse if they are +-- higher. Higher price and slower response times are negative factors for nodes. +-- This function rectifies that, and scores each node relative to the performance +-- of each of their peers. +local function recalculate_scores(state, route, opts) + -- TODO: Refactor such that this does not have `O(:facepalm:)` properties... + + -- Calculate stats for each relevant performance characteristic. + local perf_stats = calculate_stats(route.nodes, "performance") + local price_stats = calculate_stats(route.nodes, "price") + + -- Calculate the multipliers for performance and price from their weights. + local total_weight = state["performance-weight"] + state["pricing-weight"] + local perf_weight = state["performance-weight"] / total_weight + local pricing_weight = state["pricing-weight"] / total_weight + + -- Calculate the score per node. + for ix, node in ipairs(route.nodes) do + -- The performance score for the node on the route should be scaled by + -- moderated by the sampling rate. The sampling rate is used to ensure + -- that new/improving nodes (and improving nodes) are given a chance to + -- be selected. + local perf_percentile = perf_stats.percentile(node) + local perf_score = + (decay(state, perf_percentile) * (1 - state["sampling-rate"])) + + state["sampling-rate"] + -- The price score for the node on the route should be scaled by the + -- pricing weight. It is not moderated by the sampling rate, as we want + -- to ensure that the node is selected if it has a low price. New nodes + -- can improve their likelihood of being selected by lowering their price. + local price_percentile = price_stats.percentile(node) + local price_score = decay(state, price_percentile) + + -- Calculate the final weight. In order to do this we: + -- 1. Apply the factor weights to the calculated scores. + -- 2. Sum them. + node.weight = + ((perf_score * perf_weight) + (price_score * pricing_weight)) + + ao.event("debug_scores", + { + "calculated_score", { + node = ix, + prefix = node.prefix, + perf = node.performance, + perf_percentile = perf_percentile, + perf_weight = perf_weight, + perf_score = perf_score, + price = node.price, + price_percentile = price_percentile, + pricing_weight = pricing_weight, + price_score = price_score, + result = node.weight + } + } + ) + end + + return route +end + +local function add_node(state, req, opts) + local route = current_route(state.routes, req.route.template, opts) + local reference = route.reference .. "/nodes/" .. tostring(#route.nodes + 1) + table.insert(route.nodes, { + prefix = req.route.prefix, + price = req.route.price, + topup = req.route.topup, + performance = state["initial-performance"], + reference = reference, + opts = { http_reference = reference } + }) + + local new_state = ao.set(state, route.reference, route) + return new_state +end + +-- Compute the new routes, with their weights, based on the current routes and +-- a new route. +function recalculate(state, _, opts) + state = ensure_defaults(state) + + for _, r in ipairs(state.routes) do + r = recalculate_scores(state, r, opts) + end + + return "ok", state +end + +-- Register a new host to a route. +function register(state, assignment, opts) + state = ensure_defaults(state) + ao.event({"register", { state = state, assignment = assignment, opts = opts }}) + local req = assignment.body + + -- If the message is signed by an explicitly trusted peer, we can skip the + -- is-admissible check. + if state["trusted-peer"] then + local committers = ao.get("committers", req) + for _, committer in ipairs(committers) do + if committer == state["trusted-peer"] then + state = add_node(state, req) + return recalculate(state, assignment, opts) + end + end + end + + req.path = state["is-admissible"].path or "is-admissible" + local status, is_admissible = ao.resolve(state["is-admissible"], req) + + ao.event({"is-admissible result:", { status, is_admissible }}) + if status == "ok" and is_admissible == "true" then + state = add_node(state, req) + return recalculate(state, assignment, opts) + else + -- If the registration is untrusted signal the issue via and event and + -- return the state unmodified + ao.event("error", { "untrusted peer requested", req}) + return "ok", state + end +end + +-- Update the performance of a host by its reference. +function duration(state, assignment, opts) + state = ensure_defaults(state) + + local req = assignment.body + local reference = req.reference + if reference == nil then + ao.event("debug_dynrouter", + { + "ignoring duration update for request without reference: ", + req["request-path"] + } + ) + return state + end + ao.event("debug_dynrouter", {"applying_duration", req.reference}) + reference = reference .. "/performance" + local duration = req.duration + local change_factor = 1 / state["performance-period"] + + -- Get the performance of the route at `reference' + local status, performance = ao.resolve(state, reference) + + -- Modify the node's existing performance score, weighted by the change + -- factor, to give more weight to the existing performance score. Each node + -- is given a poor performance score (30000ms) to start, then will slowly + -- improve its performance score over time. + performance = + (performance * (1 - change_factor)) + (duration * change_factor) + + ao.event("debug_perf", + {"Received performance", { + reference = reference, + performance = performance, + update_duration = duration, + change_factor = change_factor, + } + }) + + state = ao.set(state, reference, performance) + + ao.event("debug_router", + { + "State after performance set", + { state = state, performance = performance } + } + ) + return "ok", state +end + +function compute(state, assignment, opts) + if assignment.body.action == "register" then + return register(state, assignment, opts) + elseif assignment.body.action == "recalculate" then + return recalculate(state, assignment, opts) + elseif assignment.body.action == "performance" then + return duration(state, assignment, opts) + else + -- If we have been called without a relevant path, simply ensure that + -- the state is initialized and return it. + state = ensure_defaults(state) + return "ok", state + end +end + +--- Tests +function register_test() + local state = {} + -- Simulate a register call upon a default state. + local req = { + path = "register", + route = { + prefix = "host1", + price = 5, + template = "/test-key" + } + } + _, state = register(state, { body = req }, {}) + + -- We must now have exactly one route in state.routes. + if #state.routes ~= 1 then + error("Expected 1 route after register, got "..tostring(#state.routes)) + end + + -- Verify the node, price and default performance. + local r = state.routes[1] + ao.event("debug_router", { "route:", r }) + if r.nodes[1].prefix ~= "host1" then + error("Expected node='host1', got "..tostring(r.nodes[1].node)) + end + if r.nodes[1].price ~= 5 then + error("Expected price=0.5, got "..tostring(r.nodes[1].price)) + end + if r.nodes[1].performance ~= state["initial-performance"] then + error("Expected performance=" .. + tostring(state["initial-performance"]) .. + ", got " .. tostring(r.nodes[1].performance) + ) + end + + -- Register another provider on the route. + req = { + path = "register", + route = { + prefix = "host2", + price = 10, + template = "/test-key" + } + } + _, state = register(state, { body = req }, {}) + + ao.event("debug_router", {"state after second registration", state}) + + if #state.routes[1].nodes ~= 2 then + error("Expected 2 nodes after second registration, got " + .. tostring(#state.routes[1].nodes)) + end + + return "ok" + end + + -- Test 2: performance updates and weight recalculation +function performance_test() + -- Create a new state with a fast performance-period, giving rapid changes + -- to the performance score of nodes. + local state = { + ["performance-period"] = 6 + } + + -- Add a node to a new route on the state + local register_req = { + path = "register", + route = { + prefix = "host1", + price = 5, + template = "/test-key" + } + } + _, state = register(state, { body = register_req }, {}) + + -- Modify the request and add another node. + register_req.route.prefix = "host2" + _, state = register(state, { body = register_req }, {}) + + -- Get the references for the nodes on the route and validate it. + local node1_ref = state.routes[1].nodes[1].reference + local node2_ref = state.routes[1].nodes[2].reference + + if node1_ref ~= "routes/1/nodes/1" then + error("Invalid reference. Received: " .. node1_ref) + end + if node2_ref ~= "routes/1/nodes/2" then + error("Invalid reference. Received: " .. node2_ref) + end + + -- Record the starting scores for the nodes + local t0_node1_score = state.routes[1].nodes[1].weight + local t0_node2_score = state.routes[1].nodes[1].weight + + if t0_node1_score ~= t0_node2_score then + error("Initial node scores should be equal. Received: " + .. tostring(t0_node1_score) .. " and " .. tostring(t0_node2_score)) + end + + -- Post 2 performance updates for the first node, improving its performance. + local perf_req = { + path = "duration", + host = "host1", + reference = node1_ref, + duration = 200 + } + _, state = duration(state, { body = perf_req }, {}) + _, state = duration(state, { body = perf_req }, {}) + -- Post a performance update for the second node, with very poor performance + perf_req.reference = node2_ref + perf_req.duration = 55500 + ao.event("debug_router", {"perf_req node 2", perf_req}) + _, state = duration(state, { body = perf_req }, {}) + + ao.event("debug_router", + {"state after performance updates", { + state = state + }} + ) + + -- now trigger a recalc + _, state = recalculate(state, { body = { path = "recalculate" } }, {}) + + ao.event("debug_router", + {"Nodes after recalculation", state.routes[1].nodes} + ) + + -- Record the starting scores for the nodes + local t1_node1_score = state.routes[1].nodes[1].weight + local t1_node2_score = state.routes[1].nodes[2].weight + + ao.event("debug_router_scores", { + t0_n1 = t0_node1_score, + t1_n1 = t1_node1_score, + t0_n2 = t0_node2_score, + t1_n2 = t1_node2_score + }) + + if t1_node1_score ~= t0_node1_score then + error("Node 1 sets the benchmark: It's score should stay the same.") + end + + if t1_node2_score >= t0_node2_score then + error("Node 2 score should have decreased!") + end + + return "ok" +end \ No newline at end of file diff --git a/scripts/fix-markdown-headers.sh b/scripts/fix-markdown-headers.sh new file mode 100755 index 000000000..3910557f6 --- /dev/null +++ b/scripts/fix-markdown-headers.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Script to fix markdown headers generated by edown +# Removes unnecessary # symbols at the end of header lines + +find doc -type f -name "*.md" -exec sed -i '' -E 's/ #+$//' {} \; + +echo "Markdown headers fixed in all documentation files." \ No newline at end of file diff --git a/scripts/hyper-token-p4-client.lua b/scripts/hyper-token-p4-client.lua new file mode 100644 index 000000000..d6a14ce9e --- /dev/null +++ b/scripts/hyper-token-p4-client.lua @@ -0,0 +1,41 @@ +--- A simple script that can be used as a `~p4@1.0` ledger device, marshalling +--- requests to a local process. + +-- Find the user's balance in the current ledger state. +function balance(base, request) + local status, res = ao.resolve({ + path = + base["ledger-path"] + .. "/now/balance/" + .. request["target"] + }) + ao.event({ "client received balance response", + { status = status, res = res, target = request["target"] } } + ) + -- If the balance request fails (most likely because the user has no balance), + -- return a balance of 0. + if status ~= "ok" then + return "ok", 0 + end + + -- We have successfully retrieved the balance, so return it. + return "ok", res +end + +-- Charge the user's balance in the current ledger state. +function charge(base, request) + ao.event("debug_charge", { + "client starting charge", + { request = request, base = base } + }) + local status, res = ao.resolve({ + path = "(" .. base["ledger-path"] .. ")/push", + method = "POST", + body = request + }) + ao.event("debug_charge", { + "client received charge response", + { status = status, res = res } + }) + return "ok", res +end \ No newline at end of file diff --git a/scripts/hyper-token-p4.lua b/scripts/hyper-token-p4.lua new file mode 100644 index 000000000..ef7e323f0 --- /dev/null +++ b/scripts/hyper-token-p4.lua @@ -0,0 +1,61 @@ +--- An extension to the `hyper-token.lua` script, for execution with the +--- `lua@5.3a` device. This script adds the ability for an `admin' account to +--- charge a user's account. This is useful for allowing a node operator to +--- collect fees from users, if they are running in a trusted execution +--- environment. +--- +--- This script must be added as after the `hyper-token.lua` script in the +--- `process-definition`s `script` field. + +-- Process an `admin' charge request: +-- 1. Verify the sender's identity. +-- 2. Ensure that the quantity and account are present in the request. +-- 3. Debit the source account. +-- 4. Increment the balance of the recipient account. +function charge(base, assignment) + ao.event("debug_charge", { "Charge received: ", { assignment = assignment } }) + local admin = base.admin + local status, res, request = validate_request(base, assignment) + if status ~= "ok" then + return status, res + end + + -- Verify that the request is signed by the admin. + local committers = ao.get("committers", {"as", "message@1.0", assignment.body}) + ao.event("debug_charge", { "Validating request: ", { + committers = committers, + admin = admin + } }) + if count_common(committers, admin) ~= 1 then + return "error", base + end + + -- Ensure that the quantity and account are present in the request. + if not request.quantity or not request.account then + ao.event({ "Failure: Quantity or account not found in request.", + { request = request } }) + base.result = { + status = "error", + error = "Quantity or account not found in request." + } + return "ok", base + end + + -- Debit the source. Note: We do not check the source balance here, because + -- the node is capable of debiting the source at-will -- even it puts the + -- source into debt. This is important because the node may estimate the + -- cost of an execution at lower than its actual cost. Subsequently, the + -- ledger should at least debit the source, even if the source may not + -- deposit to restore this balance. + ao.event({ "Debit request validated: ", { assignment = assignment } }) + base.balance = base.balance or {} + base.balance[request.account] = + (base.balance[request.account] or 0) - request.quantity + + -- Increment the balance of the recipient account. + base.balance[request.recipient] = + (base.balance[request.recipient] or 0) + request.quantity + + ao.event("debug_charge", { "Charge processed: ", { balances = base.balance } }) + return "ok", base +end diff --git a/scripts/hyper-token.lua b/scripts/hyper-token.lua new file mode 100644 index 000000000..016d9713e --- /dev/null +++ b/scripts/hyper-token.lua @@ -0,0 +1,891 @@ +--- ## HyperTokens: Networks of fungible, parallel ledgers. +--- # Version: 0.1. +--- +--- An AO token standard implementation, with support for sub-ledger networks, +--- executed with the `~lua@5.3` device. This script supports both the base token +--- blueprint's 'active' keys, as well as the mainnet sub-ledger API. +--- +--- Data access actions (e.g. `balance', `info', `total-supply') are not +--- implemented due to their redundancy. Instead, the full state of the process +--- is available via the AO-Core HTTP API, including all metadata and individual +--- account balances. +--- +--- A full description of the hyper-token standard can be found in the +--- `token.md` file in this directory. The remainder of this module document +--- provides a breif overview of its design, and focuses on its implementation +--- details. +--- +--- ## Design and Implementation +--- +--- If running as a `root' token (indicated by the absence of a `token' field), +--- the `balance' field should be initialized to a table of balances for the +--- token during spawning. The `ledgers' field holds the state of a sub-ledger's +--- own balances with other ledgers. This field is always initialized to a message +--- of zero balances during the evaluation of the first assignment of the process. +--- When the token receives a `credit-notice' message, it will interpret it as a +--- deposit from the sending ledger and update its record of its own balance with +--- the sending ledger. +--- +--- Atop the standard token transfer messages, the sub-ledger API allows for +--- `transfer' messages to specify a `route' field, which is a list of ledger +--- IDs that the transfer should be routed through in order to reach a +--- recipient on a different ledger. At each `hop' in the route, the recipient +--- ledger validates whether it trusts the sending ledger and whether it knows +--- how to route to the next hop. If the recipient ledger does not trust the +--- sending ledger, it will terminate the route with a `route-termination' +--- message. If the recipient ledger knows how to route to the next hop, it +--- will create a new `transfer' message to the next hop, with the first +--- ledger from the route removed and the remainder of the route and recipient +--- and quantity to transfer forwarded along with the message. +--- +--- There are three security checks performed on incoming messages, above the +--- standard balance transfer checks: +--- +--- 1. Assignments are evaluated against the `assess/assignment' message, if +--- present. If not, the assignment is evaluated against the process's own +--- scheduler address. +--- +--- 2. If the message does not originate from an end-user (indicated by the +--- presence of a `from-process' field), the message is evaluated against +--- the `assess/request' message, if present. If not, the message is +--- evaluated against the `authority' field. The `authority' field may +--- contain a list of addresses or messages that are considered to be +--- authorities. +--- +--- 3. `credit-notice' messages that do not originate from a sub-ledger's +--- `token' are evaluated for parity of source code with the receiving +--- ledger. This is achieved by comparing the `from-base' field of the +--- credit-notice message with `process/id&commitments=none' on the receiving +--- ledger. + +--- Utility functions: + +-- Add a message to the outbox of the given base. +local function send(base, message) + table.insert(base.results.outbox, message) + return base +end + +-- Add a log message to the results of the given base. +local function log_result(base, status, message) + ao.event("token_log", {"Token action log: ", { + status = status, + message = message + }}) + base.results = base.results or {} + base.results.status = status + + if base.results.log then + table.insert(base.results.log, message) + else + base.results.log = { message } + end + + return base +end + +-- Normalize a quantity value to ensure it is a proper integer. +-- Returns either the normalized integer value or nil and an error message. +local function normalize_int(value) + local num + -- Handle string conversion + if type(value) == "string" then + -- Check for decimal part (not allowed) + if string.find(value, "%.") then + return nil + end + -- Convert to number + num = tonumber(value) + if not num then + return nil + end + elseif type(value) == "number" then + num = value + -- Check if it's an integer + if num ~= math.floor(num) then + return nil + end + else + -- Any other type is invalid. + return nil + end + + return num +end + +-- Count the number of elements in `a' that are also in `b'. +function count_common(a, b) + -- Normalize both arguments to tables. + if type(a) ~= "table" then a = { a } end + if type(b) ~= "table" then b = { b } end + + local count = 0 + for _, v in ipairs(a) do + for _, w in ipairs(b) do + if v == w then + count = count + 1 + end + end + end + + return count +end + +-- Normalize an argument to a table if it is not already a table. +local function normalize_table(value) + -- If value is already a table, return it. If it is not a string, return + -- a table containing only the value. + if type(value) == "table" then + ao.event({ "Table already normalized", { table = value } }) + return value + elseif type(value) ~= "string" then + return { value } + end + + -- If value is a string, remove quotes and split by comma. + local t = {} + local pos = 1 + local len = #value + while pos <= len do + -- find next comma + local comma_start, comma_end = value:find(",", pos, true) + local chunk + if comma_start then + chunk = value:sub(pos, comma_start - 1) + pos = comma_end + 1 + else + chunk = value:sub(pos) + pos = len + 1 + end + + -- trim whitespace and quotes + chunk = chunk:gsub("[\"']", "") + local trimmed = chunk:match("^%s*(.-)%s*$") + -- convert to number if possible + local num = tonumber(trimmed) + table.insert(t, num or trimmed) + end + + ao.event({ "Normalized table", { table = t } }) + return t +end + +--- Security verification functions: + +-- Enforce that a given list satisfies `hyper-token's grammar constraints. +-- This function is used to check that the `authority' and `scheduler' fields +-- satisfy the constraints specified by the `authority[-*]' and `scheduler[-*]' +-- fields. The supported grammar constraints are: +-- - `X`: A list of `X`s that are admissible to match in the subject. +-- - `X-match`: A count of the number of `X`s that must be present in the subject. +-- Default: Length of `X`. +-- - `X-required`: A list of `X`s that must be present in the subject. +-- Default: `{}`. +local function satisfies_list_constraints(subject, all, required, match) + -- Normalize the fields to tables, aside from the match count. + subject = normalize_table(subject) + all = normalize_table(all) + required = normalize_table(required or {}) + -- Normalize the match count. + match = match or #all + match = normalize_int(match) + + ao.event({ "Satisfies list constraints", { + subject = subject, + all = all, + match = match + }}) + + -- Check that the subject satisfies the grammar's constraints. + -- 1. The subject must have at least `match' elements in common with `all'. + -- 2. The subject must contain all elements in `required'. + local count = count_common(subject, all) + local required_count = count_common(required, subject) + + ao.event({ "Counts", { + subject = subject, + all = all, + required = required, + match = match, + count = count, + required_count = required_count + }}) + + return (count >= match) and (required_count == #required) +end + +-- Ensure that a message satisfies the grammar's constraints, or the assessment +-- message returns true, if present. +local function satisfies_constraints(message, assess, all, required, match) + -- If the assessment message is present, run it against the message. + if assess then + ao.event({ "Running assessment message against request." }, + { assessment = assess, message = message }) + local status, result = ao.resolve(assess, message) + if (status == "ok") and (result == true) then + ao.event({ "Assessment of request passed." }, { + message = message, + status = status, + result = result + }) + return true + else + ao.event({ "Assessment of request failed.", { + message = message, + status = status, + result = result + }}) + return false + end + end + + -- If the assessment message is not present, check the signatures against + -- the requirements list and specifiers. + local satisfies_auth = satisfies_list_constraints( + ao.get("committers", message), + all, + required, + match + ) + + ao.event({ "Constraint satisfaction results", { + result = satisfies_auth, + message = message, + all_admissible = all, + required = required, + required_count = match + }}) + + return satisfies_auth +end + +-- Ensure that the `authority' field satisfies the `authority[-*]' constraints +-- (as supported by `satisfies_constraints') or that the assessment message +-- returns true. +local function is_trusted_compute(base, assignment) + return satisfies_constraints( + assignment.body, + (base.assess or {})["authority"], + base.authority, + base["authority-required"], + base["authority-match"] + ) +end + +-- Ensure that the assignment is trusted. Either by running the assessment +-- process, or by checking the signature against the process's own scheduler +-- address and those it explicitly trusts. +local function is_trusted_assignment(base, assignment) + return satisfies_constraints( + assignment, + (base.assess or {})["scheduler"], + base.scheduler, + base["scheduler-required"], + base["scheduler-match"] + ) +end + +-- Determine if the ledger indicated by `base` is the root ledger. +local function is_root(base) + return base.token == nil +end + +-- Ensure that a credit-notice from another ledger is admissible. It must either +-- be from our own root ledger, or from a sub-ledger that is precisely the same +-- as our own. +local function validate_new_peer_ledger(base, request) + ao.event({ "Validating peer ledger: ", { request = request } }) + + -- Check if the request is from the root ledger. + if is_root(base) or (base.token == request.from) then + ao.event({ "Peer is parent token. Accepting." }, { + request = request + }) + return true + end + + -- Calculate the expected base ID from the process's own `process` message, + -- modified to remove the `authority' and `scheduler' fields. + -- This ensures that the process we are receiving the `credit-notice` from + -- has the same structure as our own process. + ao.event({ "Calculating expected `base` from self", { base = base } }) + local status, proc, expected + status, proc = ao.resolve({"as", "message@1.0", base}, "process") + -- Reset the `authority' and `scheduler' fields to nil, to ensure that the + -- `base` message matches the structure created by `~push@1.0`. + proc.authority = nil + proc.scheduler = nil + status, expected = + ao.resolve( + proc, + { path = "id", commitments = "none" } + ) + ao.event({ "Expected `from-base`", { status = status, expected = expected } }) + -- Check if the `from-base' field is present in the assignment. + if not request["from-base"] then + ao.event({ "`from-base` field not found in message", { + request = request + }}) + return false + end + + -- Check if the `from-base' field matches the expected ID. + local base_matches = request["from-base"] == expected + + if not base_matches then + ao.event("debug_base", { "Peer registration messages do not match", { + expected_base = expected, + received_base = request["from-base"], + process = proc, + request = request + }}) + return false + end + + -- Check that the `from-authority' and `from-scheduler' fields match the + -- expected values, to the degree specified by the `authority-match' and + -- `scheduler-match' fields. Additionally, the `authority-required' and + -- `scheduler-required' fields may be present in the base, the members of + -- which must be present in the `from-authority' and `from-scheduler' fields + -- respectively. + local authority_matches = satisfies_list_constraints( + request["from-authority"], + base.authority, + base["authority-required"], + base["authority-match"] + ) + local scheduler_matches = satisfies_list_constraints( + request["from-scheduler"], + base.scheduler, + base["scheduler-required"], + base["scheduler-match"] + ) + if (not authority_matches) or (not scheduler_matches) then + ao.event("debug_base", { "Peer security parameters do not match", { + expected_authority = base.authority, + received_authority = request["from-authority"], + expected_scheduler = base.scheduler, + received_scheduler = request["from-scheduler"], + scheduler_matches = scheduler_matches, + authority_matches = authority_matches, + request = request + }}) + return false + end + + ao.event("Peer registration messages matches. Accepting.") + + return true +end + +-- Register a new peer ledger, if the `from-base' field matches our own. +local function register_peer(base, request) + -- Validate the registering ledger + if not validate_new_peer_ledger(base, request) then + base.results = { + status = "error", + error = "Ledger registration failed." + } + return "error", base + end + + -- Add to known ledgers + base.ledgers[request.from] = base.ledgers[request.from] or 0 + + return "ok", base +end + +-- Determine if a request is from a known ledger. Makes no assessment of whether +-- a request is otherwise trustworthy. +local function is_from_trusted_ledger(base, request) + -- We always trust the root ledger. + if request.from == base["token"] then + return true, base + end + + -- We trust any ledger that is already registered in the `ledgers' map. + if base.ledgers and (base.ledgers[request.from] ~= nil) then + return true, base + end + + -- Validate whether the request is from a new peer ledger. + local status + status, base = register_peer(base, request) + if status ~= "ok" then + return false, base + end + + return true, base +end + +-- Ensure that the ledger is initialized. +local function ensure_initialized(base, assignment) + -- Ensure that the base has a `result' field before we try to register. + base.results = base.results or {} + base.results.outbox = {} + base.results.status = "OK" + -- If the ledger is not being initialized, we can skip the rest of the + -- function. + if assignment.slot ~= 0 then + return "ok", base + end + base.balance = base.balance or {} + + -- Ensure that the `ledgers' map is initialized: present and empty. + base.ledgers = base.ledgers or {} + ao.event({ "Ledgers before initialization: ", base.ledgers }) + + for _, ledger in ipairs(base.ledgers) do + base.ledgers[ledger] = 0 + end + ao.event({ "Ledgers after initialization: ", base.ledgers }) + + if not base.token then + ao.event({ "Ledger has no source token. Skipping registration." }) + return "ok", base + end + + ao.event({ "Registering self with known token ledgers: ", { + ledgers = base.ledgers + }}) + + for _, ledger in ipairs(base.ledgers) do + -- Insert the register result into the base. + base.results = send(base, { + action = "Register", + target = ledger + }) + end + + return "ok", base +end + +-- Verify that an assignment has not been processed and that the request is +-- valid. If it is, update the `from' field to the address that signed the +-- request. +function validate_request(incoming_base, assignment) + -- Ensure that the ledger is initialized. + local status, base = ensure_initialized(incoming_base, assignment) + if status ~= "ok" then + return "error", log_result(incoming_base, "error", { + message = "Ledger initialization failed.", + assignment = assignment, + status = status, + }) + end + + -- First, ensure that the message has not already been processed. + ao.event("Deduplicating message.", { + ["history-length"] = #(base.dedup or {}) + }) + + status, base = + ao.resolve( + incoming_base, + {"as", + "dedup@1.0", + { + path = "compute", + ["subject-key"] = "body", + body = assignment.body + } + } + ) + + -- Set the device back to `process@1.0`. + base.device = "process@1.0" + if status ~= "ok" then + return "error", log_result(base, "error", { + message = "Deduplication failure.", + assignment = assignment, + status = status, + incoming_base = incoming_base, + resulting_base = base + }) + end + + -- Next, ensure that the assignment is trusted. + local trusted, details = is_trusted_assignment(base, assignment) + if not trusted then + return "error", log_result(base, "error", { + message = "Assignment is not trusted.", + details = details + }) + end + + if assignment.body["from-process"] then + -- If the request is proxied, we need to check that the source + -- computation is trusted. + trusted, details = is_trusted_compute(base, assignment) + if not trusted then + return "error", log_result(base, "error", { + message = "Message computation is not trusted.", + details = details + }) + end + assignment.body.from = assignment.body["from-process"] + return "ok", base, assignment.body + else + -- If the request is not proxied, we set the `from' field to the address + -- that signed the request. + local committers = ao.get("committers", assignment.body) + if #committers == 0 then + return "error", log_result(base, "error", { + message = "No request signers found." + }) + end + + -- Only accept single-signed requests to avoid ambiguity + if #committers > 1 then + return "error", log_result(base, "error", { + message = "Multiple signers detected, making sender ambiguous. " .. + "Only singly-signed requests are supported for end-user " .. + "requests (those that do not originate from another " .. + "computation)." + }) + end + + assignment.body.from = committers[1] + return "ok", base, assignment.body + end +end + +-- Ensure that the source has the required funds, then debit the source. Takes +-- an origin, which can be used to identify the reason for the debit in logging. +-- Returns error if the source balance is not viable, or `ok` and the updated +-- base state if the debit is successful. Does not credit any funds. +local function debit_balance(base, request) + local source = request.from + + ao.event({ "Attempting to deduct balance.", { + request = request, + balances = base.balance or {} + }}) + + -- Ensure that the `source' and `quantity' fields are present in the request. + if not source or not request.quantity then + return "error", log_result(base, "error", { + message = "Fund source or quantity not found in request.", + }) + end + + -- Normalize the quantity value. + request.quantity = normalize_int(request.quantity) + if not request.quantity then + ao.event({ "Invalid quantity value: ", { quantity = request.quantity } }) + base.results = { + status = "error", + error = "Invalid quantity value.", + quantity = request.quantity + } + return "error", base + end + + -- Ensure that the source has the required funds. + -- Check 1: The source balance is present in the ledger. + local source_balance = base.balance[source] + + if not source_balance then + return "error", log_result(base, "error", { + message = "Source balance not found.", + from = source, + quantity = request.quantity, + ["is-root"] = is_root(base) + }) + end + + -- Check 2: The source balance is a valid number. + if type(source_balance) ~= "number" then + return "error", log_result(base, "error", { + message = "Source balance is not a number.", + balance = source_balance + }) + end + + -- Check 3: Ensure that the quantity to deduct is a non-negative number. + if request.quantity < 0 then + return "error", log_result(base, "error", { + message = "Quantity to deduct is negative.", + quantity = request.quantity + }) + end + + -- Check 4: Ensure that the source has enough funds. + if source_balance < request.quantity then + return "error", log_result(base, "error", { + message = "Insufficient funds.", + from = source, + quantity = request.quantity, + balance = source_balance + }) + end + + ao.event({ "Deducting funds:", { request = request } }) + base.balance[source] = source_balance - request.quantity + ao.event({ "Balances after deduction:", + { balances = base.balance, ledgers = base.ledgers } } + ) + return "ok", base +end + +-- Transfer the specified amount from the given account to the given account, +-- optionally routing to a different sub-ledger if necessary. +-- There are four differing types of transfer requests. They have the following +-- semantics: +-- Balance == owed to X. Credit == Owed to subject by X. + +-- User on root -> User on sub-ledger: +-- Xfer in: Root = Dec User balance, Inc Sub-ledger balance +-- C-N in: Sub-ledger = Inc User balance + +-- User on sub-ledgerA -> User on sub-ledgerB: +-- Xfer in: Sub-ledgerA = Dec User balance +-- C-N in: Sub-ledgerB = Inc User balance + +-- User on sub-ledgerB -> User on sub-ledgerA: +-- Xfer in: Sub-ledgerB = Dec user balance +-- C-N in: Sub-ledgerA = Inc user balance + +-- User on A->B->C: +-- Xfer in: A = Dec User balance +-- C-N in: B = +-- C-N in: C = Inc User balance + +-- User on sub-ledger -> User on root: +-- Xfer in: Sub-ledger = Dec User balance +-- C-N in: Root = Inc User balance, Dec Sub-ledger balance +function transfer(base, assignment) + ao.event({ "Transfer request received", { assignment = assignment } }) + -- Verify the security of the request. + local status, request + status, base, request = validate_request(base, assignment) + if status ~= "ok" or not request then + return "ok", base + end + + -- Ensure that the recipient is known. + if not request.recipient then + return log_result(base, "error", { + message = "Transfer request has no recipient." + }) + end + + -- Normalize the quantity value. + local quantity = normalize_int(request.quantity) + if not quantity then + return log_result(base, "error", { + message = "Invalid quantity value.", + quantity = request.quantity + }) + end + + -- Ensure that the source has the required funds. If they do, debit them. + local debit_status + debit_status, base = debit_balance(base, request) + if debit_status ~= "ok" or base == nil then + return "ok", base + end + + if is_root(base) or not request.route then + -- We are the root ledger, or the user is sending tokens directly to + -- another user. We credit the recipient's balance, or the sub-ledger's + -- balance if the request has a `route' key. + local direct_recipient = request.route or request.recipient + base.balance[direct_recipient] = + (base.balance[direct_recipient] or 0) + quantity + base = send(base, { + action = "Credit-Notice", + target = direct_recipient, + recipient = request.recipient, + quantity = quantity, + sender = request.from + }) + return log_result(base, "ok", { + message = "Direct or root transfer processed successfully.", + from_user = request.from, + to = direct_recipient, + explicit_recipient = request.recipient, + quantity = quantity + }) + end + + if request.route == base.token then + -- The user is returning tokens to the root ledger, so we send a + -- transfer to the root ledger. + base = send(base, { + action = "Transfer", + target = base.token, + recipient = request.recipient, + quantity = string.format('%d', math.floor(request.quantity)) + }) + return log_result(base, "ok", { + message = "Ledger-root transfer processed successfully.", + from_user = request.from, + to_ledger = base.token, + to_user = request.recipient, + quantity = string.format('%d', math.floor(request.quantity)) + }) + end + + -- We are not the root ledger, and the request has a `route` key. + -- Subsequently, the target must be another ledger so we dispatch a + -- credit-notice to the peer ledger. The peer will increment the balance of + -- the recipient. + base = send(base, { + action = "Credit-Notice", + target = request.route, + recipient = request.recipient, + quantity = quantity, + sender = request.from + }) + + return log_result(base, "ok", { + message = "Ledger-ledger transfer processed successfully.", + from_user = request.from, + to_ledger = request.route, + to_user = request.recipient, + quantity = quantity + }) +end + +-- Process credit notices from other ledgers. +_G["credit-notice"] = function (base, assignment) + ao.event({ "Credit-Notice received", { assignment = assignment } }) + + -- Verify the security of the request. + local status, request + status, base, request = validate_request(base, assignment) + if status ~= "ok" or not request then + return "ok", base + end + + if is_root(base) then + -- The root ledger will not process credit notices. + return log_result(base, "error", { + message = "Credit-Notice to root ledger ignored." + }) + end + + -- Ensure that the recipient is known. + if not request.recipient then + return log_result(base, "error", { + message = "Credit-Notice request has no recipient." + }) + end + + -- Normalize the quantity value. + local quantity = normalize_int(request.quantity) + if not quantity then + return log_result(base, "error", { + message = "Invalid quantity value.", + quantity = request.quantity + }) + end + + -- Ensure that the sender is a trusted ledger peer. + local trusted + trusted, base = is_from_trusted_ledger(base, request) + if not trusted then + return log_result(base, "error", { + message = "Credit-Notice not from a trusted peer ledger." + }) + end + + -- Credit the recipient's balance. + base.balance[request.recipient] = + (base.balance[request.recipient] or 0) + quantity + + return "ok", log_result(base, "ok", { + message = "Credit-Notice processed successfully.", + from_ledger = request.from, + to_ledger = request.sender, + to_user = request.recipient, + quantity = quantity, + balance = base.balance[request.recipient] + }) +end + +-- Process registration requests from other ledgers. +function register(raw_base, assignment) + ao.event({ "Register request received", { assignment = assignment } }) + + local status, base, request = validate_request(raw_base, assignment) + if (status ~= "ok") or (type(request) ~= "table") then + return "ok", base + end + + if base.ledgers[request.from] then + ao.event({ "Ledger already registered. Ignoring registration request." }) + base.results = { + message = "Ledger already registered." + } + return "ok", base + end + + -- Validate the registering ledger + status, base = register_peer(base, request) + if status ~= "ok" then + return status, base + end + + -- Send a reciprocal registration request to the remote ledger. + base = send(base, { + target = request.from, + action = "register" + }) + + return "ok", base +end + +-- Register ourselves with a remote ledger, at the request of a user or another +-- ledger. +_G["register-remote"] = function (raw_base, assignment) + -- Validate the request. + local status, base, request = validate_request(raw_base, assignment) + if (status ~= "ok") or (type(request) ~= "table") then + return "ok", base + end + + base = log_result(base, "ok", { + message = "Register-Remote request received.", + peer = request.peer + }) + + -- Send a registration request to the remote ledger. Our request is simply + -- a `Register' message, as the recipient will be assessing our unsigned + -- process ID in order to validate that we are an appropriate peer. This is + -- added by our `push-device`, so no further action is required on our part. + base = send(base, { + target = request.peer, + action = "register" + }) + + return "ok", base +end + +--- Index function, called by the `~process@1.0` device for scheduled messages. +--- We route any `action' to the appropriate function based on the request path. +function compute(base, assignment) + ao.event({ "compute called", + { balance = base.balance, ledgers = base.ledgers } }) + + assignment.body.action = string.lower(assignment.body.action or "") + + if assignment.body.action == "credit-notice" then + return _G["credit-notice"](base, assignment) + elseif assignment.body.action == "transfer" then + return transfer(base, assignment) + elseif assignment.body.action == "register" then + return register(base, assignment) + elseif assignment.body.action == "register-remote" then + return _G["register-remote"](base, assignment) + else + -- Handle unknown `action' values. + _, base = ensure_initialized(base, assignment) + base.results = { + status = "ok" + } + ao.event({ "Process initialized.", { slot = assignment.slot } }) + return "ok", base + end +end \ No newline at end of file diff --git a/scripts/hyper-token.md b/scripts/hyper-token.md new file mode 100644 index 000000000..770bf4bb4 --- /dev/null +++ b/scripts/hyper-token.md @@ -0,0 +1,217 @@ +# HyperTokens: Networks of fungible, parallel ledgers. +## Status: Draft-1 + +This document describes the implementation details of the token ledger found in +`scripts/token.lua`. The script is built for operation with the HyperBEAM +`~lua@5.3a` and `~process@1.0` devices. + +In addition to implementing the core AO token ledger API, `hyper-ledger.lua` also +implements a `sub-ledger` standard, which allows for the creation of networks +of ledgers that are may each hold fragments of the total supply of a given token. +Each ledger may execute in paralell fully asynchronously, while ownership in their +tokens can be viewed as fungible. The fungibility of tokens across these ledgers +is created as a result of their transitively enforced security properties -- each +ledger must be a precise copy of every other ledger in the network -- as well as +the transferrability of balances from one ledger to another. Ledgers that have +`register`ed with one another are able to transfer tokens directly. A multi-hop +routing option is also available for situations in which it may be desirable to +utilize pre-existing peer relationships instead. + +This document provides a terse overview of the mechanics of this standard, and +the specifics of its implementation in `scripts/token.lua`. + +## 1. Entities and State + +### 1.1 Entities +- **User Account**: Identified by wallet address, may own tokens in `ledgers`. +- **Ledger Process**: An AO process implementing this token script. +- **Root Ledger**: The base token ledger process, from which supply scarcity is + derived. Root ledgers are differentiated from `sub-ledger`s by the absence of + a `token` field. +- **Sub-Ledger**: An independent ledger that that may own tokens in the root + ledger, holding them on behalf of users and other ledgers in the network. + All sub-ledgers must have a `token' field in their process definition, which + contains the signed process ID of the root ledger. + +### 1.2 State + +Each ledger maintain the following fields: + +- `balance`: A message containing a map of user addresses to token balances. +- `ledgers`: A message containing a map of peer-ledgers and the local ledger's + balance in each. +- `token`: (Optional) The signed process ID of the root ledger. + +Additionally, ledgers (root or sub-ledger) may maintain any other metadata fields +as needed in their process definition messages. Both metadata and necessary +fields are available via the AO-Core HTTP API. + +## 2. Message Paths and Actions + +All instances of this script support calling the following functions, as either +the `path` or `action` of scheduled messages upon them: + +- **Register**: Attempt to establish a trust relationship between ledger peers. +- **Register-Remote**: User-initiated request for the recipient ledger to + register with a specific remote peer. After registration, the user may transfer + tokens from their own account on the registrant ledger, to the remote ledger + that they specified. +- **Transfer**: Move tokens between accounts (with optional routing). +- **Credit-Notice**: Notification sent by a ledger to a recipient of credit upon + a successful transfer. +- **Debit-Notice**: A notification granted by a ledger to a sender of tokens, + upon successful transfer. +- **Credit-Notice-Rejection**: A notice sent by a ledger to the sender of tokens, + if the ledger is unable to accept the transfer (crediting its own account). + This action occurs when the recipient ledger is unable to validate the sender + as a valid peer, or when the recipient ledger is unable to validate the + quantity of tokens being transferred. Upon receipt, a process will likely + wish to reverse the transfer locally. +- **Route-Termination**: Notice, dispatched to a sender of tokens, if the + ledger is unable to forward the transfer to the next hop in the stated route. + Upon receipt, senders will typically wish to send a new transfer request to + the ledger with a different route to reach their recipient. If all other + routes are exhausted, the sender may transfer the tokens to the root ledger. + +## 3. Core Process Flows + +### 3.1 Ledger Registration and Trust Negotiation. + +1. Ledger A sends `Register` to Ledger B. The `push-device` that delivers this + message to the recipient (for example, `push@1.0`) must add the + `from-process-uncommitted` field to the message, containing the hash of the + sending process. +2. Ledger B validates: + - The admissibility of the assignment (its `scheduler` commitment), + - Whether it trusts the computation commitments upon the message, and + - Whether the message is from a process that is executing precisely the same + code as the recipient, as signified by the `from-process-uncommitted` field. +3. Ledger B adds Ledger A to its list of trusted peers, if it is not already + present, and sends a reciprocal `Register` message to the sending ledger. +4. Ledger A validates and adds Ledger B using the same mechanism, then adds it + to its list of trusted peers. +5. **Result**: Bidirectional trust relationship, and the ability to transfer + tokens directly between the two ledgers. + +### 3.2 Direct Cross-Process Transfer + +**Objective**: `Alice` wants to send tokens to `Bob`, who is on Ledger B. There +is an established peer relationship between Ledger A and Ledger B. + +1. `Alice`, with tokens resident on Ledger A, initiates transfer to `Bob`, who + would like to receive tokens on Ledger B. Ledger B already has an established + trust relationship and sufficient balance for `Alice` to send tokens to `Bob`. +2. Ledger A validates request, checks their balance for `Alice`, and debits the + sender's account. +3. Ledger A sends a `Transfer` message to Ledger B. +4. Ledger B validates sender ledger, decrements sender's balance, and credits the + recipient's balance. + +### 3.3 Multi-Hop Transfer + +**Objective**: `Alice` wants to send tokens to `Bob`, who is on Ledger N, but +there is no peer relationship between Ledger A and Ledger N. + +1. `Alice` initiates a transfer on Ledger A, with a multi-hop route + (`route=[L₁, L₂, ..., Lₙ]`) to reach `Bob` on Ledger N. +2. As with S`3.2`, Ledger A validates request, checks their balance for `Alice`, + and debits the sender's account. +3. Each intermediate ledger in the route validates the balance of the sending + ledger, and debits their account. Each ledger also removes themselves from the + list of hops remaining in the `route` parameter of the request, and forwards + it onwards to the next hop. +4. Final ledger in the route validates the balance of the sending ledger, and + credits the `Bob`'s account. + +### 3.4 Transfer with Peer Registration + +**Objective**: `Alice` wants to send tokens to `Bob`, who is on Ledger B. There +is no peer relationship between Ledger A and Ledger B, but `Alice` would rather +establish one than route the transfer through a potentially longer multi-hop +route. + +1. `Alice` sends a `Register-Remote` message to Ledger B, with a `peer` parameter + of Ledger A's signed process ID. +2. Ledger B validates the request, and adds Ledger A to its list of trusted peers. +3. `Alice` sends a `Transfer` message to Ledger B. +4. Ledger A and B validate the transfer request, as in `3.2`. + +## 4. Intended Security Properties + +1. **Code Integrity**: Each new peer relationship validates that each other + peer is executing precisely the same code as it is. Subsequently, the + security properties of the original ledger are transitively applied to all + new ledgers in the network. +2. **Conservation of Tokens**: As each peer may trust each other peer to monitor + balances as they would, the network as a whole maintains the conservation + of the total supply of tokens. +3. **Trustless Registration**: A subledger may register with any AO token process + without that process needing to be aware of the subledger protocol, nor the + security properties that it enforces. Instead, users that wish to participate + in a subledger process network with security properties that they deem + acceptable may do so at-will. Processes that do choose to participate may + maintain an index of known ledgers, allowing tokens within them to be shown + as fungible with the root ledger's tokens in user interfaces, etc. + +## 5. API Reference + +### 5.1 External API Functions + +#### `transfer(state, assignment)` + +Transfers tokens from one account to another. + +Parameters: + +- `base`: The current state of the ledger process. +- `assignment`: An assignment of the transfer message from the process's + `sheduler`. +- `assignment/body`: The transfer message. +- `assignment/body/from`: Source account (determined from signature or + `from-process`). +- `assignment/body/recipient`: The destination account. +- `assignment/body/quantity`: Amount to transfer (integer). +- `assignment/body/route`: (Optional) A list of ledger IDs that should be + traversed to reach the destination ledger. + +Returns: The updated process ledger state, and: + +- `result/status`: The status of the transfer. +- `result/outbox/1`: A message containing the `Action: Credit-Notice` field, sent + to the recipient (either an end-user or another ledger). +- `result/outbox/2`: (Optional) A message containing the `Action: Debit-Notice` + field, dispatched to the sender _if_ the transfer is not a multi-hop route. + In the case of multi-hop routes, debit notices are not sent to any intermediate + ledgers, but are sent to the initial sender upon completion of the final hop. + +In event of error, the following messages are dispatched: + +- `result/outbox/1`: (Optional) A message containing the + `Action: Credit-Notice-Rejection` field, along with a `notice` field with the + ID of the credit notice that was rejected. +- `result/outbox/1`: (Optional) A message containing `Action: Route-Termination`, + sent to the initiator of an inter-ledger transfer, if the transfer is unable to + reach the destination ledger. In this case, the sender will hold a balance on + the intermediate ledger, and may attempt to route the transfer again with a + different route. + +#### `register-remote(base, assignment)` + +Initiates a registration from the target ledger to the `peer` ledger. + +Parameters: +- `base`: The current state of the ledger +- `assignment`: The message containing: +- `assignment/body/peer`: The signed process ID of the remote ledger to + register with. + +#### `register(base, assignment)` + +Attempts to register the sending ledger with the target ledger. + +Parameters: +- `base`: The current state of the ledger +- `assignment`: The message containing: +- `assignment/body/from`: The ledger requesting registration. +- `assignment/body/from-process-uncommitted`: A hash, added by the `push-device`, + of the sending process. \ No newline at end of file diff --git a/scripts/meta-test.lua b/scripts/meta-test.lua new file mode 100644 index 000000000..6c763177c --- /dev/null +++ b/scripts/meta-test.lua @@ -0,0 +1,30 @@ +--- A module that tests the `dev_lua_test' EUnit wrapper integration. + +-- Return a simple result to the calling test suite. +function basic_test() + return "ok" +end + +-- Return a message to the calling test suite. +function return_message_test() + return "ok", { key1 = "Value1", key2 = "Value2" } +end + +-- This function returns an `{error, _}` tuple which -- if it were to be picked +-- up as a test by `dev_lua_tests' -- would cause the test suite to fail. +-- Subsequently, by virtue of the code not being executed, we gain confidence +-- that the test suite generator is differentiating between relevant and +-- irrelevant functions correctly. Its a curious mechanism, but it is useful. +function meta_test_ignored() + return "error", "This should not be picked up as a test!" +end + +-- Test that the test environment granted by the generator allows us to execute +-- calls with the `ao.resolve' function, outside of the sandbox. Currently, +-- `dev_lua_test' does not support sandboxing. +function sandbox_test() + -- Simply return an AO call to the test suite. If the router device is not + -- available, this will cause the test to fail. + local status, res = ao.resolve({ path = "/~router@1.0/routes/1/template" }) + return status, res +end \ No newline at end of file diff --git a/scripts/schema.gql b/scripts/schema.gql new file mode 100644 index 000000000..85e083b46 --- /dev/null +++ b/scripts/schema.gql @@ -0,0 +1,301 @@ +### Supported GraphQL Queries. ### + +type Query { + + # Get a message by its id or a subset of its keys. + message(id: ID, keys: [KeyInput]): Message + + # Get a transaction by its id + transaction(id: ID!): Transaction + + # Get a paginated set of matching transactions using filters. + transactions( + # Find transactions from a list of ids. + ids: [ID!] + + # Find transactions from a list of owner wallet addresses, or wallet owner public keys. + owners: [String!] + + # Find transactions from a list of recipient wallet addresses. + recipients: [String!] + + # Find transactions using tags. + tags: [TagFilter!] + + # Find data items from the given data bundles. + # See: https://github.com/ArweaveTeam/arweave-standards/blob/master/ans/ANS-104.md + bundledIn: [ID!] + + # Find transactions within a given Search Indexing Service ingestion time range. + ingested_at: RangeFilter + + # Find transactions within a given block height range. + block: RangeFilter + + # Result page size (max: 100) + first: Int = 10 + + # A pagination cursor value, for fetching subsequent pages from a result set. + after: String + + # Optionally specify the result sort order. + #sort: SortOrder = HEIGHT_DESC + ): TransactionConnection! + block(id: String): Block + blocks( + # Find blocks from a list of ids. + ids: [ID!] + + # Find blocks within a given block height range. + height: RangeFilter + + # Result page size (max: 100) + first: Int = 10 + + # A pagination cursor value, for fetching subsequent pages from a result set. + after: String + + # Optionally specify the result sort order. + #sort: SortOrder = HEIGHT_DESC + ): BlockConnection! +} + +### HyperBEAM Message Schema. ### + +input KeyInput { + name: String! + value: String! +} + +type Message { + id: ID! + keys: [Key] +} + +type Key { + name: String + value: String +} + +### Arweave GraphQL Schema. ### + +# Indicates exactly one field must be supplied and this field must not be `null`. +directive @oneOf on INPUT_OBJECT + +directive @cacheControl( + maxAge: Int + scope: CacheControlScope + inheritMaxAge: Boolean +) on FIELD_DEFINITION | OBJECT | INTERFACE | UNION + +# Representation of a transaction owner. +type Owner { + # The owner's wallet address. + address: String! + + # The owner's public key as a base64url encoded string. + key: String! +} + +# Representation of a value transfer between wallets, in both winson and ar. +type Amount { + # Amount as a winston string e.g. \`"1000000000000"\`. + winston: String! + + # Amount as an AR string e.g. \`"0.000000000001"\`. + ar: String! +} + +# Basic metadata about the transaction data payload. +type MetaData { + # Size of the associated data in bytes. + size: String! + + # Type is derived from the \`content-type\` tag on a transaction. + type: String +} + +# Tag Schema +type Tag { + # UTF-8 tag name + name: String! + + # UTF-8 tag value + value: String! +} + +# Block Schema +type Block { + # The block ID. + id: ID + + # The block timestamp (UTC). + timestamp: Int + + # The block height. + height: Int! + + # The previous block ID. + previous: ID +} + +# The parent transaction for bundled transactions, +# see: https://github.com/ArweaveTeam/arweave-standards/blob/master/ans/ANS-102.md. +type Parent { + id: ID! +} + +# The data bundle containing the current data item. +# See: https://github.com/ArweaveTeam/arweave-standards/blob/master/ans/ANS-104.md. +type Bundle { + # ID of the containing data bundle. + id: ID! +} + +# Transaction Structure +type Transaction { + id: ID! + anchor: String! + signature: String! + recipient: String! + owner: Owner! + fee: Amount! + quantity: Amount! + data: MetaData! + tags: [Tag!]! + + # When this transaction was made available for querying + ingested_at: Int + + # Transactions with a null block are recent and unconfirmed, if they aren't mined into a block within 60 minutes they will be removed from results. + block: Block + + # @deprecated Don't use, kept for backwards compatability only! + parent: Parent @deprecated(reason: "Use `bundledIn`") + + # For bundled data items this references the containing bundle ID. + # See: https://github.com/ArweaveTeam/arweave-standards/blob/master/ans/ANS-104.md + bundledIn: Bundle +} + +# Paginated page info using the GraphQL cursor spec. +type PageInfo { + hasNextPage: Boolean! +} + +# Paginated result set using the GraphQL cursor spec. +type TransactionEdge { + # The cursor value for fetching the next page. + # + # Pass this to the `after` parameter in ` transactions(after: $cursor)`, the next page will start from the next item after this. + cursor: String! + + # A transaction object. + node: Transaction! +} + +# Paginated result set using the GraphQL cursor spec, +# see: https://relay.dev/graphql/connections.htm. +type TransactionConnection { + pageInfo: PageInfo! + + # The number of transactions that match this query. + count: String + edges: [TransactionEdge!]! +} + +# Paginated result set using the GraphQL cursor spec. +type BlockEdge { + # The cursor value for fetching the next page. + # + # Pass this to the after parameter in blocks(after: $cursor), the next page will start from the next item after this. + cursor: String! + + # A block object. + node: Block! +} + +# Paginated result set using the GraphQL cursor spec, +# see: https://relay.dev/graphql/connections.htm. +type BlockConnection { + pageInfo: PageInfo! + edges: [BlockEdge!]! +} + +# Find transactions with the following tag name and value +input TagFilter { + # The tag name + name: String + + # An array of values to match against. If multiple values are passed then transactions with _any_ matching tag value from the set will be returned. + # + # e.g. + # + # `{name: "app-name", values: ["app-1"]}` + # + # Returns all transactions where the `app-name` tag has a value of `app-1`. + # + # `{name: "app-name", values: ["app-1", "app-2", "app-3"]}` + # + # Returns all transactions where the `app-name` tag has a value of either `app-1` _or_ `app-2` _or_ `app-3`. + values: [String!] + + # The operator to apply to to the tag filter. Defaults to EQ (equal). + #op: TagOperator! = EQ + + # How tag names and values are matched. Defaults to EXACT. + #match: TagMatch! = EXACT +} + +# The operator to apply to a tag value. +enum TagOperator { + # Equal + EQ + + # Not equal + NEQ +} + +# The method used to determine if tags match. +enum TagMatch { + # An exact match + EXACT + + # A wildcard match + WILDCARD + + # Fuzzy match containing all search terms + FUZZY_AND + + # Fuzzy match containing at least one search term + FUZZY_OR +} + +# Filter with a min and max +input RangeFilter { + # Minimum integer to filter from + min: Int + + # Maximum integer to filter to + max: Int +} + +# Optionally reverse the result sort order from `HEIGHT_DESC` (default) to `HEIGHT_ASC`. +enum SortOrder { + # Results are sorted by the transaction block height in ascending order, with the oldest transactions appearing first, and the most recent and pending/unconfirmed appearing last. + HEIGHT_ASC + + # Results are sorted by the transaction block height in descending order, with the most recent and unconfirmed/pending transactions appearing first. + HEIGHT_DESC + + # Results are sorted by the transaction ingestion time in descending order, with the most recently ingested transactions appearing first. + INGESTED_AT_DESC + + # Results are sorted by the transaction ingestion time in ascending order, with the oldest ingested transactions appearing first. + INGESTED_AT_ASC +} + +enum CacheControlScope { + PUBLIC + PRIVATE +} \ No newline at end of file diff --git a/src/ar_bundles.erl b/src/ar_bundles.erl index dd1406880..50257208f 100644 --- a/src/ar_bundles.erl +++ b/src/ar_bundles.erl @@ -5,22 +5,21 @@ -export([new_item/4, sign_item/2, verify_item/1]). -export([encode_tags/1, decode_tags/1]). -export([serialize/1, serialize/2, deserialize/1, deserialize/2]). --export([item_to_json_struct/1, json_struct_to_item/1]). -export([data_item_signature_data/1]). -export([normalize/1]). --export([print/1, format/1, format/2]). +-export([print/1, format/1, format/2, format/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). %%% @doc Module for creating, signing, and verifying Arweave data items and bundles. -define(BUNDLE_TAGS, [ - {<<"Bundle-Format">>, <<"Binary">>}, - {<<"Bundle-Version">>, <<"2.0.0">>} + {<<"bundle-format">>, <<"binary">>}, + {<<"bundle-version">>, <<"2.0.0">>} ]). -define(LIST_TAGS, [ - {<<"Map-Format">>, <<"List">>} + {<<"map-format">>, <<"list">>} ]). % How many bytes of a binary to print with `print/1'. @@ -35,31 +34,49 @@ print(Item) -> io:format(standard_error, "~s", [lists:flatten(format(Item))]). format(Item) -> format(Item, 0). -format(Item, Indent) when is_list(Item); is_map(Item) -> - format(normalize(Item), Indent); -format(Item, Indent) when is_record(Item, tx) -> - Valid = verify_item(Item), +format(Item, Indent) -> format(Item, Indent, #{}). +format(Item, Indent, Opts) when is_list(Item); is_map(Item) -> + format(normalize(Item), Indent, Opts); +format(Item, Indent, Opts) when is_record(Item, tx) -> + MustVerify = hb_opts:get(debug_ids, true, Opts), + Valid = + if MustVerify -> verify_item(Item); + true -> true + end, + UnsignedID = + if MustVerify -> hb_util:encode(id(Item, unsigned)); + true -> <<"[SKIPPED ID]">> + end, + SignedID = + if MustVerify -> + case id(Item, signed) of + not_signed -> <<"[NOT SIGNED]">>; + ID -> hb_util:encode(ID) + end; + true -> <<"[SKIPPED ID]">> + end, format_line( "TX ( ~s: ~s ) {", [ if - Item#tx.signature =/= ?DEFAULT_SIG -> + MustVerify andalso Item#tx.signature =/= ?DEFAULT_SIG -> lists:flatten( io_lib:format( "~s (signed) ~s (unsigned)", - [hb_util:encode(id(Item, signed)), hb_util:encode(id(Item, unsigned))] + [SignedID, UnsignedID] ) ); - true -> hb_util:encode(id(Item, unsigned)) + true -> UnsignedID end, if + not MustVerify -> "[SKIPPED VERIFICATION]"; Valid == true -> "[SIGNED+VALID]"; true -> "[UNSIGNED/INVALID]" end ], Indent ) ++ - case (not Valid) andalso Item#tx.signature =/= ?DEFAULT_SIG of + case MustVerify andalso (not Valid) andalso Item#tx.signature =/= ?DEFAULT_SIG of true -> format_line("!!! CAUTION: ITEM IS SIGNED BUT INVALID !!!", Indent + 1); false -> [] @@ -75,6 +92,12 @@ format(Item, Indent) when is_record(Item, tx) -> Target -> hb_util:id(Target) end ], Indent + 1) ++ + format_line("Last TX: ~s", [ + case Item#tx.anchor of + ?DEFAULT_LAST_TX -> "[NONE]"; + LastTX -> hb_util:encode(LastTX) + end + ], Indent + 1) ++ format_line("Tags:", Indent + 1) ++ lists:map( fun({Key, Val}) -> format_line("~s -> ~s", [Key, Val], Indent + 2) end, @@ -82,12 +105,12 @@ format(Item, Indent) when is_record(Item, tx) -> ) ++ format_line("Data:", Indent + 1) ++ format_data(Item, Indent + 2) ++ format_line("}", Indent); -format(Item, Indent) -> +format(Item, Indent, _Opts) -> % Whatever we have, its not a tx... format_line("INCORRECT ITEM: ~p", [Item], Indent). format_data(Item, Indent) when is_binary(Item#tx.data) -> - case lists:keyfind(<<"Bundle-Format">>, 1, Item#tx.tags) of + case lists:keyfind(<<"bundle-format">>, 1, Item#tx.tags) of {_, _} -> format_data(deserialize(serialize(Item)), Indent); false -> @@ -225,7 +248,7 @@ new_item(Target, Anchor, Tags, Data) -> #tx{ format = ans104, target = Target, - last_tx = Anchor, + anchor = Anchor, tags = Tags, data = Data, data_size = byte_size(Data) @@ -248,11 +271,10 @@ verify_item(DataItem) -> ValidID andalso ValidSignature andalso ValidTags. type(Item) when is_record(Item, tx) -> - lists:keyfind(<<"Bundle-Map">>, 1, Item#tx.tags), - case lists:keyfind(<<"Bundle-Map">>, 1, Item#tx.tags) of - {<<"Bundle-Map">>, _} -> - case lists:keyfind(<<"Map-Format">>, 1, Item#tx.tags) of - {<<"Map-Format">>, <<"List">>} -> list; + case lists:keyfind(<<"bundle-map">>, 1, Item#tx.tags) of + {<<"bundle-map">>, _} -> + case lists:keyfind(<<"map-format">>, 1, Item#tx.tags) of + {<<"map-format">>, <<"list">>} -> list; _ -> map end; _ -> @@ -284,7 +306,7 @@ data_item_signature_data(RawItem, signed) -> utf8_encoded("1"), <<(NormItem#tx.owner)/binary>>, <<(NormItem#tx.target)/binary>>, - <<(NormItem#tx.last_tx)/binary>>, + <<(NormItem#tx.anchor)/binary>>, encode_tags(NormItem#tx.tags), <<(NormItem#tx.data)/binary>> ]). @@ -315,11 +337,17 @@ verify_data_item_tags(DataItem) -> normalize(Item) -> reset_ids(normalize_data(Item)). -%% @doc Ensure that a data item (potentially containing a map or list) has a standard, serialized form. +%% @doc Ensure that a data item (potentially containing a map or list) has a +%% standard, serialized form. normalize_data(not_found) -> throw(not_found); +normalize_data(Item = #tx{data = Bin}) when is_binary(Bin) -> + ?event({normalize_data, binary, Item}), + normalize_data_size(Item); normalize_data(Bundle) when is_list(Bundle); is_map(Bundle) -> + ?event({normalize_data, bundle, Bundle}), normalize_data(#tx{ data = Bundle }); normalize_data(Item = #tx { data = Data }) when is_list(Data) -> + ?event({normalize_data, list, Item}), normalize_data( Item#tx{ tags = add_list_tags(Item#tx.tags), @@ -338,19 +366,19 @@ normalize_data(Item = #tx { data = Data }) when is_list(Data) -> ) } ); -normalize_data(Item = #tx{data = Bin}) when is_binary(Bin) -> - normalize_data_size(Item); normalize_data(Item = #tx{data = Data}) -> + ?event({normalize_data, map, Item}), normalize_data_size( case serialize_bundle_data(Data, Item#tx.manifest) of {Manifest, Bin} -> Item#tx{ data = Bin, manifest = Manifest, - tags = add_manifest_tags( - add_bundle_tags(Item#tx.tags), - id(Manifest, unsigned) - ) + tags = + add_manifest_tags( + add_bundle_tags(Item#tx.tags), + id(Manifest, unsigned) + ) }; DirectBin -> Item#tx{ @@ -378,14 +406,14 @@ serialize(RawTX, binary) -> (TX#tx.signature)/binary, (TX#tx.owner)/binary, (encode_optional_field(TX#tx.target))/binary, - (encode_optional_field(TX#tx.last_tx))/binary, + (encode_optional_field(TX#tx.anchor))/binary, (encode_tags_size(TX#tx.tags, EncodedTags))/binary, EncodedTags/binary, (TX#tx.data)/binary >>; serialize(TX, json) -> true = enforce_valid_tx(TX), - jiffy:encode(item_to_json_struct(TX)). + hb_json:encode(hb_message:convert(TX, <<"ans104@1.0">>, #{})). %% @doc Take an item and ensure that it is of valid form. Useful for ensuring %% that a message is viable for serialization/deserialization before execution. @@ -409,8 +437,8 @@ enforce_valid_tx(TX) -> {invalid_field, unsigned_id, TX#tx.unsigned_id} ), ok_or_throw(TX, - check_size(TX#tx.last_tx, [0, 32]), - {invalid_field, last_tx, TX#tx.last_tx} + check_size(TX#tx.anchor, [0, 32]), + {invalid_field, last_tx, TX#tx.anchor} ), ok_or_throw(TX, check_size(TX#tx.owner, [0, byte_size(?DEFAULT_OWNER)]), @@ -421,9 +449,13 @@ enforce_valid_tx(TX) -> {invalid_field, target, TX#tx.target} ), ok_or_throw(TX, - check_size(TX#tx.signature, [0, byte_size(?DEFAULT_SIG)]), + check_size(TX#tx.signature, [0, 65, byte_size(?DEFAULT_SIG)]), {invalid_field, signature, TX#tx.signature} ), + ok_or_throw(TX, + check_type(TX#tx.tags, list), + {invalid_field, tags, TX#tx.tags} + ), lists:foreach( fun({Name, Value}) -> ok_or_throw(TX, @@ -436,11 +468,11 @@ enforce_valid_tx(TX) -> ), ok_or_throw(TX, check_type(Value, binary), - {invalid_field, tag_value, Value} + {invalid_field, tag_value, {Name, Value}} ), ok_or_throw(TX, check_size(Value, {range, 0, ?MAX_TAG_VALUE_SIZE}), - {invalid_field, tag_value, Value} + {invalid_field, tag_value, {Name, Value}} ); (InvalidTagForm) -> throw({invalid_field, tag, InvalidTagForm}) @@ -461,8 +493,6 @@ check_size(Bin, {range, Start, End}) -> check_type(Bin, binary) andalso byte_size(Bin) >= Start andalso byte_size(Bin) =< End; -check_size(Bin, X) when not is_list(X) -> - check_size(Bin, [X]); check_size(Bin, Sizes) -> check_type(Bin, binary) andalso lists:member(byte_size(Bin), Sizes). @@ -525,11 +555,11 @@ add_list_tags(Tags) -> add_manifest_tags(Tags, ManifestID) -> lists:filter( fun - ({<<"Bundle-Map">>, _}) -> false; + ({<<"bundle-map">>, _}) -> false; (_) -> true end, Tags - ) ++ [{<<"Bundle-Map">>, hb_util:encode(ManifestID)}]. + ) ++ [{<<"bundle-map">>, hb_util:encode(ManifestID)}]. finalize_bundle_data(Processed) -> Length = <<(length(Processed)):256/integer>>, @@ -537,6 +567,10 @@ finalize_bundle_data(Processed) -> Items = <<<> || {_, Data} <- Processed>>, <>. +to_serialized_pair(Item) when is_binary(Item) -> + % Support bundling of bare binary payloads by wrapping them in a TX that + % is explicitly marked as a binary data item. + to_serialized_pair(#tx{ tags = [{<<"ao-type">>, <<"binary">>}], data = Item }); to_serialized_pair(Item) -> % TODO: This is a hack to get the ID of the item. We need to do this because we may not % have the ID in 'item' if it is just a map/list. We need to make this more efficient. @@ -562,22 +596,22 @@ new_manifest(Index) -> TX = normalize(#tx{ format = ans104, tags = [ - {<<"Data-Protocol">>, <<"Bundle-Map">>}, - {<<"Variant">>, <<"0.0.1">>} + {<<"data-protocol">>, <<"bundle-map">>}, + {<<"variant">>, <<"0.0.1">>} ], - data = jiffy:encode(Index) + data = hb_json:encode(Index) }), TX. manifest(Map) when is_map(Map) -> Map; manifest(#tx { manifest = undefined }) -> undefined; manifest(#tx { manifest = ManifestTX }) -> - jiffy:decode(ManifestTX#tx.data, [return_maps]). + hb_json:decode(ManifestTX#tx.data). parse_manifest(Item) when is_record(Item, tx) -> parse_manifest(Item#tx.data); parse_manifest(Bin) -> - jiffy:decode(Bin, [return_maps]). + hb_json:decode(Bin). %% @doc Only RSA 4096 is currently supported. %% Note: the signature type '1' corresponds to RSA 4096 -- but it is is written in @@ -608,7 +642,7 @@ encode_tags([]) -> encode_tags(Tags) -> EncodedBlocks = lists:flatmap( fun({Name, Value}) -> - Res = [encode_avro_string(Name), encode_avro_string(Value)], + Res = [encode_avro_name(Name), encode_avro_value(Value)], case lists:member(error, Res) of true -> throw({cannot_encode_empty_string, Name, Value}); @@ -623,13 +657,22 @@ encode_tags(Tags) -> <>. %% @doc Encode a string for Avro using ZigZag and VInt encoding. -encode_avro_string(<<>>) -> - error; -encode_avro_string(String) -> - StringBytes = unicode:characters_to_binary(String, utf8), +encode_avro_name(<<>>) -> + % Zero length names are treated as a special case, due to the Avro encoder. + << 0 >>; +encode_avro_name(String) -> + StringBytes = utf8_encoded(String), Length = byte_size(StringBytes), <<(encode_zigzag(Length))/binary, StringBytes/binary>>. +encode_avro_value(<<>>) -> + % Zero length values are treated as a special case, due to the Avro encoder. + << 0 >>; +encode_avro_value(Value) when is_binary(Value) -> + % Tag values can be raw binaries + Length = byte_size(Value), + <<(encode_zigzag(Length))/binary, Value/binary>>. + %% @doc Encode an integer using ZigZag encoding. encode_zigzag(Int) when Int >= 0 -> encode_vint(Int bsl 1); @@ -668,7 +711,7 @@ deserialize(Binary, binary) -> signature = Signature, owner = Owner, target = Target, - last_tx = Anchor, + anchor = Anchor, tags = Tags, data = Data, data_size = byte_size(Data) @@ -680,29 +723,26 @@ deserialize(Binary, binary) -> %end; deserialize(Bin, json) -> try - normalize( - maybe_unbundle( - json_struct_to_item(element(1, jiffy:decode(Bin))) - ) - ) + Map = hb_json:decode(Bin), + hb_message:convert(Map, <<"ans104@1.0">>, #{}) catch _:_:_Stack -> {error, invalid_item} end. maybe_unbundle(Item) -> - Format = lists:keyfind(<<"Bundle-Format">>, 1, Item#tx.tags), - Version = lists:keyfind(<<"Bundle-Version">>, 1, Item#tx.tags), + Format = lists:keyfind(<<"bundle-format">>, 1, Item#tx.tags), + Version = lists:keyfind(<<"bundle-version">>, 1, Item#tx.tags), case {Format, Version} of - {{<<"Bundle-Format">>, <<"Binary">>}, {<<"Bundle-Version">>, <<"2.0.0">>}} -> + {{<<"bundle-format">>, <<"binary">>}, {<<"bundle-version">>, <<"2.0.0">>}} -> maybe_map_to_list(maybe_unbundle_map(Item)); _ -> Item end. maybe_map_to_list(Item) -> - case lists:keyfind(<<"Map-Format">>, 1, Item#tx.tags) of - {<<"Map-Format">>, <<"List">>} -> + case lists:keyfind(<<"map-format">>, 1, Item#tx.tags) of + {<<"map-format">>, <<"List">>} -> unbundle_list(Item); _ -> Item @@ -720,13 +760,13 @@ unbundle_list(Item) -> }. maybe_unbundle_map(Bundle) -> - case lists:keyfind(<<"Bundle-Map">>, 1, Bundle#tx.tags) of - {<<"Bundle-Map">>, MapTXID} -> + case lists:keyfind(<<"bundle-map">>, 1, Bundle#tx.tags) of + {<<"bundle-map">>, MapTXID} -> case unbundle(Bundle) of detached -> Bundle#tx { data = detached }; Items -> MapItem = find_single_layer(hb_util:decode(MapTXID), Items), - Map = jiffy:decode(MapItem#tx.data, [return_maps]), + Map = hb_json:decode(MapItem#tx.data), Bundle#tx{ manifest = MapItem, data = @@ -781,90 +821,6 @@ decode_bundle_header(0, ItemsBin, Header) -> decode_bundle_header(Count, <>, Header) -> decode_bundle_header(Count - 1, Rest, [{ID, Size} | Header]). -item_to_json_struct( - #tx{ - id = ID, - last_tx = Last, - owner = Owner, - tags = Tags, - target = Target, - data = Data, - signature = Sig - } -) -> - % Set "From" if From-Process is Tag or set with "Owner" address - From = - case lists:filter(fun({Name, _}) -> Name =:= <<"From-Process">> end, Tags) of - [{_, FromProcess}] -> FromProcess; - [] -> hb_util:encode(ar_wallet:to_address(Owner)) - end, - Fields = [ - {<<"Id">>, hb_util:encode(ID)}, - % NOTE: In Arweave TXs, these are called "last_tx" - {<<"Anchor">>, hb_util:encode(Last)}, - % NOTE: When sent to ao "Owner" is the wallet address - {<<"Owner">>, hb_util:encode(ar_wallet:to_address(Owner))}, - {<<"From">>, From}, - {<<"Tags">>, - lists:map( - fun({Name, Value}) -> - { - [ - {name, maybe_list_to_binary(Name)}, - {value, maybe_list_to_binary(Value)} - ] - } - end, - Tags - )}, - {<<"Target">>, hb_util:encode(Target)}, - {<<"Data">>, Data}, - {<<"Signature">>, hb_util:encode(Sig)} - ], - {Fields}. - -maybe_list_to_binary(List) when is_list(List) -> - list_to_binary(List); -maybe_list_to_binary(Bin) -> - Bin. - -json_struct_to_item(Map) when is_map(Map) -> - deserialize(jiffy:encode(Map), json); -json_struct_to_item({TXStruct}) -> - json_struct_to_item(TXStruct); -json_struct_to_item(RawTXStruct) -> - TXStruct = [{string:lowercase(FieldName), Value} || {FieldName, Value} <- RawTXStruct], - Tags = - case hb_util:find_value(<<"tags">>, TXStruct) of - undefined -> - []; - Xs -> - Xs - end, - TXID = hb_util:decode(hb_util:find_value(<<"id">>, TXStruct, hb_util:encode(?DEFAULT_ID))), - #tx{ - format = ans104, - id = TXID, - last_tx = hb_util:decode(hb_util:find_value(<<"anchor">>, TXStruct, <<>>)), - owner = hb_util:decode( - hb_util:find_value(<<"owner">>, TXStruct, hb_util:encode(?DEFAULT_OWNER)) - ), - tags = - lists:map( - fun({KeyVals}) -> - {_, Name} = lists:keyfind(<<"name">>, 1, KeyVals), - {_, Value} = lists:keyfind(<<"value">>, 1, KeyVals), - {Name, Value} - end, - Tags - ), - target = hb_util:decode(hb_util:find_value(<<"target">>, TXStruct, <<>>)), - data = hb_util:find_value(<<"data">>, TXStruct, <<>>), - signature = hb_util:decode( - hb_util:find_value(<<"signature">>, TXStruct, hb_util:encode(?DEFAULT_SIG)) - ) - }. - %% @doc Decode the signature from a binary format. Only RSA 4096 is currently supported. %% Note: the signature type '1' corresponds to RSA 4096 - but it is is written in %% little-endian format which is why we match on `<<1, 0>>'. @@ -905,8 +861,9 @@ decode_avro_name(NameSize, Rest, Count) -> {ValueSize, Rest3} = decode_zigzag(Rest2), decode_avro_value(ValueSize, Name, Rest3, Count). -decode_avro_value(0, _, Rest, _) -> - {[], Rest}; +decode_avro_value(0, Name, Rest, Count) -> + {DecodedTags, NonAvroRest} = decode_avro_tags(Rest, Count - 1), + {[{Name, <<>>} | DecodedTags], NonAvroRest}; decode_avro_value(ValueSize, Name, Rest, Count) -> <> = Rest, {DecodedTags, NonAvroRest} = decode_avro_tags(Rest2, Count - 1), @@ -940,7 +897,7 @@ ar_bundles_test_() -> [ {timeout, 30, fun test_no_tags/0}, {timeout, 30, fun test_with_tags/0}, - {timeout, 30, fun test_bundle_with_zero_length_tag/0}, + {timeout, 30, fun test_with_zero_length_tag/0}, {timeout, 30, fun test_unsigned_data_item_id/0}, {timeout, 30, fun test_unsigned_data_item_normalization/0}, {timeout, 30, fun test_empty_bundle/0}, @@ -951,9 +908,48 @@ ar_bundles_test_() -> {timeout, 30, fun test_basic_member_id/0}, {timeout, 30, fun test_deep_member/0}, {timeout, 30, fun test_extremely_large_bundle/0}, - {timeout, 30, fun test_serialize_deserialize_deep_signed_bundle/0} + {timeout, 30, fun test_serialize_deserialize_deep_signed_bundle/0}, + {timeout, 30, fun test_encode_tags/0} ]. +test_encode_tags() -> + BinValue = <<1, 2, 3, 255, 254>>, + TestCases = [ + {simple_string_tags, [{<<"tag1">>, <<"value1">>}]}, + {binary_value_tag, [{<<"binary-tag">>, BinValue}]}, + {mixed_tags, + [ + {<<"string-tag">>, <<"string-value">>}, + {<<"binary-tag">>, BinValue} + ] + }, + {empty_value_tag, [{<<"empty-value-tag">>, <<>>}]}, + {unicode_tag, [{<<"unicode-tag">>, <<"你好世界">>}]} + ], + lists:foreach( + fun({Label, InputTags}) -> + Encoded = encode_tags(InputTags), + Wrapped = + << + (length(InputTags)):64/little, + (byte_size(Encoded)):64/little, + Encoded/binary + >>, + {DecodedTags, <<>>} = decode_tags(Wrapped), + ?assertEqual(InputTags, DecodedTags, Label) + end, + TestCases + ), + % Test case: Empty tags list + EmptyTags = [], + EncodedEmpty = encode_tags(EmptyTags), + ?assertEqual(<<>>, EncodedEmpty), + WrappedEmpty = <<0:64/little, 0:64/little>>, + {[], <<>>} = decode_tags(WrappedEmpty). + +run_test() -> + test_with_zero_length_tag(). + test_no_tags() -> {Priv, Pub} = ar_wallet:new(), {KeyType, Owner} = Pub, @@ -961,12 +957,9 @@ test_no_tags() -> Anchor = crypto:strong_rand_bytes(32), DataItem = new_item(Target, Anchor, [], <<"data">>), SignedDataItem = sign_item(DataItem, {Priv, Pub}), - ?assertEqual(true, verify_item(SignedDataItem)), assert_data_item(KeyType, Owner, Target, Anchor, [], <<"data">>, SignedDataItem), - SignedDataItem2 = deserialize(serialize(SignedDataItem)), - ?assertEqual(SignedDataItem, SignedDataItem2), ?assertEqual(true, verify_item(SignedDataItem2)), assert_data_item(KeyType, Owner, Target, Anchor, [], <<"data">>, SignedDataItem2). @@ -979,23 +972,26 @@ test_with_tags() -> Tags = [{<<"tag1">>, <<"value1">>}, {<<"tag2">>, <<"value2">>}], DataItem = new_item(Target, Anchor, Tags, <<"taggeddata">>), SignedDataItem = sign_item(DataItem, {Priv, Pub}), - ?assertEqual(true, verify_item(SignedDataItem)), assert_data_item(KeyType, Owner, Target, Anchor, Tags, <<"taggeddata">>, SignedDataItem), - SignedDataItem2 = deserialize(serialize(SignedDataItem)), - ?assertEqual(SignedDataItem, SignedDataItem2), ?assertEqual(true, verify_item(SignedDataItem2)), assert_data_item(KeyType, Owner, Target, Anchor, Tags, <<"taggeddata">>, SignedDataItem2). -test_bundle_with_zero_length_tag() -> - Item = #tx{ +test_with_zero_length_tag() -> + Item = normalize(#tx{ format = ans104, - tags = [{<<"tag1">>, <<"">>}], - data = <<"data">> - }, - ?assertThrow({cannot_encode_empty_string, <<"tag1">>, <<>>}, serialize([Item])). + tags = [ + {<<"normal-tag-1">>, <<"tag1">>}, + {<<"empty-tag">>, <<>>}, + {<<"normal-tag-2">>, <<"tag2">>} + ], + data = <<"Typical data field.">> + }), + Serialized = serialize(Item), + Deserialized = deserialize(Serialized), + ?assertEqual(Item, Deserialized). test_unsigned_data_item_id() -> Item1 = deserialize( @@ -1014,7 +1010,7 @@ assert_data_item(KeyType, Owner, Target, Anchor, Tags, Data, DataItem) -> ?assertEqual(KeyType, DataItem#tx.signature_type), ?assertEqual(Owner, DataItem#tx.owner), ?assertEqual(Target, DataItem#tx.target), - ?assertEqual(Anchor, DataItem#tx.last_tx), + ?assertEqual(Anchor, DataItem#tx.anchor), ?assertEqual(Tags, DataItem#tx.tags), ?assertEqual(Data, DataItem#tx.data), ?assertEqual(byte_size(Data), DataItem#tx.data_size). @@ -1022,18 +1018,21 @@ assert_data_item(KeyType, Owner, Target, Anchor, Tags, Data, DataItem) -> test_empty_bundle() -> Bundle = serialize([]), BundleItem = deserialize(Bundle), - ?assertEqual([], BundleItem#tx.data). + ?assertEqual(#{}, BundleItem#tx.data). test_bundle_with_one_item() -> Item = new_item( crypto:strong_rand_bytes(32), crypto:strong_rand_bytes(32), [], - ItemData = crypto:strong_rand_bytes(32) + ItemData = crypto:strong_rand_bytes(1000) ), + ?event({item, Item}), Bundle = serialize([Item]), + ?event({bundle, Bundle}), BundleItem = deserialize(Bundle), - ?assertEqual(ItemData, (erlang:hd(BundleItem#tx.data))#tx.data). + ?event({bundle_item, BundleItem}), + ?assertEqual(ItemData, (maps:get(<<"1">>, BundleItem#tx.data))#tx.data). test_bundle_with_two_items() -> Item1 = new_item( @@ -1050,31 +1049,31 @@ test_bundle_with_two_items() -> ), Bundle = serialize([Item1, Item2]), BundleItem = deserialize(Bundle), - ?assertEqual(ItemData1, (erlang:hd(BundleItem#tx.data))#tx.data), - ?assertEqual(ItemData2, (erlang:hd(tl(BundleItem#tx.data)))#tx.data). + ?assertEqual(ItemData1, (maps:get(<<"1">>, BundleItem#tx.data))#tx.data), + ?assertEqual(ItemData2, (maps:get(<<"2">>, BundleItem#tx.data))#tx.data). test_recursive_bundle() -> W = ar_wallet:new(), Item1 = sign_item(#tx{ id = crypto:strong_rand_bytes(32), - last_tx = crypto:strong_rand_bytes(32), + anchor = crypto:strong_rand_bytes(32), data = <<1:256/integer>> }, W), Item2 = sign_item(#tx{ id = crypto:strong_rand_bytes(32), - last_tx = crypto:strong_rand_bytes(32), + anchor = crypto:strong_rand_bytes(32), data = [Item1] }, W), Item3 = sign_item(#tx{ id = crypto:strong_rand_bytes(32), - last_tx = crypto:strong_rand_bytes(32), + anchor = crypto:strong_rand_bytes(32), data = [Item2] }, W), Bundle = serialize([Item3]), BundleItem = deserialize(Bundle), - [UnbundledItem3] = BundleItem#tx.data, - [UnbundledItem2] = UnbundledItem3#tx.data, - [UnbundledItem1] = UnbundledItem2#tx.data, + #{<<"1">> := UnbundledItem3} = BundleItem#tx.data, + #{<<"1">> := UnbundledItem2} = UnbundledItem3#tx.data, + #{<<"1">> := UnbundledItem1} = UnbundledItem2#tx.data, ?assert(verify_item(UnbundledItem1)), % TODO: Verify bundled lists... ?assertEqual(Item1#tx.data, UnbundledItem1#tx.data). @@ -1087,7 +1086,7 @@ test_bundle_map() -> }, W), Item2 = sign_item(#tx{ format = ans104, - last_tx = crypto:strong_rand_bytes(32), + anchor = crypto:strong_rand_bytes(32), data = #{<<"key1">> => Item1} }, W), Bundle = serialize(Item2), @@ -1144,7 +1143,6 @@ test_deep_member() -> ?assertEqual(false, member(crypto:strong_rand_bytes(32), Item2)). test_serialize_deserialize_deep_signed_bundle() -> - application:ensure_all_started(hb), W = ar_wallet:new(), % Test that we can serialize, deserialize, and get the same IDs back. Item1 = sign_item(#tx{data = <<"item1_data">>}, W), @@ -1161,14 +1159,4 @@ test_serialize_deserialize_deep_signed_bundle() -> % Test that we can sign an item twice and the unsigned ID is the same. Item3 = sign_item(Item2, W), ?assertEqual(id(Item3, unsigned), id(Item2, unsigned)), - ?assert(verify_item(Item3)). - % Test that we can write to disk and read back the same ID. - % hb_cache:write(hb_message:convert(Item2, converge, tx, #{}), #{}), - % {ok, MsgFromDisk} = hb_cache:read(hb_util:encode(id(Item2, unsigned)), #{}), - % FromDisk = hb_message:convert(MsgFromDisk, tx, converge, #{}), - % format(FromDisk), - % ?assertEqual(id(Item2, signed), id(FromDisk, signed)), - % % Test that normalizing the item and signing it again yields the same unsigned ID. - % NormItem2 = normalize(Item2), - % SignedNormItem2 = sign_item(NormItem2, W), - % ?assertEqual(id(SignedNormItem2, unsigned), id(Item2, unsigned)). \ No newline at end of file + ?assert(verify_item(Item3)). \ No newline at end of file diff --git a/src/ar_http.erl b/src/ar_http.erl deleted file mode 100644 index 7ccf95d33..000000000 --- a/src/ar_http.erl +++ /dev/null @@ -1,502 +0,0 @@ -%%% @doc A wrapper library for gun. --module(ar_http). --behaviour(gen_server). --include("include/hb.hrl"). --export([start_link/1, req/2]). --export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]). - --record(state, { - pid_by_peer = #{}, - status_by_pid = #{}, - opts = #{} -}). - -%%% ================================================================== -%%% Public interface. -%%% ================================================================== - -start_link(Opts) -> - ?event(debug, {start_link, Opts}), - gen_server:start_link({local, ?MODULE}, ?MODULE, Opts, []). - -req(Args, Opts) -> req(Args, false, Opts). -req(Args, ReestablishedConnection, Opts) -> - StartTime = erlang:monotonic_time(), - #{ peer := Peer, path := Path, method := Method } = Args, - Response = - case catch gen_server:call(?MODULE, {get_connection, Args}, infinity) of - {ok, PID} -> - ar_rate_limiter:throttle(Peer, Path, Opts), - case request(PID, Args, Opts) of - {error, Error} when Error == {shutdown, normal}; - Error == noproc -> - case ReestablishedConnection of - true -> - {error, client_error}; - false -> - req(Args, true, Opts) - end; - Reply -> - Reply - end; - {'EXIT', _} -> - {error, client_error}; - Error -> - Error - end, - EndTime = erlang:monotonic_time(), - %% Only log the metric for the top-level call to req/2 - not the recursive call - %% that happens when the connection is reestablished. - case ReestablishedConnection of - true -> - ok; - false -> - prometheus_histogram:observe(http_request_duration_seconds, [ - method_to_list(Method), - Path, - get_status_class(Response) - ], EndTime - StartTime) - end, - Response. -%%% ================================================================== -%%% gen_server callbacks. -%%% ================================================================== - -init(Opts) -> - prometheus_counter:new([ - {name, gun_requests_total}, - {labels, [http_method, route, status_class]}, - { - help, - "The total number of GUN requests." - } - ]), - prometheus_gauge:new([{name, outbound_connections}, - {help, "The current number of the open outbound network connections"}]), - prometheus_histogram:new([ - {name, http_request_duration_seconds}, - {buckets, [0.01, 0.1, 0.5, 1, 5, 10, 30, 60]}, - {labels, [http_method, route, status_class]}, - { - help, - "The total duration of an ar_http:req call. This includes more than" - " just the GUN request itself (e.g. establishing a connection, " - "throttling, etc...)" - } - ]), - prometheus_histogram:new([ - {name, http_client_get_chunk_duration_seconds}, - {buckets, [0.1, 1, 10, 60]}, - {labels, [status_class, peer]}, - { - help, - "The total duration of an HTTP GET chunk request made to a peer." - } - ]), - prometheus_counter:new([ - {name, http_client_downloaded_bytes_total}, - {help, "The total amount of bytes requested via HTTP, per remote endpoint"}, - {labels, [route]} - ]), - prometheus_counter:new([ - {name, http_client_uploaded_bytes_total}, - {help, "The total amount of bytes posted via HTTP, per remote endpoint"}, - {labels, [route]} - ]), - ?event(debug, started), - {ok, #state{ opts = Opts }}. - -handle_call({get_connection, Args}, From, - #state{ pid_by_peer = PIDPeer, status_by_pid = StatusByPID } = State) -> - Peer = maps:get(peer, Args), - case maps:get(Peer, PIDPeer, not_found) of - not_found -> - {ok, PID} = open_connection(Args, State#state.opts), - MonitorRef = monitor(process, PID), - PIDPeer2 = maps:put(Peer, PID, PIDPeer), - StatusByPID2 = - maps:put( - PID, - {{connecting, [{From, Args}]}, MonitorRef, Peer}, - StatusByPID - ), - { - noreply, - State#state{ - pid_by_peer = PIDPeer2, - status_by_pid = StatusByPID2 - } - }; - PID -> - case maps:get(PID, StatusByPID) of - {{connecting, PendingRequests}, MonitorRef, Peer} -> - StatusByPID2 = - maps:put(PID, - { - {connecting, [{From, Args} | PendingRequests]}, - MonitorRef, - Peer - }, - StatusByPID - ), - {noreply, State#state{ status_by_pid = StatusByPID2 }}; - {connected, _MonitorRef, Peer} -> - {reply, {ok, PID}, State} - end - end; - -handle_call(Request, _From, State) -> - ?event(warning, {unhandled_call, {module, ?MODULE}, {request, Request}}), - {reply, ok, State}. - -handle_cast(Cast, State) -> - ?event(warning, {unhandled_cast, {module, ?MODULE}, {cast, Cast}}), - {noreply, State}. - -handle_info({gun_up, PID, _Protocol}, #state{ status_by_pid = StatusByPID } = State) -> - case maps:get(PID, StatusByPID, not_found) of - not_found -> - %% A connection timeout should have occurred. - {noreply, State}; - {{connecting, PendingRequests}, MonitorRef, Peer} -> - [gen_server:reply(ReplyTo, {ok, PID}) || {ReplyTo, _} <- PendingRequests], - StatusByPID2 = maps:put(PID, {connected, MonitorRef, Peer}, StatusByPID), - prometheus_gauge:inc(outbound_connections), - {noreply, State#state{ status_by_pid = StatusByPID2 }}; - {connected, _MonitorRef, Peer} -> - ?event(warning, - {gun_up_pid_already_exists, {peer, ar_util:format_peer(Peer)}}), - {noreply, State} - end; - -handle_info({gun_error, PID, Reason}, - #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> - case maps:get(PID, StatusByPID, not_found) of - not_found -> - ?event(warning, {gun_connection_error_with_unknown_pid}), - {noreply, State}; - {Status, _MonitorRef, Peer} -> - PIDByPeer2 = maps:remove(Peer, PIDByPeer), - StatusByPID2 = maps:remove(PID, StatusByPID), - Reason2 = - case Reason of - timeout -> - connect_timeout; - {Type, _} -> - Type; - _ -> - Reason - end, - case Status of - {connecting, PendingRequests} -> - reply_error(PendingRequests, Reason2); - connected -> - prometheus_gauge:dec(outbound_connections), - ok - end, - gun:shutdown(PID), - ?event(debug, {connection_error, {reason, Reason}}), - {noreply, State#state{ status_by_pid = StatusByPID2, pid_by_peer = PIDByPeer2 }} - end; - -handle_info({gun_down, PID, Protocol, Reason, _KilledStreams, _UnprocessedStreams}, - #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> - case maps:get(PID, StatusByPID, not_found) of - not_found -> - ?event(warning, - {gun_connection_down_with_unknown_pid, {protocol, Protocol}}), - {noreply, State}; - {Status, _MonitorRef, Peer} -> - PIDByPeer2 = maps:remove(Peer, PIDByPeer), - StatusByPID2 = maps:remove(PID, StatusByPID), - Reason2 = - case Reason of - {Type, _} -> - Type; - _ -> - Reason - end, - case Status of - {connecting, PendingRequests} -> - reply_error(PendingRequests, Reason2); - _ -> - prometheus_gauge:dec(outbound_connections), - ok - end, - {noreply, - State#state{ - status_by_pid = StatusByPID2, - pid_by_peer = PIDByPeer2 - } - } - end; - -handle_info({'DOWN', _Ref, process, PID, Reason}, - #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> - case maps:get(PID, StatusByPID, not_found) of - not_found -> - {noreply, State}; - {Status, _MonitorRef, Peer} -> - PIDByPeer2 = maps:remove(Peer, PIDByPeer), - StatusByPID2 = maps:remove(PID, StatusByPID), - case Status of - {connecting, PendingRequests} -> - reply_error(PendingRequests, Reason); - _ -> - prometheus_gauge:dec(outbound_connections), - ok - end, - {noreply, - State#state{ - status_by_pid = StatusByPID2, - pid_by_peer = PIDByPeer2 - } - } - end; - -handle_info(Message, State) -> - ?event(warning, {unhandled_info, {module, ?MODULE}, {message, Message}}), - {noreply, State}. - -terminate(Reason, #state{ status_by_pid = StatusByPID }) -> - ?event(info,{http_client_terminating, {reason, Reason}}), - maps:map(fun(PID, _Status) -> gun:shutdown(PID) end, StatusByPID), - ok. - -%%% ================================================================== -%%% Private functions. -%%% ================================================================== - -open_connection(#{ peer := Peer }, Opts) -> - {Host, Port} = parse_peer(Peer, Opts), - ConnectTimeout = - hb_opts:get(http_connect_timeout, no_connect_timeout, Opts), - BaseGunOpts = - #{ - http_opts => - #{ - keepalive => - hb_opts:get( - http_keepalive, - no_keepalive_timeout, - Opts - ) - }, - retry => 0, - connect_timeout => ConnectTimeout - }, - GunOpts = - case Proto = hb_opts:get(protocol, no_proto, Opts) of - http3 -> BaseGunOpts#{protocols => [http3], transport => quic}; - http2 -> BaseGunOpts#{protocols => [http2], transport => tcp}; - http1 -> BaseGunOpts#{protocols => [http], transport => tcp} - end, - ?event(http, {gun_open, {host, Host}, {port, Port}, {protocol, Proto}}), - gun:open(Host, Port, GunOpts). - -parse_peer(Peer, Opts) -> - case binary:split(Peer, <<":">>, [global]) of - [_Protocol, <<"//", Host/binary>>, Port] -> - { - binary_to_list(Host), - parse_port(Port) - }; - [Host, Port] -> - {binary_to_list(Host), parse_port(Port)}; - [Host] -> - {binary_to_list(Host), hb_opts:get(http_port, 443, Opts)} - end. - -parse_port(Port) -> - list_to_integer(hb_util:remove_trailing_noise(binary_to_list(Port), "/")). - -reply_error([], _Reason) -> - ok; -reply_error([PendingRequest | PendingRequests], Reason) -> - ReplyTo = element(1, PendingRequest), - Args = element(2, PendingRequest), - Method = maps:get(method, Args), - Path = maps:get(path, Args), - record_response_status(Method, Path, {error, Reason}), - gen_server:reply(ReplyTo, {error, Reason}), - reply_error(PendingRequests, Reason). - -record_response_status(Method, Path, Response) -> - prometheus_counter:inc(gun_requests_total, - [ - method_to_list(Method), - Path, - get_status_class(Response) - ] - ). - -method_to_list(get) -> - "GET"; -method_to_list(post) -> - "POST"; -method_to_list(put) -> - "PUT"; -method_to_list(head) -> - "HEAD"; -method_to_list(delete) -> - "DELETE"; -method_to_list(connect) -> - "CONNECT"; -method_to_list(options) -> - "OPTIONS"; -method_to_list(trace) -> - "TRACE"; -method_to_list(patch) -> - "PATCH"; -method_to_list(_) -> - "unknown". - -request(PID, Args, Opts) -> - Timer = - inet:start_timer( - hb_opts:get(http_request_send_timeout, no_request_send_timeout, Opts) - ), - Method = maps:get(method, Args), - Path = maps:get(path, Args), - Headers = maps:get(headers, Args, []), - Body = maps:get(body, Args, <<>>), - ?event(http, {gun_request, {method, Method}, {path, Path}, {headers, Headers}, {body, Body}}), - Ref = gun:request(PID, Method, Path, Headers, Body), - ResponseArgs = - #{ - pid => PID, stream_ref => Ref, - timer => Timer, limit => maps:get(limit, Args, infinity), - counter => 0, acc => [], start => os:system_time(microsecond), - is_peer_request => maps:get(is_peer_request, Args, true) - }, - Response = await_response(maps:merge(Args, ResponseArgs), Opts), - record_response_status(Method, Path, Response), - inet:stop_timer(Timer), - Response. - -await_response(Args, Opts) -> - #{ pid := PID, stream_ref := Ref, timer := Timer, start := Start, limit := Limit, - counter := Counter, acc := Acc, method := Method, path := Path } = Args, - case gun:await(PID, Ref, inet:timeout(Timer)) of - {response, fin, Status, Headers} -> - End = os:system_time(microsecond), - upload_metric(Args), - {ok, {{integer_to_binary(Status), <<>>}, Headers, <<>>, Start, End}}; - {response, nofin, Status, Headers} -> - await_response(Args#{ status => Status, headers => Headers }, Opts); - {data, nofin, Data} -> - case Limit of - infinity -> - await_response(Args#{ acc := [Acc | Data] }, Opts); - Limit -> - Counter2 = size(Data) + Counter, - case Limit >= Counter2 of - true -> - await_response( - Args#{ - counter := Counter2, - acc := [Acc | Data] - }, - Opts - ); - false -> - log(err, http_fetched_too_much_data, Args, - <<"Fetched too much data">>, Opts), - {error, too_much_data} - end - end; - {data, fin, Data} -> - End = os:system_time(microsecond), - FinData = iolist_to_binary([Acc | Data]), - download_metric(FinData, Args), - upload_metric(Args), - {ok, - maps:get(status, Args), - maps:get(headers, Args), - FinData - }; - {error, timeout} = Response -> - record_response_status(Method, Path, Response), - gun:cancel(PID, Ref), - log(warn, gun_await_process_down, Args, Response, Opts), - Response; - {error, Reason} = Response when is_tuple(Reason) -> - record_response_status(Method, Path, Response), - log(warn, gun_await_process_down, Args, Reason, Opts), - Response; - Response -> - record_response_status(Method, Path, Response), - log(warn, gun_await_unknown, Args, Response, Opts), - Response - end. - -log(Type, Event, #{method := Method, peer := Peer, path := Path}, Reason, Opts) -> - ?event( - http, - {gun_log, - {type, Type}, - {event, Event}, - {method, Method}, - {peer, Peer}, - {path, Path}, - {reason, Reason} - }, - Opts - ), - ok. - -download_metric(Data, #{path := Path}) -> - prometheus_counter:inc( - http_client_downloaded_bytes_total, - [Path], - byte_size(Data) - ). - -upload_metric(#{method := post, path := Path, body := Body}) -> - prometheus_counter:inc( - http_client_uploaded_bytes_total, - [Path], - byte_size(Body) - ); -upload_metric(_) -> - ok. - -% @doc Return the HTTP status class label for cowboy_requests_total and -% gun_requests_total metrics. -get_status_class({ok, {{Status, _}, _, _, _, _}}) -> - get_status_class(Status); -get_status_class({error, connection_closed}) -> - "connection_closed"; -get_status_class({error, connect_timeout}) -> - "connect_timeout"; -get_status_class({error, timeout}) -> - "timeout"; -get_status_class({error,{shutdown,timeout}}) -> - "shutdown_timeout"; -get_status_class({error, econnrefused}) -> - "econnrefused"; -get_status_class({error, {shutdown,econnrefused}}) -> - "shutdown_econnrefused"; -get_status_class({error, {shutdown,ehostunreach}}) -> - "shutdown_ehostunreach"; -get_status_class({error, {shutdown,normal}}) -> - "shutdown_normal"; -get_status_class({error, {closed,_}}) -> - "closed"; -get_status_class({error, noproc}) -> - "noproc"; -get_status_class(208) -> - "already_processed"; -get_status_class(Data) when is_integer(Data), Data > 0 -> - prometheus_http:status_class(Data); -get_status_class(Data) when is_binary(Data) -> - case catch binary_to_integer(Data) of - {_, _} -> - "unknown"; - Status -> - get_status_class(Status) - end; -get_status_class(Data) when is_atom(Data) -> - atom_to_list(Data); -get_status_class(_) -> - "unknown". \ No newline at end of file diff --git a/src/ar_rate_limiter.erl b/src/ar_rate_limiter.erl index 4f097d093..81781e81c 100644 --- a/src/ar_rate_limiter.erl +++ b/src/ar_rate_limiter.erl @@ -74,10 +74,10 @@ handle_cast({throttle, Peer, Path, From}, State) -> #state{ traces = Traces, opts = Opts } = State, {Type, Limit} = hb_opts:get(throttle_rpm_by_path, Path, Opts), Now = os:system_time(millisecond), - case maps:get({Peer, Type}, Traces, not_found) of + case hb_maps:get({Peer, Type}, Traces, not_found, Opts) of not_found -> gen_server:reply(From, ok), - Traces2 = maps:put({Peer, Type}, {1, queue:from_list([Now])}, Traces), + Traces2 = hb_maps:put({Peer, Type}, {1, queue:from_list([Now])}, Traces, Opts), {noreply, State#state{ traces = Traces2 }}; {N, Trace} -> {N2, Trace2} = cut_trace(N, queue:in(Now, Trace), Now, Opts), @@ -87,7 +87,7 @@ handle_cast({throttle, Peer, Path, From}, State) -> %% Try to approach but not hit the limit. case N2 + 1 > max(1, HalfLimit * 80 / 100) of true -> - ?event(debug, + ?event( {approaching_peer_rpm_limit, {peer, Peer}, {path, Path}, @@ -103,7 +103,7 @@ handle_cast({throttle, Peer, Path, From}, State) -> {noreply, State}; false -> gen_server:reply(From, ok), - Traces2 = maps:put({Peer, Type}, {N2 + 1, Trace2}, Traces), + Traces2 = hb_maps:put({Peer, Type}, {N2 + 1, Trace2}, Traces, Opts), {noreply, State#state{ traces = Traces2 }} end end; diff --git a/src/ar_timestamp.erl b/src/ar_timestamp.erl index 1776c23b9..fcf3f8d11 100644 --- a/src/ar_timestamp.erl +++ b/src/ar_timestamp.erl @@ -57,10 +57,6 @@ cache(Current) -> %% @doc Refresh the timestamp cache periodically. refresher(TSServer) -> timer:sleep(?TIMEOUT), - TS = - case hb_opts:get(mode) of - debug -> { 0, 0, << 0:256 >> }; - prod -> hb_client:arweave_timestamp() - end, + TS = hb_client:arweave_timestamp(), TSServer ! {refresh, TS}, refresher(TSServer). \ No newline at end of file diff --git a/src/ar_tx.erl b/src/ar_tx.erl index 8b77d2410..7269c0107 100644 --- a/src/ar_tx.erl +++ b/src/ar_tx.erl @@ -14,7 +14,7 @@ new(Dest, Reward, Qty, Last) -> #tx{ id = crypto:strong_rand_bytes(32), - last_tx = Last, + anchor = Last, quantity = Qty, target = Dest, data = <<>>, @@ -25,7 +25,7 @@ new(Dest, Reward, Qty, Last) -> new(Dest, Reward, Qty, Last, SigType) -> #tx{ id = crypto:strong_rand_bytes(32), - last_tx = Last, + anchor = Last, quantity = Qty, target = Dest, data = <<>>, @@ -61,15 +61,13 @@ signature_data_segment(TX) -> << (TX#tx.target)/binary >>, << (list_to_binary(integer_to_list(TX#tx.quantity)))/binary >>, << (list_to_binary(integer_to_list(TX#tx.reward)))/binary >>, - << (TX#tx.last_tx)/binary >>, + << (TX#tx.anchor)/binary >>, << (integer_to_binary(TX#tx.data_size))/binary >>, << (TX#tx.data_root)/binary >> ], ar_deep_hash:hash(List). %% @doc Verify the transaction's signature. -verify_signature(_TX, do_not_verify_signature) -> - true; verify_signature(TX = #tx{ signature_type = SigType }, verify_signature) -> SignatureDataSegment = signature_data_segment(TX), ar_wallet:verify({SigType, TX#tx.owner}, SignatureDataSegment, TX#tx.signature). @@ -133,7 +131,7 @@ json_struct_to_tx(TXStruct) -> #tx{ format = Format, id = TXID, - last_tx = hb_util:decode(hb_util:find_value(<<"last_tx">>, TXStruct)), + anchor = hb_util:decode(hb_util:find_value(<<"anchor">>, TXStruct)), owner = hb_util:decode(hb_util:find_value(<<"owner">>, TXStruct)), tags = [{hb_util:decode(Name), hb_util:decode(Value)} %% Only the elements matching this pattern are included in the list. @@ -156,7 +154,7 @@ tx_to_json_struct( #tx{ id = ID, format = Format, - last_tx = Last, + anchor = Anchor, owner = Owner, tags = Tags, target = Target, @@ -177,7 +175,7 @@ tx_to_json_struct( Format end}, {id, hb_util:encode(ID)}, - {last_tx, hb_util:encode(Last)}, + {anchor, hb_util:encode(Anchor)}, {owner, hb_util:encode(Owner)}, {tags, lists:map( @@ -208,4 +206,4 @@ tx_to_json_struct( false -> Fields end, - maps:from_list(Fields2). \ No newline at end of file + hb_maps:from_list(Fields2). \ No newline at end of file diff --git a/src/ar_wallet.erl b/src/ar_wallet.erl index c1b6e3324..4bff3d6f7 100644 --- a/src/ar_wallet.erl +++ b/src/ar_wallet.erl @@ -1,6 +1,8 @@ -module(ar_wallet). --export([sign/2, sign/3, hmac/1, hmac/2, verify/3, verify/4, to_address/1, to_address/2, new/0, new/1]). --export([new_keyfile/2, load_keyfile/1, load_key/1]). +-export([sign/2, sign/3, hmac/1, hmac/2, verify/3, verify/4]). +-export([to_pubkey/1, to_pubkey/2, to_address/1, to_address/2, new/0, new/1]). +-export([new_keyfile/2, load_keyfile/1, load_keyfile/2, load_key/1, load_key/2]). +-export([to_json/1, from_json/1, from_json/2]). -include("include/ar.hrl"). -include_lib("public_key/include/public_key.hrl"). @@ -17,6 +19,7 @@ new(KeyType = {KeyAlg, PublicExpnt}) when KeyType =:= {rsa, 65537} -> = crypto:generate_key(KeyAlg, {4096, PublicExpnt}), {{KeyType, Priv, Pub}, {KeyType, Pub}}. + %% @doc Sign some data with a private key. sign(Key, Data) -> sign(Key, Data, sha256). @@ -32,7 +35,9 @@ sign({{rsa, PublicExpnt}, Priv, Pub}, Data, DigestType) when PublicExpnt =:= 655 modulus = binary:decode_unsigned(Pub), privateExponent = binary:decode_unsigned(Priv) } - ). + ); +sign({{KeyType, Priv, Pub}, {KeyType, Pub}}, Data, DigestType) -> + sign({KeyType, Priv, Pub}, Data, DigestType). hmac(Data) -> hmac(Data, sha256). @@ -54,17 +59,29 @@ verify({{rsa, PublicExpnt}, Pub}, Data, Sig, DigestType) when PublicExpnt =:= 65 } ). +%% @doc Find a public key from a wallet. +to_pubkey(Pubkey) -> + to_pubkey(Pubkey, ?DEFAULT_KEY_TYPE). +to_pubkey(PubKey, {rsa, 65537}) when bit_size(PubKey) == 256 -> + % Small keys are not secure, nobody is using them, the clause + % is for backwards-compatibility. + PubKey; +to_pubkey({{_, _, PubKey}, {_, PubKey}}, {rsa, 65537}) -> + PubKey; +to_pubkey(PubKey, {rsa, 65537}) -> + PubKey. + %% @doc Generate an address from a public key. to_address(Pubkey) -> to_address(Pubkey, ?DEFAULT_KEY_TYPE). to_address(PubKey, {rsa, 65537}) when bit_size(PubKey) == 256 -> - %% Small keys are not secure, nobody is using them, the clause - %% is for backwards-compatibility. PubKey; -to_address({{_, _, PubKey}, {_, PubKey}}, {rsa, 65537}) -> +to_address({{_, _, PubKey}, {_, PubKey}}, _) -> to_address(PubKey); to_address(PubKey, {rsa, 65537}) -> - to_rsa_address(PubKey). + to_rsa_address(PubKey); +to_address(PubKey, {ecdsa, 256}) -> + to_ecdsa_address(PubKey). %% @doc Generate a new wallet public and private key, with a corresponding keyfile. %% The provided key is used as part of the file name. @@ -76,56 +93,19 @@ new_keyfile(KeyType, WalletName) -> {?RSA_SIGN_ALG, PublicExpnt} -> {[Expnt, Pb], [Expnt, Pb, Prv, P1, P2, E1, E2, C]} = crypto:generate_key(rsa, {?RSA_PRIV_KEY_SZ, PublicExpnt}), - Ky = - jiffy:encode( - { - [ - {kty, <<"RSA">>}, - {ext, true}, - {e, hb_util:encode(Expnt)}, - {n, hb_util:encode(Pb)}, - {d, hb_util:encode(Prv)}, - {p, hb_util:encode(P1)}, - {q, hb_util:encode(P2)}, - {dp, hb_util:encode(E1)}, - {dq, hb_util:encode(E2)}, - {qi, hb_util:encode(C)} - ] - } - ), + PrivKey = {KeyType, Prv, Pb}, + Ky = to_json(PrivKey), {Pb, Prv, Ky}; {?ECDSA_SIGN_ALG, secp256k1} -> {OrigPub, Prv} = crypto:generate_key(ecdh, secp256k1), - <<4:8, PubPoint/binary>> = OrigPub, - PubPointMid = byte_size(PubPoint) div 2, - <> = PubPoint, - Ky = - jiffy:encode( - { - [ - {kty, <<"EC">>}, - {crv, <<"secp256k1">>}, - {x, hb_util:encode(X)}, - {y, hb_util:encode(Y)}, - {d, hb_util:encode(Prv)} - ] - } - ), - {compress_ecdsa_pubkey(OrigPub), Prv, Ky}; + CompressedPub = compress_ecdsa_pubkey(OrigPub), + PrivKey = {KeyType, Prv, CompressedPub}, + Ky = to_json(PrivKey), + {CompressedPub, Prv, Ky}; {?EDDSA_SIGN_ALG, ed25519} -> {{_, Prv, Pb}, _} = new(KeyType), - Ky = - jiffy:encode( - { - [ - {kty, <<"OKP">>}, - {alg, <<"EdDSA">>}, - {crv, <<"Ed25519">>}, - {x, hb_util:encode(Pb)}, - {d, hb_util:encode(Prv)} - ] - } - ), + PrivKey = {KeyType, Prv, Pb}, + Ky = to_json(PrivKey), {Pb, Prv, Ky} end, Filename = wallet_filepath(WalletName, Pub, KeyType), @@ -143,6 +123,12 @@ wallet_filepath2(Wallet) -> %% Return not_found if arweave_keyfile_[addr].json or [addr].json is not found %% in [data_dir]/?WALLET_DIR. load_key(Addr) -> + load_key(Addr, #{}). + +%% @doc Read the keyfile for the key with the given address from disk. +%% Return not_found if arweave_keyfile_[addr].json or [addr].json is not found +%% in [data_dir]/?WALLET_DIR. +load_key(Addr, Opts) -> Path = hb_util:encode(Addr), case filelib:is_file(Path) of false -> @@ -151,38 +137,79 @@ load_key(Addr) -> false -> not_found; true -> - load_keyfile(Path2) + load_keyfile(Path2, Opts) end; true -> - load_keyfile(Path) + load_keyfile(Path, Opts) end. %% @doc Extract the public and private key from a keyfile. load_keyfile(File) -> + load_keyfile(File, #{}). + +%% @doc Extract the public and private key from a keyfile. +load_keyfile(File, Opts) -> {ok, Body} = file:read_file(File), - {Key} = jiffy:decode(Body), + from_json(Body, Opts). + +%% @doc Convert a wallet private key to JSON (JWK) format +to_json({PrivKey, _PubKey}) -> + to_json(PrivKey); +to_json({{?RSA_SIGN_ALG, PublicExpnt}, Priv, Pub}) when PublicExpnt =:= 65537 -> + hb_json:encode(#{ + kty => <<"RSA">>, + ext => true, + e => hb_util:encode(<>), + n => hb_util:encode(Pub), + d => hb_util:encode(Priv) + }); +to_json({{?ECDSA_SIGN_ALG, secp256k1}, Priv, CompressedPub}) -> + % For ECDSA, we need to expand the compressed pubkey to get X,Y coordinates + % This is a simplified version - ideally we'd implement pubkey expansion + hb_json:encode(#{ + kty => <<"EC">>, + crv => <<"secp256k1">>, + d => hb_util:encode(Priv) + % TODO: Add x and y coordinates from expanded pubkey + }); +to_json({{?EDDSA_SIGN_ALG, ed25519}, Priv, Pub}) -> + hb_json:encode(#{ + kty => <<"OKP">>, + alg => <<"EdDSA">>, + crv => <<"Ed25519">>, + x => hb_util:encode(Pub), + d => hb_util:encode(Priv) + }). + +%% @doc Parse a wallet from JSON (JWK) format +from_json(JsonBinary) -> + from_json(JsonBinary, #{}). + +%% @doc Parse a wallet from JSON (JWK) format with options +from_json(JsonBinary, Opts) -> + Key = hb_json:decode(JsonBinary), {Pub, Priv, KeyType} = - case lists:keyfind(<<"kty">>, 1, Key) of - {<<"kty">>, <<"EC">>} -> - {<<"x">>, XEncoded} = lists:keyfind(<<"x">>, 1, Key), - {<<"y">>, YEncoded} = lists:keyfind(<<"y">>, 1, Key), - {<<"d">>, PrivEncoded} = lists:keyfind(<<"d">>, 1, Key), + case hb_maps:get(<<"kty">>, Key, undefined, Opts) of + <<"EC">> -> + XEncoded = hb_maps:get(<<"x">>, Key, undefined, Opts), + YEncoded = hb_maps:get(<<"y">>, Key, undefined, Opts), + PrivEncoded = hb_maps:get(<<"d">>, Key, undefined, Opts), OrigPub = iolist_to_binary([<<4:8>>, hb_util:decode(XEncoded), hb_util:decode(YEncoded)]), Pb = compress_ecdsa_pubkey(OrigPub), Prv = hb_util:decode(PrivEncoded), KyType = {?ECDSA_SIGN_ALG, secp256k1}, {Pb, Prv, KyType}; - {<<"kty">>, <<"OKP">>} -> - {<<"x">>, PubEncoded} = lists:keyfind(<<"x">>, 1, Key), - {<<"d">>, PrivEncoded} = lists:keyfind(<<"d">>, 1, Key), + <<"OKP">> -> + PubEncoded = hb_maps:get(<<"x">>, Key, undefined, Opts), + PrivEncoded = hb_maps:get(<<"d">>, Key, undefined, Opts), Pb = hb_util:decode(PubEncoded), Prv = hb_util:decode(PrivEncoded), KyType = {?EDDSA_SIGN_ALG, ed25519}, {Pb, Prv, KyType}; _ -> - {<<"n">>, PubEncoded} = lists:keyfind(<<"n">>, 1, Key), - {<<"d">>, PrivEncoded} = lists:keyfind(<<"d">>, 1, Key), + PubEncoded = hb_maps:get(<<"n">>, Key, undefined, Opts), + PrivEncoded = hb_maps:get(<<"d">>, Key, undefined, Opts), Pb = hb_util:decode(PubEncoded), Prv = hb_util:decode(PrivEncoded), KyType = {?RSA_SIGN_ALG, 65537}, @@ -200,6 +227,9 @@ to_rsa_address(PubKey) -> hash_address(PubKey) -> crypto:hash(sha256, PubKey). +to_ecdsa_address(PubKey) -> + hb_keccak:key_to_ethereum_address(PubKey). + %%%=================================================================== %%% Private functions. %%%=================================================================== diff --git a/src/cargo.hrl b/src/cargo.hrl new file mode 100644 index 000000000..bcbb63656 --- /dev/null +++ b/src/cargo.hrl @@ -0,0 +1,8 @@ +-cargo_header_version 1. +-ifndef(CARGO_LOAD_APP). +-define(CARGO_LOAD_APP,hb). +-endif. +-ifndef(CARGO_HRL). +-define(CARGO_HRL, 1). +-define(load_nif_from_crate(__CRATE,__INIT),(fun()->__APP=?CARGO_LOAD_APP,__PATH=filename:join([code:priv_dir(__APP),"crates",__CRATE,__CRATE]),erlang:load_nif(__PATH,__INIT)end)()). +-endif. diff --git a/src/dev_apply.erl b/src/dev_apply.erl new file mode 100644 index 000000000..d3fd3d8dd --- /dev/null +++ b/src/dev_apply.erl @@ -0,0 +1,287 @@ +%%% @doc A device that executes AO resolutions. It can be passed a key that +%%% refers to a path stored in the base message to execute upon the base or +%%% message referenced by the `source' key. +%%% +%%% Alternatively, a `base' and `request' pair can be passed to execute +%%% together via invoking the `pair' key. +%%% +%%% When given a message with a `base' and `request' key, the default handler +%%% will invoke `pair' upon it, setting the `path' in the resulting request to +%%% the key that `apply' was invoked with. +%%% +%%% Paths found in keys interpreted by this device can contain a `base:' or +%%% `request:' prefix to indicate the message from which the path should be +%%% retrieved. If no such prefix is present, the `Request' message is checked +%%% first, and the `Base' message is checked second. +-module(dev_apply). +-export([info/1, pair/3, default/4]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% @doc The device info. Forwards all keys aside `pair', `keys' and `set' are +%% resolved with the `apply/4' function. +info(_) -> + #{ + excludes => [<<"keys">>, <<"set">>, <<"set_path">>, <<"remove">>], + default => fun default/4 + }. + +%% @doc The default handler. If the `base' and `request' keys are present in +%% the given request, then the `pair' function is called. Otherwise, the `eval' +%% key is used to resolve the request. +default(Key, Base, Request, Opts) -> + ?event(debug_apply, {req, {key, Key}, {base, Base}, {request, Request}}), + FoundBase = hb_maps:get(<<"base">>, Request, not_found, Opts), + FoundRequest = hb_maps:get(<<"request">>, Request, not_found, Opts), + case {FoundBase, FoundRequest} of + {B, R} when B =/= not_found andalso R =/= not_found -> + pair(Key, Base, Request, Opts); + _ -> + eval(Base, Request#{ <<"apply-path">> => Key }, Opts) + end. + +%% @doc Apply a request. We source the `base' message for the request either +%% from the `source' key if it is present, or we assume that the entire base +%% should be used. After sourcing the base, we resolve the `apply-path' on top +%% of it as a singleton message, if it is present in the request. +eval(Base, Request, Opts) -> + maybe + ?event({eval, {base, Base}, {request, Request}}), + {ok, ApplyBase} ?= + case find_path(<<"source">>, Base, Request, Opts) of + {ok, SourcePath} -> + find_key(SourcePath, Base, Request, Opts); + {error, path_not_found, _} -> + % If the base is not found, we return the base for this + % request, minus the device (which will, inherently, be + % `apply@1.0' and cause recursion). + {ok, hb_maps:without([<<"device">>], Base, Opts)} + end, + ?event({eval, {apply_base, ApplyBase}}), + case find_path(<<"apply-path">>, Base, Request, Opts) of + {error, path_not_found, _} -> + ?event({eval, no_path_to_execute}), + {ok, ApplyBase}; + {ok, ApplyPathKey} -> + ?event({eval, {key_containing_path_to_execute, ApplyPathKey}}), + case find_key(ApplyPathKey, ApplyBase, Request, Opts) of + {error, _, _} -> + ?event({eval, path_to_execute_not_found}), + {error, + << + "Path `", + (normalize_path(ApplyPathKey))/binary, + "` to execute not found." + >> + }; + {ok, ApplyPath} -> + ApplyMsg = ApplyBase#{ <<"path">> => ApplyPath }, + ?event({executing, ApplyMsg}), + hb_ao:resolve(ApplyMsg, Opts) + end + end + else + Error -> error_to_message(Error) + end. + +%% @doc Apply the message found at `request' to the message found at `base'. +pair(Base, Request, Opts) -> + pair(<<"undefined">>, Base, Request, Opts). +pair(PathToSet, Base, Request, Opts) -> + maybe + {ok, RequestPath} ?= find_path(<<"request">>, Base, Request, Opts), + {ok, BasePath} ?= find_path(<<"base">>, Base, Request, Opts), + ?event({eval_pair, {base_source, BasePath}, {request_source, RequestPath}}), + {ok, RequestSource} ?= find_key(RequestPath, Base, Request, Opts), + {ok, BaseSource} ?= find_key(BasePath, Base, Request, Opts), + PreparedRequest = + case PathToSet of + <<"undefined">> -> RequestSource; + _ -> RequestSource#{ <<"path">> => PathToSet } + end, + ?event({eval_pair, {base, BaseSource}, {request, PreparedRequest}}), + hb_ao:resolve(BaseSource, PreparedRequest, Opts) + else + Error -> error_to_message(Error) + end. + +%% @doc Resolve the given path on the message as `message@1.0'. +find_path(Path, Base, Request, Opts) -> + Res = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, Request}, Path}, + {{as, <<"message@1.0">>, Base}, Path} + ], + path_not_found, + Opts + ), + case Res of + path_not_found -> {error, path_not_found, Path}; + Value -> {ok, Value} + end. + +%% @doc Find the value of the source key, supporting `base:' and `request:' +%% prefixes. +find_key(Path, Base, Request, Opts) -> + BaseAs = {as, <<"message@1.0">>, Base}, + RequestAs = {as, <<"message@1.0">>, Request}, + MaybeResolve = + case hb_path:term_to_path_parts(Path) of + [BinKey|RestKeys] -> + case binary:split(BinKey, <<":">>) of + [<<"base">>, <<"">>] -> + {message, Base}; + [<<"request">>, <<"">>] -> + {message, Request}; + [<<"base">>, Key] -> + {resolve, [{BaseAs, normalize_path([Key|RestKeys])}]}; + [Req, Key] when Req == <<"request">> orelse Req == <<"req">> -> + {resolve, [{RequestAs, normalize_path([Key|RestKeys])}]}; + [_] -> + {resolve, [ + {RequestAs, normalize_path(Path)}, + {BaseAs, normalize_path(Path)} + ]} + end; + _ -> {error, invalid_path, Path} + end, + case MaybeResolve of + Err = {error, _, _} -> Err; + {message, Message} -> {ok, Message}; + {resolve, Sources} -> + ?event( + {resolving_from_sources, + {path, Path}, + {sources, Sources} + } + ), + case hb_ao:get_first(Sources, source_not_found, Opts) of + source_not_found -> {error, source_not_found, Path}; + Source -> {ok, Source} + end + end. + +%% @doc Normalize the path. +normalize_path(Path) -> + case hb_path:to_binary(Path) of + <<"">> -> <<"/">>; + P -> P + end. + +%% @doc Convert an error to a message. +error_to_message({error, invalid_path, ErrPath}) -> + {error, #{ + <<"body">> => + <<"Path `", (normalize_path(ErrPath))/binary, "` is invalid.">> + }}; +error_to_message({error, source_not_found, ErrPath}) -> + {error, #{ + <<"body">> => + << + "Source path `", + (normalize_path(ErrPath))/binary, + "` to apply not found." + >> + }}; +error_to_message({error, path_not_found, ErrPath}) -> + {error, #{ + <<"body">> => + << + "Path `", + (normalize_path(ErrPath))/binary, + "` to apply not found." + >> + }}; +error_to_message(Error) -> + Error. + +%%% Tests + +resolve_key_test() -> + hb:init(), + Base = #{ + <<"device">> => <<"apply@1.0">>, + <<"body">> => <<"/~meta@1.0/build/node">>, + <<"irrelevant">> => <<"irrelevant">> + }, + Request = #{ + <<"irrelevant2">> => <<"irrelevant2">>, + <<"path">> => <<"body">> + }, + ?assertEqual({ok, <<"HyperBEAM">>}, hb_ao:resolve(Base, Request, #{})). + +resolve_pair_test() -> + Base = #{ + <<"device">> => <<"apply@1.0">>, + <<"data-container">> => #{ <<"relevant">> => <<"DATA">> }, + <<"base">> => <<"data-container">>, + <<"irrelevant">> => <<"irrelevant">> + }, + Request = #{ + <<"irrelevant2">> => <<"irrelevant2">>, + <<"data-path">> => <<"relevant">>, + <<"request">> => <<"data-path">>, + <<"path">> => <<"pair">> + }, + ?assertEqual({ok, <<"DATA">>}, hb_ao:resolve(Base, Request, #{})). + +reverse_resolve_pair_test() -> + ?assertEqual( + {ok, <<"TEST">>}, + hb_ao:resolve( + << + "/~meta@1.0/build", + "/node~apply@1.0&node=TEST&base=request:&request=base:" + >>, + #{} + ) + ). + +resolve_with_prefix_test() -> + ShortTraceLen = hb_opts:get(short_trace_len), + Node = hb_http_server:start_node(), + ?assertEqual( + {ok, ShortTraceLen}, + hb_http:request( + <<"GET">>, + Node, + <<"/~meta@1.0/info/request:debug-info~apply@1.0">>, + #{ + <<"debug-info">> => <<"short_trace_len">> + }, + #{} + ) + ). + +apply_over_http_test() -> + Node = hb_http_server:start_node(), + Signed = + hb_message:commit( + #{ + <<"device">> => <<"apply@1.0">>, + <<"user-path">> => <<"/user-request/test-key">>, + <<"user-request">> => + #{ + <<"test-key">> => <<"DATA">> + } + }, + #{ priv_wallet => hb:wallet() } + ), + ?assertEqual( + {ok, <<"DATA">>}, + hb_ao:resolve( + Signed#{ <<"path">> => <<"/user-path">> }, + #{ priv_wallet => hb:wallet() } + ) + ), + ?assertEqual( + {ok, <<"DATA">>}, + hb_http:request( + <<"GET">>, + Node, + <<"/user-path">>, + Signed, + #{ priv_wallet => hb:wallet() } + ) + ). diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl new file mode 100644 index 000000000..704937f1e --- /dev/null +++ b/src/dev_arweave.erl @@ -0,0 +1,289 @@ +%%% @doc A device that provides access to Arweave network information, relayed +%%% from a designated node. +%%% +%%% The node(s) that are used to query data may be configured by altering the +%%% `/arweave` route in the node's configuration message. +-module(dev_arweave). +-export([tx/3, block/3, current/3, status/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Proxy the `/info' endpoint from the Arweave node. +status(_Base, _Request, Opts) -> + request(<<"GET">>, <<"/info">>, Opts). + +%% @doc Returns the given transaction, if known to the client node(s), as an +%% AO-Core message. +tx(Base, Request, Opts) -> + case hb_maps:get(<<"method">>, Request, <<"GET">>, Opts) of + <<"POST">> -> post_tx(Base, Request, Opts); + <<"GET">> -> get_tx(Base, Request, Opts) + end. + +%% @doc Upload a transaction to Arweave, using the node's default bundler (see +%% `hb_client:upload/2' for more details). Ensures that uploaded transactions are +%% stored in the local cache after a successful response has been received. +post_tx(_Base, Request, Opts) -> + case hb_client:upload(Request, Opts) of + Res = {ok, _} -> + ?event(arweave, {uploaded, Request}), + CacheRes = hb_cache:write(Request, Opts), + ?event(arweave, + {cache_uploaded_message, + {msg, Request}, + {status, + case CacheRes of {ok, _} -> ok; + _ -> failed + end + } + } + ), + Res; + Res -> + Res + end. + +%% @doc Get a transaction ID from the Arweave node, as indicated by the `tx` key +%% in the request or base message. If the `data' key is present and set to +%% `false', the data is not retrieved and added to the response. If the `data' +%% key is set to `always', transactions for which the header is available but +%% the data is not will lead to an error. Otherwise, just the header will be +%% returned. +get_tx(Base, Request, Opts) -> + case find_txid(Base, Request, Opts) of + not_found -> {error, not_found}; + TXID -> + case request(<<"GET">>, <<"/tx/", TXID/binary>>, Opts) of + {ok, TXHeader} -> + ?event(arweave, {retrieved_tx_header, {tx, TXID}}), + maybe_add_data(TXID, TXHeader, Base, Request, Opts); + Other -> Other + end + end. + +%% @doc Handle the optional adding of data to the transaction header, depending +%% on the request. Semantics of the `data' key are described in the `get_tx/3' +%% documentation. +maybe_add_data(TXID, Header, Base, Request, Opts) -> + GetData = + hb_util:atom(hb_ao:get_first( + [ + {Request, <<"data">>}, + {Base, <<"data">>} + ], + true, + Opts + )), + case hb_util:atom(GetData) of + false -> + {ok, Header}; + _ -> + case data(Base, Request, Opts) of + {ok, Data} -> + FullMessage = Header#{ <<"data">> => Data }, + ?event( + arweave, + {retrieved_tx_with_data, + {id, TXID}, + {data_size, byte_size(Data)}, + {message, FullMessage} + } + ), + {ok, FullMessage}; + {error, Reason} -> + ?event(arweave, + {data_retrieval_failed_after_header, + {id, TXID}, + {error, Reason} + } + ), + if GetData =/= always -> {ok, Header}; + true -> {error, Reason} + end + end + end. + +%% @doc Retrieve the data of a transaction from Arweave. +data(Base, Request, Opts) -> + case find_txid(Base, Request, Opts) of + not_found -> {error, not_found}; + TXID -> + ?event(arweave, {retrieving_tx_data, {tx, TXID}}), + request(<<"GET">>, <<"/raw/", TXID/binary>>, Opts) + end. + +%% @doc Retrieve (and cache) block information from Arweave. If the `block' key +%% is present, it is used to look up the associated block. If it is of Arweave +%% block hash length (43 characters), it is used as an ID. If it is parsable as +%% an integer, it is used as a block height. If it is not present, the current +%% block is used. +block(Base, Request, Opts) -> + Block = + hb_ao:get_first( + [ + {Request, <<"block">>}, + {Base, <<"block">>} + ], + not_found, + Opts + ), + case Block of + <<"current">> -> current(Base, Request, Opts); + not_found -> current(Base, Request, Opts); + ID when ?IS_ID(ID) -> block({id, ID}, Opts); + MaybeHeight -> + try hb_util:int(MaybeHeight) of + Int -> block({height, Int}, Opts) + catch + _:_ -> + { + error, + <<"Invalid block reference `", MaybeHeight/binary, "`">> + } + end + end. +block({id, ID}, Opts) -> + case hb_cache:read(ID, Opts) of + {ok, Block} -> + ?event(arweave, {retrieved_block_from_cache, {id, ID}}), + {ok, Block}; + not_found -> + request(<<"GET">>, <<"/block/hash/", ID/binary>>, Opts) + end; +block({height, Height}, Opts) -> + case dev_arweave_block_cache:read(Height, Opts) of + {ok, Block} -> + ?event(arweave, {retrieved_block_from_cache, {height, Height}}), + {ok, Block}; + not_found -> + request( + <<"GET">>, + <<"/block/height/", (hb_util:bin(Height))/binary>>, + Opts + ) + end. + +%% @doc Retrieve the current block information from Arweave. +current(_Base, _Request, Opts) -> + request(<<"GET">>, <<"/block/current">>, Opts). + +%%% Internal Functions + +%% @doc Find the transaction ID to retrieve from Arweave based on the request or +%% base message. +find_txid(Base, Request, Opts) -> + hb_ao:get_first( + [ + {Request, <<"tx">>}, + {Base, <<"tx">>} + ], + not_found, + Opts + ). + +%% @doc Make a request to the Arweave node and parse the response into an +%% AO-Core message. Most Arweave API responses are in JSON format, but without +%% a `content-type' header. Subsequently, we parse the response manually and +%% pass it back as a message. +request(Method, Path, Opts) -> + ?event(arweave, {arweave_request, {method, Method}, {path, Path}}), + Res = + hb_http:request( + #{ + <<"path">> => <<"/arweave", Path/binary>>, + <<"method">> => Method + }, + Opts + ), + to_message(Path, Res, Opts). + +%% @doc Transform a response from the Arweave node into an AO-Core message. +to_message(Path = <<"/raw/", _/binary>>, {ok, #{ <<"body">> := Body }}, _Opts) -> + ?event(arweave, + {arweave_raw_response, + {path, Path}, + {data_size, byte_size(Body)} + } + ), + {ok, Body}; +to_message(Path = <<"/block/", _/binary>>, {ok, #{ <<"body">> := Body }}, Opts) -> + Block = hb_message:convert(Body, <<"structured@1.0">>, <<"json@1.0">>, Opts), + ?event(arweave, + {arweave_block_response, + {path, Path}, + {block, Block} + } + ), + CacheRes = dev_arweave_block_cache:write(Block, Opts), + ?event(arweave, + {cached_arweave_block, + {path, Path}, + {result, CacheRes} + } + ), + {ok, Block}; +to_message(Path, {ok, #{ <<"body">> := Body }}, Opts) -> + % All other responses that are `OK' status are converted from JSON to an + % AO-Core message. + ?event(arweave, + {arweave_json_response, + {path, Path}, + {body_size, byte_size(Body)} + } + ), + { + ok, + hb_message:convert( + Body, + <<"structured@1.0">>, + <<"json@1.0">>, + Opts + ) + }. + +%%% Tests + +post_ans104_tx_test() -> + ServerOpts = #{ store => [hb_test_utils:test_store()] }, + Server = hb_http_server:start_node(ServerOpts), + ClientOpts = + #{ + store => [hb_test_utils:test_store()], + priv_wallet => hb:wallet() + }, + Msg = + hb_message:commit( + #{ + <<"variant">> => <<"ao.N.1">>, + <<"type">> => <<"Process">>, + <<"data">> => <<"test-data">> + }, + ClientOpts, + #{ <<"commitment-device">> => <<"ans104@1.0">> } + ), + {ok, PostRes} = + hb_http:post( + Server, + Msg#{ + <<"path">> => <<"/~arweave@2.9-pre/tx">>, + <<"codec-device">> => <<"ans104@1.0">> + }, + ClientOpts + ), + ?assertMatch(#{ <<"status">> := 200 }, PostRes), + SignedID = hb_message:id(Msg, signed, ClientOpts), + {ok, GetRes} = + hb_http:get( + Server, <<"/", SignedID/binary>>, + ClientOpts + ), + ?assertMatch( + #{ + <<"status">> := 200, + <<"variant">> := <<"ao.N.1">>, + <<"type">> := <<"Process">>, + <<"data">> := <<"test-data">> + }, + GetRes + ), + ok. \ No newline at end of file diff --git a/src/dev_arweave_block_cache.erl b/src/dev_arweave_block_cache.erl new file mode 100644 index 000000000..4d20cb764 --- /dev/null +++ b/src/dev_arweave_block_cache.erl @@ -0,0 +1,66 @@ +%%% @doc A module that performs caching operations for the Arweave device, +%%% focused on ensuring that block metadata is queriable via pseudo-paths. +-module(dev_arweave_block_cache). +-export([latest/1, heights/1, read/2, write/2]). +-export([path/2]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc The pseudo-path prefix which the Arweave block cache should use. +-define(ARWEAVE_BLOCK_CACHE_PREFIX, <<"~arweave@2.9">>). + +%% @doc Get the latest block from the cache. +latest(Opts) -> + case heights(Opts) of + {ok, []} -> + ?event(arweave_cache, no_blocks_in_cache), + not_found; + {ok, Blocks} -> + Latest = lists:max(Blocks), + ?event(arweave_cache, {latest_block_from_cache, {latest, Latest}}), + {ok, Latest} + end. + +%% @doc Get the list of blocks from the cache. +heights(Opts) -> + AllBlocks = + hb_cache:list_numbered( + hb_store:path(hb_opts:get(store, no_viable_store, Opts), [ + ?ARWEAVE_BLOCK_CACHE_PREFIX, + <<"block">>, + <<"height">> + ]), + Opts + ), + ?event(arweave_cache, {listed_blocks, length(AllBlocks)}), + {ok, AllBlocks}. + +%% @doc Read a block from the cache. +read(Block, Opts) -> + Res = hb_cache:read(path(Block, Opts), Opts), + ?event(arweave_cache, {read_block, {reference, Block}, {result, Res}}), + Res. + +%% @doc Return the path of a block that will be used in the cache. +path(Block, Opts) when is_integer(Block) -> + hb_store:path(hb_opts:get(store, no_viable_store, Opts), [ + ?ARWEAVE_BLOCK_CACHE_PREFIX, + <<"block">>, + <<"height">>, + hb_util:bin(Block) + ]). + +%% @doc Write a block to the cache and create pseudo-paths for it. +write(Block, Opts) -> + {ok, Height} = hb_maps:find(<<"height">>, Block, Opts), + {ok, BlockID} = hb_maps:find(<<"indep_hash">>, Block, Opts), + {ok, BlockHash} = hb_maps:find(<<"hash">>, Block, Opts), + {ok, MsgID} = hb_cache:write(Block, Opts), + % Link the independent hash and the dependent hash to the written AO-Core + % message ID. + hb_cache:link(MsgID, BlockID, Opts), + hb_cache:link(MsgID, BlockHash, Opts), + % Link the block height pseudo-path to the message. + hb_cache:link(MsgID, path(Height, Opts), Opts), + ?event(arweave_cache, {wrote_block, {height, Height}, {message_id, MsgID}}), + {ok, MsgID}. \ No newline at end of file diff --git a/src/dev_auth_hook.erl b/src/dev_auth_hook.erl new file mode 100644 index 000000000..9436aa734 --- /dev/null +++ b/src/dev_auth_hook.erl @@ -0,0 +1,703 @@ +%%% @doc A device offering an on-request hook that signs incoming messages with +%%% node-hosted wallets, in accordance with the node operator's configuration. +%%% It is intended for deployment in environments where a node's users have +%%% intrinsic reasons for trusting the node outside of the scope of this device. +%%% For example, if executed on a node running in a Trusted Execution Environment +%%% with `~snp@1.0', or a node they operate or is operated by a trusted +%%% third-party. +%%% +%%% This device utilizes the `generator' interface type which other devices may +%%% implement. The generator is used to find/create a secret based on a user's +%%% request, which is then passed to the `~proxy-wallet@1.0' device and matched +%%% with a wallet which is used to sign the request. The `generator' interface +%%% may implement the following keys: +%%% +%%% 
+%%%     `generate' (optional): A key that generates a secret based on a
+%%%                            user's request. May return either the secret
+%%%                            directly, or a message with a `secret' key. If 
+%%%                            a message is returned, it is assumed to be a
+%%%                            modified version of the user's request and is
+%%%                            used for further processing.
+%%%     `finalize' (optional): A key that takes the message sequence after this
+%%%                            device has processed it and returns it in a
+%%%                            modified form.
+%%%
+%%% +%%% At present, the `~cookie-secret@1.0' and `~http-auth@1.0' devices implement +%%% the `generator' interface. For example, the following hook definition will +%%% use the `~cookie-secret@1.0' device to generate and manage wallets for +%%% users, with authentication details stored in cookies: +%%% +%%% 
+%%%   "on": {
+%%%     "request": {
+%%%       "device": "auth-hook@1.0",
+%%%       "secret-provider": {
+%%%         "device": "cookie-secret@1.0"
+%%%       }
+%%%     }
+%%%   }
+%%%
+%%% +%%% `~auth-hook@1.0' expects to receive a `secret-provider' key in the hook +%%% base message. It may optionally also take a `generate-path' and +%%% `finalize-path', which are used to generate the secret and post-process the +%%% response. If either `X-path' keys are not present, the `generate' and +%%% `finalize' paths are used upon the `secret-provider' message. If the secret +%%% provider's device does not implement these keys, the operations are skipped. +%%% +%%% Node operators may also specify a `when' message inside their hook definition +%%% which is used to determine when messages should be signed. The supported keys +%%% are: +%%% +%%% 
+%%%     `committers': always | uncommitted | [committer1, or committer2, or ...]
+%%%     `keys': always | [key1, or key2, or ...]
+%%%
+%%% +%%% Both keys are optional and can be combined to form 'and' conditions. For +%%% example, the following hook definition will sign all uncommitted requests +%%% that have the `Authorization' header: +%%% +%%% 
+%%%   "on": {
+%%%     "request": {
+%%%       "device": "auth-hook@1.0",
+%%%       "when": {
+%%%             "keys": ["authorization"],
+%%%             "committers": "uncommitted"
+%%%         }
+%%%       }
+%%%     }
+%%%
+%%% +-module(dev_auth_hook). +-export([request/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% Default key used to indicate that an individual message in the path should +%%% be signed. +-define(DEFAULT_COMMIT_KEY, <<"!">>). + +%%% Default keys to ignore when signing +-define(DEFAULT_IGNORED_KEYS, + [ + <<"secret">>, + <<"cookie">>, + <<"set-cookie">>, + <<"path">>, + <<"method">>, + <<"authorization">>, + ?DEFAULT_COMMIT_KEY + ] +). + +%% @doc Process an incoming request through a key provider. The key provider +%% should be a message optionally implementing the following keys: +%% 
+%%     `generate-path': The path to call the `generate' function.
+%%     `finalize-path': The path to call the `finalize' function.
+%%     `skip-commit': Whether to skip committing the request.
+%%     `ignored-keys': A list of keys to ignore when signing (can be overridden
+%%     by the user request).
+%%
+%% +request(Base, HookReq, Opts) -> + ?event({auth_hook_request, {base, Base}, {hook_req, HookReq}}), + maybe + % Get the key provider from options and short-circuit if none is + % provided. + {ok, Provider} ?= find_provider(Base, Opts), + % Check if the request already has signatures, or the hook base enforces + % that we should always attempt to sign the request. + {ok, Request} ?= hb_maps:find(<<"request">>, HookReq, Opts), + {ok, OrigMessages} ?= hb_maps:find(<<"body">>, HookReq, Opts), + true ?= is_relevant(Base, Request, OrigMessages, Opts), + ?event(auth_hook_is_relevant), + % Call the key provider to normalize authentication (generate if needed) + {ok, IntermediateProvider, NormReq} ?= + generate_secret(Provider, Request, Opts), + % Call `~secret@1.0' to generate a wallet if needed. Returns refreshed + % options. + {ok, NormProvider, NewOpts} ?= + generate_wallet(IntermediateProvider, NormReq, Opts), + ?event( + {auth_hook_normalized, + {intermediate_provider, IntermediateProvider}, + {norm_provider, NormProvider}, + {norm_req, NormReq} + } + ), + % Sign the full request + {ok, SignedReq} ?= sign_request(NormProvider, NormReq, NewOpts), + ?event(auth_hook_signed), + % Process individual messages if needed + {ok, MessageSequence} ?= + maybe_sign_messages( + NormProvider, + SignedReq, + NewOpts + ), + ?event(auth_hook_processed_messages), + % Call the key provider to finalize the response + {ok, FinalSequence} ?= + finalize( + NormProvider, + SignedReq, + MessageSequence, + NewOpts + ), + ?event({auth_hook_returning, FinalSequence}), + {ok, #{ <<"body">> => FinalSequence, <<"request">> => SignedReq }} + else + {error, AuthError} -> + ?event({auth_hook_auth_error, AuthError}), + {error, AuthError}; + {skip, {committers, Committers}, {keys, Keys}} -> + ?event({auth_hook_skipping, {committers, Committers}, {keys, Keys}}), + {ok, HookReq}; + error -> + ?event({auth_hook_error, no_request}), + {ok, HookReq}; + Other -> + ?event({auth_hook_unexpected_result, Other}), + Other + end. + +%% @doc Check if the request is relevant to the hook base. Node operators may +%% specify criteria for activation of the hook based on the committers of the +%% request (`always', `uncommitted', or a list of committers), or the presence +%% of certain keys (`always', or a list of keys) on any of the messages in the +%% sequence. +is_relevant(Base, Request, MessageSequence, Opts) -> + Committers = is_relevant_from_committers(Base, Request, Opts), + Keys = + lists:any( + fun(Msg) -> is_relevant_from_keys(Base, Msg, Opts) end, + [Request | MessageSequence] + ), + ?event({auth_hook_is_relevant, {committers, Committers}, {keys, Keys}}), + if Committers andalso Keys -> true; + true -> {skip, {committers, Committers}, {keys, Keys}} + end. + +%% @doc Check if the request is relevant to the hook base based on the committers +%% of the request. +is_relevant_from_committers(Base, Request, Opts) -> + Config = + hb_util:deep_get( + [<<"when">>, <<"committers">>], + Base, + <<"uncommitted">>, + Opts + ), + ?event({auth_hook_is_relevant_from_committers, {config, Config}, {base, Base}}), + case Config of + <<"always">> -> true; + <<"uncommitted">> -> hb_message:signers(Request, Opts) == []; + RelevantCommitters -> + lists:any( + fun(Signer) -> + lists:member(Signer, RelevantCommitters) + end, + hb_message:signers(Request, Opts) + ) + end. + +%% @doc Check if the request is relevant to the hook base based on the presence +%% of keys specified in the hook base. +is_relevant_from_keys(_Base, ID, _Opts) when is_binary(ID) -> + false; +is_relevant_from_keys(Base, {as, _, Msg}, Opts) -> + is_relevant_from_keys(Base, Msg, Opts); +is_relevant_from_keys(Base, {resolve, Msg}, Opts) -> + is_relevant_from_keys(Base, Msg, Opts); +is_relevant_from_keys(Base, Request, Opts) -> + Config = hb_util:deep_get([<<"when">>, <<"keys">>], Base, <<"always">>, Opts), + ?event( + { + auth_hook_is_relevant_from_keys, + {config, Config}, + {base, Base}, + {request, Request} + } + ), + case Config of + <<"always">> -> true; + RelevantKeys -> + lists:any( + fun(Key) -> + case hb_maps:find(Key, Request, Opts) of + {ok, _} -> true; + error -> false + end + end, + RelevantKeys + ) + end. + +%% @doc Normalize authentication credentials, generating new ones if needed. +generate_secret(Provider, Request, Opts) -> + case call_provider(<<"generate">>, Provider, Request, Opts) of + {error, not_found} -> + ?event({no_generate_handler, Provider}), + {ok, Provider, strip_sensitive(Request, Opts)}; + {error, Err} -> + % Forward the error. The main handler will fail to match this and + % return the error to the user. + ?event({generate_error, Err}), + {error, Err}; + {ok, Secret} when is_binary(Secret) -> + % The provider returned a direct key, calculate the committer and + % generate a wallet for it, if needed. + ?event({secret_from_provider, Secret}), + {ok, Provider#{ <<"secret">> => Secret }, strip_sensitive(Request, Opts)}; + {ok, NormalizedReq} when is_map(NormalizedReq) -> + % If there is a `wallet' field in the request, we move it to the + % provider, else continue with the existing provider. + ?event({normalized_req, NormalizedReq}), + case hb_maps:find(<<"secret">>, NormalizedReq, Opts) of + {ok, Key} -> + ?event({key_found_in_normalized_req, Key}), + { + ok, + Provider#{ <<"secret">> => Key }, + strip_sensitive(NormalizedReq, Opts) + }; + error -> + ?event({no_key_in_normalized_req, NormalizedReq}), + {ok, Provider, strip_sensitive(NormalizedReq, Opts)} + end + end. + +%% @doc Strip the `secret' field from a request. +strip_sensitive(Request, Opts) -> + hb_maps:without([<<"secret">>], Request, Opts). + +%% @doc Generate a wallet with the key if the `wallet' field is not present in +%% the provider after normalization. +generate_wallet(Provider, Request, Opts) -> + {ok, #{ <<"body">> := WalletID }} = + dev_secret:generate(Provider, Request, Opts), + ?event({generated_wallet, WalletID}), + {ok, Provider, refresh_opts(Opts)}. + +%% @doc Sign a request using the configured key provider +sign_request(Provider, Msg, Opts) -> + case hb_maps:get(<<"skip-commit">>, Provider, true, Opts) of + false -> + % Skip signing and return the normalized message. + ?event({provider_requested_signing_skip, Provider}), + {ok, Msg}; + true -> + % Wallet signs without ignored keys + IgnoredKeys = ignored_keys(Msg, Opts), + WithoutIgnored = hb_maps:without(IgnoredKeys, Msg, Opts), + % Call the wallet to sign the request. + case dev_secret:commit(WithoutIgnored, Provider, Opts) of + {ok, Signed} -> + ?event({auth_hook_signed, Signed}), + SignedWithIgnored = + hb_maps:merge( + Signed, + hb_maps:with(IgnoredKeys, Msg, Opts), + Opts + ), + {ok, SignedWithIgnored}; + {error, Err} -> + ?event({auth_hook_sign_error, Err}), + {error, Err} + end + end. + +%% @doc Process a sequence of messages, signing those marked for signing +maybe_sign_messages(Provider, SignedReq, Opts) -> + Parsed = hb_singleton:from(SignedReq, Opts), + ?event({auth_hook_parsed_messages, {sequence_length, length(Parsed)}}), + SignKey = hb_opts:get(auth_hook_commit_key, ?DEFAULT_COMMIT_KEY, Opts), + Processed = maybe_sign_messages(Provider, SignKey, Parsed, Opts), + {ok, Processed}. +maybe_sign_messages(_Provider, _Key, [], _Opts) -> []; +maybe_sign_messages(Provider, Key, [Msg | Rest], Opts) when is_map(Msg) -> + case hb_util:atom(hb_maps:get(Key, Msg, false, Opts)) of + true -> + Uncommitted = hb_message:uncommitted(Msg, Opts), + ?event({auth_hook_signing_message, {uncommitted, Msg}}), + case sign_request(Provider, Uncommitted, Opts) of + {ok, Signed} -> + [ + Signed + | + maybe_sign_messages(Provider, Key, Rest, Opts) + ]; + {error, Err} -> + ?event({auth_hook_sign_error, Err}), + [{error, Err}] + end; + _ -> + [Msg | maybe_sign_messages(Provider, Key, Rest, Opts)] + end; +maybe_sign_messages(Provider, Key, [Msg | Rest], Opts) -> + [Msg | maybe_sign_messages(Provider, Key, Rest, Opts)]. + +%% @doc Finalize the response by adding authentication state +finalize(KeyProvider, SignedReq, MessageSequence, Opts) -> + % Add the signed request and message sequence to the response, mirroring the + % structure of a normal `~hook@1.0' on-request hook. + Req = + #{ + <<"request">> => SignedReq, + <<"body">> => MessageSequence + }, + case call_provider(<<"finalize">>, KeyProvider, Req, Opts) of + {ok, Finalized} -> + ?event({auth_hook_finalized, Finalized}), + {ok, Finalized}; + {error, not_found} -> + ?event(auth_hook_no_finalize_handler), + {ok, MessageSequence} + end. + +%%% Utility functions + +%% @doc Refresh the options and log an event if they have changed. +refresh_opts(Opts) -> + NewOpts = hb_http_server:get_opts(Opts), + case NewOpts of + Opts -> ?event(auth_hook_no_opts_change); + _ -> + ?event( + {auth_hook_opts_changed, + {size_diff, + erlang:external_size(NewOpts) - + erlang:external_size(Opts) + } + } + ) + end, + NewOpts. + +%% @doc Get the key provider from the base message or the defaults. +find_provider(Base, Opts) -> + case hb_maps:get(<<"secret-provider">>, Base, no_key_provider, Opts) of + no_key_provider -> + case hb_opts:get(hook_secret_provider, no_key_provider, Opts) of + no_key_provider -> {error, no_key_provider}; + SecretProvider -> SecretProvider + end; + SecretProvider when is_binary(SecretProvider) -> + {ok, #{ <<"device">> => SecretProvider }}; + SecretProvider when is_map(SecretProvider) -> + {ok, SecretProvider}; + _ -> + {error, invalid_auth_provider} + end. + +%% @doc Find the appropriate handler for a key in the key provider. +call_provider(Key, Provider, Request, Opts) -> + ?event({call_provider, {key, Key}, {provider, Provider}, {req, Request}}), + ExecKey = hb_maps:get(<< Key/binary, "-path">>, Provider, Key, Opts), + ?event({call_provider, {exec_key, ExecKey}}), + case hb_ao:resolve(Provider, Request#{ <<"path">> => ExecKey }, Opts) of + {ok, Msg} when is_map(Msg) -> + % The result is a message. We revert the path to its original value. + case hb_maps:find(<<"path">>, Request, Opts) of + {ok, Path} -> {ok, Msg#{ <<"path">> => Path }}; + _ -> {ok, Msg} + end; + {ok, _} = Res -> + % The result is a non-message. We return it as-is. + Res; + {error, Err} -> + ?event({call_provider_error, Err}), + {error, Err} + end. + +%% @doc Default keys to ignore when signing +ignored_keys(Msg, Opts) -> + hb_maps:get( + <<"ignored-keys">>, + Msg, + hb_opts:get( + hook_auth_ignored_keys, + ?DEFAULT_IGNORED_KEYS, + Opts + ) + ). + +%%% Tests + +cookie_test() -> + % Start a node with a secret-provider that uses the cookie device. + Node = + hb_http_server:start_node( + #{ + priv_wallet => ServerWallet = ar_wallet:new(), + on => #{ + <<"request">> => #{ + <<"device">> => <<"auth-hook@1.0">>, + <<"path">> => <<"request">>, + <<"secret-provider">> => + #{ + <<"device">> => <<"cookie@1.0">> + } + } + } + } + ), + % Run a request and check that the response is signed. The cookie device + % will generate a new cookie for the client. + {ok, Response} = + hb_http:get( + Node, + #{ + <<"path">> => <<"commitments">>, + <<"body">> => <<"Test data">> + }, + #{} + ), + % Filter the response to only include signed commitments. + Signers = signers_from_commitments_response(Response, ServerWallet), + ?event( + {response, {found_signers, Signers}} + ), + ?assertEqual(1, length(Signers)), + % Generate a further request and check that the same address is used. Extract + % the cookie given in the first request and use it to sign the second. + [CookieAddress] = Signers, + #{ <<"priv">> := CookiePriv } = Response, + ?event( + {cookie_from_response, + {cookie_priv, CookiePriv}, + {cookie_address, CookieAddress} + } + ), + {ok, Response2} = + hb_http:get( + Node, + #{ + <<"path">> => <<"/commitments">>, + <<"body">> => <<"Test data2">>, + <<"priv">> => CookiePriv + }, + #{} + ), + % Check that the second request is signed with the same address as the first. + ?assertEqual( + [CookieAddress], + signers_from_commitments_response(Response2, ServerWallet) + ). + +http_auth_test() -> + % Start a node with the `~http-auth@1.0' device as the secret-provider. + Node = + hb_http_server:start_node( + #{ + priv_wallet => ServerWallet = ar_wallet:new(), + on => #{ + <<"request">> => #{ + <<"device">> => <<"auth-hook@1.0">>, + <<"path">> => <<"request">>, + <<"secret-provider">> => + #{ + <<"device">> => <<"http-auth@1.0">>, + <<"access-control">> => + #{ <<"device">> => <<"http-auth@1.0">> } + } + } + } + } + ), + % Run a request and check that the response is a 401 with the + % `www-authenticate' header. + Resp1 = + hb_http:get( + Node, + #{ + <<"path">> => <<"commitments">>, + <<"body">> => <<"Test data">> + }, + #{} + ), + ?assertMatch( + {error, #{ <<"status">> := 401, <<"www-authenticate">> := _ }}, + Resp1 + ), + % Run a request with the `Authorization' header and check that the response + % is signed. + AuthStr = << "Basic ", (base64:encode(<<"user:pass">>))/binary >>, + Resp2 = + hb_http:get( + Node, + #{ + <<"path">> => <<"commitments">>, + <<"body">> => <<"Test data">>, + <<"authorization">> => AuthStr + }, + #{} + ), + ?assertMatch( + {ok, #{ <<"status">> := 200 }}, + Resp2 + ), + % Filter the response to only include signed commitments. + Signers = signers_from_commitments_response(hb_util:ok(Resp2), ServerWallet), + ?event( + {response, {found_signers, Signers}} + ), + ?assertEqual(1, length(Signers)), + % Generate a further request and check that the same address is used. + [Signer] = Signers, + {ok, Resp3} = + hb_http:get( + Node, + #{ + <<"path">> => <<"commitments">>, + <<"body">> => <<"Test data2">>, + <<"authorization">> => AuthStr + }, + #{} + ), + ?assertEqual( + [Signer], + signers_from_commitments_response(Resp3, ServerWallet) + ). + +chained_preprocess_test() -> + % Start a node with the `~http-auth@1.0' device as the secret-provider, with + % a router chained afterwards in the request hook. + RelayWallet = ar_wallet:new(), + RelayAddress = hb_util:human_id(RelayWallet), + RelayURL = hb_http_server:start_node(#{ priv_wallet => RelayWallet }), + Node = + hb_http_server:start_node( + #{ + priv_wallet => ar_wallet:new(), + relay_allow_commit_request => true, + on => #{ + <<"request">> => + [ + #{ + <<"device">> => <<"auth-hook@1.0">>, + <<"path">> => <<"request">>, + <<"secret-provider">> => + #{ + <<"device">> => <<"http-auth@1.0">>, + <<"access-control">> => + #{ + <<"device">> => <<"http-auth@1.0">> + } + } + }, + #{ + <<"device">> => <<"router@1.0">>, + <<"path">> => <<"preprocess">>, + <<"commit-request">> => true + } + ] + }, + routes => [ + #{ + <<"template">> => <<"/~meta@1.0/info/address">>, + <<"node">> => #{ <<"prefix">> => RelayURL } + } + ] + } + ), + % Run a request with the `Authorization' header and check that the response + % is signed. + AuthStr = << "Basic ", (base64:encode(<<"user:pass">>))/binary >>, + Resp1 = + hb_http:get( + Node, + #{ + <<"path">> => <<"/~meta@1.0/info/address">>, + <<"authorization">> => AuthStr + }, + #{} + ), + ?assertMatch({ok, RelayAddress}, Resp1). + +when_test() -> + % Start a node with the `~http-auth@1.0' device as the secret-provider. Only + % request commitment with the hook if the `Authorization' header is present. + Node = + hb_http_server:start_node( + #{ + priv_wallet => ServerWallet = ar_wallet:new(), + on => #{ + <<"request">> => #{ + <<"device">> => <<"auth-hook@1.0">>, + <<"path">> => <<"request">>, + <<"when">> => #{ + <<"keys">> => [<<"authorization">>] + }, + <<"secret-provider">> => + #{ + <<"device">> => <<"http-auth@1.0">>, + <<"access-control">> => + #{ <<"device">> => <<"http-auth@1.0">> } + } + } + } + } + ), + % Run a request and check that the response is not signed, but is `status: 200'. + {ok, Resp1} = + hb_http:get( + Node, + #{ + <<"path">> => <<"~meta@1.0/info">>, + <<"body">> => <<"Test data">> + }, + #{} + ), + ?assertEqual(200, hb_maps:get(<<"status">>, Resp1, 0)), + % Run a request with the `Authorization' header and check that the response + % is signed. + AuthStr = << "Basic ", (base64:encode(<<"user:pass">>))/binary >>, + Resp2 = + hb_http:get( + Node, + #{ + <<"path">> => <<"commitments">>, + <<"body">> => <<"Test data">>, + <<"authorization">> => AuthStr + }, + #{} + ), + ?assertMatch( + {ok, #{ <<"status">> := 200 }}, + Resp2 + ), + ?assertMatch( + [_], + signers_from_commitments_response( + hb_util:ok(Resp2), + ServerWallet + ) + ). + +%% @doc The cookie hook test(s) call `GET /commitments', which returns the +%% commitments found on the client request during execution on the server. +%% This function filters the response to return only the signers of that message, +%% excluding the server's own signature. +signers_from_commitments_response(Response, ServerWallet) -> + ServerAddress = ar_wallet:to_address(ServerWallet), + hb_maps:values(hb_maps:filtermap( + fun(Key, Value) when ?IS_ID(Key) -> + Type = hb_maps:get(<<"type">>, Value, not_found, #{}), + Committer = hb_maps:get(<<"committer">>, Value, not_found, #{}), + case {Type, Committer} of + {<<"rsa-pss-sha512">>, ServerAddress} -> false; + {<<"rsa-pss-sha512">>, _} -> {true, Committer}; + _ -> false + end; + (_Key, _Value) -> + false + end, + Response, + #{} + )). \ No newline at end of file diff --git a/src/dev_cache.erl b/src/dev_cache.erl new file mode 100644 index 000000000..461965695 --- /dev/null +++ b/src/dev_cache.erl @@ -0,0 +1,350 @@ +%%% @doc A device that looks up an ID from a local store and returns it, +%%% honoring the `accept' key to return the correct format. The cache also +%%% supports writing messages to the store, if the node message has the +%%% writer's address in its `cache_writers' key. +-module(dev_cache). +-export([read/3, write/3, link/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Read data from the cache. +%% Retrieves data corresponding to a key from a local store. +%% The key is extracted from the incoming message under <<"target">>. +%% The options map may include store configuration. +%% If the "accept" header is set to <<"application/aos-2">>, the result is +%% converted to a JSON structure and encoded. +%% +%% @param M1 Ignored parameter. +%% @param M2 The request message containing the key (<<"target">>) and an +%% optional "accept" header. +%% @param Opts A map of configuration options. +%% @returns {ok, Data} on success, +%% not_found if the key does not exist, +%% {error, Reason} on failure. +read(_M1, M2, Opts) -> + Location = hb_ao:get(<<"target">>, M2, Opts), + ?event({read, {key_extracted, Location}}), + ?event(debug_gateway, cache_read), + case hb_cache:read(Location, Opts) of + {ok, Res} -> + ?event({read, {cache_result, ok, Res}}), + case hb_ao:get(<<"accept">>, M2, Opts) of + <<"application/aos-2">> -> + ?event(dev_cache, + {read, + {accept_header, <<"application/aos-2">>} + } + ), + JSONMsg = dev_json_iface:message_to_json_struct(Res, Opts), + ?event(dev_cache, {read, {json_message, JSONMsg}}), + {ok, + #{ + <<"body">> => hb_json:encode(JSONMsg), + <<"content-type">> => <<"application/aos-2">> + } + }; + _ -> + {ok, Res} + end; + not_found -> + % The cache does not have this ID,but it may still be an explicit + % `data/' path. + % Store = hb_opts:get(store, [], Opts), + Store = maps:get(store, Opts), + ?event(dev_cache, {read, {location, Location}, {store, Store}}), + hb_store:read(Store, Location) + end. + +%% @doc Write data to the cache. +%% Processes a write request by first verifying that the request comes from a +%% trusted writer (as defined by the `cache_writers' configuration in the +%% options). The write type is determined from the message ("single" or "batch") +%% and the data is stored accordingly. +%% +%% @param M1 Ignored parameter. +%% @param M2 The request message containing the data to write, the write type, +%% and any additional parameters. +%% @param Opts A map of configuration options. +%% @returns {ok, Path} on success, where Path indicates where the data was +%% stored, {error, Reason} on failure. +write(_M1, M2, Opts) -> + case is_trusted_writer(M2, Opts) of + true -> + ?event(dev_cache, {write, {trusted_writer, true}}), + Type = hb_ao:get(<<"type">>, M2, <<"single">>, Opts), + ?event(dev_cache, {write, {write_type, Type}}), + case Type of + <<"single">> -> + ?event(dev_cache, {write, {write_single_called}}), + write_single(M2, Opts); + <<"batch">> -> + ?event(dev_cache, {write, {write_batch_called}}), + hb_maps:map( + fun(_, Value) -> + ?event(dev_cache, {write, {batch_item, Value}}), + write_single(Value, Opts) + end, + hb_ao:get(<<"body">>, M2, Opts), + Opts + ); + _ -> + ?event(dev_cache, {write, {invalid_write_type, Type}}), + {error, + #{ + <<"status">> => 400, + <<"body">> => <<"Invalid write type.">> + } + } + end; + false -> + ?event(dev_cache, {write, {trusted_writer, false}}), + {error, + #{ + <<"status">> => 403, + <<"body">> => <<"Not authorized to write to the cache.">> + } + } + end. + +%% @doc Link a source to a destination in the cache. +link(_Base, Req, Opts) -> + case is_trusted_writer(Req, Opts) of + true -> + Source = hb_ao:get(<<"source">>, Req, Opts), + Destination = hb_ao:get(<<"destination">>, Req, Opts), + write_single(#{ + <<"operation">> => <<"link">>, + <<"source">> => Source, + <<"destination">> => Destination + }, Opts); + false -> + {error, not_authorized} + end. + +%% @doc Helper function to write a single data item to the cache. +%% Extracts the body, location, and operation from the message. +%% Depending on the type of data (map or binary) or if a link operation is +%% requested, it writes the data to the store using the appropriate function. +%% +%% @param Msg The message containing data to be written. +%% @param Opts A map of configuration options. +%% @returns {ok, #{status := 200, path := Path}} on success, +%% {error, Reason} on failure. +write_single(Msg, Opts) -> + Body = hb_ao:get(<<"body">>, Msg, Opts), + ?event(dev_cache, {write_single, {body_extracted, Body}}), + Location = hb_ao:get(<<"location">>, Msg, Opts), + ?event(dev_cache, {write_single, {location_extracted, Location}}), + Operation = hb_ao:get(<<"operation">>, Msg, <<"write">>, Opts), + ?event(dev_cache, {write_single, {operation, Operation}}), + case {Operation, Body, Location} of + {<<"write">>, not_found, _} -> + ?event(dev_cache, {write_single, {error, "No body to write"}}), + {error, + #{ + <<"status">> => 400, + <<"body">> => <<"No body to write.">> + } + }; + {<<"write">>, Binary, not_found} when is_binary(Binary) -> + % When asked to write only a binary, we do not calculate any + % alternative IDs. + ?event(dev_cache, + {write_single, + {processing_binary, Binary, Location} + } + ), + {ok, Path} = hb_cache:write(Binary, Opts), + ?event(dev_cache, {write_single, {binary_written, Path}}), + {ok, #{ <<"status">> => 200, <<"path">> => Path }}; + {<<"link">>, _, _} -> + ?event(dev_cache, {write_single, {processing_link}}), + Source = hb_ao:get(<<"source">>, Msg, Opts), + Destination = hb_ao:get(<<"destination">>, Msg, Opts), + ?event(dev_cache, + {write_single, + {link_params, Source, Destination} + } + ), + ok = hb_cache:link(Source, Destination, Opts), + ?event(dev_cache, {write_single, {link_success}}), + {ok, #{ <<"status">> => 200 }}; + _ -> + ?event(dev_cache, {write_single, {error, <<"Invalid write type">>}}), + {error, + #{ + <<"status">> => 400, + <<"body">> => <<"Invalid write type.">> + } + } + end. + +%% @doc Verify that the request originates from a trusted writer. +%% Checks that the single signer of the request is present in the list +%% of trusted cache writer addresses specified in the options. +%% +%% @param Req The request message. +%% @param Opts A map of configuration options. +%% @returns true if the request is from an authorized writer, false +%% otherwise. +is_trusted_writer(Req, Opts) -> + Signers = hb_message:signers(Req, Opts), + ?event(dev_cache, {is_trusted_writer, {signers, Signers}, {req, Req}}), + CacheWriters = hb_opts:get(cache_writers, [], Opts), + ?event(dev_cache, {is_trusted_writer, {cache_writers, CacheWriters}}), + AnyTrusted = lists:any(fun(Signer) -> lists:member(Signer, CacheWriters) end, Signers), + case AnyTrusted of + true -> + ?event(dev_cache, {is_trusted_writer, {trusted, true}}), + true; + _ -> + ?event(dev_cache, {is_trusted_writer, {trusted, false}}), + false + end. + +%%%-------------------------------------------------------------------- +%%% Test Helpers +%%%-------------------------------------------------------------------- + +%% @doc Create a test environment with a local store and node. +%% Ensures that the required application is started, configures a local +%% file-system store, resets the store for a clean state, creates a wallet +%% for signing requests, and starts a node with the store and trusted cache +%% writer configuration. +%% +%% @param StorePrefix A binary specifying the prefix for the local store. +%% @returns {ok, TestOpts, [LocalStore, Wallet, Address, Node]} +setup_test_env() -> + Timestamp = integer_to_binary(os:system_time(millisecond)), + StorePrefix = <<"cache-TEST/remote-", Timestamp/binary>>, + ?event(dev_cache, {setup_test_env, {start, StorePrefix}}), + application:ensure_all_started(hb), + ?event(dev_cache, {setup_test_env, {hb_started}}), + LocalStore = + #{ <<"store-module">> => hb_store_fs, <<"name">> => StorePrefix }, + ?event(dev_cache, {setup_test_env, {local_store_configured, LocalStore}}), + hb_store:reset(LocalStore), + ?event(dev_cache, {setup_test_env, {store_reset}}), + Wallet = ar_wallet:new(), + Address = hb_util:human_id(ar_wallet:to_address(Wallet)), + ?event(dev_cache, {setup_test_env, {address, Address}}), + Node = hb_http_server:start_node(#{ + cache_control => [<<"no-cache">>, <<"no-store">>], + store => LocalStore, + cache_writers => [ + Address, + hb_util:human_id(ar_wallet:to_address(hb:wallet())) + ], + store_all_signed => false + }), + ?event(dev_cache, {setup_test_env, {node_started, Node}}), + TestOpts = #{ + cache_control => [<<"no-cache">>, <<"no-store">>], + store_all_signed => false, + store => [ + #{ + <<"store-module">> => hb_store_remote_node, + <<"node">> => Node, + priv_wallet => Wallet + } + ] + }, + {ok, TestOpts, [LocalStore, Wallet, Address, Node]}. + +%% @doc Write data to the cache via HTTP. +%% Constructs a write request message with the provided data, signs it with the +%% given wallet, sends it to the node, and verifies that the response indicates +%% a successful write. +%% +%% @param Node The target node. +%% @param Data The data to be written. +%% @param Wallet The wallet used to sign the request. +%% @returns {ok, WriteResponse} on success. +write_to_cache(Node, Data, Wallet) -> + ?event(dev_cache, {write_to_cache, {start, Node}}), + WriteMsg = #{ + <<"path">> => <<"/~cache@1.0/write">>, + <<"method">> => <<"POST">>, + <<"body">> => Data + }, + ?event(dev_cache, {write_to_cache, {message_created, WriteMsg}}), + SignedMsg = hb_message:commit(WriteMsg, Wallet), + ?event(dev_cache, {write_to_cache, {message_signed}}), + WriteResult = hb_http:post(Node, SignedMsg, #{}), + ?event(dev_cache, {write_to_cache, {http_post, WriteResult}}), + {ok, WriteResponse} = WriteResult, + ?event(dev_cache, {write_to_cache, {response_received, WriteResponse}}), + Status = hb_ao:get(<<"status">>, WriteResponse, 0, #{}), + ?assertEqual(200, Status), + Path = hb_ao:get(<<"path">>, WriteResponse, not_found, #{}), + ?assertNotEqual(not_found, Path), + ?event(dev_cache, {write_to_cache, {write_success, Path}}), + {WriteResponse, Path}. + +%% @doc Read data from the cache via HTTP. +%% Constructs a GET request using the provided path, sends it to the node, +%% and returns the response. +%% +%% @param Node The target node. +%% @param Path The key or location where the data is stored. +%% @returns The response read from the cache (either binary or wrapped in +%% {ok, Response}). +read_from_cache(Node, Path) -> + ?event(dev_cache, {read_from_cache, {start, Node, Path}}), + ReadMsg = #{ + <<"path">> => <<"/~cache@1.0/read">>, + <<"method">> => <<"GET">>, + <<"target">> => Path + }, + ?event(dev_cache, {read_from_cache, {request_created, ReadMsg}}), + ?event({test_read, request, ReadMsg}), + ReadResult = hb_http:get(Node, ReadMsg, #{}), + ?event(dev_cache, {read_from_cache, {http_get, ReadResult}}), + case ReadResult of + ReadResponse when is_binary(ReadResponse) -> + ?event(dev_cache, + {read_from_cache, + {response_binary, ReadResponse} + } + ), + ReadResponse; + {ok, ReadResponse} -> + ?event(dev_cache, {read_from_cache, {response_ok, ReadResponse}}), + ReadResponse; + {error, Reason} -> + ?event(dev_cache, {read_from_cache, {response_error, Reason}}), + {error, Reason} + end. + +%%%-------------------------------------------------------------------- +%%% Tests +%%%-------------------------------------------------------------------- + +%% @doc Test that the cache can be written to and read from using the hb_cache +%% API. +cache_write_message_test() -> + ?event(dev_cache, {cache_api_test, {start}}), + {ok, Opts, _} = setup_test_env(), + TestData = #{ + <<"test_key">> => <<"test_value">> + }, + ?event(dev_cache, {cache_api_test, {opts, Opts}}), + {ok, Path} = hb_cache:write(TestData, Opts), + ?event(dev_cache, {cache_api_test, {data_written, Path}}), + {ok, ReadData} = hb_cache:read(Path, Opts), + ?event(dev_cache, {cache_api_test, {data_read, ReadData}}), + ?assert(hb_message:match(TestData, ReadData, only_present, Opts)), + ?event(dev_cache, {cache_api_test}), + ok. + +%% @doc Ensure that we can write direct binaries to the cache. +cache_write_binary_test() -> + ?event(dev_cache, {cache_api_test, {start}}), + {ok, Opts, _} = setup_test_env(), + TestData = <<"test_binary">>, + {ok, Path} = hb_cache:write(TestData, Opts), + {ok, ReadData} = hb_cache:read(Path, Opts), + ?event(dev_cache, {cache_api_test, {data_read, ReadData}}), + ?assertEqual(TestData, ReadData), + ?event(dev_cache, {cache_api_test}), + ok. \ No newline at end of file diff --git a/src/dev_cacheviz.erl b/src/dev_cacheviz.erl new file mode 100644 index 000000000..c44f0704e --- /dev/null +++ b/src/dev_cacheviz.erl @@ -0,0 +1,69 @@ +%%% @doc A device that generates renders (or renderable dot output) of a node's +%%% cache. +-module(dev_cacheviz). +-export([dot/3, svg/3, json/3, index/3, js/3]). +-include("include/hb.hrl"). + +%% @doc Output the dot representation of the cache, or a specific path within +%% the cache set by the `target' key in the request. +dot(_, Req, Opts) -> + Target = hb_ao:get(<<"target">>, Req, all, Opts), + Dot = + hb_cache_render:cache_path_to_dot( + Target, + #{ + render_data => + hb_util:atom( + hb_ao:get(<<"render-data">>, Req, false, Opts) + ) + }, + Opts + ), + {ok, #{ <<"content-type">> => <<"text/vnd.graphviz">>, <<"body">> => Dot }}. + +%% @doc Output the SVG representation of the cache, or a specific path within +%% the cache set by the `target' key in the request. +svg(Base, Req, Opts) -> + {ok, #{ <<"body">> := Dot }} = dot(Base, Req, Opts), + ?event(cacheviz, {dot, Dot}), + Svg = hb_cache_render:dot_to_svg(Dot), + {ok, #{ <<"content-type">> => <<"image/svg+xml">>, <<"body">> => Svg }}. + +%% @doc Return a JSON representation of the cache graph, suitable for use with +%% the `graph.js' library. If the request specifies a `target' key, we use that +%% target. Otherwise, we generate a new target by writing the message to the +%% cache and using the ID of the written message. +json(Base, Req, Opts) -> + ?event({json, {base, Base}, {req, Req}}), + Target = + case hb_ao:get(<<"target">>, Req, Opts) of + not_found -> + case map_size(maps:without([<<"device">>], hb_private:reset(Base))) of + 0 -> + all; + _ -> + ?event({writing_base_for_rendering, Base}), + {ok, Path} = hb_cache:write(Base, Opts), + ?event({wrote_message, Path}), + ID = hb_message:id(Base, all, Opts), + ?event({generated_id, ID}), + ID + end; + <<".">> -> all; + ReqTarget -> ReqTarget + end, + MaxSize = hb_util:int(hb_ao:get(<<"max-size">>, Req, 250, Opts)), + ?event({max_size, MaxSize}), + ?event({generating_json_for, {target, Target}}), + Res = hb_cache_render:get_graph_data(Target, MaxSize, Opts), + ?event({graph_data, Res}), + Res. + +%% @doc Return a renderer in HTML form for the JSON format. +index(Base, _, _Opts) -> + ?event({cacheviz_index, {base, Base}}), + dev_hyperbuddy:return_file(<<"cacheviz@1.0">>, <<"graph.html">>). + +%% @doc Return a JS library that can be used to render the JSON format. +js(_, _, _Opts) -> + dev_hyperbuddy:return_file(<<"cacheviz@1.0">>, <<"graph.js">>). \ No newline at end of file diff --git a/src/dev_codec_ans104.erl b/src/dev_codec_ans104.erl new file mode 100644 index 000000000..b6a514538 --- /dev/null +++ b/src/dev_codec_ans104.erl @@ -0,0 +1,428 @@ +%%% @doc Codec for managing transformations from `ar_bundles'-style Arweave TX +%%% records to and from TABMs. +-module(dev_codec_ans104). +-export([to/3, from/3, commit/3, verify/3, content_type/1]). +-export([serialize/3, deserialize/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Return the content type for the codec. +content_type(_) -> {ok, <<"application/ans104">>}. + +%% @doc Serialize a message or TX to a binary. +serialize(Msg, Req, Opts) when is_map(Msg) -> + serialize(to(Msg, Req, Opts), Req, Opts); +serialize(TX, _Req, _Opts) when is_record(TX, tx) -> + {ok, ar_bundles:serialize(TX)}. + +%% @doc Deserialize a binary ans104 message to a TABM. +deserialize(#{ <<"body">> := Binary }, Req, Opts) -> + deserialize(Binary, Req, Opts); +deserialize(Binary, Req, Opts) when is_binary(Binary) -> + deserialize(ar_bundles:deserialize(Binary), Req, Opts); +deserialize(TX, Req, Opts) when is_record(TX, tx) -> + from(TX, Req, Opts). + +%% @doc Sign a message using the `priv_wallet' key in the options. Supports both +%% the `hmac-sha256' and `rsa-pss-sha256' algorithms, offering unsigned and +%% signed commitments. +commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> + commit(Msg, Req#{ <<"type">> => <<"unsigned-sha256">> }, Opts); +commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> + commit(Msg, Req#{ <<"type">> => <<"rsa-pss-sha256">> }, Opts); +commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) -> + % Convert the given message to an ANS-104 TX record, sign it, and convert + % it back to a structured message. + {ok, TX} = to(hb_private:reset(Msg), Req, Opts), + Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), + Signed = ar_bundles:sign_item(TX, Wallet), + SignedStructured = + hb_message:convert( + Signed, + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ), + {ok, SignedStructured}; +commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> + % Remove the commitments from the message, convert it to ANS-104, then back. + % This forces the message to be normalized and the unsigned ID to be + % recalculated. + { + ok, + hb_message:convert( + hb_maps:without([<<"commitments">>], Msg, Opts), + <<"ans104@1.0">>, + <<"structured@1.0">>, + Opts + ) + }. + +%% @doc Verify an ANS-104 commitment. +verify(Msg, Req, Opts) -> + ?event({verify, {base, Msg}, {req, Req}}), + OnlyWithCommitment = + hb_private:reset( + hb_message:with_commitments( + Req, + Msg, + Opts + ) + ), + ?event({verify, {only_with_commitment, OnlyWithCommitment}}), + {ok, TX} = to(OnlyWithCommitment, Req, Opts), + ?event({verify, {encoded, TX}}), + Res = ar_bundles:verify_item(TX), + {ok, Res}. + +%% @doc Convert a #tx record into a message map recursively. +from(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, Binary}; +from(TX, Req, Opts) when is_record(TX, tx) -> + case lists:keyfind(<<"ao-type">>, 1, TX#tx.tags) of + false -> + do_from(TX, Req, Opts); + {<<"ao-type">>, <<"binary">>} -> + {ok, TX#tx.data} + end. +do_from(RawTX, Req, Opts) -> + % Ensure the TX is fully deserialized. + TX = ar_bundles:deserialize(ar_bundles:normalize(RawTX)), + ?event({parsed_tx, TX}), + % Get the fields, tags, and data from the TX. + Fields = dev_codec_ans104_from:fields(TX, Opts), + Tags = dev_codec_ans104_from:tags(TX, Opts), + Data = dev_codec_ans104_from:data(TX, Req, Tags, Opts), + ?event({parsed_components, {fields, Fields}, {tags, Tags}, {data, Data}}), + % Calculate the committed keys on from the TX. + Keys = dev_codec_ans104_from:committed(TX, Fields, Tags, Data, Opts), + ?event({determined_committed_keys, Keys}), + % Create the base message from the fields, tags, and data, filtering to + % include only the keys that are committed. Will throw if a key is missing. + Base = dev_codec_ans104_from:base(Keys, Fields, Tags, Data, Opts), + ?event({calculated_base_message, Base}), + % Add the commitments to the message if the TX has a signature. + WithCommitments = + dev_codec_ans104_from:with_commitments(TX, Tags, Base, Keys, Opts), + ?event({parsed_message, WithCommitments}), + {ok, WithCommitments}. + +%% @doc Internal helper to translate a message to its #tx record representation, +%% which can then be used by ar_bundles to serialize the message. We call the +%% message's device in order to get the keys that we will be checkpointing. We +%% do this recursively to handle nested messages. The base case is that we hit +%% a binary, which we return as is. +to(Binary, _Req, _Opts) when is_binary(Binary) -> + % ar_bundles cannot serialize just a simple binary or get an ID for it, so + % we turn it into a TX record with a special tag, tx_to_message will + % identify this tag and extract just the binary. + {ok, + #tx{ + tags = [{<<"ao-type">>, <<"binary">>}], + data = Binary + } + }; +to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX}; +to(RawTABM, Req, Opts) when is_map(RawTABM) -> + % Ensure that the TABM is fully loaded if the `bundle` key is set to true. + ?event({to, {inbound, RawTABM}, {req, Req}}), + MaybeBundle = dev_codec_ans104_to:maybe_load(RawTABM, Req, Opts), + TX0 = dev_codec_ans104_to:siginfo(MaybeBundle, Opts), + ?event({found_siginfo, TX0}), + % Calculate and normalize the `data', if applicable. + Data = dev_codec_ans104_to:data(MaybeBundle, Req, Opts), + ?event({calculated_data, Data}), + TX1 = TX0#tx { data = Data }, + % Calculate the tags for the TX. + Tags = dev_codec_ans104_to:tags(TX1, MaybeBundle, Data, Opts), + ?event({calculated_tags, Tags}), + TX2 = TX1#tx { tags = Tags }, + ?event({tx_before_id_gen, TX2}), + Res = + try ar_bundles:reset_ids(ar_bundles:normalize(TX2)) + catch + Type:Error:Stacktrace -> + ?event({{reset_ids_error, Error}, {tx_without_data, TX2}}), + ?event({prepared_tx_before_ids, + {tags, {explicit, TX2#tx.tags}}, + {data, TX2#tx.data} + }), + erlang:raise(Type, Error, Stacktrace) + end, + ?event({to_result, Res}), + {ok, Res}; +to(Other, _Req, _Opts) -> + throw({invalid_tx, Other}). + +%%% ANS-104-specific testing cases. + +normal_tags_test() -> + Msg = #{ + <<"first-tag">> => <<"first-value">>, + <<"second-tag">> => <<"second-value">> + }, + {ok, Encoded} = to(Msg, #{}, #{}), + ?event({encoded, Encoded}), + {ok, Decoded} = from(Encoded, #{}, #{}), + ?event({decoded, Decoded}), + ?assert(hb_message:match(Msg, Decoded)). + +from_maintains_tag_name_case_test() -> + TX = #tx { + tags = [ + {<<"Test-Tag">>, <<"test-value">>} + ] + }, + SignedTX = ar_bundles:sign_item(TX, hb:wallet()), + ?event({signed_tx, SignedTX}), + ?assert(ar_bundles:verify_item(SignedTX)), + TABM = hb_util:ok(from(SignedTX, #{}, #{})), + ?event({tabm, TABM}), + ConvertedTX = hb_util:ok(to(TABM, #{}, #{})), + ?event({converted_tx, ConvertedTX}), + ?assert(ar_bundles:verify_item(ConvertedTX)), + ?assertEqual(ConvertedTX, ar_bundles:normalize(SignedTX)). + +restore_tag_name_case_from_cache_test() -> + Opts = #{ store => hb_test_utils:test_store() }, + TX = #tx { + tags = [ + {<<"Test-Tag">>, <<"test-value">>}, + {<<"test-tag-2">>, <<"test-value-2">>} + ] + }, + SignedTX = ar_bundles:sign_item(TX, ar_wallet:new()), + SignedMsg = + hb_message:convert( + SignedTX, + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ), + SignedID = hb_message:id(SignedMsg, all), + ?event({signed_msg, SignedMsg}), + OnlyCommitted = hb_message:with_only_committed(SignedMsg, Opts), + ?event({only_committed, OnlyCommitted}), + {ok, ID} = hb_cache:write(SignedMsg, Opts), + ?event({id, ID}), + {ok, ReadMsg} = hb_cache:read(SignedID, Opts), + ?event({restored_msg, ReadMsg}), + {ok, ReadTX} = to(ReadMsg, #{}, Opts), + ?event({restored_tx, ReadTX}), + ?assert(hb_message:match(ReadMsg, SignedMsg)), + ?assert(ar_bundles:verify_item(ReadTX)). + +unsigned_duplicated_tag_name_test() -> + TX = ar_bundles:reset_ids(ar_bundles:normalize(#tx { + tags = [ + {<<"Test-Tag">>, <<"test-value">>}, + {<<"test-tag">>, <<"test-value-2">>} + ] + })), + Msg = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}), + ?event({msg, Msg}), + TX2 = hb_message:convert(Msg, <<"ans104@1.0">>, <<"structured@1.0">>, #{}), + ?event({tx2, TX2}), + ?assertEqual(TX, TX2). + +signed_duplicated_tag_name_test() -> + TX = ar_bundles:sign_item(#tx { + tags = [ + {<<"Test-Tag">>, <<"test-value">>}, + {<<"test-tag">>, <<"test-value-2">>} + ] + }, ar_wallet:new()), + Msg = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}), + ?event({msg, Msg}), + TX2 = hb_message:convert(Msg, <<"ans104@1.0">>, <<"structured@1.0">>, #{}), + ?event({tx2, TX2}), + ?assertEqual(TX, TX2), + ?assert(ar_bundles:verify_item(TX2)). + +simple_to_conversion_test() -> + Msg = #{ + <<"first-tag">> => <<"first-value">>, + <<"second-tag">> => <<"second-value">> + }, + {ok, Encoded} = to(Msg, #{}, #{}), + ?event({encoded, Encoded}), + {ok, Decoded} = from(Encoded, #{}, #{}), + ?event({decoded, Decoded}), + ?assert(hb_message:match(Msg, hb_message:uncommitted(Decoded, #{}))). + +% @doc Ensure that items with an explicitly defined target field lead to: +% 1. A target being set in the `target' field of the TX record on inbound. +% 2. The parsed message having a `target' field which is committed. +% 3. The target field being placed back into the record, rather than the `tags', +% on re-encoding. +external_item_with_target_field_test() -> + TX = + ar_bundles:sign_item( + #tx { + target = crypto:strong_rand_bytes(32), + tags = [ + {<<"test-tag">>, <<"test-value">>}, + {<<"test-tag-2">>, <<"test-value-2">>} + ], + data = <<"test-data">> + }, + ar_wallet:new() + ), + EncodedTarget = hb_util:encode(TX#tx.target), + ?event({tx, TX}), + Decoded = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}), + ?event({decoded, Decoded}), + ?assertEqual(EncodedTarget, hb_maps:get(<<"target">>, Decoded, undefined, #{})), + {ok, OnlyCommitted} = hb_message:with_only_committed(Decoded, #{}), + ?event({only_committed, OnlyCommitted}), + ?assertEqual(EncodedTarget, hb_maps:get(<<"target">>, OnlyCommitted, undefined, #{})), + Encoded = hb_message:convert(OnlyCommitted, <<"ans104@1.0">>, <<"structured@1.0">>, #{}), + ?assertEqual(TX#tx.target, Encoded#tx.target), + ?event({result, {initial, TX}, {result, Encoded}}), + ?assertEqual(TX, Encoded). + +% @doc Ensure that items made inside HyperBEAM use the tags to encode `target' +% values, rather than the `target' field. +generate_item_with_target_tag_test() -> + Msg = + #{ + <<"target">> => Target = <<"NON-ID-TARGET">>, + <<"other-key">> => <<"other-value">> + }, + {ok, TX} = to(Msg, #{}, #{}), + ?event({encoded_tx, TX}), + % The encoded TX should have ignored the `target' field, setting a tag instead. + ?assertEqual(?DEFAULT_TARGET, TX#tx.target), + Decoded = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}), + ?event({decoded, Decoded}), + % The decoded message should have the `target' key set to the tag value. + ?assertEqual(Target, hb_maps:get(<<"target">>, Decoded, undefined, #{})), + {ok, OnlyCommitted} = hb_message:with_only_committed(Decoded, #{}), + ?event({only_committed, OnlyCommitted}), + % The target key should have been committed. + ?assertEqual(Target, hb_maps:get(<<"target">>, OnlyCommitted, undefined, #{})), + Encoded = hb_message:convert(OnlyCommitted, <<"ans104@1.0">>, <<"structured@1.0">>, #{}), + ?event({result, {initial, TX}, {result, Encoded}}), + ?assertEqual(TX, Encoded). + +generate_item_with_target_field_test() -> + Msg = + hb_message:commit( + #{ + <<"target">> => Target = hb_util:encode(crypto:strong_rand_bytes(32)), + <<"other-key">> => <<"other-value">> + }, + #{ priv_wallet => hb:wallet() }, + <<"ans104@1.0">> + ), + {ok, TX} = to(Msg, #{}, #{}), + ?event({encoded_tx, TX}), + ?assertEqual(Target, hb_util:encode(TX#tx.target)), + Decoded = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}), + ?event({decoded, Decoded}), + ?assertEqual(Target, hb_maps:get(<<"target">>, Decoded, undefined, #{})), + {ok, OnlyCommitted} = hb_message:with_only_committed(Decoded, #{}), + ?event({only_committed, OnlyCommitted}), + ?assertEqual(Target, hb_maps:get(<<"target">>, OnlyCommitted, undefined, #{})), + Encoded = hb_message:convert(OnlyCommitted, <<"ans104@1.0">>, <<"structured@1.0">>, #{}), + ?event({result, {initial, TX}, {result, Encoded}}), + ?assertEqual(TX, Encoded). + +type_tag_test() -> + TX = + ar_bundles:sign_item( + #tx { + tags = [{<<"type">>, <<"test-value">>}] + }, + ar_wallet:new() + ), + ?event({tx, TX}), + Structured = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}), + ?event({structured, Structured}), + TX2 = hb_message:convert(Structured, <<"ans104@1.0">>, <<"structured@1.0">>, #{}), + ?event({after_conversion, TX2}), + ?assertEqual(TX, TX2). + +ao_data_key_test() -> + Msg = + hb_message:commit( + #{ + <<"other-key">> => <<"Normal value">>, + <<"body">> => <<"Body value">> + }, + #{ priv_wallet => hb:wallet() }, + <<"ans104@1.0">> + ), + ?event({msg, Msg}), + Enc = hb_message:convert(Msg, <<"ans104@1.0">>, #{}), + ?event({enc, Enc}), + ?assertEqual(<<"Body value">>, Enc#tx.data), + Dec = hb_message:convert(Enc, <<"structured@1.0">>, <<"ans104@1.0">>, #{}), + ?event({dec, Dec}), + ?assert(hb_message:verify(Dec, all, #{})). + +simple_signed_to_httpsig_test() -> + Structured = + hb_message:commit( + #{ <<"test-tag">> => <<"test-value">> }, + #{ priv_wallet => ar_wallet:new() }, + #{ + <<"commitment-device">> => <<"ans104@1.0">> + } + ), + ?event(debug_test, {msg, Structured}), + HTTPSig = + hb_message:convert( + Structured, + <<"httpsig@1.0">>, + <<"structured@1.0">>, + #{} + ), + ?event(debug_test, {httpsig, HTTPSig}), + Structured2 = + hb_message:convert( + HTTPSig, + <<"structured@1.0">>, + <<"httpsig@1.0">>, + #{} + ), + ?event(debug_test, {decoded, Structured2}), + Match = hb_message:match(Structured, Structured2, #{}), + ?assert(Match), + ?assert(hb_message:verify(Structured2, all, #{})), + HTTPSig2 = hb_message:convert(Structured2, <<"httpsig@1.0">>, <<"structured@1.0">>, #{}), + ?event(debug_test, {httpsig2, HTTPSig2}), + ?assert(hb_message:verify(HTTPSig2, all, #{})), + ?assert(hb_message:match(HTTPSig, HTTPSig2)). + +unsorted_tag_map_test() -> + TX = + ar_bundles:sign_item( + #tx{ + format = ans104, + tags = [ + {<<"z">>, <<"position-1">>}, + {<<"a">>, <<"position-2">>} + ], + data = <<"data">> + }, + ar_wallet:new() + ), + ?assert(ar_bundles:verify_item(TX)), + ?event(debug_test, {tx, TX}), + {ok, TABM} = dev_codec_ans104:from(TX, #{}, #{}), + ?event(debug_test, {tabm, TABM}), + {ok, Decoded} = dev_codec_ans104:to(TABM, #{}, #{}), + ?event(debug_test, {decoded, Decoded}), + ?assert(ar_bundles:verify_item(Decoded)). + +field_and_tag_ordering_test() -> + UnsignedTABM = #{ + <<"a">> => <<"value1">>, + <<"z">> => <<"value2">>, + <<"target">> => <<"NON-ID-TARGET">> + }, + + Wallet = hb:wallet(), + SignedTABM = hb_message:commit( + UnsignedTABM, #{priv_wallet => Wallet}, <<"ans104@1.0">>), + ?assert(hb_message:verify(SignedTABM)). \ No newline at end of file diff --git a/src/dev_codec_ans104_from.erl b/src/dev_codec_ans104_from.erl new file mode 100644 index 000000000..1ebe8524e --- /dev/null +++ b/src/dev_codec_ans104_from.erl @@ -0,0 +1,307 @@ +%%% @doc Library functions for decoding ANS-104-style data items to TABM form. +-module(dev_codec_ans104_from). +-export([fields/2, tags/2, data/4, committed/5, base/5]). +-export([with_commitments/5]). +-include("include/hb.hrl"). + +%% @doc Return a TABM message containing the fields of the given decoded +%% ANS-104 data item that should be included in the base message. +fields(Item, _Opts) -> + case Item#tx.target of + ?DEFAULT_TARGET -> #{}; + Target -> + #{ + <<"target">> => hb_util:encode(Target) + } + end. + +%% @doc Return a TABM of the raw tags of the item, including all metadata +%% (e.g. `ao-type', `ao-data-key', etc.) +tags(Item, Opts) -> + Tags = hb_ao:normalize_keys( + deduplicating_from_list(Item#tx.tags, Opts), + Opts + ), + ao_types(Tags, Opts). + +%% @doc Ensure the encoded keys in the `ao-types' field are lowercased and +%% normalized like the other keys in the tags field. +ao_types(#{ <<"ao-types">> := AoTypes } = Tags, Opts) -> + AOTypes = dev_codec_structured:decode_ao_types(AoTypes, Opts), + % Normalize all keys in the ao-types map and re-encode + NormAOTypes = + maps:fold( + fun(Key, Val, Acc) -> + NormKey = hb_util:to_lower(hb_ao:normalize_key(Key)), + Acc#{ NormKey => Val } + end, + #{}, + AOTypes + ), + EncodedAOTypes = dev_codec_structured:encode_ao_types(NormAOTypes, Opts), + Tags#{ <<"ao-types">> := EncodedAOTypes }; +ao_types(Tags, _Opts) -> + Tags. + +%% @doc Return a TABM of the keys and values found in the data field of the item. +data(Item, Req, Tags, Opts) -> + % If the data field is empty, we return an empty map. If it is a map, we + % return it as such. Otherwise, we return a map with the data key set to + % the raw data value. This handles unbundling nested messages, as well as + % applying the `ao-data-key' tag if given. + DataKey = maps:get(<<"ao-data-key">>, Tags, <<"data">>), + case {DataKey, Item#tx.data} of + {_, ?DEFAULT_DATA} -> #{}; + {DataKey, Map} when is_map(Map) -> + % If the data is a map, we need to recursively turn its children + % into messages from their tx representations. + hb_ao:normalize_keys( + hb_maps:map( + fun(_, InnerValue) -> + hb_util:ok(dev_codec_ans104:from(InnerValue, Req, Opts)) + end, + Map, + Opts + ), + Opts + ); + {DataKey, Data} -> #{ DataKey => Data } + end. + +%% @doc Calculate the list of committed keys for an item, based on its +%% components (fields, tags, and data). +committed(Item, Fields, Tags, Data, Opts) -> + hb_util:unique( + data_keys(Data, Opts) ++ + tag_keys(Item, Opts) ++ + field_keys(Fields, Tags, Data, Opts) + ). + +%% @doc Return the list of the keys from the fields TABM. +field_keys(BaseFields, Tags, Data, Opts) -> + HasTarget = + hb_maps:is_key(<<"target">>, BaseFields, Opts) orelse + hb_maps:is_key(<<"target">>, Tags, Opts) orelse + hb_maps:is_key(<<"target">>, Data, Opts), + case HasTarget of + true -> [<<"target">>]; + false -> [] + end. + +%% @doc Return the list of the keys from the data TABM. +data_keys(Data, Opts) -> + hb_util:to_sorted_keys(Data, Opts). + +%% @doc Return the list of the keys from the tags TABM. Filter all metadata +%% tags: `ao-data-key', `ao-types', `bundle-format', `bundle-version'. +tag_keys(Item, _Opts) -> + MetaTags = [ + <<"bundle-format">>, + <<"bundle-version">>, + <<"bundle-map">>, + <<"ao-data-key">> + ], + lists:filtermap( + fun({Tag, _}) -> + case lists:member(Tag, MetaTags) of + true -> false; + false -> {true, hb_util:to_lower(hb_ao:normalize_key(Tag))} + end + end, + Item#tx.tags + ). + +%% @doc Return the complete message for an item, less its commitments. The +%% precidence order for choosing fields to place into the base message is: +%% 1. Data +%% 2. Tags +%% 3. Fields +base(CommittedKeys, Fields, Tags, Data, Opts) -> + hb_maps:from_list( + lists:map( + fun(Key) -> + case hb_maps:find(Key, Data, Opts) of + error -> + case hb_maps:find(Key, Fields, Opts) of + error -> + case hb_maps:find(Key, Tags, Opts) of + error -> throw({missing_key, Key}); + {ok, Value} -> {Key, Value} + end; + {ok, Value} -> {Key, Value} + end; + {ok, Value} -> {Key, Value} + end + end, + CommittedKeys + ) + ). + +%% @doc Return a message with the appropriate commitments added to it. +with_commitments(Item, Tags, Base, CommittedKeys, Opts) -> + case Item#tx.signature of + ?DEFAULT_SIG -> + case normal_tags(Item#tx.tags) of + true -> Base; + false -> + with_unsigned_commitment(Item, Tags, Base, CommittedKeys, Opts) + end; + _ -> with_signed_commitment(Item, Tags, Base, CommittedKeys, Opts) + end. + +%% @doc Returns a commitments message for an item, containing an unsigned +%% commitment. +with_unsigned_commitment(Item, Tags, UncommittedMessage, CommittedKeys, Opts) -> + ID = hb_util:human_id(Item#tx.unsigned_id), + UncommittedMessage#{ + <<"commitments">> => #{ + ID => + filter_unset( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"committed">> => CommittedKeys, + <<"type">> => <<"unsigned-sha256">>, + <<"bundle">> => bundle_commitment_key(Tags, Opts), + <<"original-tags">> => original_tags(Item, Opts), + <<"field-target">> => + case Item#tx.target of + ?DEFAULT_TARGET -> unset; + Target -> hb_util:encode(Target) + end, + <<"field-anchor">> => + case Item#tx.anchor of + ?DEFAULT_LAST_TX -> unset; + LastTX -> LastTX + end + }, + Opts + ) + } + }. + +%% @doc Returns a commitments message for an item, containing a signed +%% commitment. +with_signed_commitment(Item, Tags, UncommittedMessage, CommittedKeys, Opts) -> + Address = hb_util:human_id(ar_wallet:to_address(Item#tx.owner)), + ID = hb_util:human_id(Item#tx.id), + Commitment = + filter_unset( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"committer">> => Address, + <<"committed">> => CommittedKeys, + <<"signature">> => hb_util:encode(Item#tx.signature), + <<"keyid">> => + <<"publickey:", (hb_util:encode(Item#tx.owner))/binary>>, + <<"type">> => <<"rsa-pss-sha256">>, + <<"bundle">> => bundle_commitment_key(Tags, Opts), + <<"original-tags">> => original_tags(Item, Opts), + <<"field-anchor">> => + case Item#tx.anchor of + ?DEFAULT_LAST_TX -> unset; + LastTX -> LastTX + end, + <<"field-target">> => + case Item#tx.target of + ?DEFAULT_TARGET -> unset; + Target -> hb_util:encode(Target) + end + }, + Opts + ), + UncommittedMessage#{ + <<"commitments">> => #{ + ID => Commitment + } + }. + +%% @doc Return the bundle key for an item. +bundle_commitment_key(Tags, Opts) -> + hb_util:bin(hb_maps:is_key(<<"bundle-format">>, Tags, Opts)). + +%% @doc Check whether a list of key-value pairs contains only normalized keys. +normal_tags(Tags) -> + lists:all( + fun({Key, _}) -> + hb_util:to_lower(hb_ao:normalize_key(Key)) =:= Key + end, + Tags + ). + +%% @doc Return the original tags of an item if it is applicable. Otherwise, +%% return `undefined'. +original_tags(Item, _Opts) -> + case normal_tags(Item#tx.tags) of + true -> unset; + false -> encoded_tags_to_map(Item#tx.tags) + end. + +%% @doc Convert an ANS-104 encoded tag list into a HyperBEAM-compatible map. +encoded_tags_to_map(Tags) -> + hb_util:list_to_numbered_message( + lists:map( + fun({Key, Value}) -> + #{ + <<"name">> => Key, + <<"value">> => Value + } + end, + Tags + ) + ). + +%% @doc Remove all undefined values from a map. +filter_unset(Map, Opts) -> + hb_maps:filter( + fun(_, Value) -> + case Value of + unset -> false; + _ -> true + end + end, + Map, + Opts + ). + +%% @doc Deduplicate a list of key-value pairs by key, generating a list of +%% values for each normalized key if there are duplicates. +deduplicating_from_list(Tags, Opts) -> + % Aggregate any duplicated tags into an ordered list of values. + Aggregated = + lists:foldl( + fun({Key, Value}, Acc) -> + NormKey = hb_util:to_lower(hb_ao:normalize_key(Key)), + case hb_maps:get(NormKey, Acc, undefined, Opts) of + undefined -> hb_maps:put(NormKey, Value, Acc, Opts); + Existing when is_list(Existing) -> + hb_maps:put(NormKey, Existing ++ [Value], Acc, Opts); + ExistingSingle -> + hb_maps:put(NormKey, [ExistingSingle, Value], Acc, Opts) + end + end, + #{}, + Tags + ), + ?event({deduplicating_from_list, {aggregated, Aggregated}}), + % Convert aggregated values into a structured-field list. + Res = + hb_maps:map( + fun(_Key, Values) when is_list(Values) -> + % Convert Erlang lists of binaries into a structured-field list. + iolist_to_binary( + hb_structured_fields:list( + [ + {item, {string, Value}, []} + || + Value <- Values + ] + ) + ); + (_Key, Value) -> + Value + end, + Aggregated, + Opts + ), + ?event({deduplicating_from_list, {result, Res}}), + Res. \ No newline at end of file diff --git a/src/dev_codec_ans104_to.erl b/src/dev_codec_ans104_to.erl new file mode 100644 index 000000000..f6ba53b59 --- /dev/null +++ b/src/dev_codec_ans104_to.erl @@ -0,0 +1,310 @@ +%%% @doc Library functions for encoding messages to the ANS-104 format. +-module(dev_codec_ans104_to). +-export([maybe_load/3, siginfo/2, data/3, tags/4]). +-include("include/hb.hrl"). + +%% @doc Determine if the message should be loaded from the cache and re-converted +%% to the TABM format. We do this if the `bundle' key is set to true. +maybe_load(RawTABM, Req, Opts) -> + case hb_util:atom(hb_ao:get(<<"bundle">>, Req, false, Opts)) of + false -> RawTABM; + true -> + % Convert back to the fully loaded structured@1.0 message, then + % convert to TABM with bundling enabled. + Structured = hb_message:convert(RawTABM, <<"structured@1.0">>, Opts), + Loaded = hb_cache:ensure_all_loaded(Structured, Opts), + % Convert to TABM with bundling enabled. + LoadedTABM = + hb_message:convert( + Loaded, + tabm, + #{ + <<"device">> => <<"structured@1.0">>, + <<"bundle">> => true + }, + Opts + ), + % Ensure the commitments from the original message are the only + % ones in the fully loaded message. + LoadedComms = maps:get(<<"commitments">>, RawTABM, #{}), + LoadedTABM#{ <<"commitments">> => LoadedComms } + end. + +%% @doc Calculate the fields for a message, returning an initial TX record. +%% One of the nuances here is that the `target' field must be set correctly. +%% If the message has a commitment, we extract the `field-target' if found and +%% place it in the `target' field. If the message does not have a commitment, +%% we check if the `target' field is set in the message. If it is encodable as +%% a valid 32-byte binary ID (assuming it is base64url encoded in the `to' call), +%% we place it in the `target' field. Otherwise, we leave it unset. +siginfo(Message, Opts) -> + MaybeCommitment = + hb_message:commitment( + #{ <<"commitment-device">> => <<"ans104@1.0">> }, + Message, + Opts + ), + case MaybeCommitment of + {ok, _, Commitment} -> commitment_to_tx(Commitment, Opts); + not_found -> + case hb_maps:find(<<"target">>, Message, Opts) of + {ok, EncodedTarget} -> + case hb_util:safe_decode(EncodedTarget) of + {ok, Target} when ?IS_ID(Target) -> + #tx{ target = Target }; + _ -> #tx{} + end; + error -> #tx{} + end; + multiple_matches -> + throw({multiple_ans104_commitments_unsupported, Message}) + end. + +%% @doc Convert a commitment to a base TX record. Extracts the owner, signature, +%% tags, and last TX from the commitment. If the value is not present, the +%% default value is used. +commitment_to_tx(Commitment, Opts) -> + Signature = + hb_util:decode( + maps:get(<<"signature">>, Commitment, hb_util:encode(?DEFAULT_SIG)) + ), + Owner = + case hb_maps:find(<<"keyid">>, Commitment, Opts) of + {ok, KeyID} -> + hb_util:decode( + dev_codec_httpsig_keyid:remove_scheme_prefix(KeyID) + ); + error -> ?DEFAULT_OWNER + end, + Tags = + case hb_maps:find(<<"original-tags">>, Commitment, Opts) of + {ok, OriginalTags} -> original_tags_to_tags(OriginalTags); + error -> [] + end, + LastTX = + case hb_maps:find(<<"field-anchor">>, Commitment, Opts) of + {ok, EncodedLastTX} -> hb_util:decode(EncodedLastTX); + error -> ?DEFAULT_LAST_TX + end, + Target = + case hb_maps:find(<<"field-target">>, Commitment, Opts) of + {ok, EncodedTarget} -> hb_util:decode(EncodedTarget); + error -> ?DEFAULT_TARGET + end, + ?event({commitment_owner, Owner}), + ?event({commitment_signature, Signature}), + ?event({commitment_tags, Tags}), + ?event({commitment_last_tx, LastTX}), + #tx{ + owner = Owner, + signature = Signature, + tags = Tags, + anchor = LastTX, + target = Target + }. + +%% @doc Calculate the data field for a message. +data(TABM, Req, Opts) -> + DataKey = inline_key(TABM), + % Translate the keys into a binary map. If a key has a value that is a map, + % we recursively turn its children into messages. + UnencodedNestedMsgs = data_messages(TABM, Opts), + NestedMsgs = + hb_maps:map( + fun(_, Msg) -> + hb_util:ok(dev_codec_ans104:to(Msg, Req, Opts)) + end, + UnencodedNestedMsgs, + Opts + ), + DataVal = hb_maps:get(DataKey, TABM, ?DEFAULT_DATA), + ?event(debug_data, {data_val, DataVal}), + case {DataVal, hb_maps:size(NestedMsgs, Opts)} of + {Binary, 0} when is_binary(Binary) -> + % There are no nested messages, so we return the binary alone. + Binary; + {?DEFAULT_DATA, _} -> + NestedMsgs; + {DataVal, _} -> + NestedMsgs#{ + DataKey => hb_util:ok(dev_codec_ans104:to(DataVal, Req, Opts)) + } + end. + +%% @doc Calculate the data value for a message. The rules are: +%% 1. There should be no more than 128 keys in the tags. +%% 2. Each key must be equal or less to 1024 bytes. +%% 3. Each value must be equal or less to 3072 bytes. +%% Presently, if we exceed these limits, we throw an error. +data_messages(TABM, Opts) when is_map(TABM) -> + UncommittedTABM = + hb_maps:without( + [<<"commitments">>, <<"data">>, <<"target">>], + hb_private:reset(TABM), + Opts + ), + % If there are too many keys in the TABM, throw an error. + if map_size(UncommittedTABM) > 128 -> + throw({too_many_keys, UncommittedTABM}); + true -> + % If there are less than 128 keys, we return those that are large, or + % are nested messages. + hb_maps:filter( + fun(Key, Value) -> + case is_map(Value) of + true -> true; + false -> byte_size(Value) > 3072 orelse byte_size(Key) > 1024 + end + end, + UncommittedTABM, + Opts + ) + end. + +%% @doc Calculate the tags field for a data item. If the TX already has tags +%% from the commitment decoding step, we use them. Otherwise we determine the +%% keys to use from the `committed' field of the TABM. +tags(#tx{ tags = ExistingTags }, _, _, _) when ExistingTags =/= [] -> + ExistingTags; +tags(TX, TABM, Data, Opts) -> + DataKey = inline_key(TABM), + MaybeCommitment = + hb_message:commitment( + #{ <<"commitment-device">> => <<"ans104@1.0">> }, + TABM, + Opts + ), + CommittedTagKeys = + case MaybeCommitment of + {ok, _, Commitment} -> + % There is already a commitment, so the tags and order are + % pre-determined. However, if the message has been bundled, + % any `+link`-suffixed keys in the committed list may need to + % be resolved to their base keys (e.g., `output+link` -> `output`). + % We normalize each committed key to whichever form actually + % exists in the current TABM to avoid missing keys. + lists:map( + fun(CommittedKey) -> + NormalizedKey = hb_ao:normalize_key(CommittedKey), + BaseKey = hb_link:remove_link_specifier(NormalizedKey), + case hb_maps:find(BaseKey, TABM, Opts) of + {ok, _} -> BaseKey; + error -> + BaseKeyLink = <>, + case hb_maps:find(BaseKeyLink, TABM, Opts) of + {ok, _} -> BaseKeyLink; + error -> BaseKey + end + end + end, + hb_util:message_to_ordered_list( + hb_util:ok( + hb_maps:find(<<"committed">>, Commitment, Opts) + ) + ) + ) -- + % If the target is set in the base TX from the + % commitment, we check if the TABM equals that value. If it does, + % we do not additionally add the target tag. If they differ, we + % include it. + case include_target_tag(TX, TABM, Opts) of + false -> [<<"target">>]; + true -> [] + end; + not_found -> + % There is no commitment, so we need to generate the tags. The + % bundle-format and bundle-version tags are added by `ar_bundles` + % so we do not add them here. The ao-data-key tag is added if it + % is set to a non-default value, followed by the keys from the + % TABM (less the data keys and target key -- see + % `include_target_tag/3` for rationale). + hb_util:list_without( + [<<"commitments">>] ++ + if is_map(Data) -> hb_maps:keys(Data, Opts); + true -> [] + end ++ + case include_target_tag(TX, TABM, Opts) of + false -> [<<"target">>]; + true -> [] + end, + hb_util:to_sorted_keys(hb_private:reset(TABM), Opts) + ); + multiple_matches -> + throw({multiple_ans104_commitments_unsupported, TABM}) + end, + ?event( + {tags_before_data_key, + {committed_tag_keys, CommittedTagKeys}, + {data_key, DataKey}, + {data, Data}, + {tabm, TABM} + }), + committed_tag_keys_to_tags(TX, TABM, DataKey, CommittedTagKeys, Opts). + +%% @doc Return whether to include the `target' tag in the tags list. +include_target_tag(TX, TABM, Opts) -> + case {TX#tx.target, hb_maps:get(<<"target">>, TABM, undefined, Opts)} of + {?DEFAULT_TARGET, _} -> true; + {FieldTarget, TagTarget} when FieldTarget =/= TagTarget -> false; + _ -> true + end. + +%% @doc Apply the `ao-data-key' to the committed keys to generate the list of +%% tags to include in the message. +committed_tag_keys_to_tags(TX, TABM, DataKey, Committed, Opts) -> + DataKeysToExclude = + case TX#tx.data of + Data when is_map(Data)-> maps:keys(Data); + _ -> [] + end, + case DataKey of + <<"data">> -> []; + _ -> [{<<"ao-data-key">>, DataKey}] + end ++ + lists:map( + fun(Key) -> + case hb_maps:find(Key, TABM, Opts) of + error -> throw({missing_committed_key, Key}); + {ok, Value} -> {Key, Value} + end + end, + hb_util:list_without( + [DataKey | DataKeysToExclude], + Committed + ) + ). + +%%% Utility functions + +%% @doc Determine if an `ao-data-key` should be added to the message. +inline_key(Msg) -> + InlineKey = maps:get(<<"ao-data-key">>, Msg, undefined), + case { + InlineKey, + maps:get(<<"data">>, Msg, ?DEFAULT_DATA) == ?DEFAULT_DATA, + maps:is_key(<<"body">>, Msg) + andalso not ?IS_LINK(maps:get(<<"body">>, Msg, undefined)) + } of + {Explicit, _, _} when Explicit =/= undefined -> + % ao-data-key already exists, so we honor it. + InlineKey; + {_, true, true} -> + % There is no specific data field set, but there is a body, so we + % use that as the `inline-key`. + <<"body">>; + _ -> + % Default: `data' resolves to `data'. + <<"data">> + end. + +%% @doc Convert a HyperBEAM-compatible map into an ANS-104 encoded tag list, +%% recreating the original order of the tags. +original_tags_to_tags(TagMap) -> + OrderedList = hb_util:message_to_ordered_list(hb_private:reset(TagMap)), + ?event({ordered_tagmap, {explicit, OrderedList}, {input, {explicit, TagMap}}}), + lists:map( + fun(#{ <<"name">> := Key, <<"value">> := Value }) -> + {Key, Value} + end, + OrderedList + ). \ No newline at end of file diff --git a/src/dev_codec_cookie.erl b/src/dev_codec_cookie.erl new file mode 100644 index 000000000..8765e0c7d --- /dev/null +++ b/src/dev_codec_cookie.erl @@ -0,0 +1,473 @@ +%%% @doc A utility device that manages setting and encoding/decoding the cookies +%%% found in requests from a caller. This device implements the `~cookie@1.0' +%%% codec, inline with the `~message@1.0' schema for conversion. +%%% +%%% Additionally, a `commit' to a message using a secret generated and stored +%%% in the cookies of the caller, and a `verify' key that validates said +%%% commitments. In addition, a `generate' key is provided to perform only the +%%% generation side of the commitment process. The `finalize' key may be +%%% employed to add a `set' operation to the end of a message sequence, which +%%% is used in hooks that need to ensure a caller always receives cookies +%%% generated outside of the normal AO-Core execution flow. In totality, these +%%% keys implement the `generator' interface type, and may be employed in +%%% various contexts. For example, `~auth-hook@1.0' may be configured to use +%%% this device to generate and store secrets in the cookies of the caller, +%%% which are then used with the `~proxy-wallet@1.0' device to sign requests. +%%% +%%% The `commit' and `verify' keys utilize the `~httpsig@1.0''s HMAC `secret' +%%% commitment scheme, which uses a secret key to commit to a message, with the +%%% `committer' being listed as a hash of the secret. +%%% +%%% This device supports the following paths: +%%% +%%% `/commit': Sets a `secret' key in the cookies of the caller. The name of +%%% the cookie is calculated as the hash of the secret. +%%% `/verify': Verifies the caller's request by checking the committer in the +%%% request matches the secret in the cookies of the base message. +%%% `/store': Sets the keys in the request message in the cookies of the caller. +%%% `/extract': Extracts the cookies from a base message. +%%% `/reset': Removes all cookie keys from the base message. +%%% `/to': Converts a message containing cookie sources (`cookie', `set-cookie', +%%% or `priv/cookie') into the format specified in the request message (e.g. +%%% `set-cookie', `cookie'). +%%% `/from': Converts a message containing encoded cookies into a message +%%% containing the cookies parsed and normalized. +-module(dev_codec_cookie). +%%% Public cookie manipulation API. +-export([get_cookie/3, store/3, extract/3, reset/2]). +%%% Public message codec API. +-export([to/3, from/3]). +%%% Public commit/verify API. +-export([commit/3, verify/3]). +%%% Generator API. +-export([generate/3, finalize/3]). +%%% Public utility functions. +-export([opts/1]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% @doc Get the private store options to use for functions in the cookie device. +opts(Opts) -> hb_private:opts(Opts). + +%%% ~message@1.0 Commitments API keys. +commit(Base, Req, RawOpts) -> dev_codec_cookie_auth:commit(Base, Req, RawOpts). +verify(Base, Req, RawOpts) -> dev_codec_cookie_auth:verify(Base, Req, RawOpts). + +%% @doc Preprocessor keys that utilize cookies and the `~secret@1.0' device to +%% sign inbound HTTP requests from users if they are not already signed. We use +%% the `~hook@1.0' authentication framework to implement this. +generate(Base, Req, Opts) -> + dev_codec_cookie_auth:generate(Base, Req, Opts). + +%% @doc Finalize an `on-request' hook by adding the `set-cookie' header to the +%% end of the message sequence. +finalize(Base, Request, Opts) -> + dev_codec_cookie_auth:finalize(Base, Request, Opts). + +%% @doc Get the cookie with the given key from the base message. The format of +%% the cookie is determined by the `format' key in the request message: +%% - `default': The cookie is returned in its raw form. It will be a message +%% if the source was a `set-cookie' header line containing attributes/flags, +%% or a binary if only the value was provided (as with the `cookie' header). +%% - `set-cookie': The cookie is normalized to a message with `value', +%% `attributes', and `flags' keys. +%% - `cookie': The cookie is normalized to a binary, ommitting any attributes +%% or flags. +%% +%% The `format' may be specified in the request message as the `req:format' key. +%% If no `format' is specified, the default is `default'. +get_cookie(Base, Req, RawOpts) -> + Opts = opts(RawOpts), + {ok, Cookies} = extract(Base, Req, Opts), + Key = hb_maps:get(<<"key">>, Req, undefined, Opts), + case hb_maps:get(Key, Cookies, undefined, Opts) of + undefined -> {error, not_found}; + Cookie -> + Format = hb_maps:get(<<"format">>, Req, <<"default">>, Opts), + case Format of + <<"default">> -> {ok, Cookie}; + <<"set-cookie">> -> {ok, normalize_cookie_value(Cookie)}; + <<"cookie">> -> {ok, value(Cookie)} + end + end. + +%% @doc Return the parsed and normalized cookies from a message. +extract(Msg, Req, Opts) -> + {ok, MsgWithCookie} = from(Msg, Req, Opts), + Cookies = hb_private:get(<<"cookie">>, MsgWithCookie, #{}, Opts), + {ok, Cookies}. + +%% @doc Set the keys in the request message in the cookies of the caller. Removes +%% a set of base keys from the request message before setting the remainder as +%% cookies. +store(Base, Req, RawOpts) -> + Opts = opts(RawOpts), + ?event({store, {base, Base}, {req, Req}}), + {ok, ExistingCookies} = extract(Base, Req, Opts), + ?event({store, {existing_cookies, ExistingCookies}}), + {ok, ResetBase} = reset(Base, Opts), + ?event({store, {reset_base, ResetBase}}), + MsgToSet = + hb_maps:without( + [ + <<"path">>, + <<"accept-bundle">>, + <<"ao-peer">>, + <<"host">>, + <<"method">>, + <<"body">> + ], + hb_private:reset(Req), + Opts + ), + ?event({store, {msg_to_set, MsgToSet}}), + NewCookies = hb_maps:merge(ExistingCookies, MsgToSet, Opts), + NewBase = hb_private:set(ResetBase, <<"cookie">>, NewCookies, Opts), + {ok, NewBase}. + +%% @doc Remove all cookie keys from the given message (including `cookie' and +%% `set-cookie' in the base, and `priv/cookie' in the request message). +reset(Base, RawOpts) -> + Opts = opts(RawOpts), + WithoutBaseCookieKeys = + hb_maps:without( + [<<"cookie">>, <<"set-cookie">>], + Base, + Opts + ), + WithoutPrivCookie = + hb_private:set( + WithoutBaseCookieKeys, + <<"cookie">>, + unset, + Opts + ), + {ok, WithoutPrivCookie}. + +%% @doc Convert a message containing cookie sources (`cookie', `set-cookie', +%% or `priv/cookie') into a message containing the cookies serialized as the +%% specified `format' (given in the request message). The `format' may take the +%% following values: +%% +%% - `set-cookie': A list of encoded cookie binary header lines (e.g. +%% `"key1=value1; attr1=value2; flag1; flag2..."'). +%% - `cookie': A single, concatenated cookie header line without attributes or +%% flags (e.g. `"key1=value1; key2=value2; ..."'). +%% +%% Note that the `format: cookie' form is information lossy: All provided +%% attributes and flags are discarded. +to(Msg, Req, Opts) -> + ?event({to, {msg, Msg}, {req, Req}}), + CookieOpts = opts(Opts), + LoadedMsg = hb_cache:ensure_all_loaded(Msg, CookieOpts), + ?event({to, {loaded_msg, LoadedMsg}}), + do_to(LoadedMsg, Req, CookieOpts). +do_to(Msg, Req = #{ <<"format">> := <<"set-cookie">> }, Opts) when is_map(Msg) -> + ?event({to_set_cookie, {msg, Msg}, {req, Req}}), + {ok, ExtractedParsedCookies} = extract(Msg, Req, Opts), + {ok, ResetBase} = reset(Msg, Opts), + SetCookieLines = + maps:values( + maps:map( + fun to_set_cookie_line/2, + ExtractedParsedCookies + ) + ), + MsgWithSetCookie = + ResetBase#{ + <<"set-cookie">> => SetCookieLines + }, + {ok, MsgWithSetCookie}; +do_to(Msg, Req = #{ <<"format">> := <<"cookie">> }, Opts) when is_map(Msg) -> + ?event({to_cookie, {msg, Msg}, {req, Req}}), + {ok, ExtractedParsedCookies} = extract(Msg, Req, Opts), + {ok, ResetBase} = reset(Msg, Opts), + CookieLines = + hb_maps:values( + hb_maps:map( + fun to_cookie_line/2, + ExtractedParsedCookies, + Opts + ), + Opts + ), + ?event({to_cookie, {cookie_lines, CookieLines}}), + CookieLine = join(CookieLines, <<"; ">>), + {ok, ResetBase#{ <<"cookie">> => CookieLine }}; +do_to(Msg, _Req, _Opts) when is_map(Msg) -> + error({cookie_to_error, {no_format_specified, Msg}}); +do_to(Msg, _Req, _Opts) -> + error({cookie_to_error, {unexpected_message_format, Msg}}). + +%% @doc Convert a single cookie into a `set-cookie' header line. The cookie +%% may come in the form of `key => binary' or `key => cookie-message', where +%% the cookie-message is a map with the following keys: +%% +%% - `value': The raw binary cookie value. +%% - `attributes': A map of cookie attribute key-value pairs. +%% - `flags`: A list of cookie flags, represented as binaries. +%% +%% If the cookie is a binary, we normalize it to a cookie-message before +%% processing. +%% Note: Assumes that the cookies have all been loaded from the cache fully. +to_set_cookie_line(Key, RawCookie) -> + Cookie = normalize_cookie_value(RawCookie), + % Encode the cookie key-value pair as a string to use as the base. + ValueBin = + << + Key/binary, "=\"", + (maps:get(<<"value">>, Cookie))/binary, + "\"" + >>, + % Encode the cookie attributes as key-value (non-quoted) pairs, separated + % by `;'. + ?event({to_line, {key, Key}, {cookie, {explicit, Cookie}}, {value, ValueBin}}), + AttributesBin = + case maps:get(<<"attributes">>, Cookie, #{}) of + EmptyAttributes when map_size(EmptyAttributes) == 0 -> + ?event({attributes, {none_in, Cookie}}), + <<>>; + Attributes -> + ?event({attributes, Attributes}), + JointAttributes = + join( + [ + << AttrKey/binary, "=", AttrValue/binary >> + || + {AttrKey, AttrValue} <- to_sorted_list(Attributes) + ], + <<"; ">> + ), + << "; ", JointAttributes/binary >> + end, + FlagsBin = + case maps:get(<<"flags">>, Cookie, []) of + [] -> <<>>; + Flags -> << "; ", (join(Flags, <<"; ">>))/binary >> + end, + << ValueBin/binary, AttributesBin/binary, FlagsBin/binary >>. + +%% @doc Convert a single cookie into a `cookie' header component. These +%% components can be joined to form a `cookie' header line. This function +%% reuses the `to_set_cookie_line' function to generate the components, but +%% unsets the `attributes' and `flags' keys first. +to_cookie_line(Key, Cookie) -> + to_set_cookie_line(Key, value(Cookie)). + +%% @doc Normalize a message containing a `cookie', `set-cookie', and potentially +%% a `priv/cookie' key into a message with only the `priv/cookie' key. +from(Msg, Req, Opts) -> + CookieOpts = opts(Opts), + LoadedMsg = hb_cache:ensure_all_loaded(Msg, Opts), + do_from(LoadedMsg, Req, CookieOpts). +do_from(Msg, Req, Opts) when is_map(Msg) -> + {ok, ResetBase} = reset(Msg, Opts), + % Get the cookies, parsed, from each available source. + {ok, FromCookie} = from_cookie(Msg, Req, Opts), + {ok, FromSetCookie} = from_set_cookie(Msg, Req, Opts), + FromPriv = hb_private:get(<<"cookie">>, Msg, #{}, Opts), + % Merge all found cookies into a single map. + MergedMsg = hb_maps:merge(FromCookie, FromSetCookie, Opts), + AllParsed = hb_maps:merge(MergedMsg, FromPriv, Opts), + % Set the cookies in the private element of the message. + {ok, hb_private:set(ResetBase, <<"cookie">>, AllParsed, Opts)}; +do_from(CookiesMsg, _Req, _Opts) -> + error({cookie_from_error, {unexpected_message_format, CookiesMsg}}). + +%% @doc Convert the `cookie' key into a parsed cookie message. `cookie' headers +%% are in the format of `key1=value1; key2=value2; ...'. There are no attributes +%% or flags, so we split on `;' and return a map of key-value pairs. We also +%% decode the values, in case they are URI-encoded. +from_cookie(#{ <<"cookie">> := Cookie }, Req, Opts) -> + from_cookie(Cookie, Req, Opts); +from_cookie(Cookies, Req, Opts) when is_list(Cookies) -> + MergedParsed = + lists:foldl( + fun(Cookie, Acc) -> + {ok, Parsed} = from_cookie(Cookie, Req, Opts), + hb_maps:merge(Acc, Parsed, Opts) + end, + #{}, + Cookies + ), + {ok, MergedParsed}; +from_cookie(Cookie, _Req, _Opts) when is_binary(Cookie) -> + BinaryCookiePairs = split(semicolon, Cookie), + KeyValList = + lists:map( + fun(BinaryCookiePair) -> + {[Key, Value], _Rest} = split(pair, BinaryCookiePair), + {Key, hb_escape:decode(Value)} + end, + BinaryCookiePairs + ), + NormalizedMessage = maps:from_list(KeyValList), + {ok, NormalizedMessage}; +from_cookie(_MsgWithoutCookie, _Req, _Opts) -> + % The cookie key is not present in the message, so we return an empty map. + {ok, #{}}. + +%% @doc Convert a `set-cookie' header line into a cookie message. The `set-cookie' +%% header has a `key=value' pair, and possibly attributes and flags. The form +%% looks as follows: `key=value; attr1=value1; attr2=value2; flag1; flag2'. +from_set_cookie(#{ <<"set-cookie">> := Cookie }, Req, Opts) -> + ?event({from_set_cookie, {cookie, Cookie}}), + from_set_cookie(Cookie, Req, Opts); +from_set_cookie(MsgWithoutSet, _Req, _Opts) when is_map(MsgWithoutSet) -> + % The set-cookie key is not present in the message, so we return an empty map. + {ok, #{}}; +from_set_cookie(Lines, Req, Opts) when is_list(Lines) -> + MergedParsed = + lists:foldl( + fun(Line, Acc) -> + {ok, Parsed} = from_set_cookie(Line, Req, Opts), + hb_maps:merge(Acc, Parsed) + end, + #{}, + Lines + ), + {ok, MergedParsed}; +from_set_cookie(Line, _Req, Opts) when is_binary(Line) -> + {[Key, Value], Rest} = split(pair, Line), + ValueDecoded = hb_escape:decode(Value), + % If there is no remaining binary after the pair, we have a simple key-value + % pair, returning just the binary as the value. Otherwise, we split the + % remaining binary into attributes and flags and return a message with the + % value and those parsed elements. + case Rest of + <<>> -> {ok, #{ Key => ValueDecoded }}; + _ -> + AllAttrs = split(semicolon, Rest), + % We partition the attributes into pairs and flags, where flags are + % any attributes that do not contain an `=' character. + {AttrPairs, Flags} = + lists:partition( + fun(Attr) -> + case hb_util:split_depth_string_aware_single($=, Attr) of + {no_match, _, _} -> false; + {_, _, _} -> true + end + end, + AllAttrs + ), + % We sort the flags and generate an attributes map from the pairs. + SortedFlags = to_sorted_list(Flags), + UnquotedFlags = lists:map(fun unquote/1, SortedFlags), + ?event( + {from_line, + {key, Key}, + {value, {explicit, Value}}, + {attrs, AttrPairs}, + {flags, UnquotedFlags} + } + ), + Attributes = + maps:from_list( + lists:map( + fun(AttrPairBin) -> + {[AttrKey, AttrValue], _} = split(pair, AttrPairBin), + AttrKeyTrimmed = trim_bin(AttrKey), + AttrValueTrimmed = trim_bin(AttrValue), + {AttrKeyTrimmed, unquote(AttrValueTrimmed)} + end, + AttrPairs + ) + ), + MaybeAttributes = + if map_size(Attributes) > 0 -> #{ <<"attributes">> => Attributes }; + true -> #{} + end, + MaybeFlags = + if length(UnquotedFlags) > 0 -> #{ <<"flags">> => UnquotedFlags }; + true -> #{} + end, + MaybeAllAttributes = hb_maps:merge(MaybeAttributes, MaybeFlags, Opts), + {ok, #{ Key => MaybeAllAttributes#{ <<"value">> => ValueDecoded }}} + end. + +%%% Internal helpers + +%% @doc Takes a message or list of binaries and returns a sorted list of key- +%% value pairs. Assumes that the message has been loaded from the cache fully. +to_sorted_list(Msg) when is_map(Msg) -> + lists:keysort( + 1, + [ + {trim_bin(hb_util:bin(K)), trim_bin(V)} + || {K, V} <- maps:to_list(Msg) + ] + ); +to_sorted_list(Binaries) when is_list(Binaries) -> + lists:sort( + lists:map( + fun(Bin) -> trim_bin(hb_util:bin(Bin)) end, + Binaries + ) + ). + +%% @doc Take a single parse cookie and return only the value (ignoring attributes +%% and flags). +value(Msg) when is_map(Msg) -> + maps:get(<<"value">>, Msg, Msg); +value(Bin) when is_binary(Bin) -> + Bin. + +%% @doc Normalize a cookie value to a map with the following keys: +%% - `value': The raw binary cookie value. +%% - `attributes': A map of cookie attribute key-value pairs. +%% - `flags`: A list of cookie flags, represented as binaries. +normalize_cookie_value(Msg) when is_map(Msg) -> + Msg#{ + <<"value">> => maps:get(<<"value">>, Msg, Msg), + <<"attributes">> => maps:get(<<"attributes">>, Msg, #{}), + <<"flags">> => maps:get(<<"flags">>, Msg, []) + }; +normalize_cookie_value(Bin) when is_binary(Bin) -> + #{ + <<"value">> => Bin, + <<"attributes">> => #{}, + <<"flags">> => [] + }. + +%%% Internal helpers + +%% @doc Trim a binary of leading and trailing whitespace. +trim_bin(Bin) when is_binary(Bin) -> + list_to_binary(string:trim(binary_to_list(Bin))). + +%% @doc Join a list of binaries into a `separator'-separated string. Abstracts +%% the complexities of converting to/from string lists, as Erlang only provides +%% a `binary:join` function as of OTP/28. +join(Binaries, Separator) -> + hb_util:bin( + string:join( + lists:map(fun hb_util:list/1, Binaries), + hb_util:list(Separator) + ) + ). + +%% @doc Split a binary by a separator type (`pair', `lines', or `attributes'). +%% Separator types that are plural return a list of all parts. Singular types +%% return a single part and the remainder of the binary. +split(pair, Bin) -> + [Key, ValueRest] = binary:split(Bin, <<"=">>), + {_, Value, Rest} = hb_util:split_depth_string_aware_single($;, ValueRest), + {[Key, unquote(Value)], trim_leading(Rest)}; +split(lines, Bin) -> + lists:map(fun trim_leading/1, hb_util:split_depth_string_aware($,, Bin)); +split(semicolon, Bin) -> + lists:map(fun trim_leading/1, hb_util:split_depth_string_aware($;, Bin)). + +%% @doc Remove leading whitespace from a binary, if present. +trim_leading(Line) when not is_binary(Line) -> + trim_leading(hb_util:bin(Line)); +trim_leading(<<>>) -> <<>>; +trim_leading(<<" ", Rest/binary>>) -> trim_leading(Rest); +trim_leading(Line) -> Line. + +%% @doc Unquote a binary if it is quoted. If it is not quoted, we return the +%% binary as is. +unquote(<< $\", Rest/binary>>) -> + {Unquoted, _} = hb_util:split_escaped_single($\", Rest), + Unquoted; +unquote(Bin) -> Bin. \ No newline at end of file diff --git a/src/dev_codec_cookie_auth.erl b/src/dev_codec_cookie_auth.erl new file mode 100644 index 000000000..bc4077875 --- /dev/null +++ b/src/dev_codec_cookie_auth.erl @@ -0,0 +1,252 @@ +%%% @doc Implements the `message@1.0' commitment interface for the `~cookie@1.0', +%%% as well as the `generator' interface type for the `~auth-hook@1.0' device. +%%% See the [cookie codec](dev_codec_cookie.html) documentation for more details. +-module(dev_codec_cookie_auth). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). +-export([commit/3, verify/3]). +-export([generate/3, finalize/3]). + +%% @doc Generate a new secret (if no `committer' specified), and use it as the +%% key for the `httpsig@1.0' commitment. If a `committer' is given, we search +%% for it in the cookie message instead of generating a new secret. See the +%% module documentation of `dev_codec_cookie' for more details on its scheme. +generate(Base, Request, Opts) -> + {WithCookie, Secrets} = + case find_secrets(Request, Opts) of + [] -> + {ok, GeneratedSecret} = generate_secret(Base, Request, Opts), + {ok, Updated} = store_secret(GeneratedSecret, Request, Opts), + {Updated, [GeneratedSecret]}; + FoundSecrets -> + {Request, FoundSecrets} + end, + ?event({normalized_cookies_found, {secrets, Secrets}}), + { + ok, + WithCookie#{ + <<"secret">> => Secrets + } + }. + +%% @doc Finalize an `on-request' hook by adding the cookie to the chain of +%% messages. The inbound request has the same structure as a normal `~hook@1.0' +%% on-request hook: The message sequence is the body of the request, and the +%% request is the request message. +finalize(Base, Request, Opts) -> + ?event(debug_auth, {finalize, {base, Base}, {request, Request}}), + maybe + {ok, SignedMsg} ?= hb_maps:find(<<"request">>, Request, Opts), + {ok, MessageSequence} ?= hb_maps:find(<<"body">>, Request, Opts), + % Cookie auth adds set-cookie to response + {ok, #{ <<"set-cookie">> := SetCookie }} = + dev_codec_cookie:to( + SignedMsg, + #{ <<"format">> => <<"set-cookie">> }, + Opts + ), + { + ok, + MessageSequence ++ + [#{ <<"path">> => <<"set">>, <<"set-cookie">> => SetCookie }] + } + else error -> + {error, no_request} + end. + +%% @doc Generate a new secret (if no `committer' specified), and use it as the +%% key for the `httpsig@1.0' commitment. If a `committer' is given, we search +%% for it in the cookie message instead of generating a new secret. See the +%% module documentation of `dev_codec_cookie' for more details on its scheme. +commit(Base, Request, RawOpts) when ?IS_LINK(Request) -> + Opts = dev_codec_cookie:opts(RawOpts), + commit(Base, hb_cache:ensure_loaded(Request, Opts), Opts); +commit(Base, Req = #{ <<"secret">> := Secret }, RawOpts) -> + Opts = dev_codec_cookie:opts(RawOpts), + commit(hb_cache:ensure_loaded(Secret, Opts), Base, Req, Opts); +commit(Base, Request, RawOpts) -> + Opts = dev_codec_cookie:opts(RawOpts), + % Calculate the key to use for the commitment. + SecretRes = + case find_secret(Request, Opts) of + {ok, RawSecret} -> + {ok, RawSecret}; + {error, no_secret} -> + generate_secret(Base, Request, Opts); + {error, not_found} -> + throw({error, <<"Necessary cookie not found in request.">>}) + end, + case SecretRes of + {ok, Secret} -> commit(Secret, Base, Request, Opts); + {error, Err} -> {error, Err} + end. + +%% @doc Given the secret key, commit the message and set the cookie. This +%% function may be used by other devices via a direct module call, in order to +%% commit a message and set the given secret key in the cookie. +commit(Secret, Base, Request, Opts) -> + {ok, CommittedMsg} = + dev_codec_httpsig_proxy:commit( + <<"cookie@1.0">>, + Secret, + Base, + Request, + Opts + ), + store_secret(Secret, CommittedMsg, Opts). + +%% @doc Update the nonces for a given secret. +store_secret(Secret, Msg, Opts) -> + CookieAddr = dev_codec_httpsig_keyid:secret_key_to_committer(Secret), + % Create the cookie parameters, using the name as the key and the secret as + % the value. + {ok, Cookies} = dev_codec_cookie:extract(Msg, #{}, Opts), + NewCookies = Cookies#{ <<"secret-", CookieAddr/binary>> => Secret }, + {ok, WithCookie} = dev_codec_cookie:store(Msg, NewCookies, Opts), + {ok, WithCookie}. + +%% @doc Verify the HMAC commitment with the key being the secret from the +%% request cookies. We find the appropriate cookie from the cookie message by +%% the committer ID given in the request message. +verify(Base, ReqLink, RawOpts) when ?IS_LINK(ReqLink) -> + Opts = dev_codec_cookie:opts(RawOpts), + verify(Base, hb_cache:ensure_loaded(ReqLink, Opts), Opts); +verify(Base, Req = #{ <<"secret">> := Secret }, RawOpts) -> + Opts = dev_codec_cookie:opts(RawOpts), + ?event({verify_with_explicit_key, {base, Base}, {request, Req}}), + dev_codec_httpsig_proxy:verify( + hb_util:decode(Secret), + Base, + Req, + Opts + ); +verify(Base, Request, RawOpts) -> + Opts = dev_codec_cookie:opts(RawOpts), + ?event({verify_finding_key, {base, Base}, {request, Request}}), + case find_secret(Request, Opts) of + {ok, Secret} -> + dev_codec_httpsig_proxy:verify( + hb_util:decode(Secret), + Base, + Request, + Opts + ); + {error, Err} -> + {error, Err} + end. + +%% @doc Generate a new secret key for the given request. The user may specify +%% a generator function in the request, which will be executed to generate the +%% secret key. If no generator is specified, the default generator is used. +%% A `generator` may be either a path or full message. If no path is present in +%% a generator message, the `generate` path is assumed. +generate_secret(_Base, Request, Opts) -> + case hb_maps:get(<<"generator">>, Request, undefined, Opts) of + undefined -> + % If no generator is specified, use the default generator. + case hb_opts:get(cookie_default_generator, <<"random">>, Opts) of + <<"random">> -> + default_generator(Opts); + Provider -> + execute_generator(Request#{<<"path">> => Provider}, Opts) + end; + Provider -> + % Execute the user's generator function. + execute_generator(Request#{<<"path">> => Provider}, Opts) + end. + +%% @doc Generate a new secret key using the default generator. +default_generator(_Opts) -> + {ok, hb_util:encode(crypto:strong_rand_bytes(64))}. + +%% @doc Execute a generator function. See `generate_secret/3' for more details. +execute_generator(GeneratorPath, Opts) when is_binary(GeneratorPath) -> + hb_ao:resolve(GeneratorPath, Opts); +execute_generator(Generator, Opts) -> + Path = hb_maps:get(<<"path">>, Generator, <<"generate">>, Opts), + hb_ao:resolve(Generator#{ <<"path">> => Path }, Opts). + +%% @doc Find all secrets in the cookie of a message. +find_secrets(Request, Opts) -> + maybe + {ok, Cookie} ?= dev_codec_cookie:extract(Request, #{}, Opts), + [ + hb_maps:get(SecretRef, Cookie, secret_unavailable, Opts) + || + SecretRef = <<"secret-", _/binary>> <- hb_maps:keys(Cookie) + ] + else error -> [] + end. + +%% @doc Find the secret key for the given committer, if it exists in the cookie. +find_secret(Request, Opts) -> + maybe + {ok, Committer} ?= hb_maps:find(<<"committer">>, Request, Opts), + find_secret(Committer, Request, Opts) + else error -> {error, no_secret} + end. +find_secret(Committer, Request, Opts) -> + maybe + {ok, Cookie} ?= dev_codec_cookie:extract(Request, #{}, Opts), + {ok, _Secret} ?= hb_maps:find(<<"secret-", Committer/binary>>, Cookie, Opts) + else error -> {error, not_found} + end. + +%%% Tests + +%% @doc Call the cookie codec's `commit' and `verify' functions directly. +directly_invoke_commit_verify_test() -> + Base = #{ <<"test-key">> => <<"test-value">> }, + CommittedMsg = + hb_message:commit( + Base, + #{}, + #{ + <<"commitment-device">> => <<"cookie@1.0">> + } + ), + ?event({committed_msg, CommittedMsg}), + ?assertEqual(1, length(hb_message:signers(CommittedMsg, #{}))), + VerifyReq = + apply_cookie( + CommittedMsg#{ + <<"committers">> => hb_message:signers(CommittedMsg, #{}) + }, + CommittedMsg, + #{} + ), + VerifyReqWithoutComms = hb_maps:without([<<"commitments">>], VerifyReq, #{}), + ?event({verify_req_without_comms, VerifyReqWithoutComms}), + ?assert(hb_message:verify(CommittedMsg, VerifyReqWithoutComms, #{})), + ok. + +%% @doc Set keys in a cookie and verify that they can be parsed into a message. +http_set_get_cookies_test() -> + Node = hb_http_server:start_node(#{}), + {ok, SetRes} = + hb_http:get( + Node, + <<"/~cookie@1.0/store?k1=v1&k2=v2">>, + #{} + ), + ?event(debug_cookie, {set_cookie_test, {set_res, SetRes}}), + ?assertMatch(#{ <<"set-cookie">> := _ }, SetRes), + Req = apply_cookie(#{ <<"path">> => <<"/~cookie@1.0/extract">> }, SetRes, #{}), + {ok, Res} = hb_http:get(Node, Req, #{}), + ?assertMatch(#{ <<"k1">> := <<"v1">>, <<"k2">> := <<"v2">> }, Res), + ok. + +%%% Test Helpers + +%% @doc Takes the cookies from the `GenerateResponse' and applies them to the +%% `Target' message. +apply_cookie(NextReq, GenerateResponse, Opts) -> + {ok, Cookie} = dev_codec_cookie:extract(GenerateResponse, #{}, Opts), + {ok, NextWithParsedCookie} = dev_codec_cookie:store(NextReq, Cookie, Opts), + {ok, NextWithCookie} = + dev_codec_cookie:to( + NextWithParsedCookie, + #{ <<"format">> => <<"cookie">> }, + Opts + ), + NextWithCookie. \ No newline at end of file diff --git a/src/dev_codec_cookie_test_vectors.erl b/src/dev_codec_cookie_test_vectors.erl new file mode 100644 index 000000000..2994cf2a4 --- /dev/null +++ b/src/dev_codec_cookie_test_vectors.erl @@ -0,0 +1,764 @@ +%%% @doc A battery of cookie parsing and encoding test vectors. +-module(dev_codec_cookie_test_vectors). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%%% Test Helpers + +%% @doc Assert that when given the inputs in the test set, the outputs are +%% all equal to the expected value when the function is applied to them. +assert_set(TestSet, Fun) -> + {Inputs, Expected} = maps:get(TestSet, test_data()), + ?event(match_cookie, {starting_group_match, {inputs, {explicit, Inputs}}}), + lists:foreach( + fun(Input) -> + Res = Fun(Input), + ?event( + match_cookie, + {matching, + {expected, {explicit, Expected}, {output, {explicit, Res}}} + } + ), + ?assertEqual(Expected, Res) + end, + Inputs + ). + +%% @doc Convert a cookie message to a string. +to_string(CookieMsg) -> + {ok, BaseMsg} = dev_codec_cookie:store(#{}, CookieMsg, #{}), + {ok, Msg} = + dev_codec_cookie:to( + BaseMsg, + #{ <<"format">> => <<"set-cookie">> }, + #{} + ), + hb_maps:get(<<"set-cookie">>, Msg, [], #{}). + +%% @doc Convert a string to a cookie message. +from_string(String) -> + {ok, BaseMsg} = + dev_codec_cookie:from( + #{ <<"set-cookie">> => String }, + #{}, + #{} + ), + {ok, Cookie} = dev_codec_cookie:extract(BaseMsg, #{}, #{}), + Cookie. + +%%% Tests + +%% @doc returns a map of tuples of the form `testset_name => {[before], after}'. +%% These sets are used to test the correctness of the parsing and serialization +%% of cookie messages. The `before` is a list of inputs for which all of the +%% outputs are expected to match the `after' value. +test_data() -> + #{ + from_string_raw_value => + { + [<<"k1=v1">>, <<"k1=\"v1\"">>], + #{ <<"k1">> => <<"v1">> } + }, + from_string_attributes => + { + [<<"k1=v1; k2=v2">>, <<"k1=\"v1\"; k2=\"v2\"">>], + #{ + <<"k1">> => + #{ + <<"value">> => <<"v1">>, + <<"attributes">> => #{ <<"k2">> => <<"v2">> } + } + } + }, + from_string_flags => + { + [<<"k1=v1; k2=v2; f1; f2">>, <<"k1=\"v1\"; k2=\"v2\"; f1; f2">>], + #{ + <<"k1">> => + #{ + <<"value">> => <<"v1">>, + <<"attributes">> => #{ <<"k2">> => <<"v2">> }, + <<"flags">> => [<<"f1">>, <<"f2">>] + } + } + }, + to_string_raw_value => + { + [ + #{ <<"k1">> => <<"v1">> }, + #{ <<"k1">> => #{ <<"value">> => <<"v1">> } }, + #{ + <<"k1">> => + #{ + <<"value">> => <<"v1">>, + <<"attributes">> => #{}, + <<"flags">> => [] + } + } + ], + [<<"k1=\"v1\"">>] + }, + to_string_attributes => + { + [ + #{ + <<"k1">> => + #{ + <<"value">> => <<"v1">>, + <<"attributes">> => #{ <<"k2">> => <<"v2">> } + } + }, + #{ + <<"k1">> => + #{ + <<"value">> => <<"v1">>, + <<"attributes">> => #{ <<"k2">> => <<"v2">> }, + <<"flags">> => [] + } + } + ], + [<<"k1=\"v1\"; k2=v2">>] + }, + to_string_flags => + { + [ + #{ + <<"k1">> => + #{ + <<"value">> => <<"v1">>, + <<"flags">> => [<<"f1">>, <<"f2">>] + } + }, + #{ + <<"k1">> => + #{ + <<"value">> => <<"v1">>, + <<"attributes">> => #{}, + <<"flags">> => [<<"f1">>, <<"f2">>] + } + } + ], + [<<"k1=\"v1\"; f1; f2">>] + }, + parse_realworld_1 => + { + [ + [ + <<"cart=110045_77895_53420; SameSite=Strict">>, + <<"affiliate=e4rt45dw; SameSite=Lax">> + ] + ], + #{ + <<"cart">> => + #{ + <<"value">> => <<"110045_77895_53420">>, + <<"attributes">> => #{ <<"SameSite">> => <<"Strict">> } + }, + <<"affiliate">> => + #{ + <<"value">> => <<"e4rt45dw">>, + <<"attributes">> => #{ <<"SameSite">> => <<"Lax">> } + } + } + }, + parse_user_settings_and_permissions => + { + [ + [ + <<"user_settings=notifications=true,privacy=strict,layout=grid; Path=/; HttpOnly; Secure">>, + <<"user_permissions=\"read;write;delete\"; Path=/; SameSite=None; Secure">> + ] + ], + #{ + <<"user_settings">> => + #{ + <<"value">> => <<"notifications=true,privacy=strict,layout=grid">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"HttpOnly">>, <<"Secure">>] + }, + <<"user_permissions">> => + #{ + <<"value">> => <<"read;write;delete">>, + <<"attributes">> => #{ <<"Path">> => <<"/">>, <<"SameSite">> => <<"None">> }, + <<"flags">> => [<<"Secure">>] + } + } + }, + parse_session_and_temp_data => + { + [ + [ + <<"SESSION_ID=abc123xyz ; path= /dashboard ; samesite=Strict ; Secure">>, + <<"temp_data=cleanup_me; Max-Age=-1; Path=/">> + ] + ], + #{ + <<"SESSION_ID">> => + #{ + <<"value">> => <<"abc123xyz ">>, + <<"attributes">> => #{ <<"path">> => <<"/dashboard">>, <<"samesite">> => <<"Strict">> }, + <<"flags">> => [<<"Secure">>] + }, + <<"temp_data">> => + #{ + <<"value">> => <<"cleanup_me">>, + <<"attributes">> => #{ <<"Max-Age">> => <<"-1">>, <<"Path">> => <<"/">> } + } + } + }, + parse_empty_and_anonymous => + { + [ + [ + <<"user_preference=; Path=/; HttpOnly">>, + <<"=anonymous_session_123; Path=/guest">> + ] + ], + #{ + <<"user_preference">> => + #{ + <<"value">> => <<"">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"HttpOnly">>] + }, + <<>> => + #{ + <<"value">> => <<"anonymous_session_123">>, + <<"attributes">> => #{ <<"Path">> => <<"/guest">> } + } + } + }, + parse_app_config_and_analytics => + { + [ + [ + <<"$app_config$=theme@dark!%20mode; Path=/">>, + <<"analytics_session_data_with_very_long_name_for_tracking_purposes=comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more; Path=/">> + ] + ], + #{ + <<"$app_config$">> => + #{ + <<"value">> => <<"theme@dark! mode">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + }, + <<"analytics_session_data_with_very_long_name_for_tracking_purposes">> => + #{ + <<"value">> => <<"comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + } + } + }, + parse_debug_and_tracking => + { + [ + [ + <<"debug_info=\\tIndented\\t\\nMultiline\\n; Path=/">>, + <<"tracking_id=user_12345; CustomAttr=CustomValue; Analytics=Enabled; Path=/; HttpOnly">> + ] + ], + #{ + <<"debug_info">> => + #{ + <<"value">> => <<"\\tIndented\\t\\nMultiline\\n">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + }, + <<"tracking_id">> => + #{ + <<"value">> => <<"user_12345">>, + <<"attributes">> => #{ + <<"CustomAttr">> => <<"CustomValue">>, + <<"Analytics">> => <<"Enabled">>, + <<"Path">> => <<"/">> + }, + <<"flags">> => [<<"HttpOnly">>] + } + } + }, + parse_cache_and_form_token => + { + [ + [ + <<"cache_bust=v1.2.3; Expires=Mon, 99 Feb 2099 25:99:99 GMT; Path=/">>, + <<"form_token=form_abc123; SameSite=Strick; Secure">> + ] + ], + #{ + <<"cache_bust">> => + #{ + <<"value">> => <<"v1.2.3">>, + <<"attributes">> => #{ + <<"Expires">> => <<"Mon, 99 Feb 2099 25:99:99 GMT">>, + <<"Path">> => <<"/">> + } + }, + <<"form_token">> => + #{ + <<"value">> => <<"form_abc123">>, + <<"attributes">> => #{ <<"SameSite">> => <<"Strick">> }, + <<"flags">> => [<<"Secure">>] + } + } + }, + parse_token_and_reactions => + { + [ + [ + <<"access_token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c; Path=/; HttpOnly; Secure">>, + <<"reaction_prefs=👍👎; Path=/; Secure">> + ] + ], + #{ + <<"access_token">> => + #{ + <<"value">> => <<"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"HttpOnly">>, <<"Secure">>] + }, + <<"reaction_prefs">> => + #{ + <<"value">> => <<"👍👎">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"Secure">>] + } + } + }, + parse_error_log_and_auth_token => + { + [ + [ + <<"error_log=\"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed\"; Path=/">>, + <<"auth_token=bearer_xyz789; Secure; Path=/api; Secure; HttpOnly">> + ] + ], + #{ + <<"error_log">> => + #{ + <<"value">> => <<"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + }, + <<"auth_token">> => + #{ + <<"value">> => <<"bearer_xyz789">>, + <<"attributes">> => #{ <<"Path">> => <<"/api">> }, + <<"flags">> => [<<"HttpOnly">>,<<"Secure">>, <<"Secure">>] + } + } + }, + parse_csrf_and_quick_setting => + { + [ + [ + <<"csrf_token=abc123; \"HttpOnly\"; Path=/">>, + <<"quick_setting=\"enabled\"">> + ] + ], + #{ + <<"csrf_token">> => + #{ + <<"value">> => <<"abc123">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"HttpOnly">>] + }, + <<"quick_setting">> => <<"enabled">> + } + }, + parse_admin_and_upload => + { + [ + [ + <<"secret_key=confidential; Path=%2Fadmin">>, + <<"admin_flag=true; Path=/">> + + ] + ], + #{ + <<"secret_key">> => + #{ + <<"value">> => <<"confidential">>, + <<"attributes">> => #{ <<"Path">> => <<"%2Fadmin">> } + }, + <<"admin_flag">> => + #{ + <<"value">> => <<"true">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + } + } + }, + parse_search_and_tags => + { + [ + [ + <<"search_history=\"query,results\"; Path=/">>, + <<"user_tags=\"work,personal\"; Path=/">> + ] + ], + #{ + <<"search_history">> => + #{ + <<"value">> => <<"query,results">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + }, + <<"user_tags">> => + #{ + <<"value">> => <<"work,personal">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + } + } + }, + to_string_realworld_1 => + { + [ + #{ + <<"cart">> => + #{ + <<"value">> => <<"110045_77895_53420">>, + <<"attributes">> => #{ <<"SameSite">> => <<"Strict">> } + }, + <<"affiliate">> => + #{ + <<"value">> => <<"e4rt45dw">>, + <<"attributes">> => #{ <<"SameSite">> => <<"Lax">> } + } + } + ], + [ + <<"affiliate=\"e4rt45dw\"; SameSite=Lax">>, + <<"cart=\"110045_77895_53420\"; SameSite=Strict">> + ] + }, + to_string_user_settings_and_permissions => + { + [ + #{ + <<"user_settings">> => + #{ + <<"value">> => <<"notifications=true,privacy=strict,layout=grid">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"HttpOnly">>, <<"Secure">>] + }, + <<"user_permissions">> => + #{ + <<"value">> => <<"read;write;delete">>, + <<"attributes">> => #{ <<"Path">> => <<"/">>, <<"SameSite">> => <<"None">> }, + <<"flags">> => [<<"Secure">>] + } + } + ], + [ + <<"user_permissions=\"read;write;delete\"; Path=/; SameSite=None; Secure">>, + <<"user_settings=\"notifications=true,privacy=strict,layout=grid\"; Path=/; HttpOnly; Secure">> + ] + }, + to_string_session_and_temp_data => + { + [ + #{ + <<"SESSION_ID">> => + #{ + <<"value">> => <<"abc123xyz ">>, + <<"attributes">> => #{ <<"path">> => <<"/dashboard">>, <<"samesite">> => <<"Strict">> }, + <<"flags">> => [<<"Secure">>] + }, + <<"temp_data">> => + #{ + <<"value">> => <<"cleanup_me">>, + <<"attributes">> => #{ <<"Max-Age">> => <<"-1">>, <<"Path">> => <<"/">> } + } + } + ], + [ + <<"SESSION_ID=\"abc123xyz \"; path=/dashboard; samesite=Strict; Secure">>, + <<"temp_data=\"cleanup_me\"; Max-Age=-1; Path=/">> + ] + }, + to_string_empty_and_anonymous => + { + [ + #{ + <<"user_preference">> => + #{ + <<"value">> => <<"">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"HttpOnly">>] + }, + <<>> => + #{ + <<"value">> => <<"anonymous_session_123">>, + <<"attributes">> => #{ <<"Path">> => <<"/guest">> } + } + } + ], + [ + <<"=\"anonymous_session_123\"; Path=/guest">>, + <<"user_preference=\"\"; Path=/; HttpOnly">> + ] + }, + to_string_app_config_and_analytics => + { + [ + #{ + <<"$app_config$">> => + #{ + <<"value">> => <<"theme@dark!%20mode">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + }, + <<"analytics_session_data_with_very_long_name_for_tracking_purposes">> => + #{ + <<"value">> => <<"comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + } + } + ], + [ + <<"$app_config$=\"theme@dark!%20mode\"; Path=/">>, + <<"analytics_session_data_with_very_long_name_for_tracking_purposes=\"comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more\"; Path=/">> + ] + }, + to_string_debug_and_tracking => + { + [ + #{ + <<"debug_info">> => + #{ + <<"value">> => <<"\\tIndented\\t\\nMultiline\\n">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + }, + <<"tracking_id">> => + #{ + <<"value">> => <<"user_12345">>, + <<"attributes">> => #{ + <<"CustomAttr">> => <<"CustomValue">>, + <<"Analytics">> => <<"Enabled">>, + <<"Path">> => <<"/">> + }, + <<"flags">> => [<<"HttpOnly">>] + } + } + ], + [ + <<"debug_info=\"\\tIndented\\t\\nMultiline\\n\"; Path=/">>, + <<"tracking_id=\"user_12345\"; Analytics=Enabled; CustomAttr=CustomValue; Path=/; HttpOnly">> + ] + }, + to_string_cache_and_form_token => + { + [ + #{ + <<"cache_bust">> => + #{ + <<"value">> => <<"v1.2.3">>, + <<"attributes">> => #{ + <<"Expires">> => <<"Mon, 99 Feb 2099 25:99:99 GMT">>, + <<"Path">> => <<"/">> + } + }, + <<"form_token">> => + #{ + <<"value">> => <<"form_abc123">>, + <<"attributes">> => #{ <<"SameSite">> => <<"Strick">> }, + <<"flags">> => [<<"Secure">>] + } + } + ], + [ + <<"cache_bust=\"v1.2.3\"; Expires=Mon, 99 Feb 2099 25:99:99 GMT; Path=/">>, + <<"form_token=\"form_abc123\"; SameSite=Strick; Secure">> + ] + }, + to_string_token_and_reactions => + { + [ + #{ + <<"access_token">> => + #{ + <<"value">> => <<"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"HttpOnly">>, <<"Secure">>] + }, + <<"reaction_prefs">> => + #{ + <<"value">> => <<"👍👎">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"Secure">>] + } + } + ], + [ + <<"access_token=\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c\"; Path=/; HttpOnly; Secure">>, + <<"reaction_prefs=\"👍👎\"; Path=/; Secure">> + ] + }, + to_string_error_log_and_auth_token => + { + [ + #{ + <<"error_log">> => + #{ + <<"value">> => <<"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + }, + <<"auth_token">> => + #{ + <<"value">> => <<"bearer_xyz789">>, + <<"attributes">> => #{ <<"Path">> => <<"/api">> }, + <<"flags">> => [<<"HttpOnly">>, <<"Secure">>, <<"Secure">>] + } + } + ], + [ + <<"auth_token=\"bearer_xyz789\"; Path=/api; HttpOnly; Secure; Secure">>, + <<"error_log=\"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed\"; Path=/">> + ] + }, + to_string_csrf_and_quick_setting => + { + [ + #{ + <<"csrf_token">> => + #{ + <<"value">> => <<"abc123">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> }, + <<"flags">> => [<<"HttpOnly">>] + }, + <<"quick_setting">> => <<"enabled">> + } + ], + [ + <<"csrf_token=\"abc123\"; Path=/; HttpOnly">>, + <<"quick_setting=\"enabled\"">> + ] + }, + to_string_admin_and_upload => + { + [ + #{ + <<"secret_key">> => + #{ + <<"value">> => <<"confidential">>, + <<"attributes">> => #{ <<"Path">> => <<"%2Fadmin">> } + }, + <<"admin_flag">> => + #{ + <<"value">> => <<"true">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + } + } + ], + [ + <<"admin_flag=\"true\"; Path=/">>, + <<"secret_key=\"confidential\"; Path=%2Fadmin">> + ] + }, + to_string_search_and_tags => + { + [ + #{ + <<"search_history">> => + #{ + <<"value">> => <<"query,results">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + }, + <<"user_tags">> => + #{ + <<"value">> => <<"work,personal">>, + <<"attributes">> => #{ <<"Path">> => <<"/">> } + } + } + ], + [ + <<"search_history=\"query,results\"; Path=/">>, + <<"user_tags=\"work,personal\"; Path=/">> + ] + } + }. + +from_string_basic_test() -> + assert_set(from_string_raw_value, fun from_string/1). + +from_string_attributes_test() -> + assert_set(from_string_attributes, fun from_string/1). + +from_string_flags_test() -> + assert_set(from_string_flags, fun from_string/1). + +to_string_basic_test() -> + assert_set(to_string_raw_value, fun to_string/1). + +to_string_attributes_test() -> + assert_set(to_string_attributes, fun to_string/1). + +to_string_flags_test() -> + assert_set(to_string_flags, fun to_string/1). + +parse_realworld_test() -> + assert_set(parse_realworld_1, fun from_string/1). + +parse_user_settings_and_permissions_test() -> + assert_set(parse_user_settings_and_permissions, fun from_string/1). + +parse_session_and_temp_data_test() -> + assert_set(parse_session_and_temp_data, fun from_string/1). + +parse_empty_and_anonymous_test() -> + assert_set(parse_empty_and_anonymous, fun from_string/1). + +parse_app_config_and_analytics_test() -> + assert_set(parse_app_config_and_analytics, fun from_string/1). + +parse_debug_and_tracking_test() -> + assert_set(parse_debug_and_tracking, fun from_string/1). + +parse_cache_and_form_token_test() -> + assert_set(parse_cache_and_form_token, fun from_string/1). + +parse_token_and_reactions_test() -> + assert_set(parse_token_and_reactions, fun from_string/1). + +parse_error_log_and_auth_token_test() -> + assert_set(parse_error_log_and_auth_token, fun from_string/1). + +parse_csrf_and_quick_setting_test() -> + assert_set(parse_csrf_and_quick_setting, fun from_string/1). + +parse_admin_and_upload_test() -> + assert_set(parse_admin_and_upload, fun from_string/1). + +parse_search_and_tags_test() -> + assert_set(parse_search_and_tags, fun from_string/1). + +to_string_realworld_1_test() -> + assert_set(to_string_realworld_1, fun to_string/1). + +to_string_user_settings_and_permissions_test() -> + assert_set(to_string_user_settings_and_permissions, fun to_string/1). + +to_string_session_and_temp_data_test() -> + assert_set(to_string_session_and_temp_data, fun to_string/1). + +to_string_empty_and_anonymous_test() -> + assert_set(to_string_empty_and_anonymous, fun to_string/1). + +to_string_app_config_and_analytics_test() -> + assert_set(to_string_app_config_and_analytics, fun to_string/1). + +to_string_debug_and_tracking_test() -> + assert_set(to_string_debug_and_tracking, fun to_string/1). + +to_string_cache_and_form_token_test() -> + assert_set(to_string_cache_and_form_token, fun to_string/1). + +to_string_token_and_reactions_test() -> + assert_set(to_string_token_and_reactions, fun to_string/1). + +to_string_error_log_and_auth_token_test() -> + assert_set(to_string_error_log_and_auth_token, fun to_string/1). + +to_string_csrf_and_quick_setting_test() -> + assert_set(to_string_csrf_and_quick_setting, fun to_string/1). + +to_string_admin_and_upload_test() -> + assert_set(to_string_admin_and_upload, fun to_string/1). + +to_string_search_and_tags_test() -> + assert_set(to_string_search_and_tags, fun to_string/1). \ No newline at end of file diff --git a/src/dev_codec_flat.erl b/src/dev_codec_flat.erl new file mode 100644 index 000000000..1d4dd9988 --- /dev/null +++ b/src/dev_codec_flat.erl @@ -0,0 +1,173 @@ +%%% @doc A codec for turning TABMs into/from flat Erlang maps that have +%%% (potentially multi-layer) paths as their keys, and a normal TABM binary as +%%% their value. +-module(dev_codec_flat). +-export([from/3, to/3, commit/3, verify/3]). +%%% Testing utilities +-export([serialize/1, serialize/2, deserialize/1]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%%% Route signature functions to the `dev_codec_httpsig' module +commit(Msg, Req, Opts) -> dev_codec_httpsig:commit(Msg, Req, Opts). +verify(Msg, Req, Opts) -> dev_codec_httpsig:verify(Msg, Req, Opts). + +%% @doc Convert a flat map to a TABM. +from(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin}; +from(Map, Req, Opts) when is_map(Map) -> + {ok, + maps:fold( + fun(Path, Value, Acc) -> + case Value of + [] -> + ?event(error, + {empty_list_value, + {path, Path}, + {value, Value}, + {map, Map} + } + ); + _ -> + ok + end, + hb_util:deep_set( + hb_path:term_to_path_parts(Path, Opts), + hb_util:ok(from(Value, Req, Opts)), + Acc, + Opts + ) + end, + #{}, + Map + ) + }. + +%% @doc Convert a TABM to a flat map. +to(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin}; +to(Map, Req, Opts) when is_map(Map) -> + Res = + maps:fold( + fun(Key, Value, Acc) -> + case to(Value, Req, Opts) of + {ok, SubMap} when is_map(SubMap) -> + maps:fold( + fun(SubKey, SubValue, InnerAcc) -> + maps:put( + hb_path:to_binary([Key, SubKey]), + SubValue, + InnerAcc + ) + end, + Acc, + SubMap + ); + {ok, SimpleValue} -> + maps:put(hb_path:to_binary([Key]), SimpleValue, Acc) + end + end, + #{}, + Map + ), + {ok, Res}. + +serialize(Map) when is_map(Map) -> + serialize(Map, #{}). + +serialize(Map, Opts) when is_map(Map) -> + Flattened = hb_message:convert(Map, <<"flat@1.0">>, #{}), + {ok, + iolist_to_binary(lists:foldl( + fun(Key, Acc) -> + [ + Acc, + hb_path:to_binary(Key), + <<": ">>, + hb_maps:get(Key, Flattened, Opts), <<"\n">> + ] + end, + <<>>, + hb_util:to_sorted_keys(Flattened, Opts) + ) + ) + }. + +deserialize(Bin) when is_binary(Bin) -> + Flat = lists:foldl( + fun(Line, Acc) -> + case binary:split(Line, <<": ">>, [global]) of + [Key, Value] -> + Acc#{ Key => Value }; + _ -> + Acc + end + end, + #{}, + binary:split(Bin, <<"\n">>, [global]) + ), + {ok, hb_message:convert(Flat, <<"structured@1.0">>, <<"flat@1.0">>, #{})}. + +%%% Tests + +simple_conversion_test() -> + Flat = #{[<<"a">>] => <<"value">>}, + Nested = #{<<"a">> => <<"value">>}, + ?assert(hb_message:match(Nested, hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})))), + ?assert(hb_message:match(Flat, hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})))). + +nested_conversion_test() -> + Flat = #{<<"a/b">> => <<"value">>}, + Nested = #{<<"a">> => #{<<"b">> => <<"value">>}}, + Unflattened = hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})), + Flattened = hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})), + ?assert(hb_message:match(Nested, Unflattened)), + ?assert(hb_message:match(Flat, Flattened)). + +multiple_paths_test() -> + Flat = #{ + <<"x/y">> => <<"1">>, + <<"x/z">> => <<"2">>, + <<"a">> => <<"3">> + }, + Nested = #{ + <<"x">> => #{ + <<"y">> => <<"1">>, + <<"z">> => <<"2">> + }, + <<"a">> => <<"3">> + }, + ?assert(hb_message:match(Nested, hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})))), + ?assert(hb_message:match(Flat, hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})))). + +path_list_test() -> + Nested = #{ + <<"x">> => #{ + [<<"y">>, <<"z">>] => #{ + <<"a">> => <<"2">> + }, + <<"a">> => <<"2">> + } + }, + Flat = hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})), + lists:foreach( + fun(Key) -> + ?assert(not lists:member($\n, binary_to_list(Key))) + end, + hb_maps:keys(Flat, #{}) + ). + +binary_passthrough_test() -> + Bin = <<"raw binary">>, + ?assertEqual(Bin, hb_util:ok(dev_codec_flat:from(Bin, #{}, #{}))), + ?assertEqual(Bin, hb_util:ok(dev_codec_flat:to(Bin, #{}, #{}))). + +deep_nesting_test() -> + Flat = #{<<"a/b/c/d">> => <<"deep">>}, + Nested = #{<<"a">> => #{<<"b">> => #{<<"c">> => #{<<"d">> => <<"deep">>}}}}, + Unflattened = hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})), + Flattened = hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})), + ?assert(hb_message:match(Nested, Unflattened)), + ?assert(hb_message:match(Flat, Flattened)). + +empty_map_test() -> + ?assertEqual(#{}, hb_util:ok(dev_codec_flat:from(#{}, #{}, #{}))), + ?assertEqual(#{}, hb_util:ok(dev_codec_flat:to(#{}, #{}, #{}))). \ No newline at end of file diff --git a/src/dev_codec_http_auth.erl b/src/dev_codec_http_auth.erl new file mode 100644 index 000000000..c221bb12f --- /dev/null +++ b/src/dev_codec_http_auth.erl @@ -0,0 +1,175 @@ +%%% @doc Implements a two-step authentication process for HTTP requests, using +%%% the `Basic' authentication scheme. This device is a viable implementation +%%% of the `generator' interface type employed by `~auth-hook@1.0', as well as +%%% the `~message@1.0' commitment scheme interface. +%%% +%%% `http-auth@1.0`'s `commit' and `verify' keys proxy to the `~httpsig@1.0' +%%% secret key HMAC commitment scheme, utilizing a secret key derived from the +%%% user's authentication information. Callers may also utilize the `generate' +%%% key directly to derive entropy from HTTP Authorization headers provided by +%%% the user. If no Authorization header is provided, the `generate' key will +%%% return a `401 Unauthorized` response, which triggers a recipient's browser +%%% to prompt the user for authentication details and resend the request. +%%% +%%% The `generate' key derives secrets for it's users by calling PBKDF2 with +%%% the user's authentication information. The parameters for the PBKDF2 +%%% algorithm are configurable, and can be specified in the request message: +%%% +%%% 
+%%%   salt:       The salt to use for the PBKDF2 algorithm. Defaults to
+%%%               `sha256("constant:ao")'.
+%%%   iterations: The number of iterations to use for the PBKDF2 algorithm.
+%%%               Defaults to `1,200,000'.
+%%%   alg:        The hashing algorithm to use with PBKDF2. Defaults to
+%%%               `sha256'.
+%%%   key-length: The length of the key to derive from PBKDF2. Defaults to
+%%%               `64'.
+%%%
+%%% +%%% The default iteration count was chosen at two times the recommendation of +%%% OWASP in 2023 (600,000), and executes at a run rate of ~5-10 key derivations +%%% per second on modern CPU hardware. Additionally, the default salt was chosen +%%% such that it is a public constant (needed in order for reproducibility +%%% between nodes), and hashed in order to provide additional entropy, in +%%% alignment with RFC 8018, Section 4.1. +-module(dev_codec_http_auth). +-export([commit/3, verify/3]). +-export([generate/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc The default salt to use for the PBKDF2 algorithm. This value must be +%% global across all nodes that intend to have a shared keyspace, although in +%% instances where this is not possible, users may specify non-standard salts +%% to the `/generate' path with the `salt' request key. +-define(DEFAULT_SALT, <<"constant:ao">>). + +%% @doc Generate or extract a new secret and commit to the message with the +%% `~httpsig@1.0/commit?type=hmac-sha256&scheme=secret' commitment mechanism. +commit(Base, Req, Opts) -> + case generate(Base, Req, Opts) of + {ok, Key} -> + {ok, CommitRes} = + dev_codec_httpsig_proxy:commit( + <<"http-auth@1.0">>, + Key, + Base, + Req, + Opts + ), + ?event({commit_result, CommitRes}), + {ok, CommitRes}; + {error, Err} -> + {error, Err} + end. + +%% @doc Verify a given `Base' message with a derived `Key' using the +%% `~httpsig@1.0' secret key HMAC commitment scheme. +verify(Base, RawReq, Opts) -> + ?event({verify_invoked, {base, Base}, {req, RawReq}}), + {ok, Key} = generate(Base, RawReq, Opts), + ?event({verify_found_key, {key, Key}, {base, Base}, {req, RawReq}}), + {ok, VerifyRes} = + dev_codec_httpsig_proxy:verify( + Key, + Base, + RawReq, + Opts + ), + ?event({verify_result, VerifyRes}), + {ok, VerifyRes}. + +%% @doc Collect authentication information from the client. If the `raw' flag +%% is set to `true', return the raw authentication information. Otherwise, +%% derive a key from the authentication information and return it. +generate(_Msg, ReqLink, Opts) when ?IS_LINK(ReqLink) -> + generate(_Msg, hb_cache:ensure_loaded(ReqLink, Opts), Opts); +generate(_Msg, #{ <<"secret">> := Secret }, _Opts) -> + {ok, Secret}; +generate(_Msg, Req, Opts) -> + case hb_maps:get(<<"authorization">>, Req, undefined, Opts) of + <<"Basic ", Auth/binary>> -> + Decoded = base64:decode(Auth), + ?event(key_gen, {generated_key, {auth, Auth}, {decoded, Decoded}}), + case hb_maps:get(<<"raw">>, Req, false, Opts) of + true -> {ok, Decoded}; + false -> derive_key(Decoded, Req, Opts) + end; + undefined -> + {error, + #{ + <<"status">> => 401, + <<"www-authenticate">> => <<"Basic">>, + <<"details">> => <<"No authorization header provided.">> + } + }; + Unrecognized -> + {error, + #{ + <<"status">> => 400, + <<"details">> => + <<"Unrecognized authorization header: ", Unrecognized/binary>> + } + } + end. + +%% @doc Derive a key from the authentication information using the PBKDF2 +%% algorithm and user specified parameters. +derive_key(Decoded, Req, Opts) -> + Alg = hb_util:atom(hb_maps:get(<<"alg">>, Req, <<"sha256">>, Opts)), + Salt = + hb_maps:get( + <<"salt">>, + Req, + hb_crypto:sha256(?DEFAULT_SALT), + Opts + ), + Iterations = hb_maps:get(<<"iterations">>, Req, 2 * 600_000, Opts), + KeyLength = hb_maps:get(<<"key-length">>, Req, 64, Opts), + ?event(key_gen, + {derive_key, + {alg, Alg}, + {salt, Salt}, + {iterations, Iterations}, + {key_length, KeyLength} + } + ), + case hb_crypto:pbkdf2(Alg, Decoded, Salt, Iterations, KeyLength) of + {ok, Key} -> + EncodedKey = hb_util:encode(Key), + {ok, EncodedKey}; + {error, Err} -> + ?event(key_gen, + {pbkdf2_error, + {alg, Alg}, + {salt, Salt}, + {iterations, Iterations}, + {key_length, KeyLength}, + {error, Err} + } + ), + {error, + #{ + <<"status">> => 500, + <<"details">> => <<"Failed to derive key.">> + } + } + end. + +%%% Tests + +benchmark_pbkdf2_test() -> + Key = crypto:strong_rand_bytes(32), + Iterations = 2 * 600_000, + KeyLength = 32, + Derivations = + hb_test_utils:benchmark( + fun() -> + hb_crypto:pbkdf2(sha256, Key, <<"salt">>, Iterations, KeyLength) + end + ), + hb_test_utils:benchmark_print( + <<"Derived">>, + <<"keys (1.2m iterations each)">>, + Derivations + ). \ No newline at end of file diff --git a/src/dev_codec_httpsig.erl b/src/dev_codec_httpsig.erl new file mode 100644 index 000000000..c104ec2ac --- /dev/null +++ b/src/dev_codec_httpsig.erl @@ -0,0 +1,629 @@ +%%% @doc This module implements HTTP Message Signatures as described in RFC-9421 +%%% (https://datatracker.ietf.org/doc/html/rfc9421), as an AO-Core device. +%%% It implements the codec standard (from/1, to/1), as well as the optional +%%% commitment functions (id/3, sign/3, verify/3). The commitment functions +%%% are found in this module, while the codec functions are relayed to the +%%% `dev_codec_httpsig_conv' module. +-module(dev_codec_httpsig). +%%% Codec API functions +-export([to/3, from/3]). +%%% Uni-directional codec support (_to_ binary/header+body components), but not +%%% back. +-export([serialize/2, serialize/3]). +%%% Commitment API functions +-export([commit/3, verify/3]). +%%% Public API functions +-export([add_content_digest/2, normalize_for_encoding/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% Routing functions for the `dev_codec_httpsig_conv' module +to(Msg, Req, Opts) -> dev_codec_httpsig_conv:to(Msg, Req, Opts). +from(Msg, Req, Opts) -> dev_codec_httpsig_conv:from(Msg, Req, Opts). + +%% @doc Generate the `Opts' to use during AO-Core operations in the codec. +opts(RawOpts) -> + RawOpts#{ + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>], + force_message => false + }. + +%% @doc A helper utility for creating a direct encoding of a HTTPSig message. +%% +%% This function supports two modes of operation: +%% 1. `format: binary`, yielding a raw binary HTTP/1.1-style response that can +%% either be stored or emitted raw accross a transport medium. +%% 2. `format: components`, yielding a message containing `headers` and `body` +%% keys, suitable for use in connecting to HTTP-response flows implemented +%% by other servers. +%% +%% Optionally, the `index` key can be set to override resolution of the default +%% index page into HTTP responses that do not contain their own `body` field. +serialize(Msg, Opts) -> serialize(Msg, #{}, Opts). +serialize(Msg, #{ <<"format">> := <<"components">> }, Opts) -> + % Convert to HTTPSig via TABM through calling `hb_message:convert` rather + % than executing `to/3` directly. This ensures that our responses are + % normalized. + {ok, EncMsg} = hb_message:convert(Msg, <<"httpsig@1.0">>, Opts), + {ok, + #{ + <<"body">> => hb_maps:get(<<"body">>, EncMsg, <<>>), + <<"headers">> => hb_maps:without([<<"body">>], EncMsg) + } + }; +serialize(Msg, _Req, Opts) -> + % We assume the default format of `binary` if none of the prior clauses + % match. + HTTPSig = hb_message:convert(Msg, <<"httpsig@1.0">>, Opts), + {ok, dev_codec_httpsig_conv:encode_http_msg(HTTPSig, Opts) }. + +verify(Base, Req, RawOpts) -> + % A rsa-pss-sha512 commitment is verified by regenerating the signature + % base and validating against the signature. + Opts = opts(RawOpts), + {ok, EncMsg, EncComm, _} = normalize_for_encoding(Base, Req, Opts), + SigBase = signature_base(EncMsg, EncComm, Opts), + KeyRes = dev_codec_httpsig_keyid:req_to_key_material(Req, Opts), + RawSignature = hb_util:decode(Signature = maps:get(<<"signature">>, Req)), + ?event(debug_httpsig, + { + httpsig_verifying, + {signature, Signature}, + {parsed_key_material, KeyRes}, + {req, Req}, + {signature_base, {string, SigBase}} + } + ), + case {KeyRes, maps:get(<<"type">>, Req)} of + {{ok, _, Key, _KeyID}, <<"rsa-pss-sha512">>} -> + ?event(httpsig_verify, {verify, {rsa_pss_sha512, {sig_base, SigBase}}}), + { + ok, + ar_wallet:verify( + {{rsa, 65537}, Key}, + SigBase, + RawSignature, + sha512 + ) + }; + {{ok, _, Key, KeyID}, <<"hmac-sha256">>} -> + % Generate the HMAC from the key and signature base. + ActualHMac = + hb_util:human_id( + crypto:mac(hmac, sha256, Key, SigBase) + ), + ?event(httpsig_verify, + {verify, + {hmac_sha256, + {keyid, KeyID}, + {sig_base, SigBase}, + {actual_hmac, {string, ActualHMac}}, + {signature, {string, Signature}}, + {matches, Signature =:= ActualHMac} + } + }), + {ok, Signature =:= ActualHMac}; + {{error, Reason}, _Type} -> + ?event(httpsig_verify, {verify, {error, Reason}}), + {ok, false}; + {{failure, Info}, _Type} -> + ?event(httpsig_verify, {verify, {failure, Info}}), + {failure, Info} + end. + +%% @doc Commit to a message using the HTTP-Signature format. We use the `type' +%% parameter to determine the type of commitment to use. If the `type' parameter +%% is `signed', we default to the rsa-pss-sha512 algorithm. If the `type' +%% parameter is `unsigned', we default to the hmac-sha256 algorithm. +commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> + commit(Msg, Req#{ <<"type">> => <<"hmac-sha256">> }, Opts); +commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> + commit(Msg, Req#{ <<"type">> => <<"rsa-pss-sha512">> }, Opts); +commit(MsgToSign, Req = #{ <<"type">> := <<"rsa-pss-sha512">> }, RawOpts) -> + ?event( + {generating_rsa_pss_sha512_commitment, {msg, MsgToSign}, {req, Req}} + ), + Opts = opts(RawOpts), + Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), + if Wallet =:= no_viable_wallet -> + throw({cannot_commit, no_viable_wallet, MsgToSign}); + true -> + ok + end, + % Utilize the hashpath, if present, as the tag for the commitment. + MaybeTagMap = + case MsgToSign of + #{ <<"priv">> := #{ <<"hashpath">> := HP }} -> #{ <<"tag">> => HP }; + _ -> #{} + end, + % Generate the unsigned commitment and signature base. + ToCommit = hb_ao:normalize_keys(keys_to_commit(MsgToSign, Req, Opts)), + ?event({to_commit, ToCommit}), + UnsignedCommitment = + maybe_bundle_tag_commitment( + MaybeTagMap#{ + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"type">> => <<"rsa-pss-sha512">>, + <<"keyid">> => + << + "publickey:", + (base64:encode(ar_wallet:to_pubkey(Wallet)))/binary + >>, + <<"committer">> => + hb_util:human_id(ar_wallet:to_address(Wallet)), + <<"committed">> => ToCommit + }, + Req, + Opts + ), + {ok, EncMsg, EncComm, ModCommittedKeys} = + normalize_for_encoding(MsgToSign, UnsignedCommitment, Opts), + ?event({encoded_to_httpsig_for_commitment, MsgToSign}), + % Generate the signature base + SignatureBase = signature_base(EncMsg, EncComm, Opts), + ?event({rsa_signature_base, {string, SignatureBase}}), + ?event({mod_committed_keys, ModCommittedKeys}), + % Sign the signature base + Signature = ar_wallet:sign(Wallet, SignatureBase, sha512), + % Generate the ID of the signature + ID = hb_util:human_id(crypto:hash(sha256, Signature)), + ?event({rsa_commit, {committed, ToCommit}}), + % Calculate the ID and place the signature into the `commitments' key of the + % message. After, we call `commit' again to add the hmac to the new + % message. + commit( + MsgToSign#{ + <<"commitments">> => + (maps:get(<<"commitments">>, MsgToSign, #{}))#{ + ID => + UnsignedCommitment#{ + <<"signature">> => hb_util:encode(Signature), + <<"committed">> => ModCommittedKeys + } + } + }, + Req#{ <<"type">> => <<"hmac-sha256">> }, + Opts + ); +commit(BaseMsg, Req = #{ <<"type">> := <<"hmac-sha256">> }, RawOpts) -> + % Extract the key material from the request. + Opts = opts(RawOpts), + ?event({req_to_key_material, {req, Req}}), + {ok, Scheme, Key, KeyID} = dev_codec_httpsig_keyid:req_to_key_material(Req, Opts), + Committer = dev_codec_httpsig_keyid:keyid_to_committer(Scheme, KeyID), + % Remove any existing hmac commitments with the given keyid before adding + % the new one. + Msg = + hb_message:without_commitments( + #{ + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"type">> => <<"hmac-sha256">>, + <<"keyid">> => KeyID + }, + BaseMsg, + Opts + ), + % Extract the base commitments from the message. + Commitments = maps:get(<<"commitments">>, Msg, #{}), + CommittedKeys = keys_to_commit(Msg, Req, Opts), + % Create the commitment with the appropriate keyid, committed keys, and + % bundle specifier. + CommitmentWithoutCommitter = #{ + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"type">> => <<"hmac-sha256">>, + <<"keyid">> => KeyID, + <<"committed">> => hb_ao:normalize_keys(CommittedKeys) + }, + % If the committer is undefined, we do not need to add the `committer' key. + BaseCommitment = + if Committer =:= undefined -> CommitmentWithoutCommitter; + true -> CommitmentWithoutCommitter#{ <<"committer">> => Committer } + end, + UnauthedCommitment = + maybe_bundle_tag_commitment( + BaseCommitment, + Req, + Opts + ), + {ok, EncMsg, EncComm, ModCommittedKeys} = + normalize_for_encoding(Msg, UnauthedCommitment, Opts), + SigBase = signature_base(EncMsg, EncComm, Opts), + HMac = hb_util:human_id(crypto:mac(hmac, sha256, Key, SigBase)), + ?event(httpsig_commit, + {hmac_commit, + {type, <<"hmac-sha256">>}, + {keyid, KeyID}, + {committer, Committer}, + {committed, CommittedKeys}, + {sig_base, SigBase}, + {hmac, HMac} + } + ), + Res = + { + ok, + Msg#{ + <<"commitments">> => + Commitments#{ + HMac => + UnauthedCommitment#{ + <<"signature">> => HMac, + <<"committed">> => ModCommittedKeys + } + } + } + }, + ?event({hmac_generation_complete, Res}), + Res. + +%% @doc Annotate the commitment with the `bundle' key if the request contains +%% it. +maybe_bundle_tag_commitment(Commitment, Req, _Opts) -> + case hb_util:atom(maps:get(<<"bundle">>, Req, false)) of + true -> Commitment#{ <<"bundle">> => <<"true">> }; + false -> Commitment + end. + +%% @doc Derive the set of keys to commit to from a `commit` request and a +%% base message. +keys_to_commit(_Base, #{ <<"committed">> := Explicit}, _Opts) -> + % Case 1: Explicitly provided keys to commit. + % Add `+link` specifiers to the user given list as necessary, in order for + % their given keys to match the HTTPSig encoded TABM form. + hb_util:list_to_numbered_message(Explicit); +keys_to_commit(Base, _Req, Opts) -> + % Extract the set of committed keys from the message. + case hb_message:committed(Base, #{ <<"committers">> => <<"all">> }, opts(Opts)) of + [] -> + % Case 3: Default to all keys in the TABM-encoded message, aside + % metadata. + hb_util:list_to_numbered_message( + lists:map( + fun hb_link:remove_link_specifier/1, + hb_util:to_sorted_keys(Base, Opts) + -- [<<"commitments">>, <<"priv">>] + ) + ); + Keys -> + % Case 2: Replicate the raw keys that the existing commitments have + % used. This leads to a message whose commitments can be 'stacked' + % and represented together in HTTPSig format. + hb_util:list_to_numbered_message(Keys) + end. + +%% @doc If the `body' key is present and a binary, replace it with a +%% content-digest. +add_content_digest(Msg, _Opts) -> + case maps:get(<<"body">>, Msg, not_found) of + Body when is_binary(Body) -> + % Remove the body from the message and add the content-digest, + % encoded as a structured field. + (maps:without([<<"body">>], Msg))#{ + <<"content-digest">> => + hb_util:bin(hb_structured_fields:dictionary( + #{ + <<"sha-256">> => + {item, {binary, hb_crypto:sha256(Body)}, []} + } + )) + }; + _ -> Msg + end. + +%% @doc Given a base message and a commitment, derive the message and commitment +%% normalized for encoding. +normalize_for_encoding(Msg, Commitment, Opts) -> + % Extract the requested keys to include in the signature base. + RawInputs = + hb_util:message_to_ordered_list( + maps:get(<<"committed">>, Commitment, []), + Opts + ), + % Normalize the keys to their maybe-linked form, adding `+link` if necessary. + Inputs = + lists:map( + fun(Key) -> + NormalizedKey = hb_ao:normalize_key(Key), + case maps:is_key(NormalizedKey, Msg) of + true -> NormalizedKey; + false -> + case maps:is_key(<>, Msg) of + true -> <>; + false -> NormalizedKey + end + end + end, + RawInputs + ), + ?event({inputs, {list, Inputs}}), + % Filter the message down to only the requested keys, then encode it. + MsgWithOnlyInputs = maps:with(Inputs, Msg), + ?event({msg_with_only_inputs, maps:without([<<"commitments">>], MsgWithOnlyInputs)}), + {ok, EncodedWithSigInfo} = + to( + maps:without([<<"commitments">>], MsgWithOnlyInputs), + #{ + <<"bundle">> => + hb_util:atom(maps:get(<<"bundle">>, Commitment, false)) + }, + Opts + ), + % Remove the signature and signature-input keys from the encoded message, + % convert the `body' key to a `content-digest' key, if present. + Encoded = add_content_digest(EncodedWithSigInfo, Opts), + % Transform the list of requested keys to their `httpsig@1.0' equivalents. + EncodedKeys = maps:keys(Encoded), + EncodedKeysWithBodyKey = + case hb_maps:get(<<"ao-body-key">>, EncodedWithSigInfo, not_found) of + not_found -> + EncodedKeys; + AOBodyKey -> + hb_util:list_replace( + EncodedKeys, + AOBodyKey, + [<<"body">>, <<"ao-body-key">>] + ) + end, + % The keys to be used in encodings of the message: + KeysForEncoding = + hb_util:list_replace( + EncodedKeysWithBodyKey, + <<"body">>, + <<"content-digest">> + ), + % Calculate the keys that have been removed from the message, as a result + % of being added to the body. These keys will need to be removed from the + % `committed' list and re-added where the `content-digest' was. + BodyKeys = + lists:filter( + fun(Key) -> not key_present(Key, Encoded) end, + RawInputs + ), + KeysForCommitment = + dev_codec_httpsig_siginfo:from_siginfo_keys( + EncodedWithSigInfo, + BodyKeys, + KeysForEncoding + ), + ?event(debug_httpsig, + {normalized_for_encoding, + {raw_inputs, Inputs}, + {inputs_for_encoding, KeysForEncoding}, + {final_for_commitment_message, KeysForCommitment}, + {encoded_message, Encoded} + } + ), + { + ok, + Encoded, + Commitment#{ <<"committed">> => KeysForEncoding }, + KeysForCommitment + }. + +%% @doc Calculate if a key or its `+link' TABM variant is present in a message. +key_present(Key, Msg) -> + NormalizedKey = hb_ao:normalize_key(Key), + maps:is_key(NormalizedKey, Msg) + orelse maps:is_key(<>, Msg). + +%% @doc create the signature base that will be signed in order to create the +%% Signature and SignatureInput. +%% +%% This implements a portion of RFC-9421 see: +%% https://datatracker.ietf.org/doc/html/rfc9421#name-creating-the-signature-base +signature_base(EncodedMsg, Commitment, Opts) -> + ComponentsLines = + signature_components_line( + EncodedMsg, + Commitment, + Opts + ), + ?event({component_identifiers_for_sig_base, ComponentsLines}), + ParamsLine = signature_params_line(Commitment, Opts), + SignatureBase = + << + ComponentsLines/binary, "\n", + "\"@signature-params\": ", ParamsLine/binary + >>, + ?event(signature_base, {signature_base, {string, SignatureBase}}), + SignatureBase. + +%% @doc Given a list of Component Identifiers and a Request/Response Message +%% context, create the "signature-base-line" portion of the signature base +%% TODO: catch duplicate identifier: +%% https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.1 +%% +%% See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.1 +signature_components_line(Req, Commitment, _Opts) -> + ComponentsLines = + lists:map( + fun(Name) -> + case maps:get(Name, Req, not_found) of + not_found -> + throw( + { + missing_key_for_signature_component_line, + Name, + {message, Req}, + {commitment, Commitment} + } + ); + Value -> + << <<"\"">>/binary, Name/binary, <<"\"">>/binary, <<": ">>/binary, Value/binary>> + end + end, + maps:get(<<"committed">>, Commitment) + ), + iolist_to_binary(lists:join(<<"\n">>, ComponentsLines)). + +%% @doc construct the "signature-params-line" part of the signature base. +%% +%% See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.3.2.4 +signature_params_line(RawCommitment, Opts) -> + Commitment = + maps:without( + [<<"signature">>, <<"signature-input">>], + RawCommitment + ), + ?event(debug_enc, {signature_params_line, {commitment, Commitment}}), + hb_util:bin( + hb_structured_fields:list( + [ + { + list, + lists:map( + fun(Key) -> {item, {string, Key}, []} end, + dev_codec_httpsig_siginfo:add_derived_specifiers( + hb_util:message_to_ordered_list( + maps:get(<<"committed">>, Commitment), + Opts + ) + ) + ), + lists:map( + fun ({<<"alg">>, Param}) when is_binary(Param) -> + {<<"alg">>, {string, Param}}; + ({Name, Param}) when is_binary(Param) -> + {Name, {string, Param}}; + ({Name, Param}) when is_integer(Param) -> + {Name, Param} + end, + lists:sort(maps:to_list( + maps:with( + [ + <<"created">>, + <<"expires">>, + <<"nonce">>, + <<"alg">>, + <<"keyid">>, + <<"tag">>, + <<"bundle">> + ], + Commitment#{ <<"alg">> => maps:get(<<"type">>, Commitment) } + ) + )) + ) + } + ] + ) + ). + +%%% +%%% TESTS +%%% + +%%% Integration Tests + +%% @doc Ensure that we can validate a signature on an extremely large and complex +%% message that is sent over HTTP, signed with the codec. +validate_large_message_from_http_test() -> + Node = hb_http_server:start_node(Opts = #{ + force_signed => true, + commitment_device => <<"httpsig@1.0">>, + extra => + [ + [ + [ + #{ + <<"n">> => N, + <<"m">> => M, + <<"o">> => O + } + || + O <- lists:seq(1, 3) + ] + || + M <- lists:seq(1, 3) + ] + || + N <- lists:seq(1, 3) + ] + }), + {ok, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, Opts), + Signers = hb_message:signers(Res, Opts), + ?event({received, {signers, Signers}, {res, Res}}), + ?assert(length(Signers) == 1), + ?assert(hb_message:verify(Res, Signers, Opts)), + ?event({sig_verifies, Signers}), + ?assert(hb_message:verify(Res, all, Opts)), + ?event({hmac_verifies, <<"hmac-sha256">>}), + {ok, OnlyCommitted} = hb_message:with_only_committed(Res, Opts), + ?event({msg_with_only_committed, OnlyCommitted}), + ?assert(hb_message:verify(OnlyCommitted, Signers, Opts)), + ?event({msg_with_only_committed_verifies, Signers}), + ?assert(hb_message:verify(OnlyCommitted, all, Opts)), + ?event({msg_with_only_committed_verifies_hmac, <<"hmac-sha256">>}). + +committed_id_test() -> + Msg = #{ <<"basic">> => <<"value">> }, + Signed = hb_message:commit(Msg, hb:wallet()), + ?assert(hb_message:verify(Signed, all, #{})), + ?event({signed_msg, Signed}), + UnsignedID = hb_message:id(Signed, none), + SignedID = hb_message:id(Signed, all), + ?event({ids, {unsigned_id, UnsignedID}, {signed_id, SignedID}}), + ?assertNotEqual(UnsignedID, SignedID). + +commit_secret_key_test() -> + Msg = #{ <<"basic">> => <<"value">> }, + CommittedMsg = + hb_message:commit( + Msg, + #{}, + #{ + <<"type">> => <<"hmac-sha256">>, + <<"secret">> => <<"test-secret">>, + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"scheme">> => <<"secret">> + } + ), + ?event({committed_msg, CommittedMsg}), + Committers = hb_message:signers(CommittedMsg, #{}), + ?assert(length(Committers) == 1), + ?event({committers, Committers}), + ?assert( + hb_message:verify( + CommittedMsg, + #{ <<"committers">> => Committers, <<"secret">> => <<"test-secret">> }, + #{} + ) + ), + ?assertNot( + hb_message:verify( + CommittedMsg, + #{ <<"committers">> => Committers, <<"secret">> => <<"bad-secret">> }, + #{} + ) + ). + +multicommitted_id_test() -> + Msg = #{ <<"basic">> => <<"value">> }, + Signed1 = hb_message:commit(Msg, Wallet1 = ar_wallet:new()), + Signed2 = hb_message:commit(Signed1, Wallet2 = ar_wallet:new()), + Addr1 = hb_util:human_id(ar_wallet:to_address(Wallet1)), + Addr2 = hb_util:human_id(ar_wallet:to_address(Wallet2)), + ?event({signed_msg, Signed2}), + UnsignedID = hb_message:id(Signed2, none), + SignedID = hb_message:id(Signed2, all), + ?event({ids, {unsigned_id, UnsignedID}, {signed_id, SignedID}}), + ?assertNotEqual(UnsignedID, SignedID), + ?assert(hb_message:verify(Signed2, [])), + ?assert(hb_message:verify(Signed2, [Addr1])), + ?assert(hb_message:verify(Signed2, [Addr2])), + ?assert(hb_message:verify(Signed2, [Addr1, Addr2])), + ?assert(hb_message:verify(Signed2, [Addr2, Addr1])), + ?assert(hb_message:verify(Signed2, all)). + +%% @doc Test that we can sign and verify a message with a link. We use +sign_and_verify_link_test() -> + Msg = #{ + <<"normal">> => <<"typical-value">>, + <<"untyped">> => #{ <<"inner-untyped">> => <<"inner-value">> }, + <<"typed">> => #{ <<"inner-typed">> => 123 } + }, + NormMsg = hb_message:convert(Msg, <<"structured@1.0">>, #{}), + ?event({msg, NormMsg}), + Signed = hb_message:commit(NormMsg, hb:wallet()), + ?event({signed_msg, Signed}), + ?assert(hb_message:verify(Signed)). diff --git a/src/dev_codec_httpsig_conv.erl b/src/dev_codec_httpsig_conv.erl new file mode 100644 index 000000000..cc08b85d0 --- /dev/null +++ b/src/dev_codec_httpsig_conv.erl @@ -0,0 +1,941 @@ + +%%% @doc A codec that marshals TABM encoded messages to and from the "HTTP" +%%% message structure. +%%% +%%% Every HTTP message is an HTTP multipart message. +%%% See https://datatracker.ietf.org/doc/html/rfc7578 +%%% +%%% For each TABM Key: +%%% +%%% The Key/Value Pair will be encoded according to the following rules: +%%% "signatures" -> {SignatureInput, Signature} header Tuples, each encoded +%%% as a Structured Field Dictionary +%%% "body" -> +%%% - if a map, then recursively encode as its own HyperBEAM message +%%% - otherwise encode as a normal field +%%% _ -> encode as a normal field +%%% +%%% Each field will be mapped to the HTTP Message according to the following +%%% rules: +%%% "body" -> always encoded part of the body as with Content-Disposition +%%% type of "inline" +%%% _ -> +%%% - If the byte size of the value is less than the ?MAX_TAG_VALUE, +%%% then encode as a header, also attempting to encode as a +%%% structured field. +%%% - Otherwise encode the value as a part in the multipart response +%%% +-module(dev_codec_httpsig_conv). +-export([to/3, from/3, encode_http_msg/2]). +%%% Helper utilities +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +% The max header length is 4KB +-define(MAX_HEADER_LENGTH, 4096). +% https://datatracker.ietf.org/doc/html/rfc7231#section-3.1.1.4 +-define(CRLF, <<"\r\n">>). +-define(DOUBLE_CRLF, <>). + +%% @doc Convert a HTTP Message into a TABM. +%% HTTP Structured Field is encoded into it's equivalent TABM encoding. +from(Bin, _Req, _Opts) when is_binary(Bin) -> {ok, Bin}; +from(Link, _Req, _Opts) when ?IS_LINK(Link) -> {ok, Link}; +from(HTTP, _Req, Opts) -> + % First, parse all headers excluding the signature-related headers, as they + % are handled separately. + Headers = hb_maps:without([<<"body">>], HTTP, Opts), + % Next, we need to potentially parse the body, get the ordering of the body + % parts, and add them to the TABM. + {OrderedBodyKeys, BodyTABM} = body_to_tabm(HTTP, Opts), + % Merge the body keys with the headers. + WithBodyKeys = maps:merge(Headers, BodyTABM), + % Decode the `ao-ids' key into a map. `ao-ids' is an encoding of literal + % binaries whose keys (given that they are IDs) cannot be distributed as + % HTTP headers. + WithIDs = ungroup_ids(WithBodyKeys, Opts), + % Remove the signature-related headers, such that they can be reconstructed + % from the commitments. + MsgWithoutSigs = hb_maps:without( + [<<"signature">>, <<"signature-input">>, <<"commitments">>], + WithIDs, + Opts + ), + % Finally, we need to add the signatures to the TABM. + Commitments = + dev_codec_httpsig_siginfo:siginfo_to_commitments( + WithIDs, + OrderedBodyKeys, + Opts + ), + MsgWithSigs = + case ?IS_EMPTY_MESSAGE(Commitments) of + false -> MsgWithoutSigs#{ <<"commitments">> => Commitments }; + true -> MsgWithoutSigs + end, + ?event({message_with_commitments, MsgWithSigs}), + Res = + hb_maps:without( + Removed = + hb_maps:keys(Commitments) ++ + [<<"content-digest">>] ++ + case maps:get(<<"content-type">>, MsgWithSigs, undefined) of + <<"multipart/", _/binary>> -> [<<"content-type">>]; + _ -> [] + end ++ + case hb_message:is_signed_key(<<"ao-body-key">>, MsgWithSigs, Opts) of + true -> []; + false -> [<<"ao-body-key">>] + end, + MsgWithSigs, + Opts + ), + ?event({message_without_commitments, Res, Removed}), + {ok, Res}. + +%% @doc Generate the body TABM from the `body' key of the encoded message. +body_to_tabm(HTTP, Opts) -> + % Extract the body and content-type from the HTTP message. + Body = hb_maps:get(<<"body">>, HTTP, no_body, Opts), + ContentType = hb_maps:get(<<"content-type">>, HTTP, undefined, Opts), + {_, InlinedKey} = inline_key(HTTP), + ?event({inlined_body_key, InlinedKey}), + % Parse the body into a TABM. + {OrderedBodyKeys, BodyTABM} = + case body_to_parts(ContentType, Body, Opts) of + no_body -> {[], #{}}; + {normal, RawBody} -> + % The body is not a multipart, so we just return the inlined key. + {[InlinedKey], #{ InlinedKey => RawBody }}; + {multipart, Parts} -> + % Parse each part of the multipart body into an individual TABM, + % with its associated key. + OrderedBodyTABMs = + lists:map( + fun(Part) -> + from_body_part(InlinedKey, Part, Opts) + end, + Parts + ), + % Merge all of the parts into a single TABM. + {ok, MergedParts} = + dev_codec_flat:from( + maps:from_list(OrderedBodyTABMs), + #{}, + Opts + ), + % Calculate the ordered body keys of the multipart data. The + % nested body parts are labelled by `path`, rather than `key`: + % That is, a body part may contain a `/` in its key, representing + % that the nested form is not a direct child of the parent + % message. Subsequently, we need to take just the first + % `path part' of the key and return the unique'd list. + {MessagePaths, _} = lists:unzip(OrderedBodyTABMs), + Keys = + hb_util:unique( + lists:map( + fun(Path) -> + hd(binary:split(Path, <<"/">>, [global])) + end, + MessagePaths + ) + ), + % Return both as a pair. + {Keys, MergedParts} + end, + {OrderedBodyKeys, BodyTABM}. + +%% @doc Split the body into parts, if it is a multipart. +body_to_parts(_ContentType, no_body, _Opts) -> no_body; +body_to_parts(ContentType, Body, _Opts) -> + ?event( + {from_body, + {content_type, {explicit, ContentType}}, + {body, Body} + } + ), + Params = + case ContentType of + undefined -> []; + _ -> + {item, {_, _XT}, XParams} = + hb_structured_fields:parse_item(ContentType), + XParams + end, + case lists:keyfind(<<"boundary">>, 1, Params) of + false -> + % The body is not a multipart, so just set as is to the Inlined key on + % the TABM. + {normal, Body}; + {_, {_Type, Boundary}} -> + % We need to manually parse the multipart body into key/values on the + % TABM. + % + % Find the sub-part of the body within the boundary. + % We also make sure to account for the CRLF at end and beginning + % of the starting and terminating part boundary, respectively + % + % ie. + % --foo-boundary\r\n + % My-Awesome: Part + % + % an awesome body\r\n + % --foo-boundary-- + BegPat = <<"--", Boundary/binary, ?CRLF/binary>>, + EndPat = <>, + {Start, SL} = binary:match(Body, BegPat), + {End, _} = binary:match(Body, EndPat), + BodyPart = binary:part(Body, Start + SL, End - (Start + SL)), + % By taking into account all parts of the surrounding boundary above, + % we get precisely the sub-part that we're interested without any + % additional parsing + {multipart, binary:split( + BodyPart, + [<>], + [global] + )} + end. + +%% @doc Parse a single part of a multipart body into a TABM. +from_body_part(InlinedKey, Part, Opts) -> + % Extract the Headers block and Body. Only split on the FIRST double CRLF + {RawHeadersBlock, RawBody} = + case binary:split(Part, [?DOUBLE_CRLF], []) of + [XRawHeadersBlock] -> + % The message has no body. + {XRawHeadersBlock, <<>>}; + [XRawHeadersBlock, XRawBody] -> + {XRawHeadersBlock, XRawBody} + end, + % Extract individual headers + RawHeaders = binary:split(RawHeadersBlock, ?CRLF, [global]), + % Now we parse each header, splitting into {Key, Value} + Headers = + hb_maps:from_list(lists:filtermap( + fun(<<>>) -> false; + (RawHeader) -> + case binary:split(RawHeader, [<<": ">>]) of + [Name, Value] -> {true, {Name, Value}}; + _ -> + % skip lines that aren't properly formatted headers + false + end + end, + RawHeaders + )), + % The Content-Disposition is from the parent message, + % so we separate off from the rest of the headers + case hb_maps:get(<<"content-disposition">>, Headers, undefined, Opts) of + undefined -> + % A Content-Disposition header is required for each part + % in the multipart body + throw({error, no_content_disposition_in_multipart, Headers}); + RawDisposition when is_binary(RawDisposition) -> + % Extract the name + {item, {_, Disposition}, DispositionParams} = + hb_structured_fields:parse_item(RawDisposition), + {ok, PartName} = + case Disposition of + <<"inline">> -> + {ok, InlinedKey}; + _ -> + % Otherwise, we need to extract the name of the part + % from the Content-Disposition parameters + case lists:keyfind(<<"name">>, 1, DispositionParams) of + {_, {_type, PN}} -> {ok, PN}; + false -> no_part_name_found + end + end, + Commitments = + dev_codec_httpsig_siginfo:siginfo_to_commitments( + Headers#{ PartName => RawBody }, + [PartName], + Opts + ), + RestHeaders = + hb_maps:without( + [ + <<"content-disposition">>, + <<"content-type">>, + <<"ao-body-key">>, + <<"content-digest">> + ], + Headers, + Opts + ), + PartNameSplit = binary:split(PartName, <<"/">>, [global]), + NestedPartName = lists:last(PartNameSplit), + ParsedPart = + case hb_maps:size(Commitments, Opts) of + 0 -> + WithoutTypes = maps:without([<<"ao-types">>], RestHeaders), + Types = + hb_maps:get( + <<"ao-types">>, + RestHeaders, + <<>>, + Opts + ), + case {hb_maps:size(WithoutTypes, Opts), Types, RawBody} of + {0, <<"empty-message">>, <<>>} -> + % The message is empty, so we return an empty + % map. + #{}; + {_, _, <<>>} -> + % There is no body to the message, so we return + % just the headers. + RestHeaders; + {0, _, _} -> + % There are no headers besides content-disposition, + % so we return the body as is. + RawBody; + {_, _, _} -> + % There are other headers, so we need to parse + % the body as a TABM. + {_, RawBodyKey} = inline_key(Headers), + RestHeaders#{ RawBodyKey => RawBody } + end; + _ -> maps:get(NestedPartName, Commitments, #{}) + end, + {PartName, ParsedPart} + end. + +%%% @doc Convert a TABM into an HTTP Message. The HTTP Message is a simple Erlang Map +%%% that can translated to a given web server Response API +to(TABM, Req, Opts) -> to(TABM, Req, [], Opts). +to(Bin, _Req, _FormatOpts, _Opts) when is_binary(Bin) -> {ok, Bin}; +to(Link, _Req, _FormatOpts, _Opts) when ?IS_LINK(Link) -> {ok, Link}; +to(TABM, Req = #{ <<"index">> := true }, _FormatOpts, Opts) -> + % If the caller has specified that an `index` page is requested, we: + % 1. Convert the message to HTTPSig as usual. + % 2. Check if the `body` and `content-type` keys are set. If either are, + % we return the message as normal. + % 3. If they are not, we convert the given message back to its original + % form and resolve `path = index` upon it. + % 4. If this yields a result, we convert it to TABM and merge it with the + % original HTTP-Sig encoded message. We prefer keys from the original + % if conflicts arise. + % 5. The resulting combined message is returned to the user. + {ok, EncOriginal} = to(TABM, maps:without([<<"index">>], Req), Opts), + OrigBody = hb_maps:get(<<"body">>, EncOriginal, <<>>, Opts), + OrigContentType = hb_maps:get(<<"content-type">>, EncOriginal, <<>>, Opts), + case {OrigBody, OrigContentType} of + {<<>>, <<>>} -> + % The message has no body or content-type set. Resolve the `index` + % key upon it to derive it. + Structured = hb_message:convert(TABM, <<"structured@1.0">>, Opts), + try hb_ao:resolve(Structured, #{ <<"path">> => <<"index">> }, Opts) of + {ok, IndexMsg} -> + % The index message has been calculated successfully. Convert + % it to TABM format. + IndexTABM = hb_message:convert(IndexMsg, tabm, Opts), + % Merge the index message with the original, favoring the + % keys of the original in the event of conflict. Remove the + % `priv` message, if present. + Merged = + hb_maps:merge( + hb_private:reset(IndexTABM), + hb_maps:without( + [<<"body">>, <<"content-type">>], + EncOriginal, + Opts + ) + ), + % Return the merged result. + {ok, Merged}; + Err -> + % The index resolution executed without error, but the result + % was not a valid message. We log a warning for the operator + % and return the original message to the caller. + ?event(warning, {invalid_index_result, Err}), + {ok, EncOriginal} + catch + Err:Details:Stacktrace -> + % There was an error while generating the index page. We + % log a warning for the operator and return the modified + % message to the caller. + ?event(warning, + {error_generating_index, + {type, Err}, + {details, Details}, + {stacktrace, Stacktrace} + } + ), + {ok, EncOriginal} + end; + _ -> + % Return the encoded HTTPSig message without modification. + {ok, EncOriginal} + end; +to(TABM, Req, FormatOpts, Opts) when is_map(TABM) -> + % Ensure that the material for the message is loaded, if the request is + % asking for a bundle. + Msg = + case hb_util:atom(hb_maps:get(<<"bundle">>, Req, false, Opts)) of + false -> TABM; + true -> + % Convert back to the fully loaded structured@1.0 message, then + % convert to TABM with bundling enabled. + Structured = hb_message:convert(TABM, <<"structured@1.0">>, Opts), + Loaded = hb_cache:ensure_all_loaded(Structured, Opts), + hb_message:convert( + Loaded, + tabm, + #{ + <<"device">> => <<"structured@1.0">>, + <<"bundle">> => true + }, + Opts + ) + end, + % Group the IDs into a dictionary, so that they can be distributed as + % HTTP headers. If we did not do this, ID keys would be lower-cased and + % their comparability against the original keys would be lost. + Stripped = + hb_maps:without( + [ + <<"commitments">>, + <<"signature">>, + <<"signature-input">>, + <<"priv">> + ], + Msg, + Opts + ), + WithGroupedIDs = group_ids(Stripped), + ?event({grouped, WithGroupedIDs}), + {InlineFieldHdrs, InlineKey} = inline_key(WithGroupedIDs), + Intermediate = + do_to( + WithGroupedIDs, + FormatOpts ++ [{inline, InlineFieldHdrs, InlineKey}], + Opts + ), + % Finally, add the signatures to the encoded HTTP message with the + % commitments from the original message. + CommitmentsMap = case maps:get(<<"commitments">>, Msg, undefined) of + undefined -> + case maps:get(<<"signature">>, Msg, undefined) of + undefined -> #{}; + Signature -> + MaybeBundleTag = maps:with([<<"bundle">>], Msg), + #{ + Signature => MaybeBundleTag#{ + <<"signature">> => Signature, + <<"committed">> => maps:get(<<"committed">>, Msg, #{}), + <<"keyid">> => maps:get(<<"keyid">>, Msg, <<>>), + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"type">> => maps:get(<<"type">>, Msg, <<>>) + } + } + end; + Commitments -> + Commitments + end, + ?event({converting_commitments_to_siginfo, Msg}), + {ok, + maps:merge( + Intermediate, + dev_codec_httpsig_siginfo:commitments_to_siginfo( + TABM, + CommitmentsMap, + Opts + ) + ) + }. + +do_to(Binary, _FormatOpts, _Opts) when is_binary(Binary) -> Binary; +do_to(TABM, FormatOpts, Opts) when is_map(TABM) -> + InlineKey = + case lists:keyfind(inline, 1, FormatOpts) of + {inline, _InlineFieldHdrs, Key} -> Key; + _ -> not_set + end, + % Calculate the initial encoding from the TABM + Enc0 = + maps:fold( + fun(<<"body">>, Value, AccMap) -> + OldBody = maps:get(<<"body">>, AccMap, #{}), + AccMap#{ <<"body">> => OldBody#{ <<"body">> => Value } }; + (Key, Value, AccMap) when Key =:= InlineKey andalso InlineKey =/= not_set -> + OldBody = maps:get(<<"body">>, AccMap, #{}), + AccMap#{ <<"body">> => OldBody#{ InlineKey => Value } }; + (Key, Value, AccMap) -> + field_to_http(AccMap, {Key, Value}, #{}) + end, + % Add any inline field denotations to the HTTP message + case lists:keyfind(inline, 1, FormatOpts) of + {inline, InlineFieldHdrs, _InlineKey} -> InlineFieldHdrs; + _ -> #{} + end, + maps:without([<<"priv">>], TABM) + ), + ?event({prepared_body_map, {msg, Enc0}}), + BodyMap = maps:get(<<"body">>, Enc0, #{}), + GroupedBodyMap = group_maps(BodyMap, <<>>, #{}, Opts), + Enc1 = + case GroupedBodyMap of + EmptyBody when map_size(EmptyBody) =:= 0 -> + % If the body map is empty, then simply set the body to be a + % corresponding empty binary. + ?event({encoding_empty_body, {msg, Enc0}}), + Enc0; + #{ InlineKey := UserBody } + when map_size(GroupedBodyMap) =:= 1 andalso is_binary(UserBody) -> + % Simply set the sole body binary as the body of the + % HTTP message, no further encoding required + % + % NOTE: this may only be done for the top most message as + % sub-messages MUST be encoded as sub-parts, in order to preserve + % the nested hierarchy of messages, even in the case of a sole + % body binary. + % + % In all other cases, the mapping fallsthrough to the case below + % that properly encodes a nested body within a sub-part + ?event({encoding_single_body, {body, UserBody}, {http, Enc0}}), + hb_maps:put(<<"body">>, UserBody, Enc0, Opts); + _ -> + % Otherwise, we need to encode the body map as the + % multipart body of the HTTP message + ?event({encoding_multipart, {bodymap, {explicit, GroupedBodyMap}}}), + PartList = hb_util:to_sorted_list( + hb_maps:map( + fun(Key, M = #{ <<"body">> := _ }) when map_size(M) =:= 1 -> + % If the map has only one key, and it is `body', + % then we must encode part name with the additional + % `/body' suffix. This is because otherwise, the `body' + % element will be assumed to be an inline part, removing + % the necessary hierarchy. + encode_body_part( + <>, + M, + <<"body">>, + Opts + ); + (Key, Value) -> + encode_body_part(Key, Value, InlineKey, Opts) + end, + GroupedBodyMap, + Opts + ), + Opts + ), + Boundary = boundary_from_parts(PartList), + % Transform body into a binary, delimiting each part with the + % boundary + BodyList = lists:foldl( + fun ({_PartName, BodyPart}, Acc) -> + [ + << + "--", Boundary/binary, ?CRLF/binary, + BodyPart/binary + >> + | + Acc + ] + end, + [], + PartList + ), + % Finally, join each part of the multipart body into a single binary + % to be used as the body of the Http Message + FinalBody = iolist_to_binary(lists:join(?CRLF, lists:reverse(BodyList))), + % Ensure we append the Content-Type to be a multipart response + Enc0#{ + <<"content-type">> => + <<"multipart/form-data; boundary=", "\"" , Boundary/binary, "\"">>, + <<"body">> => <> + } + end, + % Add the content-digest to the HTTP message. `add_content_digest/1' + % will return a map with the `content-digest' key set, but the body removed, + % so we merge the two maps together to maintain the body and the content-digest. + Enc2 = case hb_maps:get(<<"body">>, Enc1, <<>>, Opts) of + <<>> -> Enc1; + _ -> + ?event({adding_content_digest, {msg, Enc1}}), + hb_maps:merge( + Enc1, + dev_codec_httpsig:add_content_digest(Enc1, Opts), + Opts + ) + end, + ?event({final_body_map, {msg, Enc2}}), + Enc2. + +%% @doc Group all elements with: +%% 1. A key that ?IS_ID returns true for, and +%% 2. A value that is immediate +%% into a combined SF dict-_like_ structure. If not encoded, these keys would +%% be sent as headers and lower-cased, losing their comparability against the +%% original keys. The structure follows all SF dict rules, except that it allows +%% for keys to contain capitals. The HyperBEAM SF parser will accept these keys, +%% but standard RFC 8741 parsers will not. Subsequently, the resulting `ao-cased' +%% key is not added to the `ao-types' map. +group_ids(Map) -> + % Find all keys that are IDs. + IDDict = maps:filter(fun(K, V) -> ?IS_ID(K) andalso is_binary(V) end, Map), + % Convert the dictionary into a list of key-value pairs + IDDictStruct = + lists:map( + fun({K, V}) -> + {K, {item, {string, V}, []}} + end, + maps:to_list(IDDict) + ), + % Convert the list of key-value pairs into a binary + IDBin = iolist_to_binary(hb_structured_fields:dictionary(IDDictStruct)), + % Remove the encoded keys from the map + Stripped = maps:without(maps:keys(IDDict), Map), + % Add the ID binary to the map if it is not empty + case map_size(IDDict) of + 0 -> Stripped; + _ -> Stripped#{ <<"ao-ids">> => IDBin } + end. + +%% @doc Decode the `ao-ids' key into a map. +ungroup_ids(Msg = #{ <<"ao-ids">> := IDBin }, Opts) -> + % Extract the ID binary from the Map + EncodedIDsMap = hb_structured_fields:parse_dictionary(IDBin), + % Convert the value back into a raw binary + IDsMap = + lists:map( + fun({K, {item, {string, Bin}, _}}) -> {K, Bin} end, + EncodedIDsMap + ), + % Add the decoded IDs to the Map and remove the `ao-ids' key + hb_maps:merge(hb_maps:without([<<"ao-ids">>], Msg, Opts), hb_maps:from_list(IDsMap), Opts); + +ungroup_ids(Msg, _Opts) -> Msg. + +%% @doc Merge maps at the same level, if possible. +group_maps(Map) -> + group_maps(Map, <<>>, #{}, #{}). +group_maps(Map, Parent, Top, Opts) when is_map(Map) -> + ?event({group_maps, {map, Map}, {parent, Parent}, {top, Top}}), + {Flattened, NewTop} = hb_maps:fold( + fun(Key, Value, {CurMap, CurTop}) -> + ?event({group_maps, {key, Key}, {value, Value}}), + NormKey = hb_ao:normalize_key(Key), + FlatK = + case Parent of + <<>> -> NormKey; + _ -> <> + end, + case Value of + _ when is_map(Value) orelse is_list(Value) -> + NormMsg = + if is_list(Value) -> + hb_message:convert( + Value, + tabm, + <<"structured@1.0">>, + Opts + ); + true -> + Value + end, + case hb_maps:size(NormMsg, Opts) of + 0 -> + { + CurMap, + hb_maps:put( + FlatK, + #{ <<"ao-types">> => <<"empty-message">> }, + CurTop, + Opts + ) + }; + _ -> + NewTop = group_maps(NormMsg, FlatK, CurTop, Opts), + {CurMap, NewTop} + end; + _ -> + ?event({group_maps, {norm_key, NormKey}, {value, Value}}), + case byte_size(Value) > ?MAX_HEADER_LENGTH of + % the value is too large to be encoded as a header + % within a part, so instead lift it to be a top level + % part + true -> + NewTop = hb_maps:put(FlatK, Value, CurTop, Opts), + {CurMap, NewTop}; + % Encode the value in the current part + false -> + NewCurMap = hb_maps:put(NormKey, Value, CurMap, Opts), + {NewCurMap, CurTop} + end + end + end, + {#{}, Top}, + Map, + Opts + ), + case hb_maps:size(Flattened, Opts) of + 0 -> NewTop; + _ -> case Parent of + <<>> -> hb_maps:merge(NewTop, Flattened, Opts); + _ -> + Res = NewTop#{ Parent => Flattened }, + ?event({returning_res, {res, Res}}), + Res + end + end. + +%% @doc Generate a unique, reproducible boundary for the +%% multipart body, however we cannot use the id of the message as +%% the boundary, as the id is not known until the message is +%% encoded. Subsequently, we generate each body part individually, +%% concatenate them, and apply a SHA2-256 hash to the result. +%% This ensures that the boundary is unique, reproducible, and +%% secure. +boundary_from_parts(PartList) -> + BodyBin = + iolist_to_binary( + lists:join(?CRLF, + lists:map( + fun ({_PartName, PartBin}) -> PartBin end, + PartList + ) + ) + ), + RawBoundary = crypto:hash(sha256, BodyBin), + hb_util:encode(RawBoundary). + +%% @doc Encode a multipart body part to a flat binary. +encode_body_part(PartName, BodyPart, InlineKey, Opts) -> + % We'll need to prepend a Content-Disposition header + % to the part, using the field name as the form part + % name. + % (See https://www.rfc-editor.org/rfc/rfc7578#section-4.2). + Disposition = + case PartName of + % The body is always made the inline part of + % the multipart body + InlineKey -> <<"inline">>; + _ -> <<"form-data;name=", "\"", PartName/binary, "\"">> + end, + % Sub-parts MUST have at least one header, according to the + % multipart spec. Adding the Content-Disposition not only + % satisfies that requirement, but also encodes the + % HB message field that resolves to the sub-message + case BodyPart of + BPMap when is_map(BPMap) -> + WithDisposition = + hb_maps:put( + <<"content-disposition">>, + Disposition, + BPMap, + Opts + ), + encode_http_flat_msg(WithDisposition, Opts); + BPBin when is_binary(BPBin) -> + % A properly encoded inlined body part MUST have a CRLF between + % it and the header block, so we MUST use two CRLF: + % - first to signal end of the Content-Disposition header + % - second to signal the end of the header block + << + "content-disposition: ", Disposition/binary, ?CRLF/binary, + ?CRLF/binary, + BPBin/binary + >> + end. + +%% @doc given a message, returns a binary tuple: +%% - A list of pairs to add to the msg, if any +%% - the field name for the inlined key +%% +%% In order to preserve the field name of the inlined +%% part, an additional field may need to be added +inline_key(Msg) -> + inline_key(Msg, #{}). + +inline_key(Msg, Opts) -> + % The message can name a key whose value will be placed in the body as the + % inline part. Otherwise, the Msg <<"body">> is used. If not present, the + % Msg <<"data">> is used. + InlineBodyKey = hb_maps:get(<<"ao-body-key">>, Msg, false, Opts), + ?event({inlined, InlineBodyKey}), + case { + InlineBodyKey, + hb_maps:is_key(<<"body">>, Msg, Opts) + andalso not ?IS_LINK(maps:get(<<"body">>, Msg, Opts)), + hb_maps:is_key(<<"data">>, Msg, Opts) + andalso not ?IS_LINK(maps:get(<<"data">>, Msg, Opts)) + } of + % ao-body-key already exists, so no need to add one + {Explicit, _, _} when Explicit =/= false -> {#{}, InlineBodyKey}; + % ao-body-key defaults to <<"body">> (see below) + % So no need to add one + {_, true, _} -> {#{}, <<"body">>}; + % We need to preserve the ao-body-key, as the <<"data">> field, + % so that it is preserved during encoding and decoding + {_, _, true} -> {#{<<"ao-body-key">> => <<"data">>}, <<"data">>}; + % default to body being the inlined part. + % This makes this utility compatible for both encoding + % and decoding httpsig@1.0 messages + _ -> {#{}, <<"body">>} + end. + +%% @doc Encode a HTTP message into a binary, converting it to `httpsig@1.0' +%% first. +encode_http_msg(Msg, Opts) -> + % Convert the message to a HTTP-Sig encoded output. + Httpsig = hb_message:convert(Msg, <<"httpsig@1.0">>, Opts), + encode_http_flat_msg(Httpsig, Opts). + +%% @doc Encode a HTTP message into a binary. The input *must* be a raw map of +%% binary keys and values. +encode_http_flat_msg(Httpsig, Opts) -> + % Serialize the headers, to be included in the part of the multipart response + HeaderList = + lists:foldl( + fun ({HeaderName, RawHeaderVal}, Acc) -> + HVal = hb_cache:ensure_loaded(RawHeaderVal, Opts), + ?event({encoding_http_header, {header, HeaderName}, {value, HVal}}), + [<> | Acc] + end, + [], + hb_maps:to_list(hb_maps:without([<<"body">>, <<"priv">>], Httpsig, Opts), Opts) + ), + EncodedHeaders = iolist_to_binary(lists:join(?CRLF, lists:reverse(HeaderList))), + case hb_maps:get(<<"body">>, Httpsig, <<>>, Opts) of + <<>> -> EncodedHeaders; + % Some-Headers: some-value + % content-type: image/png + % + % + SubBody -> <> + end. + +%% @doc All maps are encoded into the body of the HTTP message +%% to be further encoded later. +field_to_http(Httpsig, {Name, Value}, Opts) when is_map(Value) -> + NormalizedName = hb_ao:normalize_key(Name), + OldBody = hb_maps:get(<<"body">>, Httpsig, #{}, Opts), + Httpsig#{ <<"body">> => OldBody#{ NormalizedName => Value } }; +field_to_http(Httpsig, {Name, Value}, Opts) when is_binary(Value) -> + NormalizedName = hb_ao:normalize_key(Name), + % The default location where the value is encoded within the HTTP + % message depends on its size. + % + % So we check whether the size of the value is within the threshold + % to encode as a header, and otherwise default to encoding in the body. + % + % Note that a "where" Opts may force the location of the encoded + % value -- this is only a default location if not specified in Opts + DefaultWhere = + case {maps:get(where, Opts, headers), byte_size(Value)} of + {headers, Fits} when Fits =< ?MAX_HEADER_LENGTH -> headers; + _ -> body + end, + case maps:get(where, Opts, DefaultWhere) of + headers -> + Httpsig#{ NormalizedName => Value }; + body -> + OldBody = hb_maps:get(<<"body">>, Httpsig, #{}, Opts), + Httpsig#{ <<"body">> => OldBody#{ NormalizedName => Value } } + end. + +group_maps_test() -> + Map = #{ + <<"a">> => <<"1">>, + <<"b">> => #{ + <<"x">> => <<"10">>, + <<"y">> => #{ + <<"z">> => <<"20">> + }, + <<"foo">> => #{ + <<"bar">> => #{ + <<"fizz">> => <<"buzz">> + } + } + }, + <<"c">> => #{ + <<"d">> => <<"30">> + }, + <<"e">> => <<"2">>, + <<"buf">> => <<"hello">>, + <<"nested">> => #{ + <<"foo">> => <<"iiiiii">>, + <<"here">> => #{ + <<"bar">> => <<"baz">>, + <<"fizz">> => <<"buzz">>, + <<"pop">> => #{ + <<"very-fizzy">> => <<"very-buzzy">> + } + } + } + }, + Lifted = group_maps(Map), + ?assertEqual( + Lifted, + #{ + <<"a">> => <<"1">>, + <<"b">> => #{<<"x">> => <<"10">>}, + <<"b/foo/bar">> => #{<<"fizz">> => <<"buzz">>}, + <<"b/y">> => #{<<"z">> => <<"20">>}, + <<"buf">> => <<"hello">>, + <<"c">> => #{<<"d">> => <<"30">>}, + <<"e">> => <<"2">>, + <<"nested">> => #{<<"foo">> => <<"iiiiii">>}, + <<"nested/here">> => #{<<"bar">> => <<"baz">>, <<"fizz">> => <<"buzz">>}, + <<"nested/here/pop">> => #{<<"very-fizzy">> => <<"very-buzzy">>} + } + ), + ok. + +%% @doc The grouped maps encoding is a subset of the flat encoding, +%% where on keys with maps values are flattened. +%% +%% So despite needing a special encoder to produce it +%% We can simply apply the flat encoder to it to get back +%% the original message. +%% +%% The test asserts that is indeed the case. +group_maps_flat_compatible_test() -> + Map = #{ + <<"a">> => <<"1">>, + <<"b">> => #{ + <<"x">> => <<"10">>, + <<"y">> => #{ + <<"z">> => <<"20">> + }, + <<"foo">> => #{ + <<"bar">> => #{ + <<"fizz">> => <<"buzz">> + } + } + }, + <<"c">> => #{ + <<"d">> => <<"30">> + }, + <<"e">> => <<"2">>, + <<"buf">> => <<"hello">>, + <<"nested">> => #{ + <<"foo">> => <<"iiiiii">>, + <<"here">> => #{ + <<"bar">> => <<"baz">>, + <<"fizz">> => <<"buzz">> + } + } + }, + Lifted = group_maps(Map), + ?assertEqual(dev_codec_flat:from(Lifted, #{}, #{}), {ok, Map}), + ok. + +encode_message_with_links_test() -> + Msg = #{ + <<"immediate-key">> => <<"immediate-value">>, + <<"typed-key">> => 4 + }, + {ok, Path} = hb_cache:write(Msg, #{}), + {ok, Read} = hb_cache:read(Path, #{}), + % Ensure that the message now has a lazy link + ?assertMatch({link, _, _}, maps:get(<<"typed-key">>, Read, #{})), + % Encode and decode the message as `httpsig@1.0` + Enc = hb_message:convert(Msg, <<"httpsig@1.0">>, #{}), + ?event({encoded, Enc}), + Dec = hb_message:convert(Enc, <<"structured@1.0">>, <<"httpsig@1.0">>, #{}), + % Ensure that the result is the same as the original message + ?event({decoded, Dec}), + ?assert(hb_message:match(Msg, Dec, strict, #{})). \ No newline at end of file diff --git a/src/dev_codec_httpsig_keyid.erl b/src/dev_codec_httpsig_keyid.erl new file mode 100644 index 000000000..8c3d5f84b --- /dev/null +++ b/src/dev_codec_httpsig_keyid.erl @@ -0,0 +1,151 @@ +%%% @doc A library for extracting and validating key material for `httpsig@1.0' +%%% requests. Offers support for the following keyid schemes: +%%% - `publickey': The keyid is an encoded public key with the `publickey:' prefix. +%%% - `constant': The key is simply the keyid itself, including the `public:' +%%% prefix if given. +%%% - `secret': The key is hashed and the `secret:' prefix is added to the +%%% result in order to generate a keyid. +%%% +%%% These functions are abstracted in order to allow for the addition of new +%%% schemes in the future. +-module(dev_codec_httpsig_keyid). +-export([req_to_key_material/2, keyid_to_committer/1, keyid_to_committer/2]). +-export([secret_key_to_committer/1, remove_scheme_prefix/1]). +-include_lib("include/hb.hrl"). + +%%% The supported schemes for HMAC keys. +-define(KEYID_SCHEMES, [constant, publickey, secret]). +%%% The default schemes for each request type. +-define(DEFAULT_SCHEMES_BY_TYPE, #{ + <<"rsa-pss-sha512">> => publickey, + <<"hmac-sha256">> => constant +}). +%%% Default key to use for HMAC commitments. +-define(HMAC_DEFAULT_KEY, <<"constant:ao">>). + +%% @doc Extract the key and keyid from a request, returning +%% `{ok, Scheme, Key, KeyID}' or `{error, Reason}'. +req_to_key_material(Req, Opts) -> + ?event({req_to_key_material, {req, Req}}), + KeyID = maps:get(<<"keyid">>, Req, undefined), + ?event({keyid_to_key_material, {keyid, KeyID}}), + case find_scheme(KeyID, Req, Opts) of + {ok, Scheme} -> + ?event({scheme_found, {scheme, Scheme}}), + ApplyRes = apply_scheme(Scheme, KeyID, Req), + ?event({apply_scheme_result, {apply_res, ApplyRes}}), + case ApplyRes of + {ok, _, CalcKeyID} when KeyID /= undefined, CalcKeyID /= KeyID -> + {error, key_mismatch}; + {ok, Key, CalcKeyID} -> + {ok, Scheme, Key, CalcKeyID}; + {error, Reason} -> + {error, Reason} + end; + {error, undefined_scheme} -> + {ok, DefaultScheme} = req_to_default_scheme(Req, Opts), + req_to_key_material(Req#{ <<"scheme">> => DefaultScheme }, Opts); + {error, Reason} -> + {error, Reason} + end. + +%% @doc Find the scheme from a keyid or request. Returns `{ok, Scheme}' or +%% `{error, Reason}'. If no scheme is provided in either the request message +%% or the keyid (as a `scheme:' prefix), we default to the scheme specified in +%% the request type. If a scheme is provided in the request, it must match the +%% scheme in the keyid if also present. +find_scheme(KeyID, Req = #{ <<"scheme">> := RawScheme }, Opts) -> + Scheme = hb_util:atom(RawScheme), + % Validate that the scheme in the request matches the scheme in the keyid. + case find_scheme(KeyID, maps:without([<<"scheme">>], Req), Opts) of + {ok, Scheme} -> {ok, Scheme}; + {error, undefined_scheme} -> {ok, Scheme}; + _OtherScheme -> {error, scheme_mismatch} + end; +find_scheme(undefined, _Req, _Opts) -> + {error, undefined_scheme}; +find_scheme(KeyID, Req, Opts) -> + SchemeRes = + case binary:split(KeyID, <<":">>) of + [SchemeBin, _KeyID] -> {ok, SchemeBin}; + [_NoSchemeKeyID] -> + % Determine the default scheme based on the `type' of the request. + req_to_default_scheme(Req, Opts) + end, + case SchemeRes of + {ok, Scheme} -> + case lists:member(SchemeAtom = hb_util:atom(Scheme), ?KEYID_SCHEMES) of + true -> {ok, SchemeAtom}; + false -> {error, unknown_scheme} + end; + {error, Reason} -> + {error, Reason} + end. + +%% @doc Determine the default scheme based on the `type' of the request. +req_to_default_scheme(Req, _Opts) -> + case maps:find(<<"type">>, Req) of + {ok, Type} -> + case maps:find(Type, ?DEFAULT_SCHEMES_BY_TYPE) of + {ok, Scheme} -> {ok, Scheme}; + error -> {error, unsupported_scheme} + end; + error -> + {error, no_request_type} + end. + +%% @doc Apply the requested scheme to generate the key material (key and keyid). +apply_scheme(publickey, KeyID, _Req) -> + % Remove the `publickey:' prefix from the keyid and return the key. + PubKey = base64:decode(remove_scheme_prefix(KeyID)), + {ok, PubKey, << "publickey:", (base64:encode(PubKey))/binary >>}; +apply_scheme(constant, RawKeyID, _Req) -> + % In the `constant' scheme, the key is simply the key itself, including the + % `constant:' prefix if given. + KeyID = + if RawKeyID == undefined -> ?HMAC_DEFAULT_KEY; + true -> RawKeyID + end, + {ok, KeyID, KeyID}; +apply_scheme(secret, _KeyID, Req) -> + % In the `secret' scheme, the key is hashed to generate a keyid. + Secret = maps:get(<<"secret">>, Req, undefined), + Committer = secret_key_to_committer(Secret), + {ok, Secret, << "secret:", Committer/binary >>}; +apply_scheme(_Scheme, _Key, _KeyID) -> + {error, unsupported_scheme}. + +%% @doc Given a keyid and a scheme, generate the committer value for a commitment. +%% Returns `BinaryAddress' or `undefined' if the keyid implies no committer. +keyid_to_committer(KeyID) -> + case find_scheme(KeyID, #{}, #{}) of + {ok, Scheme} -> keyid_to_committer(Scheme, KeyID); + {error, _} -> undefined + end. +keyid_to_committer(publickey, KeyID) -> + % Note: There is a subtlety here. The `KeyID' is decoded with the + % `hb_util:decode' function rather than `base64:decode'. The reason for this + % is that certain codecs (e.g. `ans104@1.0') encode the public key with + % `base64url' encoding, rather than the standard `base64' encoding in + % HTTPSig. Our `hb_util:decode' function handles both cases returning the + % same raw bytes, and is subsequently safe. + hb_util:human_id( + ar_wallet:to_address( + hb_util:decode(remove_scheme_prefix(KeyID)) + ) + ); +keyid_to_committer(secret, KeyID) -> + remove_scheme_prefix(KeyID); +keyid_to_committer(constant, _KeyID) -> + undefined. + +%% @doc Given a secret key, generate the committer value for a commitment. +secret_key_to_committer(Key) -> + hb_util:human_id(hb_crypto:sha256(Key)). + +%% @doc Remove the `scheme:' prefix from a keyid. +remove_scheme_prefix(KeyID) -> + case binary:split(KeyID, <<":">>) of + [_Scheme, Key] -> Key; + [Key] -> Key + end. diff --git a/src/dev_codec_httpsig_proxy.erl b/src/dev_codec_httpsig_proxy.erl new file mode 100644 index 000000000..501f52def --- /dev/null +++ b/src/dev_codec_httpsig_proxy.erl @@ -0,0 +1,80 @@ +%%% @doc A utility module that contains proxy functions for calling the +%%% `~httpsig@1.0' codec's HMAC commitment functions with secret keys. +%%% +%%% These tools are helpful for implementing a standardized pattern: +%%% 1. A device verifies a user's request/derives a secret key for them. +%%% 2. The device then wants to commit a message with the user's secret key +%%% using the `secret:[h(secret)]' commitment scheme. +%%% 3. The commitment must then be modified to reference a different device +%%% as the `commitment-device' key. +%%% 4. When `/verify' is called, the `~httpsig@1.0' codec is used under-the-hood +%%% to validate the commitment on the re-derived secret key. +%%% +%%% This module is currently used by the `~cookie@1.0' and `~http-auth@1.0' +%%% devices. +-module(dev_codec_httpsig_proxy). +-export([commit/5, verify/4]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Commit to a given `Base' message with a given `Secret', setting the +%% `commitment-device' key to `Device' afterwards. +commit(Device, Secret, Base, Req, Opts) -> + % If there are no existing commitments, we use the unmodified base message. + % If there are, we remove the uncommitted parts of the message. + ExistingComms = hb_maps:get(<<"commitments">>, Base, #{}, Opts), + OnlyCommittedBase = + case map_size(ExistingComms) of + 0 -> Base; + _ -> + hb_message:uncommitted( + hb_message:with_only_committed(Base, Opts), + Opts + ) + end, + % Commit the message with the given key. + CommittedMsg = + hb_message:commit( + OnlyCommittedBase, + Opts, + Req#{ + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"type">> => <<"hmac-sha256">>, + <<"scheme">> => <<"secret">>, + <<"secret">> => Secret + } + ), + {ok, CommitmentID, Commitment} = + hb_message:commitment( + #{ + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"type">> => <<"hmac-sha256">> + }, + CommittedMsg, + Opts + ), + % Modify the commitment device to the given device. + ModCommittedMsg = + CommittedMsg#{ + <<"commitments">> => + ExistingComms#{ + CommitmentID => + Commitment#{ + <<"commitment-device">> => Device + } + } + }, + ?event({cookie_commitment, {id, CommitmentID}, {commitment, ModCommittedMsg}}), + {ok, ModCommittedMsg}. + +%% @doc Verify a given `Base' message with a given `Secret' using the `~httpsig@1.0' +%% HMAC commitment scheme. +verify(Secret, Base, RawReq, Opts) -> + ProxyRequest = + RawReq#{ + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"path">> => <<"verify">>, + <<"secret">> => Secret + }, + ?event({proxy_request, ProxyRequest}), + {ok, hb_message:verify(Base, ProxyRequest, Opts)}. \ No newline at end of file diff --git a/src/dev_codec_httpsig_siginfo.erl b/src/dev_codec_httpsig_siginfo.erl new file mode 100644 index 000000000..4ce6d051b --- /dev/null +++ b/src/dev_codec_httpsig_siginfo.erl @@ -0,0 +1,510 @@ +%% @doc A module for converting between commitments and their encoded `signature' +%% and `signature-input' keys. +-module(dev_codec_httpsig_siginfo). +-export([commitments_to_siginfo/3, siginfo_to_commitments/3]). +-export([committed_keys_to_siginfo/1, to_siginfo_keys/3, from_siginfo_keys/3]). +-export([add_derived_specifiers/1, remove_derived_specifiers/1]). +-export([commitment_to_sig_name/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% A list of components that are `derived' in the context of RFC-9421 from the +%%% request message. +-define(DERIVED_COMPONENTS, [ + <<"method">>, + <<"target-uri">>, + <<"authority">>, + <<"scheme">>, + <<"request-target">>, + <<"path">>, + <<"query">>, + <<"query-param">> + % <<"status">> % Some libraries does not support it +]). + +%% @doc Generate a `signature' and `signature-input' key pair from a commitment +%% map. +commitments_to_siginfo(_Msg, Comms, _Opts) when ?IS_EMPTY_MESSAGE(Comms) -> + #{}; +commitments_to_siginfo(Msg, Comms, Opts) -> + % Generate a SF item for each commitment's signature and signature-input. + {Sigs, SigInputs} = + maps:fold( + fun(_CommID, Commitment, {Sigs, SigInputs}) -> + {ok, SigNameRaw, SFSig, SFSigInput} = + commitment_to_sf_siginfo(Msg, Commitment, Opts), + SigName = <<"comm-", SigNameRaw/binary>>, + { + Sigs#{ SigName => SFSig }, + SigInputs#{ SigName => SFSigInput } + } + end, + {#{}, #{}}, + Comms + ), + #{ + <<"signature">> => + hb_util:bin(hb_structured_fields:dictionary(Sigs)), + <<"signature-input">> => + hb_util:bin(hb_structured_fields:dictionary(SigInputs)) + }. + +%% @doc Generate a `signature' and `signature-input' key pair from a given +%% commitment. +commitment_to_sf_siginfo(Msg, Commitment, Opts) -> + % Generate the `alg' key from the commitment. + Alg = commitment_to_alg(Commitment, Opts), + % Find the public key from the commitment, which we will use as the + % `keyid' in the `signature-input' keys. + KeyID = maps:get(<<"keyid">>, Commitment, <<>>), + % Extract the signature from the commitment. + Signature = hb_util:decode(maps:get(<<"signature">>, Commitment)), + % Extract the keys present in the commitment. + CommittedKeys = to_siginfo_keys(Msg, Commitment, Opts), + ?event({normalized_for_enc, CommittedKeys, {commitment, Commitment}}), + % Extract the hashpath, used as a tag, from the commitment. + Tag = maps:get(<<"tag">>, Commitment, undefined), + % Extract other permissible values, if present. + Nonce = maps:get(<<"nonce">>, Commitment, undefined), + Created = maps:get(<<"created">>, Commitment, undefined), + Expires = maps:get(<<"expires">>, Commitment, undefined), + % Generate the name of the signature. + SigName = hb_util:to_lower(hb_util:human_id(crypto:hash(sha256, Signature))), + % Generate the signature input and signature structured-fields. These can + % then be placed into a dictionary with other commitments and transformed + % into their binary representations. + SFSig = {item, {binary, Signature}, []}, + AdditionalParams = get_additional_params(Commitment), + Params = + lists:filter( + fun({_Key, undefined}) -> + false; + ({_Key, {_, Val}}) -> + Val =/= undefined + end, + [ + {<<"alg">>, {string, Alg}}, + {<<"keyid">>, {string, KeyID}}, + {<<"tag">>, {string, Tag}}, + {<<"created">>, Created}, + {<<"expires">>, Expires}, + {<<"nonce">>, {string, Nonce}} + ] ++ AdditionalParams + ), + SFSigInput = + {list, + [ + {item, {string, Key}, []} + || + Key <- CommittedKeys + ], + Params + }, + ?event( + {sig_input, + {string, + hb_util:bin( + hb_structured_fields:dictionary( + #{ <<"comm">> => SFSigInput } + ) + ) + } + } + ), + {ok, SigName, SFSig, SFSigInput}. + +get_additional_params(Commitment) -> + AdditionalParams = + sets:to_list( + sets:subtract( + sets:from_list(maps:keys(Commitment)), + sets:from_list( + [ + <<"alg">>, + <<"keyid">>, + <<"tag">>, + <<"created">>, + <<"expires">>, + <<"nonce">>, + <<"committed">>, + <<"signature">>, + <<"type">>, + <<"commitment-device">>, + <<"committer">> + ] + ) + ) + ), + lists:map(fun(Param) -> + ParamValue = maps:get(Param, Commitment), + case ParamValue of + Val when is_atom(Val) -> + {Param, {string, atom_to_binary(Val, utf8)}}; + Val when is_binary(Val) -> + {Param, {string, Val}}; + Val when is_list(Val) -> + {Param, {string, list_to_binary(lists:join(<<", ">>, Val))}}; + Val when is_map(Val) -> + Map = nested_map_to_string(Val), + {Param, {string, list_to_binary(lists:join(<<", ">>, Map))} } + end + end, AdditionalParams). + +nested_map_to_string(Map) -> + lists:map(fun(I) -> + case maps:get(I, Map) of + Val when is_map(Val) -> + Name = maps:get(<<"name">>, Val), + Value = maps:get(<<"value">>, Val), + <>; + Val -> + Val + end + end, maps:keys(Map)). + +%% @doc Take a message with a `signature' and `signature-input' key pair and +%% return a map of commitments. +siginfo_to_commitments( + Msg = + #{ + <<"signature">> := <<"comm-", SFSigBin/binary>>, + <<"signature-input">> := <<"comm-", SFSigInputBin/binary>> + }, + BodyKeys, + Opts) -> + % Parse the signature and signature-input structured-fields. + SFSigs = hb_structured_fields:parse_dictionary(SFSigBin), + SFSigsInputs = hb_structured_fields:parse_dictionary(SFSigInputBin), + % Group parsed signature inputs and signatures into tuple pairs by their + % name. + CommitmentSFs = + [ + {SFSig, element(2, lists:keyfind(SFSigName, 1, SFSigsInputs))} + || + {SFSigName, SFSig} <- SFSigs + ], + % Convert each tuple into a commitment and its ID. + CommitmentMessages = + lists:map( + fun ({SFSig, SFSigInput}) -> + {ok, ID, Commitment} = + sf_siginfo_to_commitment( + Msg, + BodyKeys, + SFSig, + SFSigInput, + Opts + ), + {ID, Commitment} + end, + CommitmentSFs + ), + % Convert the list of commitments into a map. + maps:from_list(CommitmentMessages); +siginfo_to_commitments(_Msg, _BodyKeys, _Opts) -> + % If the message does not contain a `signature' or `signature-input' key, + % we return an empty map. + #{}. + +%% @doc Take a signature and signature-input as parsed structured-fields and +%% return a commitment. +sf_siginfo_to_commitment(Msg, BodyKeys, SFSig, SFSigInput, Opts) -> + % Extract the signature and signature-input from the structured-fields. + {item, {binary, Sig}, []} = SFSig, + {list, SigInput, ParamsKV} = SFSigInput, + % Generate a commitment message from the signature-input parameters. + Commitment1 = + maps:from_list( + lists:map( + fun ({Key, {binary, Bin}}) -> {Key, hb_util:encode(Bin)}; + ({Key, BareItem}) -> + Item = + case hb_structured_fields:from_bare_item(BareItem) of + Res when is_binary(Res) -> + decoding_nested_map_binary(Res); + Res -> + Res + end, + {Key, Item} + end, + ParamsKV + ) + ), + % Generate the `commitment-device' key and optionally, its `type' key from + % the `alg' key. + CommitmentDeviceKeys = commitment_to_device_specifiers(Commitment1, Opts), + % Merge the commitment parameters with the commitment device, removing the + % `alg' key. + Commitment2 = + maps:merge( + CommitmentDeviceKeys, + maps:remove(<<"alg">>, Commitment1) + ), + % Generate the committed keys by parsing the signature-input list. + RawCommittedKeys = + [ + Key + || + {item, {string, Key}, []} <- SigInput + ], + CommittedKeys = from_siginfo_keys(Msg, BodyKeys, RawCommittedKeys), + % Merge and cleanup the output. + % 1. Decode the `keyid` (typically a public key) to its raw byte form. + % 2. Decode the `signature` to its raw byte form. + % 3. Filter undefined keys. + % 4. Generate the ID for the commitment from the signature. We use a SHA2-256 + % hash of the signature, unless the signature is 32 bytes, in which case we + % use the signature directly as the ID. + % 5. If the `keyid' is a public key (determined by length >= 32 bytes), set + % the `committer' to its hash. + Commitment3 = + Commitment2#{ + <<"signature">> => hb_util:encode(Sig), + <<"committed">> => CommittedKeys + }, + KeyID = maps:get(<<"keyid">>, Commitment3, <<>>), + Commitment5 = + case dev_codec_httpsig_keyid:keyid_to_committer(KeyID) of + undefined -> + Commitment3; + Committer -> + Commitment3#{ + <<"committer">> => Committer + } + end, + ID = + if byte_size(Sig) == 32 -> hb_util:human_id(Sig); + true -> hb_util:human_id(crypto:hash(sha256, Sig)) + end, + % Return the commitment and calculated ID. + {ok, ID, Commitment5}. + +decoding_nested_map_binary(Bin) -> + MapBinary = + lists:foldl( + fun (X, Acc) -> + case binary:split(X, <<":">>, [global]) of + [ID, Key, Value] -> + Acc#{ + ID => #{ <<"name">> => Key, <<"value">> => Value } + }; + _ -> + X + end + end, + #{}, + binary:split(Bin, <<", ">>, [global]) + ), + case MapBinary of + Res when is_map(Res) -> + Res; + Res -> + Res + end. + +%% @doc Normalize a list of AO-Core keys to their equivalents in `httpsig@1.0' +%% format. This involves: +%% - If the HTTPSig message given has an `ao-body-key' key and the committed keys +%% list contains it, we replace it in the list with the `body' key and add the +%% `ao-body-key' key. +%% - If the list contains a `body' key, we replace it with the `content-digest' +%% key. +%% - Otherwise, we return the list unchanged. +to_siginfo_keys(Msg, Commitment, Opts) -> + {ok, _EncMsg, EncComm, _} = + dev_codec_httpsig:normalize_for_encoding(Msg, Commitment, Opts), + maps:get(<<"committed">>, EncComm). + +%% @doc Normalize a list of `httpsig@1.0' keys to their equivalents in AO-Core +%% format. There are three stages: +%% 1. Remove the @ prefix from the component identifiers, if present. +%% 2. Replace `content-digest' with the body keys, if present. +%% 3. Replace the `body' key again with the value of the `ao-body-key' key, if +%% present. This is possible because the keys derived from the body often +%% contain the `body' key itself. +%% 4. If the `content-type' starts with `multipart/', we remove it. +from_siginfo_keys(HTTPEncMsg, BodyKeys, SigInfoCommitted) -> + % 1. Remove specifiers from the list. + BaseCommitted = remove_derived_specifiers(SigInfoCommitted), + % 2. Replace the `content-digest' key with the `body' key, if present. + WithBody = + hb_util:list_replace(BaseCommitted, <<"content-digest">>, BodyKeys), + % 3. Replace the `body' key again with the value of the `ao-body-key' key, + % if present. + ?event( + {from_siginfo_keys, + {body_keys, BodyKeys}, + {raw_committed, SigInfoCommitted}, + {with_body, {explicit, WithBody}} + } + ), + ListWithoutBodyKey = + case lists:member(<<"ao-body-key">>, WithBody) of + true -> + WithOrigBodyKey = + hb_util:list_replace( + WithBody, + <<"body">>, + maps:get(<<"ao-body-key">>, HTTPEncMsg) + ), + ?event( + {with_orig_body_key, WithOrigBodyKey} + ), + WithOrigBodyKey -- [<<"ao-body-key">>]; + false -> + WithBody + end, + % 4. If the `content-type' starts with `multipart/', we remove it. + ListWithoutContentType = + case maps:get(<<"content-type">>, HTTPEncMsg, undefined) of + <<"multipart/", _/binary>> -> + hb_util:list_replace(ListWithoutBodyKey, <<"content-type">>, []); + _ -> + ListWithoutBodyKey + end, + Final = + hb_ao:normalize_keys( + lists:map( + fun hb_link:remove_link_specifier/1, + ListWithoutContentType + ) + ), + ?event({from_siginfo_keys, {list, Final}}), + Final. + +%% @doc Convert committed keys to their siginfo format. This involves removing +%% the `body' key from the committed keys, if present, and replacing it with +%% the `content-digest' key. +committed_keys_to_siginfo(Msg) when is_map(Msg) -> + committed_keys_to_siginfo(hb_util:message_to_ordered_list(Msg)); +committed_keys_to_siginfo([]) -> []; +committed_keys_to_siginfo([<<"body">> | Rest]) -> + [<<"content-digest">> | Rest]; +committed_keys_to_siginfo([Key | Rest]) -> + [Key | committed_keys_to_siginfo(Rest)]. + +%% @doc Convert an `alg` to a commitment device. If the `alg' has the form of +%% a device specifier (`x@y.z...[/type]'), return the device. Otherwise, we +%% assume that the `alg' is a `type' of the `httpsig@1.0' algorithm. +%% `type' is an optional key that allows for subtyping of the algorithm. When +%% provided, in the `alg' it is parsed and returned as the `type' key in the +%% commitment message. +commitment_to_device_specifiers(Commitment, Opts) when is_map(Commitment) -> + commitment_to_device_specifiers(maps:get(<<"alg">>, Commitment), Opts); +commitment_to_device_specifiers(Alg, _Opts) -> + case binary:split(Alg, <<"@">>) of + [Type] -> + % The `alg' is not a device specifier, so we assume that it is a + % type of the `httpsig@1.0' algorithm. + #{ + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"type">> => Type + }; + [DevName, Specifiers] -> + % The `alg' is a device specifier. We determine if a type is present + % by splitting on the `/` character. + case binary:split(Specifiers, <<"/">>) of + [_Version] -> + % The `alg' is a device specifier without a type. + #{ + <<"commitment-device">> => Alg + }; + [Version, Type] -> + % The `alg' is a device specifier with a type. + #{ + <<"commitment-device">> => + <>, + <<"type">> => Type + } + end + end. + +%% @doc Calculate an `alg' string from a commitment message, using its +%% `commitment-device' and optionally, its `type' key. +commitment_to_alg(#{ <<"commitment-device">> := <<"httpsig@1.0">>, <<"type">> := Type }, _Opts) -> + Type; +commitment_to_alg(Commitment, _Opts) -> + Type = + case maps:get(<<"type">>, Commitment, undefined) of + undefined -> <<>>; + TypeSpecifier -> <<"/", TypeSpecifier/binary>> + end, + CommitmentDevice = maps:get(<<"commitment-device">>, Commitment), + <>. + +%% @doc Generate a signature name from a commitment. The commitment message is +%% not expected to be complete: Only the `commitment-device`, and the +%% `committer' or `keyid' keys are required. +commitment_to_sig_name(Commitment) -> + BaseStr = + case maps:get(<<"committer">>, Commitment, undefined) of + undefined -> maps:get(<<"keyid">>, Commitment); + Committer -> + << + (hb_util:to_hex(binary:part(hb_util:native_id(Committer), 1, 8))) + /binary + >> + end, + DeviceStr = + binary:replace( + maps:get( + <<"commitment-device">>, + Commitment + ), + <<"@">>, + <<"-">> + ), + <>. + +%% @doc Normalize key parameters to ensure their names are correct for inclusion +%% in the `signature-input' and associated keys. +add_derived_specifiers(ComponentIdentifiers) -> + % Remove the @ prefix from the component identifiers, if present. + Stripped = + lists:map( + fun(<<"@", Key/binary>>) -> Key; (Key) -> Key end, + ComponentIdentifiers + ), + % Add the @ prefix to the component identifiers, if they are derived. + lists:flatten( + lists:map( + fun(Key) -> + case lists:member(Key, ?DERIVED_COMPONENTS) of + true -> << "@", Key/binary >>; + false -> Key + end + end, + Stripped + ) + ). + +%% @doc Remove derived specifiers from a list of component identifiers. +remove_derived_specifiers(ComponentIdentifiers) -> + lists:map( + fun(<<"@", Key/binary>>) -> + Key; + (Key) -> + Key + end, + ComponentIdentifiers + ). + +%%% Tests. + +parse_alg_test() -> + ?assertEqual( + commitment_to_device_specifiers(#{ <<"alg">> => <<"rsa-pss-sha512">> }, #{}), + #{ + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"type">> => <<"rsa-pss-sha512">> + } + ), + ?assertEqual( + commitment_to_device_specifiers( + #{ <<"alg">> => <<"ans104@1.0/rsa-pss-sha256">> }, + #{}), + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + } + ). \ No newline at end of file diff --git a/src/dev_codec_json.erl b/src/dev_codec_json.erl new file mode 100644 index 000000000..7dd29ebd1 --- /dev/null +++ b/src/dev_codec_json.erl @@ -0,0 +1,124 @@ +%%% @doc A simple JSON codec for HyperBEAM's message format. Takes a +%%% message as TABM and returns an encoded JSON string representation. +%%% This codec utilizes the httpsig@1.0 codec for signing and verifying. +-module(dev_codec_json). +-export([to/3, from/3, commit/3, verify/3, committed/3, content_type/1]). +-export([deserialize/3, serialize/3]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% @doc Return the content type for the codec. +content_type(_) -> {ok, <<"application/json">>}. + +%% @doc Encode a message to a JSON string, using JSON-native typing. +to(Msg, _Req, _Opts) when is_binary(Msg) -> + {ok, hb_util:bin(json:encode(Msg))}; +to(Msg, Req, Opts) -> + % The input to this function will be a TABM message, so we: + % 1. Convert it to a structured message. + % 2. Load any linked items if we are in `bundle' mode. + % 3. Convert it back to a TABM message, this time preserving all types + % aside `atom's -- for which JSON has no native support. + Restructured = + hb_message:convert( + hb_private:reset(Msg), + <<"structured@1.0">>, + tabm, + Opts + ), + Loaded = + case hb_maps:get(<<"bundle">>, Req, false, Opts) of + true -> + hb_cache:ensure_all_loaded(Restructured, Opts); + false -> + Restructured + end, + {ok, JSONStructured} = + dev_codec_structured:from( + Loaded, + Req#{ <<"encode-types">> => [<<"atom">>] }, + Opts + ), + {ok, hb_json:encode(JSONStructured)}. + +%% @doc Decode a JSON string to a message. +from(Map, _Req, _Opts) when is_map(Map) -> {ok, Map}; +from(JSON, _Req, Opts) -> + % The JSON string will be a partially-TABM encoded message: Rich number + % and list types, but no `atom's. Subsequently, we convert it to a fully + % structured message after decoding, then turn the result back into a TABM. + % This is resource-intensive and could be improved, but ensures that the + % results are fully normalized. + Decoded = json:decode(JSON), + {ok, Structured} = + dev_codec_structured:to( + Decoded, + #{}, + Opts + ), + {ok, TABM} = dev_codec_structured:from(Structured, #{}, Opts), + {ok, TABM}. + +commit(Msg, Req, Opts) -> dev_codec_httpsig:commit(Msg, Req, Opts). + +verify(Msg, Req, Opts) -> dev_codec_httpsig:verify(Msg, Req, Opts). + +committed(Msg, Req, Opts) when is_binary(Msg) -> + committed(hb_util:ok(from(Msg, Req, Opts)), Req, Opts); +committed(Msg, _Req, Opts) -> + hb_message:committed(Msg, all, Opts). + +%% @doc Deserialize the JSON string found at the given path. +deserialize(Base, Req, Opts) -> + Payload = + hb_ao:get( + Target = + hb_ao:get( + <<"target">>, + Req, + <<"body">>, + Opts + ), + Base, + Opts + ), + case Payload of + not_found -> {error, #{ + <<"status">> => 404, + <<"body">> => + << + "JSON payload not found in the base message.", + "Searched for: ", Target/binary + >> + }}; + _ -> + from(Payload, Req, Opts) + end. + +%% @doc Serialize a message to a JSON string. +serialize(Base, Msg, Opts) -> + {ok, + #{ + <<"content-type">> => <<"application/json">>, + <<"body">> => hb_util:ok(to(Base, Msg, Opts)) + } + }. + +%%% Tests + +decode_with_atom_test() -> + JSON = + <<""" + [ + { + "store-module": "hb_store_fs", + "name": "cache-TEST/json-test-store", + "ao-types": "store-module=\"atom\"" + } + ] + """>>, + Msg = hb_message:convert(JSON, <<"structured@1.0">>, <<"json@1.0">>, #{}), + ?assertMatch( + [#{ <<"store-module">> := hb_store_fs }|_], + hb_cache:ensure_all_loaded(Msg, #{}) + ). \ No newline at end of file diff --git a/src/dev_codec_structured.erl b/src/dev_codec_structured.erl new file mode 100644 index 000000000..6a7ed84aa --- /dev/null +++ b/src/dev_codec_structured.erl @@ -0,0 +1,385 @@ +%%% @doc A device implementing the codec interface (to/1, from/1) for +%%% HyperBEAM's internal, richly typed message format. Supported rich types are: +%%% - `integer' +%%% - `float' +%%% - `atom' +%%% - `list' +%%% +%%% Encoding to TABM can be limited to a subset of types (with other types +%%% passing through in their rich representation) by specifying the types +%%% that should be encoded with the `encode-types' request key. +%%% +%%% This format mirrors HTTP Structured Fields, aside from its limitations of +%%% compound type depths, as well as limited floating point representations. +%%% +%%% As with all AO-Core codecs, its target format (the format it expects to +%%% receive in the `to/1' function, and give in `from/1') is TABM. +%%% +%%% For more details, see the HTTP Structured Fields (RFC-9651) specification. +-module(dev_codec_structured). +-export([to/3, from/3, commit/3, verify/3]). +-export([encode_ao_types/2, decode_ao_types/2, is_list_from_ao_types/2]). +-export([decode_value/2, encode_value/1, implicit_keys/2]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(SUPPORTED_TYPES, [<<"integer">>, <<"float">>, <<"atom">>, <<"list">>]). + +%%% Route signature functions to the `dev_codec_httpsig' module +commit(Msg, Req, Opts) -> dev_codec_httpsig:commit(Msg, Req, Opts). +verify(Msg, Req, Opts) -> dev_codec_httpsig:verify(Msg, Req, Opts). + +%% @doc Convert a rich message into a 'Type-Annotated-Binary-Message' (TABM). +from(Bin, _Req, _Opts) when is_binary(Bin) -> {ok, Bin}; +from(List, Req, Opts) when is_list(List) -> + % Encode the list as a map, then -- if our request indicates that we are + % encoding lists -- add the `.' key to the `ao-types' field, indicating + % that this message is a list and return. Otherwise, if the downstream + % encoding did not set its own `ao-types' field, we convert the message + % back to a list. + {ok, DecodedAsMap} = + from( + hb_util:list_to_numbered_message(List), + Req, + Opts + ), + EncodingLists = lists:member(<<"list">>, find_encode_types(Req, Opts)), + EncodingHasAOTypes = hb_maps:is_key(<<"ao-types">>, DecodedAsMap, Opts), + case EncodingLists orelse EncodingHasAOTypes of + true -> + AOTypes = decode_ao_types(DecodedAsMap, Opts), + {ok, DecodedAsMap#{ + <<"ao-types">> => + encode_ao_types( + AOTypes#{ + <<".">> => <<"list">> + }, + Opts + ) + } + }; + false -> + % If the downstream encoding did not set its own `ao-types' field + % we return the message as a list. + {ok, hb_util:numbered_keys_to_list(DecodedAsMap, Opts)} + end; +from(Msg, Req, Opts) when is_map(Msg) -> + % Normalize the message, offloading links to the cache. + NormLinks = hb_link:normalize(Msg, linkify_mode(Req, Opts), Opts), + NormKeysMap = hb_ao:normalize_keys(NormLinks, Opts), + EncodeTypes = find_encode_types(Req, Opts), + {Types, Values} = lists:foldl( + fun (Key, {Types, Values}) -> + case hb_maps:find(Key, NormKeysMap, Opts) of + {ok, Value} when is_binary(Value) -> + {Types, [{Key, Value} | Values]}; + {ok, Nested} when is_map(Nested) or is_list(Nested) -> + ?event({from_recursing, {nested, Nested}}), + {Types, [{Key, hb_util:ok(from(Nested, Req, Opts))} | Values]}; + {ok, Value} when + is_atom(Value) or is_integer(Value) or is_float(Value) -> + BinKey = hb_ao:normalize_key(Key), + ?event({encode_value, Value}), + case maybe_encode_value(Value, EncodeTypes) of + {Type, BinValue} -> + { + [{BinKey, Type} | Types], + [{BinKey, BinValue} | Values] + }; + skip -> + {Types, [{Key, Value} | Values]} + end; + {ok, {resolve, Operations}} when is_list(Operations) -> + {Types, [{Key, {resolve, Operations}} | Values]}; + {ok, Function} when is_function(Function) -> + % We have a function. Convert to a binary string representation. + % This value is unique to the specific byte code of the module + % that generated the function, so it is reproducible (assuming + % the same module is used) but cannot be used to resolve the + % function at runtime. + FuncRef = list_to_binary(erlang:fun_to_list(Function)), + {Types, [{Key, FuncRef} | Values]}; + {ok, _UnsupportedValue} -> + {Types, Values} + end + end, + {[],[]}, + lists:filter( + fun(Key) -> + % Filter keys that the user could set directly, but + % should be regenerated when converting. Additionally, we remove + % the `commitments' submessage, if applicable, as it should not + % be modified during encoding. + not lists:member(Key, ?REGEN_KEYS) andalso + not hb_private:is_private(Key) andalso + not (Key == <<"commitments">>) + end, + hb_util:to_sorted_keys(NormKeysMap, Opts) + ) + ), + % Encode the AoTypes as a structured dictionary + % And include as a field on the produced TABM + WithTypes = + hb_maps:from_list(case Types of + [] -> Values; + T -> + AoTypes = iolist_to_binary(hb_structured_fields:dictionary( + lists:map( + fun({Key, Value}) -> + {ok, Item} = hb_structured_fields:to_item(Value), + {hb_escape:encode(Key), Item} + end, + lists:reverse(T) + ) + )), + [{<<"ao-types">>, AoTypes} | Values] + end), + % If the message has a `commitments' field, add it to the TABM unmodified. + {ok, + case maps:get(<<"commitments">>, Msg, not_found) of + not_found -> + WithTypes; + Commitments -> + WithTypes#{ + <<"commitments">> => Commitments + } + end + }; +from(Other, _Req, _Opts) -> {ok, hb_path:to_binary(Other)}. + +%% @doc Find the types that should be encoded from the request and options. +find_encode_types(Req, Opts) -> + hb_maps:get(<<"encode-types">>, Req, ?SUPPORTED_TYPES, Opts). + +%% @doc Determine the type for a value. +type(Int) when is_integer(Int) -> <<"integer">>; +type(Float) when is_float(Float) -> <<"float">>; +type(Atom) when is_atom(Atom) -> <<"atom">>; +type(List) when is_list(List) -> <<"list">>; +type(Other) -> Other. + +%% @doc Discern the linkify mode from the request and the options. +linkify_mode(Req, Opts) -> + case hb_maps:get(<<"bundle">>, Req, not_found, Opts) of + not_found -> hb_opts:get(linkify_mode, offload, Opts); + true -> + % The request is asking for a bundle, so we should _not_ linkify. + false; + false -> + % The request is asking for a flat message, so we should linkify. + true + end. + +%% @doc Convert a TABM into a native HyperBEAM message. +to(Bin, _Req, _Opts) when is_binary(Bin) -> {ok, Bin}; +to(TABM0, Req, Opts) when is_list(TABM0) -> + % If we receive a list, we convert it to a message and run `to/3' on it. + % Finally, we convert the result back to a list. + {ok, TABM1} = to(hb_util:list_to_numbered_message(TABM0), Req, Opts), + {ok, hb_util:numbered_keys_to_list(TABM1, Opts)}; +to(TABM0, Req, Opts) -> + Types = decode_ao_types(TABM0, Opts), + % Decode all links to their HyperBEAM-native, resolvable form. + TABM1 = hb_link:decode_all_links(TABM0), + % 1. Remove 'ao-types' field + % 2. Decode any binary values that have a type; + % 3. Recursively decode any maps that we encounter; + % 4. Return the remaining keys and values as a map. + ResMsg = + maps:fold( + fun (<<"ao-types">>, _Value, Acc) -> Acc; + (RawKey, BinValue, Acc) when is_binary(BinValue) -> + case hb_maps:find(hb_ao:normalize_key(RawKey), Types, Opts) of + % The value is a binary, no parsing required + error -> Acc#{ RawKey => BinValue }; + % Parse according to its type + {ok, Type} -> + Acc#{ RawKey => decode_value(Type, BinValue) } + end; + (RawKey, ChildTABM, Acc) when is_map(ChildTABM) -> + % Decode the child TABM + Acc#{ + RawKey => hb_util:ok(to(ChildTABM, Req, Opts)) + }; + (RawKey, Value, Acc) -> + % We encountered a key that already has a converted type. + % We can just return it as is. + Acc#{ RawKey => Value } + end, + #{}, + TABM1 + ), + % If the message is a list, we need to convert it back. + case maps:get(<<".">>, Types, not_found) of + not_found -> {ok, ResMsg}; + <<"list">> -> {ok, hb_util:message_to_ordered_list(ResMsg, Opts)} + end. + +%% @doc Generate an `ao-types' structured field from a map of keys and their +%% types. +encode_ao_types(Types, _Opts) -> + iolist_to_binary(hb_structured_fields:dictionary( + lists:map( + fun(Key) -> + {ok, Item} = hb_structured_fields:to_item(maps:get(Key, Types)), + {hb_escape:encode(Key), Item} + end, + hb_util:to_sorted_keys(Types) + ) + )). + +%% @doc Parse the `ao-types' field of a TABM if present, and return a map of +%% keys and their types. If the given value is a list, we return an empty map +%% as there can be no `ao-types'. +decode_ao_types(List, _Opts) when is_list(List) -> #{}; +decode_ao_types(Msg, Opts) when is_map(Msg) -> + decode_ao_types(hb_maps:get(<<"ao-types">>, Msg, <<>>, Opts), Opts); +decode_ao_types(Bin, _Opts) when is_binary(Bin) -> + hb_maps:from_list( + lists:map( + fun({Key, {item, {_, Value}, _}}) -> + {hb_escape:decode(Key), Value} + end, + hb_structured_fields:parse_dictionary(Bin) + ) + ). + +%% @doc Determine if the `ao-types' field of a TABM indicates that the message +%% is a list. +is_list_from_ao_types(Types, Opts) when is_binary(Types) -> + is_list_from_ao_types(decode_ao_types(Types, Opts), Opts); +is_list_from_ao_types(Types, _Opts) -> + case maps:find(<<".">>, Types) of + {ok, <<"list">>} -> true; + _ -> false + end. + +%% @doc Find the implicit keys of a TABM. +implicit_keys(Req, Opts) -> + hb_maps:keys( + hb_maps:filtermap( + fun(_Key, Val = <<"empty-", _/binary>>) -> {true, Val}; + (_Key, _Val) -> false + end, + decode_ao_types(Req, Opts), + Opts + ), + Opts + ). + +%% @doc Encode a value if it is in the list of supported types. +maybe_encode_value(Value, EncodeTypes) -> + case lists:member(type(Value), EncodeTypes) of + true -> encode_value(Value); + false -> skip + end. + +%% @doc Convert a term to a binary representation, emitting its type for +%% serialization as a separate tag. +encode_value(Value) when is_integer(Value) -> + [Encoded, _] = hb_structured_fields:item({item, Value, []}), + {<<"integer">>, Encoded}; +encode_value(Value) when is_float(Value) -> + ?no_prod("Must use structured field representation for floats!"), + {<<"float">>, float_to_binary(Value)}; +encode_value(Value) when is_atom(Value) -> + EncodedIOList = + hb_structured_fields:item({item, {token, hb_util:bin(Value)}, []}), + Encoded = hb_util:bin(EncodedIOList), + {<<"atom">>, Encoded}; +encode_value(Values) when is_list(Values) -> + EncodedValues = + lists:map( + fun(Bin) when is_binary(Bin) -> {item, {string, Bin}, []}; + (Item) -> + {RawType, Encoded} = encode_value(Item), + Type = hb_ao:normalize_key(RawType), + { + item, + { + string, + << + "(ao-type-", Type/binary, ") ", + Encoded/binary + >> + }, + [] + } + end, + Values + ), + EncodedList = hb_structured_fields:list(EncodedValues), + {<<"list">>, iolist_to_binary(EncodedList)}; +encode_value(Value) when is_binary(Value) -> + {<<"binary">>, Value}; +encode_value(Value) -> + Value. + +%% @doc Convert non-binary values to binary for serialization. +decode_value(Type, Value) when is_list(Type) -> + decode_value(list_to_binary(Type), Value); +decode_value(Type, Value) when is_binary(Type) -> + ?event({decoding, {type, Type}, {value, Value}}), + decode_value( + binary_to_existing_atom( + list_to_binary(string:to_lower(binary_to_list(Type))), + latin1 + ), + Value + ); +decode_value(integer, Value) -> + {item, Number, _} = hb_structured_fields:parse_item(Value), + Number; +decode_value(float, Value) -> + binary_to_float(Value); +decode_value(atom, Value) -> + {item, {_, AtomString}, _} = + hb_structured_fields:parse_item(Value), + hb_util:atom(AtomString); +decode_value(list, Value) when is_binary(Value) -> + lists:map( + fun({item, {string, <<"(ao-type-", Rest/binary>>}, _}) -> + [Type, Item] = binary:split(Rest, <<") ">>), + decode_value(Type, Item); + ({item, Item, _}) -> hb_structured_fields:from_bare_item(Item) + end, + hb_structured_fields:parse_list(iolist_to_binary(Value)) + ); +decode_value(list, Value) when is_map(Value) -> + hb_util:message_to_ordered_list(Value); +decode_value(map, Value) -> + hb_maps:from_list( + lists:map( + fun({Key, {item, Item, _}}) -> + ?event({decoded_item, {explicit, Key}, Item}), + {Key, hb_structured_fields:from_bare_item(Item)} + end, + hb_structured_fields:parse_dictionary(iolist_to_binary(Value)) + ) + ); +decode_value(BinType, Value) when is_binary(BinType) -> + decode_value( + list_to_existing_atom( + string:to_lower( + binary_to_list(BinType) + ) + ), + Value + ); +decode_value(OtherType, Value) -> + ?event({unexpected_type, OtherType, Value}), + throw({unexpected_type, OtherType, Value}). + +%%% Tests + +list_encoding_test() -> + % Test that we can encode and decode a list of integers. + {<<"list">>, Encoded} = encode_value(List1 = [1, 2, 3]), + Decoded = decode_value(list, Encoded), + ?assertEqual(List1, Decoded), + % Test that we can encode and decode a list of binaries. + {<<"list">>, Encoded2} = encode_value(List2 = [<<"1">>, <<"2">>, <<"3">>]), + ?assertEqual(List2, decode_value(list, Encoded2)), + % Test that we can encode and decode a mixed list. + {<<"list">>, Encoded3} = encode_value(List3 = [1, <<"2">>, 3]), + ?assertEqual(List3, decode_value(list, Encoded3)). \ No newline at end of file diff --git a/src/dev_copycat.erl b/src/dev_copycat.erl new file mode 100644 index 000000000..61126a93d --- /dev/null +++ b/src/dev_copycat.erl @@ -0,0 +1,20 @@ +%%% @doc A device for orchestrating indexing of messages from foreign sources +%%% into a HyperBEAM node's caches. +%%% +%%% Supported sources of messages are as follows: +%%% - A remote Arweave GraphQL endpoint. +%%% - A remote Arweave node. +%%% Each source is implemented as a separate engine, with `dev_copycat_[ENGINE]' +%%% as the module name. +-module(dev_copycat). +-export([graphql/3, arweave/3]). + +%% @doc Fetch data from a GraphQL endpoint for replication. See +%% `dev_copycat_graphql' for implementation details. +graphql(Base, Request, Opts) -> + dev_copycat_graphql:graphql(Base, Request, Opts). + +%% @doc Fetch data from an Arweave node for replication. See `dev_copycat_arweave' +%% for implementation details. +arweave(Base, Request, Opts) -> + dev_copycat_arweave:arweave(Base, Request, Opts). \ No newline at end of file diff --git a/src/dev_copycat_arweave.erl b/src/dev_copycat_arweave.erl new file mode 100644 index 000000000..6ff8f822f --- /dev/null +++ b/src/dev_copycat_arweave.erl @@ -0,0 +1,77 @@ +%%% @doc A `~copycat@1.0' engine that fetches block data from an Arweave node for +%%% replication. This engine works in _reverse_ chronological order by default, +%%% fetching blocks from the latest known block towards the Genesis block. The +%%% node avoids retrieving blocks that are already present in the cache using +%%% `~arweave@2.9-pre''s built-in caching mechanism. +-module(dev_copycat_arweave). +-export([arweave/3]). +-include_lib("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(ARWEAVE_DEVICE, <<"~arweave@2.9-pre">>). + +%% @doc Fetch blocks from an Arweave node between a given range, or from the +%% latest known block towards the Genesis block. If no range is provided, we +%% fetch blocks from the latest known block towards the Genesis block. +arweave(_Base, Request, Opts) -> + {From, To} = parse_range(Request, Opts), + fetch_blocks(Request, From, To, Opts). + +%% @doc Parse the range from the request. +parse_range(Request, Opts) -> + From = + case hb_maps:find(<<"from">>, Request, Opts) of + {ok, Height} -> Height; + error -> + {ok, LatestHeight} = + hb_ao:resolve( + <>, + Opts + ), + LatestHeight + end, + To = hb_maps:get(<<"to">>, Request, 0, Opts), + {From, To}. + +%% @doc Fetch blocks from an Arweave node between a given range. +fetch_blocks(Req, Current, Current, _Opts) -> + ?event(copycat_arweave, + {arweave_block_indexing_completed, + {reached_target, Current}, + {initial_request, Req} + } + ), + {ok, Current}; +fetch_blocks(Req, Current, To, Opts) -> + BlockRes = + hb_ao:resolve( + << + ?ARWEAVE_DEVICE/binary, + "/block=", + (hb_util:bin(Current))/binary + >>, + Opts + ), + process_block(BlockRes, Req, Current, To, Opts), + fetch_blocks(Req, Current - 1, To, Opts). + +%% @doc Process a block. +process_block(BlockRes, _Req, Current, To, _Opts) -> + case BlockRes of + {ok, _} -> + ?event( + copycat_short, + {arweave_block_cached, + {height, Current}, + {target, To} + } + ); + {error, not_found} -> + ?event( + copycat_short, + {arweave_block_not_found, + {height, Current}, + {target, To} + } + ) + end. \ No newline at end of file diff --git a/src/dev_copycat_graphql.erl b/src/dev_copycat_graphql.erl new file mode 100644 index 000000000..58c2b824a --- /dev/null +++ b/src/dev_copycat_graphql.erl @@ -0,0 +1,238 @@ +%%% @doc A `~copycat@1.0' engine that fetches data from a GraphQL endpoint for +%%% replication. +-module(dev_copycat_graphql). +-export([graphql/3]). +-include_lib("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(SUPPORTED_FILTERS, + [<<"query">>, <<"tag">>, <<"owner">>, <<"recipient">>, <<"all">>] +). + +%% @doc Takes a GraphQL query, optionally with a node address, and curses through +%% each of the messages returned by the query, indexing them into the node's +%% caches. +graphql(Base, Req, Opts) -> + case parse_query(Base, Req, Opts) of + {ok, Query} -> + Node = maps:get(<<"node">>, Opts, undefined), + OpName = hb_maps:get(<<"operationName">>, Req, undefined, Opts), + Vars = hb_maps:get(<<"variables">>, Req, #{}, Opts), + index_graphql(0, Query, Vars, Node, OpName, Opts); + Other -> + Other + end. + +%% @doc Index a GraphQL query into the node's caches. +index_graphql(Total, Query, Vars, Node, OpName, Opts) -> + maybe + ?event( + {graphql_run_called, + {query, {string, Query}}, + {operation, OpName}, + {variables, Vars} + } + ), + {ok, RawRes} ?= hb_gateway_client:query(Query, Vars, Node, OpName, Opts), + Res = hb_util:deep_get(<<"data/transactions">>, RawRes, #{}, Opts), + NodeStructs = hb_util:deep_get(<<"edges">>, Res, [], Opts), + ?event({graphql_request_returned_items, length(NodeStructs)}), + ?event( + {graphql_indexing_responses, + {query, {string, Query}}, + {variables, Vars}, + {result, Res} + } + ), + ParsedMsgs = + lists:filtermap( + fun(NodeStruct) -> + Struct = hb_maps:get(<<"node">>, NodeStruct, not_found, Opts), + try + {ok, ParsedMsg} = + hb_gateway_client:result_to_message( + Struct, + Opts + ), + {true, ParsedMsg} + catch + error:Reason -> + ?event( + warning, + {indexer_graphql_parse_failed, + {struct, NodeStruct}, + {reason, Reason} + } + ), + false + end + end, + NodeStructs + ), + ?event({graphql_parsed_msgs, length(ParsedMsgs)}), + WrittenMsgs = + lists:filter( + fun(ParsedMsg) -> + try + {ok, _} = hb_cache:write(ParsedMsg, Opts), + true + catch + error:Reason -> + ?event( + warning, + {indexer_graphql_write_failed, + {reason, Reason}, + {msg, ParsedMsg} + } + ), + false + end + end, + ParsedMsgs + ), + NewTotal = Total + length(WrittenMsgs), + ?event(copycat_short, + {indexer_graphql_wrote, + {total, NewTotal}, + {batch, length(WrittenMsgs)}, + {batch_failures, length(ParsedMsgs) - length(WrittenMsgs)} + } + ), + HasNextPage = hb_util:deep_get(<<"pageInfo/hasNextPage">>, Res, false, Opts), + case HasNextPage of + true -> + % Get the last cursor from the node structures and recurse. + {ok, Cursor} = + hb_maps:find( + <<"cursor">>, + lists:last(NodeStructs), + Opts + ), + index_graphql( + NewTotal, + Query, + Vars#{ <<"after">> => Cursor }, + Node, + OpName, + Opts + ); + false -> + {ok, NewTotal} + end + else + {error, Reason} -> + {error, Reason} + end. + +%% @doc Find or create a GraphQL query from a given base and request. We expect +%% to find either a `query' field, a `tags' field, a `tag' and `value' field, +%% an `owner' field, or a `recipient' field. If none of these fields are found, +%% we return a query that will match all results known to an Arweave gateway. +parse_query(Base, Req, Opts) -> + % Merge the keys of the base and request maps, and remove duplicates. + Merged = hb_maps:merge(Base, Req, Opts), + Keys = hb_maps:keys(Merged, Opts), + SupportedKeys = ?SUPPORTED_FILTERS, + ?event({finding_query, {supported, SupportedKeys}, {merged_req, Merged}}), + case lists:filter(fun(K) -> lists:member(K, SupportedKeys) end, Keys) of + [<<"query">>|_] -> + % Find the query in either the `query' field or the `body'. + case hb_maps:find(<<"query">>, Merged, Opts) of + {ok, QueryKeys} when is_map(QueryKeys) -> + default_query(<<"tags">>, QueryKeys, Opts); + {ok, Bin} when is_binary(Bin) -> + {ok, Bin}; + _ -> + case hb_maps:find(<<"body">>, Merged, Opts) of + {ok, Bin} when is_binary(Bin) -> + {ok, Bin}; + _ -> + {error, + #{ + <<"body">> => + <<"No query found in the request.">> + } + } + end + end; + [<<"tag">>|_] -> + Key = hb_maps:get(<<"tag">>, Merged, <<>>, Opts), + Value = hb_maps:get(<<"value">>, Merged, <<>>, Opts), + default_query(<<"tag">>, {Key, Value}, Opts); + [FilterKey|_] -> + default_query(FilterKey, Merged, Opts); + [] -> + {error, + #{ + <<"body">> => + <<"No supported filter fields found. Supported filters: ", + ( + lists:join( + <<", ">>, + lists:map( + fun(K) -> <<"\"", (K)/binary, "\"">> end, + SupportedKeys + ) + ) + )/binary + >> + } + } + end. + +%% @doc Return a default query for a given filter type. +default_query(<<"tags">>, RawMessage, Opts) -> + Message = hb_cache:ensure_all_loaded(RawMessage, Opts), + BinaryPairs = + lists:map( + fun({Key, Value}) -> {hb_util:bin(Key), hb_util:bin(Value)} end, + hb_maps:to_list(Message, Opts) + ), + TagsQueryStr = + hb_util:bin( + [ + <<"{name: \"", Key/binary, "\", values: [\"", Value/binary, "\"]}">> + || + {Key, Value} <- BinaryPairs + ] + ), + ?event({tags_query, + {message, Message}, + {binary_pairs, BinaryPairs}, + {tags_query_str, {string, TagsQueryStr}} + }), + {ok, <<"query($after: String) { ", + "transactions(after: $after, tags: [", + TagsQueryStr/binary, + "]) { ", + "edges { ", (hb_gateway_client:item_spec())/binary , " } ", + "pageInfo { hasNextPage }", + "} }">>}; +default_query(<<"tag">>, {Key, Value}, _Opts) -> + {ok, <<"query($after: String) { ", + "transactions(after: $after, tags: [", + "{name: \"", Key/binary, "\", values: [\"", Value/binary, "\"]}", + "]) { ", + "edges { ", (hb_gateway_client:item_spec())/binary , " } ", + "pageInfo { hasNextPage }", + "} }">>}; +default_query(<<"recipient">>, Merged, Opts) -> + Recipient = hb_maps:get(<<"recipient">>, Merged, <<>>, Opts), + {ok, <<"query($after: String) { ", + "transactions(after: $after, recipients: [\"", Recipient/binary, "\"]) { ", + "edges { ", (hb_gateway_client:item_spec())/binary , " } ", + "pageInfo { hasNextPage }", + "} }">>}; +default_query(<<"owner">>, Merged, Opts) -> + Owner = hb_maps:get(<<"owner">>, Merged, <<>>, Opts), + {ok, <<"query($after: String) { ", + "transactions(after: $after, owner: \"", Owner/binary, "\") { ", + "edges { ", (hb_gateway_client:item_spec())/binary , " } ", + "pageInfo { hasNextPage }", + "} }">>}; +default_query(<<"all">>, _Merged, _Opts) -> + {ok, <<"query($after: String) { ", + "transactions(after: $after) { ", + "edges { ", (hb_gateway_client:item_spec())/binary , " } ", + "pageInfo { hasNextPage }", + "} }">>}. \ No newline at end of file diff --git a/src/dev_cron.erl b/src/dev_cron.erl index 96ccb9959..60836f9d7 100644 --- a/src/dev_cron.erl +++ b/src/dev_cron.erl @@ -1,75 +1,330 @@ +%%% @doc A device that inserts new messages into the schedule to allow processes +%%% to passively 'call' themselves without user interaction. -module(dev_cron). --export([init/2, execute/2, uses/0]). +-export([once/3, every/3, stop/3, info/1, info/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). -%%% A device that inserts new messages into the schedule to allow processes -%%% to passively 'call' themselves without user interaction. +%% @doc Exported function for getting device info. +info(_) -> + #{ exports => [info, once, every, stop] }. --include("include/hb.hrl"). +info(_Msg1, _Msg2, _Opts) -> + InfoBody = #{ + <<"description">> => <<"Cron device for scheduling messages">>, + <<"version">> => <<"1.0">>, + <<"paths">> => #{ + <<"info">> => <<"Get device info">>, + <<"once">> => <<"Schedule a one-time message">>, + <<"every">> => <<"Schedule a recurring message">>, + <<"stop">> => <<"Stop a scheduled task {task}">> + } + }, + {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. --record(state, { - time, - last_run -}). - -init(State = #{ process := ProcM }, Params) -> - case lists:keyfind(<<"Time">>, 1, Params) of - {<<"Time">>, CronTime} -> - MilliSecs = parse_time(CronTime), - %% TODO: What's the most sensible way to initialize the last_run? - %% Current behavior: Timer starts after _first_ message. - {ok, State#{ cron => #state { time = MilliSecs, last_run = timestamp(ProcM) } }}; - false -> - {ok, State#{ cron => inactive }} - end. +%% @doc Exported function for scheduling a one-time message. +once(_Msg1, Msg2, Opts) -> + case hb_ao:get(<<"cron-path">>, Msg2, Opts) of + not_found -> + {error, <<"No cron path found in message.">>}; + CronPath -> + ReqMsgID = hb_message:id(Msg2, all, Opts), + % make the path specific for the end device to be used + ModifiedMsg2 = + maps:remove( + <<"cron-path">>, + maps:put(<<"path">>, CronPath, Msg2) + ), + Name = {<<"cron@1.0">>, ReqMsgID}, + Pid = spawn(fun() -> once_worker(CronPath, ModifiedMsg2, Opts) end), + hb_name:register(Name, Pid), + {ok, ReqMsgID} + end. -parse_time(BinString) -> - [AmountStr, UnitStr] = binary:split(BinString, <<"-">>), - Amount = binary_to_integer(AmountStr), - Unit = string:lowercase(binary_to_list(UnitStr)), - case Unit of - "millisecond" ++ _ -> Amount; - "second" ++ _ -> Amount * 1000; - "minute" ++ _ -> Amount * 60 * 1000; - "hour" ++ _ -> Amount * 60 * 60 * 1000; - "day" ++ _ -> Amount * 24 * 60 * 60 * 1000; - _ -> throw({error, invalid_time_unit, UnitStr}) - end. - -execute(_M, State = #{ cron := inactive }) -> - {ok, State}; -execute(M, State = #{ pass := 1, cron := #state { last_run = undefined } }) -> - {ok, State#{ cron := #state { last_run = timestamp(M) } }}; -execute(Message, State = #{ pass := 1, cron := #state { time = MilliSecs, last_run = LastRun }, schedule := Sched }) -> - case timestamp(Message) - LastRun of - Time when Time > MilliSecs -> - NextCronMsg = create_cron(State, CronTime = timestamp(Message) + MilliSecs), - {pass, - State#{ - cron := #state { last_run = CronTime }, - schedule := [NextCronMsg | Sched] +%% @doc Internal function for scheduling a one-time message. +once_worker(Path, Req, Opts) -> + % Directly call the meta device on the newly constructed 'singleton', just + % as hb_http_server does. + TracePID = hb_tracer:start_trace(), + try + dev_meta:handle(Opts#{ trace => TracePID }, Req#{ <<"path">> => Path}) + catch + Class:Reason:Stacktrace -> + ?event( + {cron_every_worker_error, + {path, Path}, + {error, Class, Reason, Stacktrace} } - }; - _ -> - {ok, State} - end; -execute(_, S) -> - {ok, S}. - -timestamp(M) -> - % TODO: Process this properly - case lists:keyfind(<<"Timestamp">>, 1, M#tx.tags) of - {<<"Timestamp">>, TSBin} -> - list_to_integer(binary_to_list(TSBin)); - false -> - 0 - end. - -create_cron(_State, CronTime) -> - #tx{ - tags = [ - {<<"Action">>, <<"Cron">>}, - {<<"Timestamp">>, list_to_binary(integer_to_list(CronTime))} - ] - }. - -uses() -> all. \ No newline at end of file + ), + throw({error, Class, Reason, Stacktrace}) + end. + + +%% @doc Exported function for scheduling a recurring message. +every(_Msg1, Msg2, Opts) -> + case { + hb_ao:get(<<"cron-path">>, Msg2, Opts), + hb_ao:get(<<"interval">>, Msg2, Opts) + } of + {not_found, _} -> + {error, <<"No cron path found in message.">>}; + {_, not_found} -> + {error, <<"No interval found in message.">>}; + {CronPath, IntervalString} -> + try + IntervalMillis = parse_time(IntervalString), + if IntervalMillis =< 0 -> + throw({error, invalid_interval_value}); + true -> + ok + end, + ReqMsgID = hb_message:id(Msg2, all, Opts), + ModifiedMsg2 = + maps:remove( + <<"cron-path">>, + maps:remove(<<"interval">>, Msg2) + ), + TracePID = hb_tracer:start_trace(), + Pid = + spawn( + fun() -> + every_worker_loop( + CronPath, + ModifiedMsg2, + Opts#{ trace => TracePID }, + IntervalMillis + ) + end + ), + Name = {<<"cron@1.0">>, ReqMsgID}, + hb_name:register(Name, Pid), + {ok, ReqMsgID} + catch + error:{invalid_time_unit, Unit} -> + {error, <<"Invalid time unit: ", Unit/binary>>}; + error:{invalid_interval_value} -> + {error, <<"Invalid interval value.">>}; + error:{Reason, _Stack} -> + {error, {<<"Error parsing interval">>, Reason}} + end + end. + +%% @doc Exported function for stopping a scheduled task. +stop(_Msg1, Msg2, Opts) -> + case hb_ao:get(<<"task">>, Msg2, Opts) of + not_found -> + {error, <<"No task ID found in message.">>}; + TaskID -> + Name = {<<"cron@1.0">>, TaskID}, + case hb_name:lookup(Name) of + Pid when is_pid(Pid) -> + ?event({cron_stopping_task, {task_id, TaskID}, {pid, Pid}}), + exit(Pid, kill), + hb_name:unregister(Name), + {ok, #{<<"status">> => 200, <<"body">> => #{ + <<"message">> => <<"Task stopped successfully">>, + <<"task_id">> => TaskID + }}}; + undefined -> + {error, <<"Task not found.">>}; + Error -> + ?event({cron_stop_lookup_error, {task_id, TaskID}, {error, Error}}), + {error, #{ + <<"error">> => + <<"Failed to lookup task or unexpected result">>, + <<"details">> => Error + }} + end + end. + +every_worker_loop(CronPath, Req, Opts, IntervalMillis) -> + Req1 = Req#{<<"path">> => CronPath}, + ?event( + {cron_every_worker_executing, + {path, CronPath}, + {req_id, hb_message:id(Req, all, Opts)} + } + ), + try + dev_meta:handle(Opts, Req1), + ?event({cron_every_worker_executed, {path, CronPath}}) + catch + Class:Reason:Stack -> + ?event(cron_error, {cron_every_worker_error, + {path, CronPath}, + {error, Class, Reason, Stack}}) + end, + timer:sleep(IntervalMillis), + every_worker_loop(CronPath, Req, Opts, IntervalMillis). + +%% @doc Parse a time string into milliseconds. +parse_time(BinString) -> + [AmountStr, UnitStr] = binary:split(BinString, <<"-">>), + Amount = binary_to_integer(AmountStr), + Unit = string:lowercase(binary_to_list(UnitStr)), + case Unit of + "millisecond" ++ _ -> Amount; + "second" ++ _ -> Amount * 1000; + "minute" ++ _ -> Amount * 60 * 1000; + "hour" ++ _ -> Amount * 60 * 60 * 1000; + "day" ++ _ -> Amount * 24 * 60 * 60 * 1000; + _ -> throw({error, invalid_time_unit, UnitStr}) + end. + +%%% Tests + +stop_once_test() -> + % Start a new node + Node = hb_http_server:start_node(), + % Set up a standard test worker (even though delay doesn't use its state) + TestWorkerPid = spawn(fun test_worker/0), + TestWorkerNameId = hb_util:human_id(crypto:strong_rand_bytes(32)), + hb_name:register({<<"test">>, TestWorkerNameId}, TestWorkerPid), + % Create a "once" task targeting the delay function + OnceUrlPath = <<"/~cron@1.0/once?test-id=", TestWorkerNameId/binary, + "&cron-path=/~test-device@1.0/delay">>, + {ok, OnceTaskID} = hb_http:get(Node, OnceUrlPath, #{}), + ?event({'cron:stop_once:test:created', {task_id, OnceTaskID}}), + % Give a short delay to ensure the task has started and called handle, + % entering the sleep + timer:sleep(200), + % Verify the once task worker process is registered and alive + OncePid = hb_name:lookup({<<"cron@1.0">>, OnceTaskID}), + ?assert(is_pid(OncePid), "Lookup did not return a PID"), + ?assert(erlang:is_process_alive(OncePid), "OnceWorker process died prematurely"), + % Call stop on the once task while it's sleeping + OnceStopPath = <<"/~cron@1.0/stop?task=", OnceTaskID/binary>>, + {ok, OnceStopResult} = hb_http:get(Node, OnceStopPath, #{}), + ?event({'cron:stop_once:test:stopped', {result, OnceStopResult}}), + % Verify success response from stop + ?assertMatch(#{<<"status">> := 200}, OnceStopResult), + % Verify name is unregistered + ?assertEqual(undefined, hb_name:lookup({<<"cron@1.0">>, OnceTaskID})), + % Allow a moment for the kill signal to be processed + timer:sleep(100), + % Verify process termination + ?assertNot(erlang:is_process_alive(OncePid), "Process not killed by stop"), + + % Call stop again to verify 404 response + {error, <<"Task not found.">>} = hb_http:get(Node, OnceStopPath, #{}). + + +%% @doc This test verifies that a recurring task can be stopped by +%% calling the stop function with the task ID. +stop_every_test() -> + % Start a new node + Node = hb_http_server:start_node(), + % Set up a test worker process to hold state (counter) + TestWorkerPid = spawn(fun test_worker/0), + TestWorkerNameId = hb_util:human_id(crypto:strong_rand_bytes(32)), + hb_name:register({<<"test">>, TestWorkerNameId}, TestWorkerPid), + % Create an "every" task that calls the test worker + EveryUrlPath = <<"/~cron@1.0/every?test-id=", TestWorkerNameId/binary, + "&interval=500-milliseconds", + "&cron-path=/~test-device@1.0/increment_counter">>, + {ok, CronTaskID} = hb_http:get(Node, EveryUrlPath, #{}), + ?event({'cron:stop_every:test:created', {task_id, CronTaskID}}), + % Verify the cron worker process was registered and is alive + CronWorkerPid = hb_name:lookup({<<"cron@1.0">>, CronTaskID}), + ?assert(is_pid(CronWorkerPid)), + ?assert(erlang:is_process_alive(CronWorkerPid)), + % Wait a bit to ensure the cron worker has run a few times + timer:sleep(1000), + % Call stop on the cron task using its ID + EveryStopPath = <<"/~cron@1.0/stop?task=", CronTaskID/binary>>, + {ok, EveryStopResult} = hb_http:get(Node, EveryStopPath, #{}), + ?event({'cron:stop_every:test:stopped', {result, EveryStopResult}}), + % Verify success response + ?assertMatch(#{<<"status">> := 200}, EveryStopResult), + % Verify the cron task name is unregistered (lookup returns undefined) + ?assertEqual(undefined, hb_name:lookup({<<"cron@1.0">>, CronTaskID})), + % Allow a moment for the process termination signal to be processed + timer:sleep(100), + % Verify the cron worker process is terminated + ?assertNot(erlang:is_process_alive(CronWorkerPid)), + % Check the counter in the original test worker was incremented + TestWorkerPid ! {get, self()}, + receive + {state, State = #{count := Count}} -> + ?event({'cron:stop_every:test:counter_state', {state, State}}), + ?assert(Count > 0) + after 1000 -> + throw(no_response_from_worker) + end, + % Call stop again using the same CronTaskID to verify the error + {error, <<"Task not found.">>} = hb_http:get(Node, EveryStopPath, #{}). + + +%% @doc This test verifies that a one-time task can be scheduled and executed. +once_executed_test() -> + % start a new node + Node = hb_http_server:start_node(), + % spawn a worker on the new node that calls test_worker/0 which inits + % test_worker/1 with a state of undefined + PID = spawn(fun test_worker/0), + % generate a random id that we can then use later to lookup the worker + ID = hb_util:human_id(crypto:strong_rand_bytes(32)), + % register the worker with the id + hb_name:register({<<"test">>, ID}, PID), + % Construct the URL path with the dynamic ID + UrlPath = <<"/~cron@1.0/once?test-id=", ID/binary, + "&cron-path=/~test-device@1.0/update_state">>, + % this should call the worker via the test device + % the test device should look up the worker via the id given + {ok, _ReqMsgId} = hb_http:get(Node, UrlPath, #{}), + % wait for the request to be processed + timer:sleep(1000), + % send a message to the worker to get the state + PID ! {get, self()}, + % receive the state from the worker + receive + {state, State} -> + ?event({once_executed_test_received_state, {state, State}}), + ?assertMatch(#{ <<"test-id">> := ID }, State) + after 1000 -> + FinalLookup = hb_name:lookup({<<"test">>, ID}), + ?event({timeout_waiting_for_worker, {pid, PID}, {lookup_result, FinalLookup}}), + throw(no_response_from_worker) + end. + +%% @doc This test verifies that a recurring task can be scheduled and executed. +every_worker_loop_test() -> + Node = hb_http_server:start_node(), + PID = spawn(fun test_worker/0), + ID = hb_util:human_id(crypto:strong_rand_bytes(32)), + hb_name:register({<<"test">>, ID}, PID), + UrlPath = <<"/~cron@1.0/every?test-id=", ID/binary, + "&interval=500-milliseconds", + "&cron-path=/~test-device@1.0/increment_counter">>, + ?event({'cron:every:test:sendUrl', {url_path, UrlPath}}), + {ok, ReqMsgId} = hb_http:get(Node, UrlPath, #{}), + ?event({'cron:every:test:get_done', {req_id, ReqMsgId}}), + timer:sleep(1500), + PID ! {get, self()}, + % receive the state from the worker + receive + {state, State = #{count := C}} -> + ?event({'cron:every:test:received_state', {state, State}}), + ?assert(C >= 3) + after 1000 -> + FinalLookup = hb_name:lookup({<<"test">>, ID}), + ?event({'cron:every:test:timeout', {pid, PID}, {lookup_result, FinalLookup}}), + throw({test_timeout_waiting_for_state, {id, ID}}) + end. + +%% @doc This is a helper function that is used to test the cron device. +%% It is used to increment a counter and update the state of the worker. +test_worker() -> test_worker(#{count => 0}). +test_worker(State) -> + receive + {increment} -> + NewCount = maps:get(count, State, 0) + 1, + ?event({'test_worker:incremented', {new_count, NewCount}}), + test_worker(State#{count := NewCount}); + {update, NewState} -> + ?event({'test_worker:updated', {new_state, NewState}}), + test_worker(NewState); + {get, Pid} -> + Pid ! {state, State}, + test_worker(State) + end. \ No newline at end of file diff --git a/src/dev_cu.erl b/src/dev_cu.erl index 2ca704045..5dcd07c52 100644 --- a/src/dev_cu.erl +++ b/src/dev_cu.erl @@ -25,15 +25,15 @@ execute(CarrierMsg, S) -> Wallet = hb:wallet(), {ok, Results} = case MaybeBundle of - #tx{data = #{ <<"Message">> := _Msg, <<"Assignment">> := Assignment }} -> + #tx{data = #{ <<"body">> := _Msg, <<"assignment">> := Assignment }} -> % TODO: Execute without needing to call the SU unnecessarily. - {_, ProcID} = lists:keyfind(<<"Process">>, 1, Assignment#tx.tags), + {_, ProcID} = lists:keyfind(<<"process">>, 1, Assignment#tx.tags), ?event({dev_cu_computing_from_full_assignment, {process, ProcID}, {slot, hb_util:id(Assignment, signed)}}), hb_process:result(ProcID, hb_util:id(Assignment, signed), Store, Wallet); _ -> - case lists:keyfind(<<"Process">>, 1, CarrierMsg#tx.tags) of + case lists:keyfind(<<"process">>, 1, CarrierMsg#tx.tags) of {_, Process} -> - {_, Slot} = lists:keyfind(<<"Slot">>, 1, CarrierMsg#tx.tags), + {_, Slot} = lists:keyfind(<<"slot">>, 1, CarrierMsg#tx.tags), ?event({dev_cu_computing_from_slot_ref, {process, Process}, {slot, Slot}}), hb_process:result(Process, Slot, Store, Wallet); false -> @@ -41,30 +41,30 @@ execute(CarrierMsg, S) -> end end, {ResType, ModState = #{ results := _ModResults }} = - case lists:keyfind(<<"Attest-To">>, 1, CarrierMsg#tx.tags) of - {_, RawAttestTo} -> - AttestTo = hb_util:decode(RawAttestTo), - ?event({attest_to_only_message, RawAttestTo}), - case ar_bundles:find(AttestTo, Results) of + case lists:keyfind(<<"commit-to">>, 1, CarrierMsg#tx.tags) of + {_, RawCommitTo} -> + CommitTo = hb_util:decode(RawCommitTo), + ?event({commit_to_only_message, RawCommitTo}), + case ar_bundles:find(CommitTo, Results) of not_found -> - ?event(message_to_attest_to_not_found), + ?event(message_to_commit_to_not_found), {ok, S#{ results => #tx { - tags = [{<<"Status">>, <<"404">>}], - data = <<"Requested message to attest to not in results bundle.">> + tags = [{<<"status">>, 404}], + data = <<"Requested message to commit to not in results bundle.">> } } }; _ -> - ?event(message_to_attest_to_found), + ?event(message_to_commit_to_found), {ok, S#{ results => ar_bundles:sign_item( #tx { tags = [ - {<<"Status">>, <<"200">>}, - {<<"Attestation-For">>, RawAttestTo} + {<<"status">>, 200}, + {<<"commitment-for">>, RawCommitTo} ], data = <<>> }, diff --git a/src/dev_dedup.erl b/src/dev_dedup.erl index 9c3842054..f19c1996b 100644 --- a/src/dev_dedup.erl +++ b/src/dev_dedup.erl @@ -1,101 +1,171 @@ -%%% @moduledoc A device that deduplicates messages send to a process. -%%% Only runs on the first pass of the `compute` key call if executed -%%% in a stack. Currently the device stores its list of already seen -%%% items in memory, but at some point it will likely make sense to -%%% drop them in the cache. +%%% @doc A device that deduplicates messages in an evaluation stream, returning +%%% status `skip' if the message has already been seen. +%%% +%%% This device is typically used to ensure that a message is only executed +%%% once, even if assigned multiple times, upon a `~process@1.0' evaluation. +%%% It can, however, be used in many other contexts. +%%% +%%% This device honors the `pass' key if it is present in the message. If so, +%%% it will only run on the first pass. Additionally, the device supports +%%% a `subject-key' key that allows the caller to specify the key whose ID +%%% should be used for deduplication. If the `subject-key' key is not present, +%%% the device will use the `body' of the request as the subject. If the key is +%%% set to `request', the device will use the entire request itself as the +%%% subject. +%%% +%%% This device runs on the first pass of the `compute' key call if executed +%%% in a stack, and not in subsequent passes. Currently the device stores its +%%% list of already seen items in memory, but at some point it will likely make +%%% sense to drop them in the cache. -module(dev_dedup). -export([info/1]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). -info(M1) -> +info(_M1) -> #{ - handler => fun handle/4 + default => fun handle/4, + exclude => [keys, set, id, commit] }. -%% @doc Forward the keys function to the message device, handle all others -%% with deduplication. We only act on the first pass. -handle(keys, M1, _M2, _Opts) -> +%% @doc Forward the keys and `set' functions to the message device, handle all +%% others with deduplication. This allows the device to be used in any context +%% where a key is called. If the `dedup-key +handle(<<"keys">>, M1, _M2, _Opts) -> dev_message:keys(M1); -handle(set, M1, M2, Opts) -> +handle(<<"set">>, M1, M2, Opts) -> dev_message:set(M1, M2, Opts); handle(Key, M1, M2, Opts) -> ?event({dedup_handle, {key, Key}, {msg1, M1}, {msg2, M2}}), - case hb_converge:get(<<"Pass">>, {as, dev_message, M1}, 1, Opts) of - 1 -> - Msg2ID = hb_converge:get(<<"id">>, M2, Opts), - Dedup = hb_converge:get(<<"Dedup">>, {as, dev_message, M1}, [], Opts), - ?event({dedup_checking, {existing, Dedup}}), - case lists:member(Msg2ID, Dedup) of + % Find the relevant parameters from the messages. We search for the + % `dedup-key' key in the first message, and use that value as the key to + % look for in the second message. + SubjectKey = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, M1}, <<"dedup-subject">>}, + {{as, <<"message@1.0">>, M2}, <<"dedup-subject">>} + ], + <<"body">>, + Opts + ), + % Get the subject of the second message. + Subject = + if SubjectKey == <<"request">> -> + % The subject is the request itself. + M2; + true -> + % The subject is the value of the subject key, which will have + % defaulted to the `body' key if not set in the base message. + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, M1}, SubjectKey}, + {{as, <<"message@1.0">>, M2}, SubjectKey} + ], + Opts + ) + end, + % Is this the first pass, if we are executing in a stack? + FirstPass = hb_ao:get(<<"pass">>, {as, <<"message@1.0">>, M1}, 1, Opts) == 1, + % Get the list of already seen subjects. + DedupList = hb_ao:get(<<"dedup">>, {as, <<"message@1.0">>, M1}, [], Opts), + ?event({dedup_handle, + {key, Key}, + {msg1, M1}, + {msg2, M2}, + {subject_key, SubjectKey}, + {subject, Subject} + }), + case {FirstPass, Subject} of + {false, _} -> + % If this is not the first pass, we can skip the deduplication + % check. + {ok, M1}; + {true, not_found} -> + % If the subject key is not present, we can skip the deduplication + % check. + {ok, M1}; + {true, _} -> + % If this is the first pass, we need to check if the subject has + % already been seen. + SubjectID = hb_message:id(Subject, all), + ?event({dedup_checking, {existing, DedupList}}), + case lists:member(SubjectID, DedupList) of true -> - ?event({already_seen, Msg2ID}), + ?event({already_seen, SubjectID}), {skip, M1}; false -> - ?event({not_seen, Msg2ID}), - M3 = hb_converge:set( - M1, - #{ <<"Dedup">> => [Msg2ID|Dedup] } - ), + ?event({not_seen, SubjectID}), + M3 = + hb_ao:set( + M1, + #{ <<"dedup">> => [SubjectID|DedupList] }, + Opts + ), ?event({dedup_updated, M3}), {ok, M3} - end; - Pass -> - ?event({multipass_detected, skipping_dedup, {pass, Pass}}), - {ok, M1} + end end. %%% Tests dedup_test() -> + hb:init(), % Create a stack with a dedup device and 2 devices that will append to a - % `Result` key. + % `Result' key. Msg = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"dedup-subject">> => <<"request">>, + <<"device-stack">> => #{ - <<"1">> => <<"Dedup/1.0">>, + <<"1">> => <<"dedup@1.0">>, <<"2">> => dev_stack:generate_append_device(<<"+D2">>), <<"3">> => dev_stack:generate_append_device(<<"+D3">>) }, - result => <<"INIT">> + <<"result">> => <<"INIT">> }, % Send the same message twice, with the same binary. - {ok, Msg2} = hb_converge:resolve(Msg, #{ path => append, bin => <<"_">> }, #{}), - {ok, Msg3} = hb_converge:resolve(Msg2, #{ path => append, bin => <<"_">> }, #{}), + {ok, Msg2} = hb_ao:resolve(Msg, + #{ <<"path">> => <<"append">>, <<"bin">> => <<"_">> }, #{}), + {ok, Msg3} = hb_ao:resolve(Msg2, + #{ <<"path">> => <<"append">>, <<"bin">> => <<"_">> }, #{}), % Send the same message twice, with another binary. - {ok, Msg4} = hb_converge:resolve(Msg3, #{ path => append, bin => <<"/">> }, #{}), - {ok, Msg5} = hb_converge:resolve(Msg4, #{ path => append, bin => <<"/">> }, #{}), + {ok, Msg4} = hb_ao:resolve(Msg3, + #{ <<"path">> => <<"append">>, <<"bin">> => <<"/">> }, #{}), + {ok, Msg5} = hb_ao:resolve(Msg4, + #{ <<"path">> => <<"append">>, <<"bin">> => <<"/">> }, #{}), % Ensure that downstream devices have only seen each message once. ?assertMatch( - #{ result := <<"INIT+D2_+D3_+D2/+D3/">> }, + #{ <<"result">> := <<"INIT+D2_+D3_+D2/+D3/">> }, Msg5 ). dedup_with_multipass_test() -> % Create a stack with a dedup device and 2 devices that will append to a - % `Result` key and a `Multipass` device that will repeat the message for + % `Result' key and a `Multipass' device that will repeat the message for % an additional pass. We want to ensure that Multipass is not hindered by % the dedup device. Msg = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"dedup-subject">> => <<"request">>, + <<"device-stack">> => #{ - <<"1">> => <<"Dedup/1.0">>, + <<"1">> => <<"dedup@1.0">>, <<"2">> => dev_stack:generate_append_device(<<"+D2">>), <<"3">> => dev_stack:generate_append_device(<<"+D3">>), - <<"4">> => <<"Multipass/1.0">> + <<"4">> => <<"multipass@1.0">> }, - result => <<"INIT">>, - <<"Passes">> => 2 + <<"result">> => <<"INIT">>, + <<"passes">> => 2 }, % Send the same message twice, with the same binary. - {ok, Msg2} = hb_converge:resolve(Msg, #{ path => append, bin => <<"_">> }, #{}), - {ok, Msg3} = hb_converge:resolve(Msg2, #{ path => append, bin => <<"_">> }, #{}), + {ok, Msg2} = hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"_">> }, #{}), + {ok, Msg3} = hb_ao:resolve(Msg2, #{ <<"path">> => <<"append">>, <<"bin">> => <<"_">> }, #{}), % Send the same message twice, with another binary. - {ok, Msg4} = hb_converge:resolve(Msg3, #{ path => append, bin => <<"/">> }, #{}), - {ok, Msg5} = hb_converge:resolve(Msg4, #{ path => append, bin => <<"/">> }, #{}), + {ok, Msg4} = hb_ao:resolve(Msg3, #{ <<"path">> => <<"append">>, <<"bin">> => <<"/">> }, #{}), + {ok, Msg5} = hb_ao:resolve(Msg4, #{ <<"path">> => <<"append">>, <<"bin">> => <<"/">> }, #{}), % Ensure that downstream devices have only seen each message once. ?assertMatch( - #{ result := <<"INIT+D2_+D3_+D2_+D3_+D2/+D3/+D2/+D3/">> }, + #{ <<"result">> := <<"INIT+D2_+D3_+D2_+D3_+D2/+D3/+D2/+D3/">> }, Msg5 - ). + ). \ No newline at end of file diff --git a/src/dev_delegated_compute.erl b/src/dev_delegated_compute.erl new file mode 100644 index 000000000..315500f95 --- /dev/null +++ b/src/dev_delegated_compute.erl @@ -0,0 +1,200 @@ +%%% @doc Simple wrapper module that enables compute on remote machines, +%%% implementing the JSON-Iface. This can be used either as a standalone, to +%%% bring trusted results into the local node, or as the `Execution-Device' of +%%% an AO process. +-module(dev_delegated_compute). +-export([init/3, compute/3, normalize/3, snapshot/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Initialize or normalize the compute-lite device. For now, we don't +%% need to do anything special here. +init(Msg1, _Msg2, _Opts) -> + {ok, Msg1}. + +%% @doc We assume that the compute engine stores its own internal state, +%% with snapshots triggered only when HyperBEAM requests them. Subsequently, +%% to load a snapshot, we just need to return the original message. +normalize(Msg1, _Msg2, Opts) -> + hb_ao:set(Msg1, #{ <<"snapshot">> => unset }, Opts). + +%% @doc Call the delegated server to compute the result. The endpoint is +%% `POST /compute' and the body is the JSON-encoded message that we want to +%% evaluate. +compute(Msg1, Msg2, Opts) -> + OutputPrefix = dev_stack:prefix(Msg1, Msg2, Opts), + % Extract the process ID - this identifies which process to run compute + % against. + ProcessID = get_process_id(Msg1, Msg2, Opts), + % If request is an assignment, we will compute the result + % Otherwise, it is a dryrun + Type = hb_ao:get(<<"type">>, Msg2, not_found, Opts), + ?event({doing_delegated_compute, {msg2, Msg2}, {type, Type}}), + % Execute the compute via external CU + case Type of + <<"assignment">> -> + Slot = hb_ao:get(<<"slot">>, Msg2, Opts), + Res = do_compute(ProcessID, Msg2, Opts); + _ -> + Slot = dryrun, + Res = do_dryrun(ProcessID, Msg2, Opts) + end, + handle_relay_response(Msg1, Msg2, Opts, Res, OutputPrefix, ProcessID, Slot). + +%% @doc Execute computation on a remote machine via relay and the JSON-Iface. +do_compute(ProcID, Msg2, Opts) -> + ?event({do_compute_msg, {req, Msg2}}), + Slot = hb_ao:get(<<"slot">>, Msg2, Opts), + {ok, AOS2 = #{ <<"body">> := Body }} = + dev_scheduler_formats:assignments_to_aos2( + ProcID, + #{ + Slot => Msg2 + }, + false, + Opts + ), + ?event({do_compute_body, {aos2, {string, Body}}}), + % Send to external CU via relay using /result endpoint + Response = + do_relay( + <<"POST">>, + <<"/result/", (hb_util:bin(Slot))/binary, "?process-id=", ProcID/binary>>, + Body, + AOS2, + Opts#{ + hashpath => ignore, + cache_control => [<<"no-store">>, <<"no-cache">>] + } + ), + extract_json_res(Response, Opts). + +%% @doc Execute dry-run computation on a remote machine via relay and use +%% the JSON-Iface to decode the response. +do_dryrun(ProcID, Msg2, Opts) -> + ?event({do_dryrun_msg, {req, Msg2}}), + % Remove commitments from the message before sending to the external CU + Body = + hb_json:encode( + dev_json_iface:message_to_json_struct( + hb_maps:without([<<"commitments">>], Msg2, Opts), + Opts + ) + ), + ?event({do_dryrun_body, {string, Body}}), + % Send to external CU via relay using /dry-run endpoint + Response = do_relay( + <<"POST">>, + <<"/dry-run?process-id=", ProcID/binary>>, + Body, + #{}, + Opts#{ + hashpath => ignore, + cache_control => [<<"no-store">>, <<"no-cache">>] + } + ), + extract_json_res(Response, Opts). + +do_relay(Method, Path, Body, AOS2, Opts) -> + hb_ao:resolve( + #{ + <<"device">> => <<"relay@1.0">>, + <<"content-type">> => <<"application/json">> + }, + AOS2#{ + <<"path">> => <<"call">>, + <<"relay-method">> => Method, + <<"relay-body">> => Body, + <<"relay-path">> => Path, + <<"content-type">> => <<"application/json">> + }, + Opts + ). + +%% @doc Extract the JSON response from the delegated compute response. +extract_json_res(Response, Opts) -> + case Response of + {ok, Res} -> + JSONRes = hb_ao:get(<<"body">>, Res, Opts), + ?event({ + delegated_compute_res_metadata, + {req, hb_maps:without([<<"body">>], Res, Opts)} + }), + {ok, JSONRes}; + {Err, Error} when Err == error; Err == failure -> + {error, Error} + end. + +get_process_id(Msg1, Msg2, Opts) -> + RawProcessID = dev_process:process_id(Msg1, #{}, Opts), + case RawProcessID of + not_found -> hb_ao:get(<<"process-id">>, Msg2, Opts); + ProcID -> ProcID + end. + +%% @doc Handle the response from the delegated compute server. Assumes that the +%% response is in AOS2-style format, decoding with the JSON-Iface. +handle_relay_response(Msg1, Msg2, Opts, Response, OutputPrefix, ProcessID, Slot) -> + case Response of + {ok, JSONRes} -> + ?event( + {compute_lite_res, + {process_id, ProcessID}, + {slot, Slot}, + {json_res, {string, JSONRes}}, + {req, Msg2} + } + ), + {ok, Msg} = dev_json_iface:json_to_message(JSONRes, Opts), + {ok, + hb_ao:set( + Msg1, + #{ + <> => Msg, + <> => + #{ + <<"content-type">> => <<"application/json">>, + <<"body">> => JSONRes + } + }, + Opts + ) + }; + {error, Error} -> + {error, Error} + end. + +%% @doc Generate a snapshot of a running computation by calling the +%% `GET /snapshot' endpoint. +snapshot(Msg, Msg2, Opts) -> + ?event({snapshotting, {req, Msg2}}), + ProcID = dev_process:process_id(Msg, #{}, Opts), + Res = + hb_ao:resolve( + #{ + <<"device">> => <<"relay@1.0">>, + <<"content-type">> => <<"application/json">> + }, + #{ + <<"path">> => <<"call">>, + <<"relay-method">> => <<"POST">>, + <<"relay-path">> => <<"/snapshot/", ProcID/binary>>, + <<"content-type">> => <<"application/json">>, + <<"body">> => <<"{}">> + }, + Opts#{ + hashpath => ignore, + cache_control => [<<"no-store">>, <<"no-cache">>] + } + ), + ?event({snapshotting_result, Res}), + case Res of + {ok, Response} -> + {ok, Response}; + {error, Error} -> + {ok, + #{ + <<"error">> => <<"No checkpoint produced.">>, + <<"error-details">> => Error + }} + end. \ No newline at end of file diff --git a/src/dev_faff.erl b/src/dev_faff.erl new file mode 100644 index 000000000..e047262e4 --- /dev/null +++ b/src/dev_faff.erl @@ -0,0 +1,46 @@ +%%% @doc A module that implements a 'friends and family' pricing policy. +%%% It will allow users to process requests only if their addresses are +%%% in the allow-list for the node. +%%% +%%% Fundamentally against the spirit of permissionlessness, but it is useful if +%%% you are running a node for your own purposes and would not like to allow +%%% others to make use of it -- even for a fee. It also serves as a useful +%%% example of how to implement a custom pricing policy, as it implements stubs +%%% for both the pricing and ledger P4 APIs. +-module(dev_faff). +%%% Pricing API +%%% We only implement `estimate/3' as we do not want to charge for requests, so +%%% we are fine with the estimate being the same as the price. +-export([estimate/3]). +%%% Ledger API +%%% We need to implement `debit/3' as it is required by the ledger API, but we +%%% do not want to charge for requests, so we return `ok' and do not actually +%%% debit the user's account. Similarly, we are not interested in taking payments +%%% from users, so we do not implement `credit/3'. +-export([charge/3]). +-include("include/hb.hrl"). + +%% @doc Decide whether or not to service a request from a given address. +estimate(_, Msg, NodeMsg) -> + ?event(payment, {estimate, {msg, Msg}}), + % Check if the address is in the allow-list. + case is_admissible(Msg, NodeMsg) of + true -> {ok, 0}; + false -> {ok, <<"infinity">>} + end. + +%% @doc Check whether all of the signers of the request are in the allow-list. +is_admissible(Msg, NodeMsg) -> + AllowList = hb_opts:get(faff_allow_list, [], NodeMsg), + Req = hb_ao:get(<<"request">>, Msg, NodeMsg), + Signers = hb_message:signers(Req, NodeMsg), + ?event(payment, {is_admissible, {signers, Signers}, {allow_list, AllowList}}), + lists:all( + fun(Signer) -> lists:member(Signer, AllowList) end, + Signers + ). + +%% @doc Charge the user's account if the request is allowed. +charge(_, Req, _NodeMsg) -> + ?event(payment, {charge, Req}), + {ok, true}. diff --git a/src/dev_genesis_wasm.erl b/src/dev_genesis_wasm.erl new file mode 100644 index 000000000..d7ad73731 --- /dev/null +++ b/src/dev_genesis_wasm.erl @@ -0,0 +1,797 @@ +%%% @doc A device that mimics an environment suitable for `legacynet' AO +%%% processes, using HyperBEAM infrastructure. This allows existing `legacynet' +%%% AO process definitions to be used in HyperBEAM. +-module(dev_genesis_wasm). +-export([init/3, compute/3, normalize/3, snapshot/3]). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("include/hb.hrl"). + +%%% Timeout for legacy CU status check. +-define(STATUS_TIMEOUT, 100). + +%% @doc Initialize the device. +init(Msg, _Msg2, _Opts) -> {ok, Msg}. + +%% @doc Normalize the device. +normalize(Msg, Msg2, Opts) -> + dev_delegated_compute:normalize(Msg, Msg2, Opts). + +%% @doc Genesis-wasm device compute handler. +%% Normal compute execution through external CU with state persistence +compute(Msg, Msg2, Opts) -> + % Validate whether the genesis-wasm feature is enabled. + case delegate_request(Msg, Msg2, Opts) of + {ok, Msg3} -> + % Resolve the `patch@1.0' device. + {ok, Msg4} = + hb_ao:resolve( + Msg3, + { + as, + <<"patch@1.0">>, + Msg2#{ <<"patch-from">> => <<"/results/outbox">> } + }, + Opts + ), + % Return the patched message. + {ok, Msg4}; + {error, Error} -> + % Return the error. + {error, Error} + end. + +%% @doc Snapshot the state of the process via the `delegated-compute@1.0' device. +snapshot(Msg, Msg2, Opts) -> + delegate_request(Msg, Msg2, Opts). + +%% @doc Proxy a request to the delegated-compute@1.0 device, ensuring that +%% the server is running. +delegate_request(Msg, Msg2, Opts) -> + % Validate whether the genesis-wasm feature is enabled. + case ensure_started(Opts) of + true -> + do_compute(Msg, Msg2, Opts); + false -> + % Return an error if the genesis-wasm feature is disabled. + {error, #{ + <<"status">> => 500, + <<"message">> => + <<"HyperBEAM was not compiled with genesis-wasm@1.0 on " + "this node.">> + }} + end. + + +%% @doc Handle normal compute execution with state persistence (GET method). +do_compute(Msg, Msg2, Opts) -> + % Resolve the `delegated-compute@1.0' device. + case hb_ao:resolve(Msg, {as, <<"delegated-compute@1.0">>, Msg2}, Opts) of + {ok, Msg3} -> + PatchResult = + hb_ao:resolve( + Msg3, + { + as, + <<"patch@1.0">>, + Msg2#{ <<"patch-from">> => <<"/results/outbox">> } + }, + Opts + ), + % Resolve the `patch@1.0' device. + case PatchResult of + {ok, Msg4} -> + % Return the patched message. + {ok, Msg4}; + {error, Error} -> + % Return the error. + {error, Error} + end; + {error, Error} -> + % Return the error. + {error, Error} + end. + +%% @doc Ensure the local `genesis-wasm@1.0' is live. If it not, start it. +ensure_started(Opts) -> + % Check if the `genesis-wasm@1.0' device is already running. The presence + % of the registered name implies its availability. + {ok, Cwd} = file:get_cwd(), + ?event({ensure_started, cwd, Cwd}), + % Determine path based on whether we're in a release or development + GenesisWasmServerDir = + case init:get_argument(mode) of + {ok, [["embedded"]]} -> + % We're in release mode - genesis-wasm-server is in the release root + filename:join([Cwd, "genesis-wasm-server"]); + _ -> + % We're in development mode - look in the build directory + DevPath = + filename:join( + [ + Cwd, + "_build", + "genesis_wasm", + "genesis-wasm-server" + ] + ), + case filelib:is_dir(DevPath) of + true -> DevPath; + false -> filename:join([Cwd, "genesis-wasm-server"]) % Fallback + end + end, + ?event({ensure_started, genesis_wasm_server_dir, GenesisWasmServerDir}), + ?event({ensure_started, genesis_wasm, self()}), + IsRunning = is_genesis_wasm_server_running(Opts), + IsCompiled = hb_features:genesis_wasm(), + GenWASMProc = is_pid(hb_name:lookup(<<"genesis-wasm@1.0">>)), + case IsRunning orelse (IsCompiled andalso GenWASMProc) of + true -> + % If it is, do nothing. + true; + false -> + % The device is not running, so we need to start it. + PID = + spawn( + fun() -> + ?event({genesis_wasm_booting, {pid, self()}}), + NodeURL = + "http://localhost:" ++ + integer_to_list(hb_opts:get(port, no_port, Opts)), + RelativeDBDir = + hb_util:list( + hb_opts:get( + genesis_wasm_db_dir, + "cache-mainnet/genesis-wasm", + Opts + ) + ), + DBDir = + filename:absname(RelativeDBDir), + CheckpointDir = + filename:absname( + hb_util:list( + hb_opts:get( + genesis_wasm_checkpoints_dir, + RelativeDBDir ++ "/checkpoints", + Opts + ) + ) + ), + DatabaseUrl = filename:absname(DBDir ++ "/genesis-wasm-db"), + filelib:ensure_path(DBDir), + filelib:ensure_path(CheckpointDir), + Port = + open_port( + {spawn_executable, + filename:join( + [ + GenesisWasmServerDir, + "launch-monitored.sh" + ] + ) + }, + [ + binary, use_stdio, stderr_to_stdout, + {args, Args = [ + "npm", + "--prefix", + GenesisWasmServerDir, + "run", + "start" + ]}, + {env, + Env = [ + {"UNIT_MODE", "hbu"}, + {"HB_URL", NodeURL}, + {"PORT", + integer_to_list( + hb_opts:get( + genesis_wasm_port, + 6363, + Opts + ) + ) + }, + {"DB_URL", DatabaseUrl}, + {"NODE_CONFIG_ENV", "production"}, + {"DEFAULT_LOG_LEVEL", + hb_util:list( + hb_opts:get( + genesis_wasm_log_level, + "error", + Opts + ) + ) + }, + {"WALLET_FILE", + filename:absname( + hb_util:list( + hb_opts:get( + priv_key_location, + no_key, + Opts + ) + ) + ) + }, + {"DISABLE_PROCESS_FILE_CHECKPOINT_CREATION", "false"}, + {"PROCESS_MEMORY_FILE_CHECKPOINTS_DIR", CheckpointDir} + ] + } + ] + ), + ?event({genesis_wasm_port_opened, {port, Port}}), + ?event( + debug_genesis, + {started_genesis_wasm, + {args, Args}, + {env, maps:from_list(Env)} + } + ), + collect_events(Port) + end + ), + hb_name:register(<<"genesis-wasm@1.0">>, PID), + ?event({genesis_wasm_starting, {pid, PID}}), + % Wait for the device to start. + hb_util:until( + fun() -> + receive after 2000 -> ok end, + Status = is_genesis_wasm_server_running(Opts), + ?event({genesis_wasm_boot_wait, {received_status, Status}}), + Status + end + ), + ?event({genesis_wasm_started, {pid, PID}}), + true + end. + +%% @doc Check if the genesis-wasm server is running, using the cached process ID +%% if available. +is_genesis_wasm_server_running(Opts) -> + case get(genesis_wasm_pid) of + undefined -> + ?event(genesis_wasm_pinging_server), + Parent = self(), + PID = spawn( + fun() -> + ?event({genesis_wasm_get_info_endpoint, {worker, self()}}), + Parent ! {ok, self(), status(Opts)} + end + ), + receive + {ok, PID, Status} -> + put(genesis_wasm_pid, Status), + ?event({genesis_wasm_received_status, Status}), + Status + after ?STATUS_TIMEOUT -> + ?event({genesis_wasm_status_check, timeout}), + erlang:exit(PID, kill), + false + end; + _ -> true + end. + +%% @doc Check if the genesis-wasm server is running by requesting its status +%% endpoint. +status(Opts) -> + ServerPort = + integer_to_binary( + hb_opts:get( + genesis_wasm_port, + 6363, + Opts + ) + ), + try hb_http:get(<<"http://localhost:", ServerPort/binary, "/status">>, Opts) of + {ok, Res} -> + ?event({genesis_wasm_status_check, {res, Res}}), + true; + Err -> + ?event({genesis_wasm_status_check, {err, Err}}), + false + catch + _:Err -> + ?event({genesis_wasm_status_check, {error, Err}}), + false + end. + +%% @doc Collect events from the port and log them. +collect_events(Port) -> + collect_events(Port, <<>>). +collect_events(Port, Acc) -> + receive + {Port, {data, Data}} -> + collect_events(Port, + log_server_events(<>) + ); + stop -> + port_close(Port), + ?event(genesis_wasm_stopped, {pid, self()}), + ok + end. + +%% @doc Log lines of output from the genesis-wasm server. +log_server_events(Bin) when is_binary(Bin) -> + log_server_events(binary:split(Bin, <<"\n">>, [global])); +log_server_events([Remaining]) -> Remaining; +log_server_events([Line | Rest]) -> + ?event(genesis_wasm_server, {server_logged, {string, Line}}), + log_server_events(Rest). + +%%% Tests + +-ifdef(ENABLE_GENESIS_WASM). +test_base_process() -> + test_base_process(#{}). +test_base_process(Opts) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + Address = hb_util:human_id(ar_wallet:to_address(Wallet)), + hb_message:commit(#{ + <<"device">> => <<"process@1.0">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"scheduler-location">> => Address, + <<"type">> => <<"Process">>, + <<"test-random-seed">> => rand:uniform(1337) + }, #{ priv_wallet => Wallet }). + +test_wasm_process(WASMImage) -> + test_wasm_process(WASMImage, #{}). +test_wasm_process(WASMImage, Opts) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + #{ <<"image">> := WASMImageID } = dev_wasm:cache_wasm_image(WASMImage, Opts), + hb_message:commit( + maps:merge( + hb_message:uncommitted(test_base_process(Opts)), + #{ + <<"execution-device">> => <<"stack@1.0">>, + <<"device-stack">> => [<<"WASM-64@1.0">>], + <<"image">> => WASMImageID + } + ), + #{ priv_wallet => Wallet } + ). + +test_wasm_stack_process(Opts, Stack) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + Address = hb_util:human_id(ar_wallet:to_address(Wallet)), + WASMProc = test_wasm_process(<<"test/aos-2-pure-xs.wasm">>, Opts), + hb_message:commit( + maps:merge( + hb_message:uncommitted(WASMProc), + #{ + <<"device-stack">> => Stack, + <<"execution-device">> => <<"genesis-wasm@1.0">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"patch-from">> => <<"/results/outbox">>, + <<"passes">> => 2, + <<"stack-keys">> => + [ + <<"init">>, + <<"compute">>, + <<"snapshot">>, + <<"normalize">>, + <<"compute">> + ], + <<"scheduler">> => Address, + <<"authority">> => Address, + <<"module">> => <<"URgYpPQzvxxfYQtjrIQ116bl3YBfcImo3JEnNo8Hlrk">>, + <<"data-protocol">> => <<"ao">>, + <<"type">> => <<"Process">> + } + ), + #{ priv_wallet => Wallet } + ). + +test_genesis_wasm_process() -> + Opts = #{ + genesis_wasm_db_dir => "cache-mainnet-test/genesis-wasm", + genesis_wasm_checkpoints_dir => "cache-mainnet-test/genesis-wasm/checkpoints", + genesis_wasm_log_level => "error", + genesis_wasm_port => 6363, + execution_device => <<"genesis-wasm@1.0">> + }, + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + Address = hb_util:human_id(ar_wallet:to_address(Wallet)), + WASMProc = test_wasm_process(<<"test/aos-2-pure-xs.wasm">>, Opts), + hb_message:commit( + maps:merge( + hb_message:uncommitted(WASMProc), + #{ + <<"execution-device">> => <<"genesis-wasm@1.0">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"push-device">> => <<"push@1.0">>, + <<"patch-from">> => <<"/results/outbox">>, + <<"passes">> => 1, + <<"scheduler">> => Address, + <<"authority">> => Address, + <<"module">> => <<"URgYpPQzvxxfYQtjrIQ116bl3YBfcImo3JEnNo8Hlrk">>, + <<"data-protocol">> => <<"ao">>, + <<"type">> => <<"Process">> + }), + #{ priv_wallet => Wallet } + ). + +schedule_test_message(Msg1, Text) -> + schedule_test_message(Msg1, Text, #{}). +schedule_test_message(Msg1, Text, MsgBase) -> + Wallet = hb:wallet(), + UncommittedBase = hb_message:uncommitted(MsgBase), + Msg2 = + hb_message:commit(#{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + UncommittedBase#{ + <<"type">> => <<"Message">>, + <<"test-label">> => Text + }, + #{ priv_wallet => Wallet } + ) + }, + #{ priv_wallet => Wallet } + ), + hb_ao:resolve(Msg1, Msg2, #{}). + +schedule_aos_call(Msg1, Code) -> + schedule_aos_call(Msg1, Code, <<"Eval">>, #{}). +schedule_aos_call(Msg1, Code, Action) -> + schedule_aos_call(Msg1, Code, Action, #{}). +schedule_aos_call(Msg1, Code, Action, Opts) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + ProcID = hb_message:id(Msg1, all), + Msg2 = + hb_message:commit( + #{ + <<"action">> => Action, + <<"data">> => Code, + <<"target">> => ProcID + }, + #{ priv_wallet => Wallet } + ), + schedule_test_message(Msg1, <<"TEST MSG">>, Msg2). + +spawn_and_execute_slot_test_() -> + { timeout, 900, fun spawn_and_execute_slot/0 }. +spawn_and_execute_slot() -> + application:ensure_all_started(hb), + Opts = #{ + priv_wallet => hb:wallet(), + cache_control => <<"always">>, + store => hb_opts:get(store) + }, + Msg1 = test_genesis_wasm_process(), + hb_cache:write(Msg1, Opts), + {ok, _SchedInit} = + hb_ao:resolve( + Msg1, + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => Msg1 + }, + Opts + ), + {ok, _} = schedule_aos_call(Msg1, <<"return 1+1">>), + {ok, _} = schedule_aos_call(Msg1, <<"return 2+2">>), + {ok, SchedulerRes} = + hb_ao:resolve(Msg1, #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"schedule">> + }, Opts), + % Verify process message is scheduled first + ?assertMatch( + <<"Process">>, + hb_ao:get(<<"assignments/0/body/type">>, SchedulerRes) + ), + % Verify messages are scheduled + ?assertMatch( + <<"return 1+1">>, + hb_ao:get(<<"assignments/1/body/data">>, SchedulerRes) + ), + ?assertMatch( + <<"return 2+2">>, + hb_ao:get(<<"assignments/2/body/data">>, SchedulerRes) + ), + {ok, Result} = hb_ao:resolve(Msg1, #{ <<"path">> => <<"now">> }, Opts), + ?assertEqual(<<"4">>, hb_ao:get(<<"results/data">>, Result)). + +compare_result_genesis_wasm_and_wasm_test_() -> + { timeout, 900, fun compare_result_genesis_wasm_and_wasm/0 }. +compare_result_genesis_wasm_and_wasm() -> + application:ensure_all_started(hb), + Opts = #{ + priv_wallet => hb:wallet(), + cache_control => <<"always">>, + store => hb_opts:get(store) + }, + % Test with genesis-wasm + MsgGenesisWasm = test_genesis_wasm_process(), + hb_cache:write(MsgGenesisWasm, Opts), + {ok, _SchedInitGenesisWasm} = + hb_ao:resolve( + MsgGenesisWasm, + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => MsgGenesisWasm + }, + Opts + ), + % Test with wasm + MsgWasm = test_wasm_stack_process(Opts, [ + <<"WASI@1.0">>, + <<"JSON-Iface@1.0">>, + <<"WASM-64@1.0">>, + <<"Multipass@1.0">> + ]), + hb_cache:write(MsgWasm, Opts), + {ok, _SchedInitWasm} = + hb_ao:resolve( + MsgWasm, + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => MsgWasm + }, + Opts + ), + % Schedule messages + {ok, _} = schedule_aos_call(MsgGenesisWasm, <<"return 1+1">>), + {ok, _} = schedule_aos_call(MsgGenesisWasm, <<"return 2+2">>), + {ok, _} = schedule_aos_call(MsgWasm, <<"return 1+1">>), + {ok, _} = schedule_aos_call(MsgWasm, <<"return 2+2">>), + % Get results + {ok, ResultGenesisWasm} = + hb_ao:resolve( + MsgGenesisWasm, + #{ <<"path">> => <<"now">> }, + Opts + ), + {ok, ResultWasm} = + hb_ao:resolve( + MsgWasm, + #{ <<"path">> => <<"now">> }, + Opts + ), + ?assertEqual( + hb_ao:get(<<"results/data">>, ResultGenesisWasm), + hb_ao:get(<<"results/data">>, ResultWasm) + ). + +send_message_between_genesis_wasm_processes_test_() -> + { timeout, 900, fun send_message_between_genesis_wasm_processes/0 }. +send_message_between_genesis_wasm_processes() -> + application:ensure_all_started(hb), + Opts = #{ + priv_wallet => hb:wallet(), + cache_control => <<"always">>, + store => hb_opts:get(store) + }, + % Create receiver process with handler + MsgReceiver = test_genesis_wasm_process(), + hb_cache:write(MsgReceiver, Opts), + ProcId = dev_process:process_id(MsgReceiver, #{}, #{}), + {ok, _SchedInitReceiver} = + hb_ao:resolve( + MsgReceiver, + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => MsgReceiver + }, + Opts + ), + schedule_aos_call(MsgReceiver, <<"Number = 10">>), + schedule_aos_call(MsgReceiver, <<" + Handlers.add('foo', function(msg) + print(\"Number: \" .. Number * 2) + return Number * 2 end) + ">>), + schedule_aos_call(MsgReceiver, <<"return Number">>), + {ok, ResultReceiver} = hb_ao:resolve(MsgReceiver, <<"now">>, Opts), + ?assertEqual(<<"10">>, hb_ao:get(<<"results/data">>, ResultReceiver)), + % Create sender process to send message to receiver + MsgSender = test_genesis_wasm_process(), + hb_cache:write(MsgSender, Opts), + {ok, _SchedInitSender} = + hb_ao:resolve( + MsgSender, + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => MsgSender + }, + Opts + ), + {ok, SendMsgToReceiver} = + schedule_aos_call( + MsgSender, + <<"Send({ Target = \"", ProcId/binary, "\", Action = \"foo\" })">> + ), + {ok, ResultSender} = hb_ao:resolve(MsgSender, <<"now">>, Opts), + {ok, Slot} = hb_ao:resolve(SendMsgToReceiver, <<"slot">>, Opts), + {ok, Res} = + hb_ao:resolve( + MsgSender, + #{ + <<"path">> => <<"push">>, + <<"slot">> => Slot, + <<"result-depth">> => 1 + }, + Opts + ), + % Get schedule for receiver + {ok, ScheduleReceiver} = + hb_ao:resolve( + MsgReceiver, + #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"schedule">> + }, + Opts + ), + ?assertEqual( + <<"foo">>, + hb_ao:get(<<"assignments/4/body/action">>, ScheduleReceiver) + ), + {ok, NewResultReceiver} = hb_ao:resolve(MsgReceiver, <<"now">>, Opts), + ?assertEqual( + <<"Number: 20">>, + hb_ao:get(<<"results/data">>, NewResultReceiver) + ). + +dryrun_genesis_wasm_test_() -> + { timeout, 900, fun dryrun_genesis_wasm/0 }. +dryrun_genesis_wasm() -> + application:ensure_all_started(hb), + Opts = #{ + priv_wallet => hb:wallet(), + cache_control => <<"always">>, + store => hb_opts:get(store) + }, + % Set up process with increment handler to receive messages + ProcReceiver = test_genesis_wasm_process(), + hb_cache:write(ProcReceiver, #{}), + {ok, _SchedInit1} = + hb_ao:resolve( + ProcReceiver, + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => ProcReceiver + }, + Opts + ), + ProcReceiverId = dev_process:process_id(ProcReceiver, #{}, #{}), + % Initialize increment handler + {ok, _} = schedule_aos_call(ProcReceiver, <<" + Number = Number or 5 + Handlers.add('Increment', function(msg) + Number = Number + 1 + ao.send({ Target = msg.From, Data = 'The current number is ' .. Number .. '!' }) + return 'The current number is ' .. Number .. '!' + end) + ">>), + % Ensure Handlers were properly added + schedule_aos_call(ProcReceiver, <<"return #Handlers.list">>), + {ok, NumHandlers} = + hb_ao:resolve( + ProcReceiver, + <<"now/results/data">>, + Opts + ), + % _eval, _default, Increment + ?assertEqual(<<"3">>, NumHandlers), + + schedule_aos_call(ProcReceiver, <<"return Number">>), + {ok, InitialNumber} = + hb_ao:resolve( + ProcReceiver, + <<"now/results/data">>, + Opts + ), + % Number is initialized to 5 + ?assertEqual(<<"5">>, InitialNumber), + % Set up sender process to send Action: Increment to receiver + ProcSender = test_genesis_wasm_process(), + hb_cache:write(ProcSender, #{}), + {ok, _SchedInit2} = hb_ao:resolve( + ProcSender, + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => ProcSender + }, + Opts + ), + % First increment + push + {ok, ToPush} = + schedule_aos_call( + ProcSender, + << + "Send({ Target = \"", + (ProcReceiverId)/binary, + "\", Action = \"Increment\" })" + >> + ), + SlotToPush = hb_ao:get(<<"slot">>, ToPush, Opts), + ?assertEqual(1, SlotToPush), + {ok, PushRes1} = + hb_ao:resolve( + ProcSender, + #{ + <<"path">> => <<"push">>, + <<"slot">> => SlotToPush, + <<"result-depth">> => 1 + }, + Opts + ), + % Check that number incremented normally + schedule_aos_call(ProcReceiver, <<"return Number">>), + {ok, AfterIncrementResult} = + hb_ao:resolve( + ProcReceiver, + <<"now/results/data">>, + Opts + ), + ?assertEqual(<<"6">>, AfterIncrementResult), + + % Send another increment and push it + {ok, ToPush2} = + schedule_aos_call( + ProcSender, + << + "Send({ Target = \"", + (ProcReceiverId)/binary, + "\", Action = \"Increment\" })" + >> + ), + SlotToPush2 = hb_ao:get(<<"slot">>, ToPush2, Opts), + ?assertEqual(3, SlotToPush2), + {ok, PushRes2} = + hb_ao:resolve( + ProcSender, + #{ + <<"path">> => <<"push">>, + <<"slot">> => SlotToPush2, + <<"result-depth">> => 1 + }, + Opts + ), + % Check that number incremented normally + schedule_aos_call(ProcReceiver, <<"return Number">>), + {ok, AfterIncrementResult2} = + hb_ao:resolve( + ProcReceiver, + <<"now/results/data">>, + Opts + ), + ?assertEqual(<<"7">>, AfterIncrementResult2), + % Test dryrun by calling compute with no assignment + % Should return result without changing state + DryrunMsg = + hb_message:commit( + #{ + <<"path">> => <<"as/compute">>, + <<"as-device">> => <<"execution">>, + <<"action">> => <<"Increment">>, + <<"target">> => ProcReceiverId + }, + Opts + ), + {ok, DryrunResult} = hb_ao:resolve(ProcReceiver, DryrunMsg, Opts), + {ok, DryrunData} = + hb_ao:resolve(DryrunResult, <<"results/outbox/1/Data">>, Opts), + ?assertEqual(<<"The current number is 8!">>, DryrunData), + % Ensure that number did not increment + schedule_aos_call(ProcReceiver, <<"return Number">>), + {ok, AfterDryrunResult} = + hb_ao:resolve( + ProcReceiver, + <<"now/results/data">>, + Opts + ), + ?assertEqual(<<"7">>, AfterDryrunResult). +-endif. \ No newline at end of file diff --git a/src/dev_green_zone.erl b/src/dev_green_zone.erl new file mode 100644 index 000000000..1669b23b2 --- /dev/null +++ b/src/dev_green_zone.erl @@ -0,0 +1,785 @@ +%%% @doc The green zone device, which provides secure communication and identity +%%% management between trusted nodes. +%%% +%%% It handles node initialization, joining existing green zones, key exchange, +%%% and node identity cloning. All operations are protected by hardware +%%% commitment and encryption. +-module(dev_green_zone). +-export([info/1, info/3, join/3, init/3, become/3, key/3, is_trusted/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +%% @doc Controls which functions are exposed via the device API. +%% +%% This function defines the security boundary for the green zone device by +%% explicitly listing which functions are available through the API. +%% +%% @param _ Ignored parameter +%% @returns A map with the `exports' key containing a list of allowed functions +info(_) -> + #{ exports => [info, init, join, become, key, is_trusted] }. + +%% @doc Provides information about the green zone device and its API. +%% +%% This function returns detailed documentation about the device, including: +%% 1. A high-level description of the device's purpose +%% 2. Version information +%% 3. Available API endpoints with their parameters and descriptions +%% +%% @param _Msg1 Ignored parameter +%% @param _Msg2 Ignored parameter +%% @param _Opts A map of configuration options +%% @returns {ok, Map} containing the device information and documentation +info(_Msg1, _Msg2, _Opts) -> + InfoBody = #{ + <<"description">> => + <<"Green Zone secure communication and identity management for trusted nodes">>, + <<"version">> => <<"1.0">>, + <<"api">> => #{ + <<"info">> => #{ + <<"description">> => <<"Get device info">> + }, + <<"init">> => #{ + <<"description">> => <<"Initialize the green zone">>, + <<"details">> => + <<"Sets up the node's cryptographic identity with wallet and AES key">> + }, + <<"join">> => #{ + <<"description">> => <<"Join an existing green zone">>, + <<"required_node_opts">> => #{ + <<"green_zone_peer_location">> => <<"Target peer's address">>, + <<"green_zone_peer_id">> => <<"Target peer's unique identifier">> + } + }, + <<"key">> => #{ + <<"description">> => <<"Retrieve and encrypt the node's private key">>, + <<"details">> => + <<"Returns the node's private key encrypted with the shared AES key">> + }, + <<"become">> => #{ + <<"description">> => <<"Clone the identity of a target node">>, + <<"required_node_opts">> => #{ + <<"green_zone_peer_location">> => <<"Target peer's address">>, + <<"green_zone_peer_id">> => <<"Target peer's unique identifier">> + } + } + } + }, + {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. + +%% @doc Provides the default required options for a green zone. +%% +%% This function defines the baseline security requirements for nodes in a green zone: +%% 1. Restricts loading of remote devices and only allows trusted signers +%% 2. Limits to preloaded devices from the initiating machine +%% 3. Enforces specific store configuration +%% 4. Prevents route changes from the defaults +%% 5. Requires matching hooks across all peers +%% 6. Disables message scheduling to prevent conflicts +%% 7. Enforces a permanent state to prevent further configuration changes +%% +%% @param Opts A map of configuration options from which to derive defaults +%% @returns A map of required configuration options for the green zone +-spec default_zone_required_opts(Opts :: map()) -> map(). +default_zone_required_opts(Opts) -> + #{ + % trusted_device_signers => hb_opts:get(trusted_device_signers, [], Opts), + % load_remote_devices => hb_opts:get(load_remote_devices, false, Opts), + % preload_devices => hb_opts:get(preload_devices, [], Opts), + % % store => hb_opts:get(store, [], Opts), + % routes => hb_opts:get(routes, [], Opts), + % on => hb_opts:get(on, undefined, Opts), + % scheduling_mode => disabled, + % initialized => permanent + }. + +%% @doc Replace values of <<"self">> in a configuration map with corresponding values from Opts. +%% +%% This function iterates through all key-value pairs in the configuration map. +%% If a value is <<"self">>, it replaces that value with the result of +%% hb_opts:get(Key, not_found, Opts) where Key is the corresponding key. +%% +%% @param Config The configuration map to process +%% @param Opts The options map to fetch replacement values from +%% @returns A new map with <<"self">> values replaced +-spec replace_self_values(Config :: map(), Opts :: map()) -> map(). +replace_self_values(Config, Opts) -> + maps:map( + fun(Key, Value) -> + case Value of + <<"self">> -> + hb_opts:get(Key, not_found, Opts); + _ -> + Value + end + end, + Config + ). + +%% @doc Returns `true' if the request is signed by a trusted node. +is_trusted(_M1, Req, Opts) -> + Signers = hb_message:signers(Req, Opts), + {ok, + hb_util:bin( + lists:any( + fun(Signer) -> + lists:member( + Signer, + maps:keys(hb_opts:get(trusted_nodes, #{}, Opts)) + ) + end, + Signers + ) + ) + }. + +%% @doc Initialize the green zone for a node. +%% +%% This function performs the following operations: +%% 1. Validates the node's history to ensure this is a valid initialization +%% 2. Retrieves or creates a required configuration for the green zone +%% 3. Ensures a wallet (keypair) exists or creates a new one +%% 4. Generates a new 256-bit AES key for secure communication +%% 5. Updates the node's configuration with these cryptographic identities +%% +%% Config options in Opts map: +%% - green_zone_required_config: (Optional) Custom configuration requirements +%% - priv_wallet: (Optional) Existing wallet to use instead of creating a new one +%% - priv_green_zone_aes: (Optional) Existing AES key, if already part of a zone +%% +%% @param _M1 Ignored parameter +%% @param _M2 May contain a `required-config' map for custom requirements +%% @param Opts A map of configuration options +%% @returns `{ok, Binary}' on success with confirmation message, or +%% `{error, Binary}' on failure with error message. +-spec init(M1 :: term(), M2 :: term(), Opts :: map()) -> {ok, binary()} | {error, binary()}. +init(_M1, _M2, Opts) -> + ?event(green_zone, {init, start}), + case hb_opts:get(green_zone_initialized, false, Opts) of + true -> + {error, <<"Green zone already initialized.">>}; + false -> + RequiredConfig = hb_opts:get( + <<"green_zone_required_config">>, + default_zone_required_opts(Opts), + Opts + ), + % Process RequiredConfig to replace <<"self">> values with actual values from Opts + ProcessedRequiredConfig = replace_self_values(RequiredConfig, Opts), + ?event(green_zone, {init, required_config, ProcessedRequiredConfig}), + % Check if a wallet exists; create one if absent. + NodeWallet = case hb_opts:get(priv_wallet, undefined, Opts) of + undefined -> + ?event(green_zone, {init, wallet, missing}), + hb:wallet(); + ExistingWallet -> + ?event(green_zone, {init, wallet, found}), + ExistingWallet + end, + % Generate a new 256-bit AES key if we have not already joined + % a green zone. + GreenZoneAES = + case hb_opts:get(priv_green_zone_aes, undefined, Opts) of + undefined -> + ?event(green_zone, {init, aes_key, generated}), + crypto:strong_rand_bytes(32); + ExistingAES -> + ?event(green_zone, {init, aes_key, found}), + ExistingAES + end, + % Store the wallet, AES key, and an empty trusted nodes map. + hb_http_server:set_opts(NewOpts =Opts#{ + priv_wallet => NodeWallet, + priv_green_zone_aes => GreenZoneAES, + trusted_nodes => #{}, + green_zone_required_opts => ProcessedRequiredConfig, + green_zone_initialized => true + }), + try_mount_encrypted_volume(GreenZoneAES, NewOpts), + ?event(green_zone, {init, complete}), + {ok, <<"Green zone initialized successfully.">>} + end. + + +%% @doc Initiates the join process for a node to enter an existing green zone. +%% +%% This function performs the following operations depending on the state: +%% 1. Validates the node's history to ensure proper initialization +%% 2. Checks for target peer information (location and ID) +%% 3. If target peer is specified: +%% a. Generates a commitment report for the peer +%% b. Prepares and sends a POST request to the target peer +%% c. Verifies the response and decrypts the returned zone key +%% d. Updates local configuration with the shared AES key +%% 4. If no peer is specified, processes the join request locally +%% +%% Config options in Opts map: +%% - green_zone_peer_location: Target peer's address +%% - green_zone_peer_id: Target peer's unique identifier +%% - green_zone_adopt_config: +%% (Optional) Whether to adopt peer's configuration (default: true) +%% +%% @param M1 The join request message with target peer information +%% @param M2 Additional request details, may include adoption preferences +%% @param Opts A map of configuration options for join operations +%% @returns `{ok, Map}' on success with join response details, or +%% `{error, Binary}' on failure with error message. +-spec join(M1 :: term(), M2 :: term(), Opts :: map()) -> + {ok, map()} | {error, binary()}. +join(M1, M2, Opts) -> + ?event(green_zone, {join, start}), + PeerLocation = hb_opts:get(<<"green_zone_peer_location">>, undefined, Opts), + PeerID = hb_opts:get(<<"green_zone_peer_id">>, undefined, Opts), + Identities = hb_opts:get(identities, #{}, Opts), + HasGreenZoneIdentity = maps:is_key(<<"green-zone">>, Identities), + ?event(green_zone, {join_peer, PeerLocation, PeerID, HasGreenZoneIdentity}), + if (not HasGreenZoneIdentity) andalso (PeerLocation =/= undefined) andalso (PeerID =/= undefined) -> + join_peer(PeerLocation, PeerID, M1, M2, Opts); + true -> + validate_join(M1, M2, hb_cache:ensure_all_loaded(Opts, Opts)) + end. + +%% @doc Encrypts and provides the node's private key for secure sharing. +%% +%% This function performs the following operations: +%% 1. Retrieves the shared AES key and the node's wallet +%% 2. Verifies that the node is part of a green zone (has a shared AES key) +%% 3. Generates a random initialization vector (IV) for encryption +%% 4. Encrypts the node's private key using AES-256-GCM with the shared key +%% 5. Returns the encrypted key and IV for secure transmission +%% +%% Required configuration in Opts map: +%% - priv_green_zone_aes: The shared AES key for the green zone +%% - priv_wallet: The node's wallet containing the private key to encrypt +%% +%% @param _M1 Ignored parameter +%% @param _M2 Ignored parameter +%% @param Opts A map of configuration options +%% @returns `{ok, Map}' containing the encrypted key and IV on success, or +%% `{error, Binary}' if the node is not part of a green zone +-spec key(M1 :: term(), M2 :: term(), Opts :: map()) -> + {ok, map()} | {error, binary()}. +key(_M1, _M2, Opts) -> + ?event(green_zone, {get_key, start}), + % Retrieve the shared AES key and the node's wallet. + GreenZoneAES = hb_opts:get(priv_green_zone_aes, undefined, Opts), + Identities = hb_opts:get(identities, #{}, Opts), + Wallet = case maps:find(<<"green-zone">>, Identities) of + {ok, #{priv_wallet := GreenZoneWallet}} -> GreenZoneWallet; + _ -> hb_opts:get(priv_wallet, undefined, Opts) + end, + {{KeyType, Priv, Pub}, _PubKey} = Wallet, + ?event(green_zone, + {get_key, wallet, hb_util:human_id(ar_wallet:to_address(Pub))}), + case GreenZoneAES of + undefined -> + % Log error if no shared AES key is found. + ?event(green_zone, {get_key, error, <<"no aes key">>}), + {error, <<"Node not part of a green zone.">>}; + _ -> + % Generate an IV and encrypt the node's private key using AES-256-GCM. + IV = crypto:strong_rand_bytes(16), + {EncryptedKey, Tag} = crypto:crypto_one_time_aead( + aes_256_gcm, + GreenZoneAES, + IV, + term_to_binary({KeyType, Priv, Pub}), + <<>>, + true + ), + + % Log successful encryption of the private key. + ?event(green_zone, {get_key, encrypt, complete}), + {ok, #{ + <<"status">> => 200, + <<"encrypted_key">> => + base64:encode(<>), + <<"iv">> => base64:encode(IV) + }} + end. + +%% @doc Clones the identity of a target node in the green zone. +%% +%% This function performs the following operations: +%% 1. Retrieves target node location and ID from the configuration +%% 2. Verifies that the local node has a valid shared AES key +%% 3. Requests the target node's encrypted key via its key endpoint +%% 4. Verifies the response is from the expected peer +%% 5. Decrypts the target node's private key using the shared AES key +%% 6. Updates the local node's wallet with the target node's identity +%% +%% Required configuration in Opts map: +%% - green_zone_peer_location: Target node's address +%% - green_zone_peer_id: Target node's unique identifier +%% - priv_green_zone_aes: The shared AES key for the green zone +%% +%% @param _M1 Ignored parameter +%% @param _M2 Ignored parameter +%% @param Opts A map of configuration options +%% @returns `{ok, Map}' on success with confirmation details, or +%% `{error, Binary}' if the node is not part of a green zone or +%% identity adoption fails. +-spec become(M1 :: term(), M2 :: term(), Opts :: map()) -> + {ok, map()} | {error, binary()}. +become(_M1, _M2, Opts) -> + ?event(green_zone, {become, start}), + % 1. Retrieve the target node's address from the incoming message. + NodeLocation = hb_opts:get(<<"green_zone_peer_location">>, undefined, Opts), + NodeID = hb_opts:get(<<"green_zone_peer_id">>, undefined, Opts), + % 2. Check if the local node has a valid shared AES key. + GreenZoneAES = hb_opts:get(priv_green_zone_aes, undefined, Opts), + case GreenZoneAES of + undefined -> + % Shared AES key not found: node is not part of a green zone. + ?event(green_zone, {become, error, <<"no aes key">>}), + {error, <<"Node not part of a green zone.">>}; + _ -> + % 3. Request the target node's encrypted key from its key endpoint. + ?event(green_zone, {become, getting_key, NodeLocation, NodeID}), + {ok, KeyResp} = hb_http:get(NodeLocation, + <<"/~greenzone@1.0/key">>, Opts), + Signers = hb_message:signers(KeyResp, Opts), + case hb_message:verify(KeyResp, Signers, Opts) and + lists:member(NodeID, Signers) of + false -> + % The response is not from the expected peer. + {error, <<"Received incorrect response from peer!">>}; + true -> + finalize_become(KeyResp, NodeLocation, NodeID, + GreenZoneAES, Opts) + end + end. + +finalize_become(KeyResp, NodeLocation, NodeID, GreenZoneAES, Opts) -> + % 4. Decode the response to obtain the encrypted key and IV. + Combined = + base64:decode( + hb_ao:get(<<"encrypted_key">>, KeyResp, Opts)), + IV = base64:decode(hb_ao:get(<<"iv">>, KeyResp, Opts)), + % 5. Separate the ciphertext and the authentication tag. + CipherLen = byte_size(Combined) - 16, + <> = Combined, + % 6. Decrypt the ciphertext using AES-256-GCM with the shared AES + % key and IV. + DecryptedBin = crypto:crypto_one_time_aead( + aes_256_gcm, + GreenZoneAES, + IV, + Ciphertext, + <<>>, + Tag, + false + ), + OldWallet = hb_opts:get(priv_wallet, undefined, Opts), + OldWalletAddr = hb_util:human_id(ar_wallet:to_address(OldWallet)), + ?event(green_zone, {become, old_wallet, OldWalletAddr}), + % Print the decrypted binary + ?event(green_zone, {become, decrypted_bin, DecryptedBin}), + % 7. Convert the decrypted binary into the target node's keypair. + {KeyType, Priv, Pub} = binary_to_term(DecryptedBin), + % Print the keypair + ?event(green_zone, {become, keypair, Pub}), + % 8. Add the target node's keypair to the local node's identities. + GreenZoneWallet = {{KeyType, Priv, Pub}, {KeyType, Pub}}, + Identities = hb_opts:get(identities, #{}, Opts), + UpdatedIdentities = Identities#{ + <<"green-zone">> => #{ + priv_wallet => GreenZoneWallet + } + }, + NewOpts = Opts#{ + identities => UpdatedIdentities + }, + ok = + hb_http_server:set_opts( + NewOpts + ), + try_mount_encrypted_volume(GreenZoneWallet, NewOpts), + ?event(green_zone, {become, update_wallet, complete}), + {ok, #{ + <<"body">> => #{ + <<"message">> => <<"Successfully adopted target node identity">>, + <<"peer-location">> => NodeLocation, + <<"peer-id">> => NodeID + } + }}. + +%% @doc Processes a join request to a specific peer node. +%% +%% This function handles the client-side join flow when connecting to a peer: +%% 1. Verifies the node is not already in a green zone +%% 2. Optionally adopts configuration from the target peer +%% 3. Generates a hardware-backed commitment report +%% 4. Sends a POST request to the peer's join endpoint +%% 5. Verifies the response signature +%% 6. Decrypts the returned AES key +%% 7. Updates local configuration with the shared key +%% 8. Optionally mounts an encrypted volume using the shared key +%% +%% @param PeerLocation The target peer's address +%% @param PeerID The target peer's unique identifier +%% @param _M1 Ignored parameter +%% @param M2 May contain ShouldMount flag to enable encrypted volume mounting +%% @param InitOpts A map of initial configuration options +%% @returns `{ok, Map}' on success with confirmation message, or +%% `{error, Map|Binary}' on failure with error details +-spec join_peer( + PeerLocation :: binary(), + PeerID :: binary(), + M1 :: term(), + M2 :: term(), + Opts :: map()) -> {ok, map()} | {error, map() | binary()}. +join_peer(PeerLocation, PeerID, _M1, _M2, InitOpts) -> + % Check here if the node is already part of a green zone. + GreenZoneAES = hb_opts:get(priv_green_zone_aes, undefined, InitOpts), + case GreenZoneAES == undefined of + true -> + Wallet = hb_opts:get(priv_wallet, undefined, InitOpts), + {ok, Report} = dev_snp:generate(#{}, #{}, InitOpts), + WalletPub = element(2, Wallet), + ?event(green_zone, {remove_uncommitted, Report}), + MergedReq = hb_ao:set( + Report, + <<"public_key">>, + base64:encode(term_to_binary(WalletPub)), + InitOpts + ), + % Create an committed join request using the wallet. + Req = hb_cache:ensure_all_loaded( + hb_message:commit(MergedReq, Wallet), + InitOpts + ), + ?event({join_req, {explicit, Req}}), + ?event({verify_res, hb_message:verify(Req)}), + % Log that the commitment report is being sent to the peer. + ?event(green_zone, {join, sending_commitment, PeerLocation, PeerID, Req}), + case hb_http:post(PeerLocation, <<"/~greenzone@1.0/join">>, Req, InitOpts) of + {ok, Resp} -> + % Log the response received from the peer. + ?event(green_zone, {join, join_response, PeerLocation, PeerID, Resp}), + % Ensure that the response is from the expected peer, avoiding + % the risk of a man-in-the-middle attack. + Signers = hb_message:signers(Resp, InitOpts), + ?event(green_zone, {join, signers, Signers}), + IsVerified = hb_message:verify(Resp, Signers, InitOpts), + ?event(green_zone, {join, verify, IsVerified}), + IsPeerSigner = lists:member(PeerID, Signers), + ?event(green_zone, {join, peer_is_signer, IsPeerSigner, PeerID}), + case IsPeerSigner andalso IsVerified of + false -> + % The response is not from the expected peer. + {error, <<"Received incorrect response from peer!">>}; + true -> + % Extract the encrypted shared AES key (zone-key) + % from the response. + ZoneKey = hb_ao:get(<<"zone-key">>, Resp, InitOpts), + % Decrypt the zone key using the local node's + % private key. + {ok, AESKey} = decrypt_zone_key(ZoneKey, InitOpts), + % Update local configuration with the retrieved + % shared AES key. + ?event(green_zone, {opts, {explicit, InitOpts}}), + NewOpts = InitOpts#{ + priv_green_zone_aes => AESKey + }, + hb_http_server:set_opts(NewOpts), + {ok, #{ + <<"body">> => + <<"Node joined green zone successfully.">>, + <<"status">> => 200 + }} + end; + {error, Reason} -> + {error, #{<<"status">> => 400, <<"reason">> => Reason}}; + {unavailable, Reason} -> + ?event(green_zone, { + join_error, + peer_unavailable, + PeerLocation, + PeerID, + Reason + }), + {error, #{ + <<"status">> => 503, + <<"body">> => <<"Peer node is unreachable.">> + }} + end; + false -> + ?event(green_zone, {join, already_joined}), + {error, <<"Node already part of green zone.">>}; + {error, Reason} -> + % Log the error and return the initial options. + ?event(green_zone, {join, error, Reason}), + {error, Reason} + end. + +%%%-------------------------------------------------------------------- +%%% Internal Functions +%%%-------------------------------------------------------------------- + +%% @doc Validates an incoming join request from another node. +%% +%% This function handles the server-side join flow when receiving a connection +%% request: +%% 1. Validates the peer's configuration meets required standards +%% 2. Extracts the commitment report and public key from the request +%% 3. Verifies the hardware-backed commitment report +%% 4. Adds the joining node to the trusted nodes list +%% 5. Encrypts the shared AES key with the peer's public key +%% 6. Returns the encrypted key to the requesting node +%% +%% @param M1 Ignored parameter +%% @param Req The join request containing commitment report and public key +%% @param Opts A map of configuration options +%% @returns `{ok, Map}' on success with encrypted AES key, or +%% `{error, Binary}' on failure with error message +-spec validate_join(M1 :: term(), Req :: map(), Opts :: map()) -> + {ok, map()} | {error, binary()}. +validate_join(M1, Req, Opts) -> + case validate_peer_opts(Req, Opts) of + true -> do_nothing; + false -> throw(invalid_join_request) + end, + ?event(green_zone, {join, start}), + % Retrieve the commitment report and address from the join request. + Report = hb_ao:get(<<"report">>, Req, Opts), + NodeAddr = hb_ao:get(<<"address">>, Req, Opts), + ?event(green_zone, {join, extract, {node_addr, NodeAddr}}), + % Retrieve and decode the joining node's public key. + ?event(green_zone, {m1, {explicit, M1}}), + ?event(green_zone, {req, {explicit, Req}}), + EncodedPubKey = hb_ao:get(<<"public_key">>, Req, Opts), + ?event(green_zone, {encoded_pub_key, {explicit, EncodedPubKey}}), + RequesterPubKey = case EncodedPubKey of + not_found -> not_found; + Encoded -> binary_to_term(base64:decode(Encoded)) + end, + ?event(green_zone, {public_key, {explicit, RequesterPubKey}}), + % Verify the commitment report provided in the join request. + case dev_snp:verify(M1, Req, Opts) of + {ok, <<"true">>} -> + % Commitment verified. + ?event(green_zone, {join, commitment, verified}), + % Retrieve the shared AES key used for encryption. + GreenZoneAES = hb_opts:get(priv_green_zone_aes, undefined, Opts), + ?event(green_zone, {green_zone_aes, {explicit, GreenZoneAES}}), + % Retrieve the local node's wallet to extract its public key. + {WalletPubKey, _} = hb_opts:get(priv_wallet, undefined, Opts), + % Add the joining node's details to the trusted nodes list. + add_trusted_node(NodeAddr, Report, RequesterPubKey, Opts), + % Log the update of trusted nodes. + ?event(green_zone, {join, update, trusted_nodes, ok}), + % Encrypt the shared AES key with the joining node's public key. + EncryptedPayload = encrypt_payload(GreenZoneAES, RequesterPubKey), + % Log completion of AES key encryption. + ?event(green_zone, {join, encrypt, aes_key, complete}), + {ok, #{ + <<"body">> => <<"Node joined green zone successfully.">>, + <<"node-address">> => NodeAddr, + <<"zone-key">> => base64:encode(EncryptedPayload), + <<"public_key">> => WalletPubKey + }}; + {ok, <<"false">>} -> + % Commitment failed. + ?event(green_zone, {join, commitment, failed}), + {error, <<"Received invalid commitment report.">>}; + Error -> + % Error during commitment verification. + ?event(green_zone, {join, commitment, error, Error}), + Error + end. + +%% @doc Validates that a peer's configuration matches required options. +%% +%% This function ensures the peer node meets configuration requirements: +%% 1. Retrieves the local node's required configuration +%% 2. Gets the peer's options from its message +%% 3. Adds required configuration to peer's required options list +%% 4. Verifies the peer's node history is valid +%% 5. Checks that the peer's options match the required configuration +%% +%% @param Req The request message containing the peer's configuration +%% @param Opts A map of the local node's configuration options +%% @returns true if the peer's configuration is valid, false otherwise +-spec validate_peer_opts(Req :: map(), Opts :: map()) -> boolean(). +validate_peer_opts(Req, Opts) -> + ?event(green_zone, {validate_peer_opts, start, Req}), + % Get the required config from the local node's configuration. + RequiredConfig = + hb_ao:normalize_keys( + hb_opts:get(green_zone_required_opts, #{}, Opts)), + ConvertedRequiredConfig = + hb_message:uncommitted( + hb_cache:ensure_all_loaded( + hb_message:commit(RequiredConfig, Opts), + Opts + ) + ), + ?event(green_zone, {validate_peer_opts, required_config, ConvertedRequiredConfig}), + PeerOpts = + hb_ao:normalize_keys( + hb_ao:get(<<"node-message">>, Req, undefined, Opts)), + % Validate each item in node_history has required options + Result = try + case hb_opts:ensure_node_history(PeerOpts, ConvertedRequiredConfig) of + {ok, _} -> + ?event(green_zone, {validate_peer_opts, history_items_check, valid}), + true; + {error, ErrorMsg} -> + ?event(green_zone, {validate_peer_opts, history_items_check, {invalid, ErrorMsg}}), + false + end + catch + HistError:HistReason:HistStacktrace -> + ?event(green_zone, {validate_peer_opts, history_items_error, + {HistError, HistReason, HistStacktrace}}), + false + end, + ?event(green_zone, {validate_peer_opts, final_result, Result}), + Result. + +%% @doc Adds a node to the trusted nodes list with its commitment report. +%% +%% This function updates the trusted nodes configuration: +%% 1. Retrieves the current trusted nodes map +%% 2. Adds the new node with its report and public key +%% 3. Updates the node configuration with the new trusted nodes list +%% +%% @param NodeAddr The joining node's address +%% @param Report The commitment report provided by the joining node +%% @param RequesterPubKey The joining node's public key +%% @param Opts A map of configuration options +%% @returns ok +-spec add_trusted_node( + NodeAddr :: binary(), + Report :: map(), + RequesterPubKey :: term(), Opts :: map()) -> ok. +add_trusted_node(NodeAddr, Report, RequesterPubKey, Opts) -> + % Retrieve the current trusted nodes map. + TrustedNodes = hb_opts:get(trusted_nodes, #{}, Opts), + % Add the joining node's details to the trusted nodes. + UpdatedTrustedNodes = maps:put(NodeAddr, #{ + report => Report, + public_key => RequesterPubKey + }, TrustedNodes), + % Update configuration with the new trusted nodes and AES key. + ok = hb_http_server:set_opts(Opts#{ + trusted_nodes => UpdatedTrustedNodes + }). + +%% @doc Encrypts an AES key with a node's RSA public key. +%% +%% This function securely encrypts the shared key for transmission: +%% 1. Extracts the RSA public key components +%% 2. Creates an RSA public key record +%% 3. Performs public key encryption on the AES key +%% +%% @param AESKey The shared AES key (256-bit binary) +%% @param RequesterPubKey The node's public RSA key +%% @returns The encrypted AES key +-spec encrypt_payload(AESKey :: binary(), RequesterPubKey :: term()) -> binary(). +encrypt_payload(AESKey, RequesterPubKey) -> + ?event(green_zone, {encrypt_payload, start}), + %% Expect RequesterPubKey in the form: { {rsa, E}, Pub } + { {rsa, E}, Pub } = RequesterPubKey, + RSAPubKey = #'RSAPublicKey'{ + publicExponent = E, + modulus = crypto:bytes_to_integer(Pub) + }, + Encrypted = public_key:encrypt_public(AESKey, RSAPubKey), + ?event(green_zone, {encrypt_payload, complete}), + Encrypted. + +%% @doc Decrypts an AES key using the node's RSA private key. +%% +%% This function handles decryption of the zone key: +%% 1. Decodes the encrypted key if it's in Base64 format +%% 2. Extracts the RSA private key components from the wallet +%% 3. Creates an RSA private key record +%% 4. Performs private key decryption on the encrypted key +%% +%% @param EncZoneKey The encrypted zone AES key (Base64 encoded or binary) +%% @param Opts A map of configuration options +%% @returns {ok, DecryptedKey} on success with the decrypted AES key +-spec decrypt_zone_key(EncZoneKey :: binary(), Opts :: map()) -> + {ok, binary()} | {error, binary()}. +decrypt_zone_key(EncZoneKey, Opts) -> + % Decode if necessary + RawEncKey = case is_binary(EncZoneKey) of + true -> base64:decode(EncZoneKey); + false -> EncZoneKey + end, + % Get wallet and extract key components + {{_KeyType = {rsa, E}, Priv, Pub}, _PubKey} = + hb_opts:get(priv_wallet, #{}, Opts), + % Create RSA private key record + RSAPrivKey = #'RSAPrivateKey'{ + publicExponent = E, + modulus = crypto:bytes_to_integer(Pub), + privateExponent = crypto:bytes_to_integer(Priv) + }, + DecryptedKey = public_key:decrypt_private(RawEncKey, RSAPrivKey), + ?event(green_zone, {decrypt_zone_key, complete}), + {ok, DecryptedKey}. + +%% @doc Attempts to mount an encrypted volume using the green zone AES key. +%% +%% This function handles the complete process of secure storage setup by +%% delegating to the dev_volume module, which provides a unified interface +%% for volume management. +%% +%% The encryption key used for the volume is the same AES key used for green zone +%% communication, ensuring that only nodes in the green zone can access the data. +%% +%% @param Key The password for the encrypted volume. +%% @param Opts A map of configuration options. +%% @returns ok (implicit) in all cases, with detailed event logs of the results. +try_mount_encrypted_volume(Key, Opts) -> + ?event(debug_volume, {try_mount_encrypted_volume, start}), + % Set up options for volume mounting with default paths + VolumeOpts = Opts#{ + priv_volume_key => Key, + volume_skip_decryption => <<"true">> + }, + % Call the dev_volume:mount function to handle the complete process + case dev_volume:mount(undefined, undefined, VolumeOpts) of + {ok, Result} -> + ?event(debug_volume, {volume_mount, success, Result}), + ok; + {error, Error} -> + ?event(debug_volume, {volume_mount, error, Error}), + ok % Still return ok as this is an optional operation + end. + +%% @doc Test RSA operations with the existing wallet structure. +%% +%% This test function verifies that encryption and decryption using the RSA keys +%% from the wallet work correctly. It creates a new wallet, encrypts a test +%% message with the RSA public key, and then decrypts it with the RSA private +%% key, asserting that the decrypted message matches the original. +rsa_wallet_integration_test() -> + % Create a new wallet using ar_wallet + Wallet = ar_wallet:new(), + {{KeyType, Priv, Pub}, {KeyType, Pub}} = Wallet, + % Create test message + PlainText = <<"HyperBEAM integration test message.">>, + % Create RSA public key record for encryption + RsaPubKey = #'RSAPublicKey'{ + publicExponent = 65537, + modulus = crypto:bytes_to_integer(Pub) + }, + % Encrypt using public key + Encrypted = public_key:encrypt_public(PlainText, RsaPubKey), + % Create RSA private key record for decryption + RSAPrivKey = #'RSAPrivateKey'{ + publicExponent = 65537, + modulus = crypto:bytes_to_integer(Pub), + privateExponent = crypto:bytes_to_integer(Priv) + }, + % Verify decryption works + Decrypted = public_key:decrypt_private(Encrypted, RSAPrivKey), + % Verify roundtrip + ?assertEqual(PlainText, Decrypted), + % Verify wallet structure + ?assertEqual(KeyType, {rsa, 65537}). \ No newline at end of file diff --git a/src/dev_hook.erl b/src/dev_hook.erl new file mode 100644 index 000000000..043d23f3f --- /dev/null +++ b/src/dev_hook.erl @@ -0,0 +1,305 @@ +%%% @doc A generalized interface for `hooking' into HyperBEAM nodes. +%%% +%%% This module allows users to define `hooks' that are executed at various +%%% points in the lifecycle of nodes and message evaluations. +%%% +%%% Hooks are maintained in the `node message' options, under the key `on' +%%% key. Each `hook' may have zero or many `handlers' which their request is +%%% executed against. A new `handler' of a hook can be registered by simply +%%% adding a new key to that message. If multiple hooks need to be executed for +%%% a single event, the key's value can be set to a list of hooks. +%%% +%%% `hook's themselves do not need to be added explicitly. Any device can add +%%% a hook by simply executing `dev_hook:on(HookName, Req, Opts)`. This +%%% function is does not affect the hashpath of a message and is not exported on +%%% the device's API, such that it is not possible to call it directly with +%%% AO-Core resolution. +%%% +%%% All handlers are expressed in the form of a message, upon which the hook's +%%% request is evaluated: +%%% +%%% AO(HookMsg, Req, Opts) => {Status, Result} +%%% +%%% The `Status' and `Result' of the evaluation can be used at the `hook' caller's +%%% discretion. If multiple handlers are to be executed for a single `hook', the +%%% result of each is used as the input to the next, on the assumption that the +%%% status of the previous is `ok'. If a non-`ok' status is encountered, the +%%% evaluation is halted and the result is returned to the caller. This means +%%% that in most cases, hooks take the form of chainable pipelines of functions, +%%% passing the most pertinent data in the `body' key of both the request and +%%% result. Hook definitions can also set the `hook/result' key to `ignore', if +%%% the result of the execution should be discarded and the prior value (the +%%% input to the hook) should be used instead. The `hook/commit-request' key can +%%% also be set to `true' if the request should be committed by the node before +%%% execution of the hook. +%%% +%%% The default HyperBEAM node implements several useful hooks. They include: +%%% +%%% start: Executed when the node starts. +%%% Req/body: The node's initial configuration. +%%% Result/body: The node's possibly updated configuration. +%%% request: Executed when a request is received via the HTTP API. +%%% Req/body: The sequence of messages that the node will evaluate. +%%% Req/request: The raw, unparsed singleton request. +%%% Result/body: The sequence of messages that the node will evaluate. +%%% step: Executed after each message in a sequence has been evaluated. +%%% Req/body: The result of the evaluation. +%%% Result/body: The result of the evaluation. +%%% response: Executed when a response is sent via the HTTP API. +%%% Req/body: The result of the evaluation. +%%% Req/request: The raw, unparsed singleton request that was used to +%%% generate the response. +%%% Result/body: The message to be sent in response to the request. +%%% +%%% Additionally, this module implements a traditional device API, allowing the +%%% node operator to register hooks to the node and find those that are +%%% currently active. +-module(dev_hook). +%%% Backend API for calling hooks, used by devices as well as AO-Core. +-export([info/1, on/3, find/2, find/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Device API information +info(_) -> + #{ excludes => [<<"on">>] }. + +%% @doc Execute a named hook with the provided request and options +%% This function finds all handlers for the hook and evaluates them in sequence. +%% The result of each handler is used as input to the next handler. +on(HookName, Req, Opts) -> + ?event(hook, {attempting_execution_for_hook, HookName}), + % Get all handlers for this hook from the options + Handlers = find(HookName, Opts), + % If no handlers are found, return the original request with ok status + case Handlers of + [] -> + ?event(hook, {no_handlers_for_hook, HookName}), + {ok, Req}; + _ -> + % Execute each handler in sequence, passing the result of each to + % the next as input. + execute_handlers(HookName, Handlers, Req, Opts) + end. + +%% @doc Get all handlers for a specific hook from the node message options. +%% Handlers are stored in the `on' key of this message. The `find/2' variant of +%% this function only takes a hook name and node message, and is not called +%% directly via the device API. Instead it is used by `on/3' and other internal +%% functionality to find handlers when necessary. The `find/3' variant can, +%% however, be called directly via the device API. +find(HookName, Opts) -> + find(#{}, #{ <<"target">> => <<"body">>, <<"body">> => HookName }, Opts). +find(_Base, Req, Opts) -> + HookName = maps:get(maps:get(<<"target">>, Req, <<"body">>), Req), + case maps:get(HookName, hb_opts:get(on, #{}, Opts), []) of + Handler when is_map(Handler) -> + % If a single handler is found, wrap it in a list. + [Handler]; + Handlers when is_list(Handlers) -> + % If multiple handlers are found, return them as is + Handlers; + _ -> + % If no handlers are found or the value is invalid, return an empty + % list. + [] + end. + +%% @doc Execute a list of handlers in sequence. +%% The result of each handler is used as input to the next handler. +%% If a handler returns a non-ok status, execution is halted. +execute_handlers(_HookName, [], Req, _Opts) -> + % If no handlers remain, return the final request with ok status + {ok, Req}; +execute_handlers(HookName, [Handler|Rest], Req, Opts) -> + % Execute the current handler + ?event(hook, {executing_handler, HookName, Handler, Req}), + % Check the status of the execution + case execute_handler(HookName, Handler, Req, Opts) of + {ok, NewReq} -> + % If status is ok, continue with the next handler + ?event(hook, {handler_executed_successfully, HookName, NewReq}), + execute_handlers(HookName, Rest, NewReq, Opts); + {Status, Res} -> + % If status is error, halt execution and return the error + {Status, Res}; + Other -> + % If status is unknown, convert to error and halt execution + ?event(hook_error, {unexpected_handler_result, HookName, Other}), + {failure, + << + "Handler for hook `", + (hb_ao:normalize_key(HookName))/binary, + "` returned unexpected result." + >> + } + end. + +%% @doc Execute a single handler +%% Handlers are expressed as messages that can be resolved via AO. +execute_handler(<<"step">>, Handler, Req, Opts = #{ on := On = #{ <<"step">> := _ }}) -> + % The `step' hook is a special case: It is executed during the course of + % a resolution, and as such, the key must be removed from the node message + % before execution of the handler. Failure to do so will result in infinite + % recursion. + execute_handler( + <<"step">>, + maps:remove(<<"step">>, Handler), + Req, + Opts#{ on => maps:remove(<<"step">>, On) } + ); +execute_handler(HookName, Handler, Req, Opts) -> + try + % Resolve the handler message, setting the path to the handler name if + % it is not already set. We ensure to ignore the hashpath such that the + % handler does not affect the hashpath of a request's output. If the + % `hook/commit` key is set to `true`, the handler request will be + % committed before execution. + BaseReq = + Req#{ + <<"path">> => hb_ao:get(<<"path">>, Handler, HookName, Opts), + <<"method">> => hb_ao:get(<<"method">>, Handler, <<"GET">>, Opts) + }, + CommitReqBin = + hb_util:bin( + hb_ao:get(<<"hook/commit-request">>, Handler, <<"false">>, Opts) + ), + {PreparedBase, PreparedReq} = + case CommitReqBin of + <<"true">> -> + { + case hb_message:signers(Handler, Opts) of + [] -> hb_message:commit(Handler, Opts); + _ -> Handler + end, + hb_message:commit(BaseReq, Opts) + }; + <<"false">> -> {Handler, BaseReq} + end, + ?event(hook, + {resolving_handler, + {name, HookName}, + {handler, Handler}, + {req, {explicit, PreparedReq}} + } + ), + % Resolve the prepared request upon the handler. + {Status, Res} = + hb_ao:resolve( + PreparedBase, + PreparedReq, + Opts#{ hashpath => ignore } + ), + ?event(hook, + {handler_result, + {name, HookName}, + {status, Status}, + {res, Res} + } + ), + case {Status, hb_ao:get(<<"hook/result">>, Handler, <<"return">>, Opts)} of + {ok, <<"ignore">>} -> {Status, Req}; + {ok, <<"return">>} -> {Status, Res}; + {ok, <<"error">>} -> {error, Res}; + _ -> {Status, Res} + end + catch + Error:Reason:Stacktrace -> + % If an exception occurs during execution, log it and return an error. + ?event(hook_error, + {handler_exception, + {while_executing, HookName}, + {error, Error}, + {reason, Reason}, + {stacktrace, {trace, Stacktrace}} + } + ), + {failure, << + "Handler for hook `", + (hb_ao:normalize_key(HookName))/binary, + "` raised an exception: ", + (iolist_to_binary(io_lib:format("~p:~p", [Error, Reason])))/binary + >>} + end. + +%%% Tests + +%% @doc Test that hooks with no handlers return the original request +no_handlers_test() -> + Req = #{ <<"test">> => <<"value">> }, + Opts = #{}, + {ok, Result} = on(<<"test_hook">>, Req, Opts), + ?assertEqual(Req, Result). + +%% @doc Test that a single handler is executed correctly +single_handler_test() -> + % Create a message with a mock handler that adds a key to the request. + Handler = #{ + <<"device">> => #{ + <<"test-hook">> => + fun(_, Req, _) -> + {ok, Req#{ <<"handler_executed">> => true }} + end + } + }, + Req = #{ <<"test">> => <<"value">> }, + Opts = #{ on => #{ <<"test-hook">> => Handler }}, + {ok, Result} = on(<<"test-hook">>, Req, Opts), + ?assertEqual(true, maps:get(<<"handler_executed">>, Result)). + +%% @doc Test that multiple handlers form a pipeline +multiple_handlers_test() -> + % Create mock handlers that modify the request in sequence + Handler1 = #{ + <<"device">> => #{ + <<"test-hook">> => + fun(_, Req, _) -> + {ok, Req#{ <<"handler1">> => true }} + end + } + }, + Handler2 = #{ + <<"device">> => #{ + <<"test-hook">> => + fun(_, Req, _) -> + {ok, Req#{ <<"handler2">> => true }} + end + } + }, + Req = #{ <<"test">> => <<"value">> }, + Opts = #{ on => #{ <<"test-hook">> => [Handler1, Handler2] }}, + {ok, Result} = on(<<"test-hook">>, Req, Opts), + ?assertEqual(true, maps:get(<<"handler1">>, Result)), + ?assertEqual(true, maps:get(<<"handler2">>, Result)). + +%% @doc Test that pipeline execution halts on error +halt_on_error_test() -> + % Create handlers where the second one returns an error + Handler1 = #{ + <<"device">> => #{ + <<"test-hook">> => + fun(_, Req, _) -> + {ok, Req#{ <<"handler1">> => true }} + end + } + }, + Handler2 = #{ + <<"device">> => #{ + <<"test-hook">> => + fun(_, _, _) -> + {error, <<"Error in handler2">>} + end + } + }, + Handler3 = #{ + <<"device">> => #{ + <<"test-hook">> => + fun(_, Req, _) -> + {ok, Req#{ <<"handler3">> => true }} + end + } + }, + Req = #{ <<"test">> => <<"value">> }, + Opts = #{ on => #{ <<"test-hook">> => [Handler1, Handler2, Handler3] }}, + {error, Result} = on(<<"test-hook">>, Req, Opts), + ?assertEqual(<<"Error in handler2">>, Result). \ No newline at end of file diff --git a/src/dev_hyperbuddy.erl b/src/dev_hyperbuddy.erl new file mode 100644 index 000000000..66e194438 --- /dev/null +++ b/src/dev_hyperbuddy.erl @@ -0,0 +1,182 @@ +%%% @doc A device that renders a REPL-like interface for AO-Core via HTML. +-module(dev_hyperbuddy). +-export([info/0, format/3, return_file/2, return_error/2]). +-export([metrics/3, events/3]). +-export([throw/3]). +-include_lib("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Export an explicit list of files via http. +info() -> + #{ + default => fun serve/4, + routes => #{ + % Default message viewer page: + <<"index">> => <<"index.html">>, + % HyperBEAM default homepage: + <<"dashboard">> => <<"dashboard.html">>, + % Interactive REPL: + <<"console">> => <<"console.html">>, + <<"graph">> => <<"graph.html">>, + % Styling and scripts: + <<"styles.css">> => <<"styles.css">>, + <<"metrics.js">> => <<"metrics.js">>, + <<"devices.js">> => <<"devices.js">>, + <<"utils.js">> => <<"utils.js">>, + <<"dashboard.js">> => <<"dashboard.js">>, + <<"graph.js">> => <<"graph.js">>, + <<"404.html">> => <<"404.html">> + }, + excludes => [<<"return_file">>] + }. + +%% @doc The main HTML page for the REPL device. +metrics(_, Req, Opts) -> + case hb_opts:get(prometheus, not hb_features:test(), Opts) of + true -> + {_, HeaderList, Body} = + prometheus_http_impl:reply( + #{path => true, + headers => + fun(Name, Default) -> + hb_ao:get(Name, Req, Default, Opts) + end, + registry => prometheus_registry:exists(<<"default">>), + standalone => false} + ), + RawHeaderMap = + hb_maps:from_list( + prometheus_cowboy:to_cowboy_headers(HeaderList) + ), + Headers = + hb_maps:map( + fun(_, Value) -> hb_util:bin(Value) end, + RawHeaderMap, + Opts + ), + {ok, Headers#{ <<"body">> => Body }}; + false -> + {ok, #{ <<"body">> => <<"Prometheus metrics disabled.">> }} + end. + +%% @doc Return the current event counters as a message. +events(_, _Req, _Opts) -> + {ok, hb_event:counters()}. + +%% @doc Employ HyperBEAM's internal pretty printer to format a message. +format(Base, Req, Opts) -> + LoadedBase = hb_cache:ensure_all_loaded(Base, Opts), + LoadedReq = hb_cache:ensure_all_loaded(Req, Opts), + {ok, + #{ + <<"body">> => + hb_util:bin( + hb_format:message( + #{ + <<"base">> => + maps:without( + [<<"device">>], + hb_private:reset(LoadedBase)), + <<"request">> => + maps:without( + [<<"path">>], + hb_private:reset(LoadedReq) + ) + }, + Opts#{ + linkify_mode => discard, + cache_control => [<<"no-cache">>, <<"no-store">>] + } + ) + ) + } + }. + +%% @doc Test key for validating the behavior of the `500` HTTP response. +throw(_Msg, _Req, Opts) -> + case hb_opts:get(mode, prod, Opts) of + prod -> {error, <<"Forced-throw unavailable in `prod` mode.">>}; + debug -> throw({intentional_error, Opts}) + end. + +%% @doc Serve a file from the priv directory. Only serves files that are explicitly +%% listed in the `routes' field of the `info/0' return value. +serve(<<"keys">>, M1, _M2, Opts) -> dev_message:keys(M1, Opts); +serve(<<"set">>, M1, M2, Opts) -> dev_message:set(M1, M2, Opts); +serve(Key, _, _, Opts) -> + ?event({hyperbuddy_serving, Key}), + Routes = hb_maps:get(routes, info(), no_routes, Opts), + case hb_maps:get(Key, Routes, undefined, Opts) of + undefined -> {error, not_found}; + Filename -> return_file(Filename) + end. + +%% @doc Read a file from disk and serve it as a static HTML page. +return_file(Name) -> + return_file(<<"hyperbuddy@1.0">>, Name, #{}). +return_file(Device, Name) -> + return_file(Device, Name, #{}). +return_file(Device, Name, Template) -> + Base = hb_util:bin(code:priv_dir(hb)), + Filename = <>, + ?event({hyperbuddy_serving, Filename}), + case file:read_file(Filename) of + {ok, RawBody} -> + Body = apply_template(RawBody, Template), + {ok, #{ + <<"body">> => Body, + <<"content-type">> => + case filename:extension(Filename) of + <<".html">> -> <<"text/html">>; + <<".js">> -> <<"text/javascript">>; + <<".css">> -> <<"text/css">>; + <<".png">> -> <<"image/png">>; + <<".ico">> -> <<"image/x-icon">> + end + } + }; + {error, _} -> + {error, not_found} + end. + +%% @doc Return an error page, with the `{{error}}` template variable replaced. +return_error(Error, Opts) when not is_map(Error) -> + return_error(#{ <<"body">> => Error }, Opts); +return_error(ErrorMsg, Opts) -> + return_file( + <<"hyperbuddy@1.0">>, + <<"500.html">>, + #{ <<"error">> => hb_format:error(ErrorMsg, Opts) } + ). + +%% @doc Apply a template to a body. +apply_template(Body, Template) when is_map(Template) -> + apply_template(Body, maps:to_list(Template)); +apply_template(Body, []) -> + Body; +apply_template(Body, [{Key, Value} | Rest]) -> + apply_template( + re:replace( + Body, + <<"\\{\\{", Key/binary, "\\}\\}">>, + hb_util:bin(Value), + [global, {return, binary}] + ), + Rest + ). + +%%% Tests + +return_templated_file_test() -> + {ok, #{ <<"body">> := Body }} = + return_file( + <<"hyperbuddy@1.0">>, + <<"500.html">>, + #{ + <<"error">> => <<"This is an error message.">> + } + ), + ?assertNotEqual( + binary:match(Body, <<"This is an error message.">>), + nomatch + ). \ No newline at end of file diff --git a/src/dev_json_iface.erl b/src/dev_json_iface.erl index 85eb7f196..3b8401cd8 100644 --- a/src/dev_json_iface.erl +++ b/src/dev_json_iface.erl @@ -11,41 +11,44 @@ %%% message. %%% %%% The device has the following requirements and interface: -%%% ``` +%%% 
 %%%     M1/Computed when /Pass == 1 ->
 %%%         Assumes:
-%%%             M1/priv/WASM/Instance
+%%%             M1/priv/wasm/instance
 %%%             M1/Process
 %%%             M2/Message
 %%%             M2/Assignment/Block-Height
 %%%         Generates:
-%%%             /WASM/Handler
-%%%             /WASM/Params
+%%%             /wasm/handler
+%%%             /wasm/params
 %%%         Side-effects:
 %%%             Writes the process and message as JSON representations into the
 %%%             WASM environment.
 %%% 
 %%%     M1/Computed when M2/Pass == 2 ->
 %%%         Assumes:
-%%%             M1/priv/WASM/Instance
+%%%             M1/priv/wasm/instance
 %%%             M2/Results
 %%%             M2/Process
 %%%         Generates:
 %%%             /Results/Outbox
-%%%             /Results/Data'''
-
+%%%             /Results/Data
-module(dev_json_iface). -export([init/3, compute/3]). +%%% Public interface helpers: +-export([message_to_json_struct/2, json_to_message/2]). +%%% Test helper exports: +-export([generate_stack/1, generate_stack/2, generate_stack/3, generate_aos_msg/2]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). %% @doc Initialize the device. -init(M1, _M2, _Opts) -> - {ok, hb_converge:set(M1, #{<<"WASM-Function">> => <<"handle">>})}. +init(M1, _M2, Opts) -> + {ok, hb_ao:set(M1, #{<<"function">> => <<"handle">>}, Opts)}. %% @doc On first pass prepare the call, on second pass get the results. compute(M1, M2, Opts) -> - case hb_converge:get(<<"Pass">>, M1, Opts) of + case hb_ao:get(<<"pass">>, M1, Opts) of 1 -> prep_call(M1, M2, Opts); 2 -> results(M1, M2, Opts); _ -> {ok, M1} @@ -53,106 +56,250 @@ compute(M1, M2, Opts) -> %% @doc Prepare the WASM environment for execution by writing the process string and %% the message as JSON representations into the WASM environment. -prep_call(M1, M2, Opts) -> - Instance = hb_private:get(<<"priv/WASM/Instance">>, M1, Opts), - Process = hb_converge:get(<<"Process">>, M1, Opts#{ hashpath => ignore }), - Assignment = hb_converge:get(<<"Assignment">>, M2, Opts#{ hashpath => ignore }), - Message = hb_converge:get(<<"Message">>, M2, Opts#{ hashpath => ignore }), - Image = hb_converge:get(<<"Process/WASM-Image">>, M1, Opts), - BlockHeight = hb_converge:get(<<"Block-Height">>, Assignment, Opts), - RawMsgJson = - ar_bundles:item_to_json_struct( - hb_message:convert(Message, tx, converge, #{}) +prep_call(RawM1, RawM2, Opts) -> + M1 = hb_cache:ensure_all_loaded(RawM1, Opts), + M2 = hb_cache:ensure_all_loaded(RawM2, Opts), + ?event({prep_call, M1, M2, Opts}), + Process = hb_ao:get(<<"process">>, M1, Opts#{ hashpath => ignore }), + Message = hb_ao:get(<<"body">>, M2, Opts#{ hashpath => ignore }), + Image = hb_ao:get(<<"process/image">>, M1, Opts), + BlockHeight = hb_ao:get(<<"block-height">>, M2, Opts), + Props = message_to_json_struct(denormalize_message(Message, Opts), Opts), + MsgProps = + Props#{ + <<"Module">> => Image, + <<"Block-Height">> => BlockHeight + }, + MsgJson = hb_json:encode(MsgProps), + ProcessProps = + #{ + <<"Process">> => message_to_json_struct(Process, Opts) + }, + ProcessJson = hb_json:encode(ProcessProps), + env_write(ProcessJson, MsgJson, M1, M2, Opts). + +%% @doc Normalize a message for AOS-compatibility. +denormalize_message(Message, Opts) -> + NormOwnerMsg = + case hb_message:signers(Message, Opts) of + [] -> Message; + [PrimarySigner|_] -> + {ok, _, Commitment} = hb_message:commitment(PrimarySigner, Message, Opts), + Message#{ + <<"owner">> => hb_util:human_id(PrimarySigner), + <<"signature">> => + hb_ao:get(<<"signature">>, Commitment, <<>>, Opts) + } + end, + NormOwnerMsg#{ + <<"id">> => hb_message:id(Message, all, Opts) + }. + +message_to_json_struct(RawMsg, Opts) -> + message_to_json_struct(RawMsg, [owner_as_address], Opts). +message_to_json_struct(RawMsg, Features, Opts) -> + TABM = + hb_message:convert( + hb_private:reset(RawMsg), + tabm, + Opts ), - {Props} = RawMsgJson, - MsgJson = jiffy:encode({ - Props ++ - [ - {<<"Module">>, Image}, - {<<"Block-Height">>, BlockHeight} - ] - }), - {ok, MsgJsonPtr} = hb_beamr_io:write_string(Instance, MsgJson), - ProcessJson = - jiffy:encode( - { - [ - {<<"Process">>, - ar_bundles:item_to_json_struct( - hb_message:convert(Process, tx, converge, #{}) + MsgWithoutCommitments = hb_maps:without([<<"commitments">>], TABM, Opts), + ID = hb_message:id(RawMsg, all), + ?event({encoding, {id, ID}, {msg, RawMsg}}), + Last = hb_ao:get(<<"anchor">>, {as, <<"message@1.0">>, MsgWithoutCommitments}, <<>>, Opts), + Owner = + case hb_message:signers(RawMsg, Opts) of + [] -> <<>>; + [Signer|_] -> + case lists:member(owner_as_address, Features) of + true -> hb_util:native_id(Signer); + false -> + {ok, Commitment} = + hb_message:commitment(Signer, RawMsg, Opts), + hb_ao:get_first( + [ + {Commitment, <<"key">>}, + {Commitment, <<"owner">>} + ], + no_signing_public_key_found_in_commitment, + Opts ) - } - ] - } + end + end, + Data = hb_ao:get(<<"data">>, {as, <<"message@1.0">>, MsgWithoutCommitments}, <<>>, Opts), + Target = hb_ao:get(<<"target">>, {as, <<"message@1.0">>, MsgWithoutCommitments}, <<>>, Opts), + % Set "From" if From-Process is Tag or set with "Owner" address + From = + hb_ao:get( + <<"from-process">>, + {as, <<"message@1.0">>, MsgWithoutCommitments}, + hb_util:encode(Owner), + Opts ), - {ok, ProcessJsonPtr} = hb_beamr_io:write_string(Instance, ProcessJson), - {ok, - hb_converge:set( - M1, + Sig = hb_ao:get(<<"signature">>, {as, <<"message@1.0">>, MsgWithoutCommitments}, <<>>, Opts), + #{ + <<"Id">> => safe_to_id(ID), + % NOTE: In Arweave TXs, these are called "last_tx" + <<"Anchor">> => Last, + % NOTE: When sent to ao "Owner" is the wallet address + <<"Owner">> => hb_util:encode(Owner), + <<"From">> => case ?IS_ID(From) of true -> safe_to_id(From); false -> From end, + <<"Tags">> => prepare_tags(TABM, Opts), + <<"Target">> => safe_to_id(Target), + <<"Data">> => Data, + <<"Signature">> => + case byte_size(Sig) of + 0 -> <<>>; + 512 -> hb_util:encode(Sig); + _ -> Sig + end + }. + +%% @doc Prepare the tags of a message as a key-value list, for use in the +%% construction of the JSON-Struct message. +prepare_tags(Msg, Opts) -> + % Prepare an ANS-104 message for JSON-Struct construction. + case hb_message:commitment(#{ <<"commitment-device">> => <<"ans104@1.0">> }, Msg, Opts) of + {ok, _, Commitment} -> + case hb_maps:find(<<"original-tags">>, Commitment, Opts) of + {ok, OriginalTags} -> + Res = hb_util:message_to_ordered_list(OriginalTags), + ?event({using_original_tags, Res}), + Res; + error -> + prepare_header_case_tags(Msg, Opts) + end; + _ -> + prepare_header_case_tags(Msg, Opts) + end. + +%% @doc Convert a message without an `original-tags' field into a list of +%% key-value pairs, with the keys in HTTP header-case. +prepare_header_case_tags(TABM, Opts) -> + % Prepare a non-ANS-104 message for JSON-Struct construction. + lists:map( + fun({Name, Value}) -> #{ - <<"WASM-Function">> => <<"handle">>, - <<"WASM-Params">> => [MsgJsonPtr, ProcessJsonPtr] - }, - Opts + <<"name">> => header_case_string(maybe_list_to_binary(Name)), + <<"value">> => maybe_list_to_binary(Value) + } + end, + hb_maps:to_list( + hb_maps:without( + [ + <<"id">>, <<"anchor">>, <<"owner">>, <<"data">>, + <<"target">>, <<"signature">>, <<"commitments">> + ], + TABM, + Opts + ), + Opts ) + ). + +%% @doc Translates a compute result -- either from a WASM execution using the +%% JSON-Iface, or from a `Legacy' CU -- and transforms it into a result message. +json_to_message(JSON, Opts) when is_binary(JSON) -> + json_to_message(hb_json:decode(JSON), Opts); +json_to_message(Resp, Opts) when is_map(Resp) -> + {ok, Data, Messages, Patches} = normalize_results(Resp), + Output = + #{ + <<"outbox">> => + hb_maps:from_list( + [ + {MessageNum, preprocess_results(Msg, Opts)} + || + {MessageNum, Msg} <- + lists:zip( + lists:seq(1, length(Messages)), + Messages + ) + ] + ), + <<"patches">> => lists:map(fun(Patch) -> tags_to_map(Patch, Opts) end, Patches), + <<"data">> => Data + }, + {ok, Output}; +json_to_message(#{ <<"ok">> := false, <<"error">> := Error }, _Opts) -> + {error, Error}; +json_to_message(Other, _Opts) -> + {error, + #{ + <<"error">> => <<"Invalid JSON message input.">>, + <<"received">> => Other + } }. +safe_to_id(<<>>) -> <<>>; +safe_to_id(ID) -> hb_util:human_id(ID). + +maybe_list_to_binary(List) when is_list(List) -> + list_to_binary(List); +maybe_list_to_binary(Bin) -> + Bin. + +header_case_string(Key) -> + NormKey = hb_ao:normalize_key(Key), + Words = string:lexemes(NormKey, "-"), + TitleCaseWords = + lists:map( + fun binary_to_list/1, + lists:map( + fun string:titlecase/1, + Words + ) + ), + TitleCaseKey = list_to_binary(string:join(TitleCaseWords, "-")), + TitleCaseKey. + %% @doc Read the computed results out of the WASM environment, assuming that %% the environment has been set up by `prep_call/3' and that the WASM executor %% has been called with `computed{pass=1}'. -results(M1, _M2, Opts) -> - Instance = hb_private:get(<<"priv/WASM/Instance">>, M1, Opts), - Type = hb_converge:get(<<"Results/WASM/Type">>, M1, Opts), - Proc = hb_converge:get(<<"Process">>, M1, Opts), - case hb_converge:to_key(Type) of - error -> +results(M1, M2, Opts) -> + Prefix = dev_stack:prefix(M1, M2, Opts), + Type = hb_ao:get(<<"results/", Prefix/binary, "/type">>, M1, Opts), + Proc = hb_ao:get(<<"process">>, M1, Opts), + case hb_ao:normalize_key(Type) of + <<"error">> -> {error, - hb_converge:set( + hb_ao:set( M1, #{ - <<"Outbox">> => undefined, - <<"Results">> => + <<"outbox">> => undefined, + <<"results">> => #{ - <<"Body">> => <<"WASM execution error.">> + <<"body">> => <<"WASM execution error.">> } }, Opts ) }; - ok -> - [Ptr] = hb_converge:get(<<"Results/WASM/Output">>, M1, Opts), - {ok, Str} = hb_beamr_io:read_string(Instance, Ptr), - try jiffy:decode(Str, [return_maps]) of + <<"ok">> -> + {ok, Str} = env_read(M1, M2, Opts), + try hb_json:decode(Str) of #{<<"ok">> := true, <<"response">> := Resp} -> - {ok, Data, Messages} = normalize_results(Resp), - Output = - hb_converge:set( - M1, - #{ - <<"Results/Outbox">> => - maps:from_list([ - {MessageNum, preprocess_results(Msg, Proc, Opts)} - || - {MessageNum, Msg} <- - lists:zip( - lists:seq(1, length(Messages)), - Messages - ) - ]), - <<"Results/Data">> => Data - }, - Opts - ), - {ok, Output} + {ok, ProcessedResults} = json_to_message(Resp, Opts), + PostProcessed = postprocess_outbox(ProcessedResults, Proc, Opts), + Out = hb_ao:set( + M1, + <<"results">>, + PostProcessed, + Opts + ), + ?event(debug_iface, {results, {processed, ProcessedResults}, {out, Out}}), + {ok, Out} catch _:_ -> + ?event(error, {json_error, Str}), {error, - hb_converge:set( + hb_ao:set( M1, #{ - <<"Results/Outbox">> => undefined, - <<"Results/Body">> => - <<"JSON error parsing WASM result output.">> + <<"results/outbox">> => undefined, + <<"results/body">> => + <<"JSON error parsing result output.">> }, Opts ) @@ -160,127 +307,212 @@ results(M1, _M2, Opts) -> end end. +%% @doc Read the results out of the execution environment. +env_read(M1, M2, Opts) -> + Prefix = dev_stack:prefix(M1, M2, Opts), + Output = hb_ao:get(<<"results/", Prefix/binary, "/output">>, M1, Opts), + case hb_private:get(<>, M1, Opts) of + not_found -> + {ok, Output}; + ReadFn -> + {ok, Read} = ReadFn(Output), + {ok, Read} + end. + +%% @doc Write the message and process into the execution environment. +env_write(ProcessStr, MsgStr, Base, Req, Opts) -> + Prefix = dev_stack:prefix(Base, Req, Opts), + Params = + case hb_private:get(<>, Base, Opts) of + not_found -> + [MsgStr, ProcessStr]; + WriteFn -> + {ok, MsgJsonPtr} = WriteFn(MsgStr), + {ok, ProcessJsonPtr} = WriteFn(ProcessStr), + [MsgJsonPtr, ProcessJsonPtr] + end, + {ok, + hb_ao:set( + Base, + #{ + <<"function">> => <<"handle">>, + <<"parameters">> => Params + }, + Opts + ) + }. + %% @doc Normalize the results of an evaluation. -normalize_results( - #{ <<"Output">> := #{<<"data">> := Data}, <<"Messages">> := Messages }) -> - {ok, Data, Messages}; normalize_results(#{ <<"Error">> := Error }) -> - {ok, Error, []}. + {ok, Error, [], []}; +normalize_results(Msg) -> + try + Output = maps:get(<<"Output">>, Msg, #{}), + Data = maps:get(<<"data">>, Output, maps:get(<<"Data">>, Msg, <<>>)), + {ok, + Data, + maps:get(<<"Messages">>, Msg, []), + maps:get(<<"patches">>, Msg, []) + } + catch + _:_ -> + {ok, <<>>, [], []} + end. %% @doc After the process returns messages from an evaluation, the %% signing node needs to add some tags to each message and spawn such that %% the target process knows these messages are created by a process. -preprocess_results(Msg, Proc, Opts) -> - RawTags = maps:get(<<"Tags">>, Msg, []), - TagList = - [ - {maps:get(<<"name">>, Tag), maps:get(<<"value">>, Tag)} - || - Tag <- RawTags ], - Tags = maps:from_list(TagList), +preprocess_results(Msg, Opts) -> + Tags = tags_to_map(Msg, Opts), FilteredMsg = - maps:without( - [<<"From-Process">>, <<"From-Image">>, <<"Anchor">>, <<"Tags">>], - Msg + hb_maps:without( + [<<"from-process">>, <<"from-image">>, <<"anchor">>, <<"tags">>], + Msg, + Opts ), - maps:merge( - maps:from_list( + hb_maps:merge( + hb_maps:from_list( lists:map( fun({Key, Value}) -> - {hb_converge:to_key(Key), Value} + {hb_ao:normalize_key(Key), Value} end, - maps:to_list(FilteredMsg) + hb_maps:to_list(FilteredMsg, Opts) ) ), - Tags#{ - <<"From-Process">> => hb_converge:get(id, Proc, Opts), - <<"From-Image">> => hb_converge:get(<<"Image">>, Proc, Opts) - } + Tags, + Opts ). +%% @doc Convert a message with tags into a map of their key-value pairs. +tags_to_map(Msg, Opts) -> + NormMsg = hb_util:lower_case_key_map( + hb_ao:normalize_keys(Msg, Opts), + Opts), + RawTags = hb_maps:get(<<"tags">>, NormMsg, [], Opts), + TagList = + [ + {hb_maps:get(<<"name">>, Tag, Opts), hb_maps:get(<<"value">>, Tag, Opts)} + || + Tag <- RawTags + ], + hb_maps:from_list(TagList). + +%% @doc Post-process messages in the outbox to add the correct `from-process' +%% and `from-image' tags. +postprocess_outbox(Msg, Proc, Opts) -> + AdjustedOutbox = + hb_maps:map( + fun(_Key, XMsg) -> + XMsg#{ + <<"from-process">> => hb_ao:get(id, Proc, Opts), + <<"from-image">> => hb_ao:get(<<"image">>, Proc, Opts) + } + end, + hb_ao:get(<<"outbox">>, Msg, #{}, Opts), + Opts + ), + hb_ao:set(Msg, <<"outbox">>, AdjustedOutbox, Opts). + %%% Tests +normalize_test_opts(Opts) -> + Opts#{ + priv_wallet => hb_opts:get(priv_wallet, hb:wallet(), Opts) + }. + test_init() -> application:ensure_all_started(hb). generate_stack(File) -> + generate_stack(File, <<"WASM">>). +generate_stack(File, Mode) -> + generate_stack(File, Mode, #{}). +generate_stack(File, _Mode, RawOpts) -> + Opts = normalize_test_opts(RawOpts), test_init(), - Wallet = hb:wallet(), - Msg0 = dev_wasm:cache_wasm_image(File), - Image = hb_converge:get(<<"Image">>, Msg0, #{}), + Msg0 = dev_wasm:cache_wasm_image(File, Opts), + Image = hb_ao:get(<<"image">>, Msg0, Opts), Msg1 = Msg0#{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => [ - <<"WASI/1.0">>, - <<"JSON-Iface/1.0">>, - <<"WASM-64/1.0">>, - <<"Multipass/1.0">> + <<"wasi@1.0">>, + <<"json-iface@1.0">>, + <<"wasm-64@1.0">>, + <<"multipass@1.0">> ], - <<"Input-Prefix">> => <<"Process">>, - <<"Output-Prefix">> => <<"WASM">>, - <<"Passes">> => 2, - <<"Stack-Keys">> => [<<"Init">>, <<"Compute">>], - <<"Process">> => - hb_message:sign(#{ - <<"Type">> => <<"Process">>, - <<"Image">> => Image, - <<"Scheduler">> => hb:address(), - <<"Authority">> => hb:address() - }, Wallet) + <<"input-prefix">> => <<"process">>, + <<"output-prefix">> => <<"wasm">>, + <<"passes">> => 2, + <<"stack-keys">> => [<<"init">>, <<"compute">>], + <<"process">> => + hb_message:commit(#{ + <<"type">> => <<"Process">>, + <<"image">> => Image, + <<"scheduler">> => hb:address(), + <<"authority">> => hb:address() + }, Opts) }, - {ok, Msg2} = hb_converge:resolve(Msg1, <<"Init">>, #{}), + {ok, Msg2} = hb_ao:resolve(Msg1, <<"init">>, Opts), Msg2. generate_aos_msg(ProcID, Code) -> - Wallet = hb:wallet(), - #{ - path => <<"Compute">>, - <<"Message">> => - hb_message:sign(#{ - <<"Action">> => <<"Eval">>, - data => Code, - target => ProcID - }, Wallet), - <<"Assignment">> => - hb_message:sign(#{ <<"Block-Height">> => 1 }, Wallet) - }. + generate_aos_msg(ProcID, Code, #{}). +generate_aos_msg(ProcID, Code, RawOpts) -> + Opts = normalize_test_opts(RawOpts), + hb_message:commit(#{ + <<"path">> => <<"compute">>, + <<"body">> => + hb_message:commit(#{ + <<"action">> => <<"Eval">>, + <<"data">> => Code, + <<"target">> => ProcID + }, Opts), + <<"block-height">> => 1 + }, Opts). -basic_aos_call_test() -> - Msg = generate_stack("test/aos-2-pure-xs.wasm"), - Proc = hb_converge:get(<<"Process">>, Msg, #{ hashpath => ignore }), - ProcID = hb_converge:get(id, Proc, #{}), - {ok, Msg3} = - hb_converge:resolve( - Msg, - generate_aos_msg(ProcID, <<"return 1+1">>), - #{} - ), - Data = hb_converge:get(<<"Results/Data">>, Msg3, #{}), - ?assertEqual(<<"2">>, Data). +basic_aos_call_test_() -> + {timeout, 20, fun() -> + Msg = generate_stack("test/aos-2-pure-xs.wasm"), + Proc = hb_ao:get(<<"process">>, Msg, #{ hashpath => ignore }), + ProcID = hb_message:id(Proc, all), + {ok, Msg3} = + hb_ao:resolve( + Msg, + generate_aos_msg(ProcID, <<"return 1+1">>), + #{} + ), + ?event({res, Msg3}), + Data = hb_ao:get(<<"results/data">>, Msg3, #{}), + ?assertEqual(<<"2">>, Data) + end}. aos_stack_benchmark_test_() -> {timeout, 20, fun() -> - BenchTime = 3, - RawWASMMsg = generate_stack("test/aos-2-pure-xs.wasm"), - Proc = hb_converge:get(<<"Process">>, RawWASMMsg, #{ hashpath => ignore }), - ProcID = hb_converge:get(id, Proc, #{}), + BenchTime = 5, + Opts = #{ store => hb_test_utils:test_store() }, + RawWASMMsg = generate_stack("test/aos-2-pure-xs.wasm", <<"WASM">>, Opts), + Proc = hb_ao:get(<<"process">>, RawWASMMsg, Opts#{ hashpath => ignore }), + ProcID = hb_ao:get(id, Proc, Opts), + Msg = generate_aos_msg(ProcID, <<"return 1">>, Opts), {ok, Initialized} = - hb_converge:resolve( - RawWASMMsg, - generate_aos_msg(ProcID, <<"return 1">>), - #{} - ), - Msg = generate_aos_msg(ProcID, <<"return 1+1">>), + hb_ao:resolve( + RawWASMMsg, + Msg, + Opts + ), + Msg2 = generate_aos_msg(ProcID, <<"return 1+1">>, Opts), Iterations = - hb:benchmark( - fun() -> hb_converge:resolve(Initialized, Msg, #{}) end, + hb_test_utils:benchmark( + fun() -> hb_ao:resolve(Initialized, Msg2, Opts) end, BenchTime ), - hb_util:eunit_print( - "Evaluated ~p AOS messages (minimal stack) in ~p sec (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] + hb_test_utils:benchmark_print( + <<"(Minimal AOS stack:) Evaluated">>, + <<"messages">>, + Iterations, + BenchTime ), - ?assert(Iterations > 10), + ?assert(Iterations >= 10), ok - end}. + end}. \ No newline at end of file diff --git a/src/dev_local_name.erl b/src/dev_local_name.erl new file mode 100644 index 000000000..71a9563ab --- /dev/null +++ b/src/dev_local_name.erl @@ -0,0 +1,189 @@ +%%% @doc A device for registering and looking up local names. This device uses +%%% the node message to store a local cache of its known names, and the typical +%%% non-volatile storage of the node message to store the names long-term. +-module(dev_local_name). +-export([info/1, lookup/3, register/3]). +%%% HyperBEAM public (non-AO resolvable) functions. +-export([direct_register/2]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The location that the device should use in the store for its links. +-define(DEV_CACHE, <<"local-name@1.0">>). + +%% @doc Export only the `lookup' and `register' functions. +info(_Opts) -> + #{ + excludes => [<<"direct_register">>, <<"keys">>, <<"set">>], + default => fun default_lookup/4 + }. + +%% @doc Takes a `key' argument and returns the value of the name, if it exists. +lookup(_, Req, Opts) -> + Key = hb_ao:get(<<"key">>, Req, no_key_specified, Opts), + ?event(local_name, {lookup, Key}), + hb_ao:resolve( + find_names(Opts), + Key, + Opts + ). + +%% @doc Handle all other requests by delegating to the lookup function. +default_lookup(Key, _, Req, Opts) -> + lookup(Key, Req#{ <<"key">> => Key }, Opts). + +%% @doc Takes a `key' and `value' argument and registers the name. The caller +%% must be the node operator in order to register a name. +register(_, Req, Opts) -> + case dev_meta:is(admin, Req, Opts) of + false -> + {error, + #{ + <<"status">> => 403, + <<"message">> => <<"Unauthorized.">> + } + }; + true -> + direct_register(Req, Opts) + end. + +%% @doc Register a name without checking if the caller is an operator. Exported +%% for use by other devices, but not publicly available. +direct_register(Req, Opts) -> + case hb_cache:write(hb_ao:get(<<"value">>, Req, Opts), Opts) of + {ok, MsgPath} -> + NormKey = hb_ao:normalize_key(hb_ao:get(<<"key">>, Req, Opts)), + hb_cache:link( + MsgPath, + LinkPath = << ?DEV_CACHE/binary, "/", NormKey/binary >>, + Opts + ), + load_names(Opts), + ?event( + local_name, + {registered, + {key, NormKey}, + {msg, MsgPath}, + {path, LinkPath} + } + ), + {ok, <<"Registered.">>}; + {error, _} -> + not_found + end. + +%% @doc Returns a message containing all known names. +find_names(Opts) -> + case hb_opts:get(local_names, not_found, Opts#{ only => local }) of + not_found -> + find_names(load_names(Opts)); + LocalNames -> + LocalNames + end. + +%% @doc Loads all known names from the cache and returns the new `node message' +%% with those names loaded into it. +load_names(Opts) -> + LocalNames = + maps:from_list(lists:map( + fun(Key) -> + NormKey = hb_ao:normalize_key(Key), + Path = << ?DEV_CACHE/binary, "/", NormKey/binary >>, + ?event(local_name, {loading, Path}), + case hb_cache:read(Path, Opts) of + {ok, Value} -> + {Key, Value}; + _ -> + {Key, not_found} + end + end, + hb_cache:list(?DEV_CACHE, Opts) + )), + ?event(local_name, {found_cache_keys, LocalNames}), + update_names(LocalNames, Opts). + +%% @doc Updates the node message with the new names. Further HTTP requests will +%% use this new message, removing the need to look up the names from non-volatile +%% storage. +update_names(LocalNames, Opts) -> + hb_http_server:set_opts(NewOpts = Opts#{ local_names => LocalNames }), + NewOpts. + +%%% Tests + +generate_test_opts() -> + Opts = #{ + priv_wallet => ar_wallet:new() + }, + Opts. + +no_names_test() -> + ?assertEqual( + {error, not_found}, + lookup(#{}, #{ <<"key">> => <<"name1">> }, #{}) + ). + +lookup_opts_name_test() -> + ?assertEqual( + {ok, <<"value1">>}, + lookup( + #{}, + #{ <<"key">> => <<"name1">> }, + #{ local_names => #{ <<"name1">> => <<"value1">>} } + ) + ). + +register_test() -> + TestName = <<"TEST-", (integer_to_binary(os:system_time(millisecond)))/binary>>, + Value = <<"TEST-VALUE-", (integer_to_binary(os:system_time(millisecond)))/binary>>, + Opts = generate_test_opts(), + ?assertEqual( + {ok, <<"Registered.">>}, + register( + #{}, + hb_message:commit( + #{ <<"key">> => TestName, <<"value">> => Value }, + Opts + ), + Opts + ) + ), + ?assertEqual( + {ok, Value}, + lookup(#{}, #{ <<"key">> => TestName, <<"load">> => false }, Opts) + ). + +unauthorized_test() -> + Opts = generate_test_opts(), + ?assertEqual( + {error, #{ <<"status">> => 403, <<"message">> => <<"Unauthorized.">> }}, + register( + #{}, + hb_message:commit( + #{ <<"key">> => <<"name1">>, <<"value">> => <<"value1">> }, + Opts#{ priv_wallet => ar_wallet:new() } + ), + Opts + ) + ). + +http_test() -> + Opts = generate_test_opts(), + Node = hb_http_server:start_node(Opts), + hb_http:post( + Node, + <<"/~local-name@1.0/register">>, + hb_message:commit( + #{ <<"key">> => <<"name1">>, <<"value">> => <<"value1">> }, + Opts + ), + Opts + ), + ?assertEqual( + {ok, <<"value1">>}, + hb_http:get( + Node, + <<"/~local-name@1.0/lookup?key=name1">>, + Opts + ) + ). \ No newline at end of file diff --git a/src/dev_lookup.erl b/src/dev_lookup.erl index 7e023725f..bd6958be7 100644 --- a/src/dev_lookup.erl +++ b/src/dev_lookup.erl @@ -1,13 +1,78 @@ +%%% @doc A device that looks up an ID from a local store and returns it, honoring +%%% the `accept' key to return the correct format. -module(dev_lookup). --export([read/1]). +-export([read/3]). -include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). -%%% The lookup device: Look up an ID by name and return it. +%%% @doc Fetch a resource from the cache using "target" ID extracted from the message +read(_M1, M2, Opts) -> + ID = hb_ao:get(<<"target">>, M2, Opts), + ?event({lookup, {id, ID}, {opts, Opts}}), + case hb_cache:read(ID, Opts) of + {ok, RawRes} -> + % We are sending the result over the wire, so make sure it is + % fully loaded, to save the recipient latency. + ?event({lookup_result, RawRes}), + case hb_ao:get(<<"accept">>, M2, Opts) of + <<"application/aos-2">> -> + Res = hb_cache:ensure_all_loaded(RawRes), + Struct = dev_json_iface:message_to_json_struct(Res, Opts), + {ok, + #{ + <<"body">> => hb_json:encode(Struct), + <<"content-type">> => <<"application/aos-2">> + }}; + _ -> + {ok, RawRes} + end; + not_found -> + ?event({lookup_not_found, ID}), + {error, not_found} + end. -read(#tx { tags = Tags }) -> - % Note: Use the local store -- do not attempt to reach remotely. - % hb_cache:read should return {ok, Val} or an error tuple, so we can return - % the value directly. - {<<"Subpath">>, Subpath} = lists:keyfind(<<"Subpath">>, 1, Tags), - ?event({looking_up_for_remote_peer, Subpath}), - hb_cache:read_message(hb_store:scope(hb_opts:get(store), local), Subpath). \ No newline at end of file +%%% Tests + +binary_lookup_test() -> + Bin = <<"Simple unsigned data item">>, + {ok, ID} = hb_cache:write(Bin, #{}), + {ok, RetrievedBin} = read(#{}, #{ <<"target">> => ID }, #{}), + ?assertEqual(Bin, RetrievedBin). + +message_lookup_test() -> + Msg = #{ <<"test-key">> => <<"test-value">>, <<"data">> => <<"test-data">> }, + {ok, ID} = hb_cache:write(Msg, #{}), + {ok, RetrievedMsg} = read(#{}, #{ <<"target">> => ID }, #{}), + ?assert(hb_message:match(Msg, RetrievedMsg)). + +aos2_message_lookup_test() -> + Msg = #{ <<"test-key">> => <<"test-value">>, <<"data">> => <<"test-data">> }, + {ok, ID} = hb_cache:write(Msg, #{}), + {ok, RetrievedMsg} = + read( + #{}, + #{ <<"target">> => ID, <<"accept">> => <<"application/aos-2">> }, + #{} + ), + + {ok, Decoded} = dev_json_iface:json_to_message(hb_ao:get(<<"body">>, RetrievedMsg, #{}), #{}), + ?assertEqual(<<"test-data">>, hb_ao:get(<<"data">>, Decoded, #{})). + +http_lookup_test() -> + Store = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-mainnet">> + }, + Opts = #{ store => [Store] }, + Msg = #{ <<"test-key">> => <<"test-value">>, <<"data">> => <<"test-data">> }, + {ok, ID} = hb_cache:write(Msg, Opts), + Node = hb_http_server:start_node(Opts), + Wallet = hb:wallet(), + Req = hb_message:commit(#{ + <<"path">> => <<"/~lookup@1.0/read?target=", ID/binary>>, + <<"device">> => <<"lookup@1.0">>, + <<"accept">> => <<"application/aos-2">> + }, Wallet), + {ok, Res} = hb_http:post(Node, Req, Opts), + {ok, Decoded} = dev_json_iface:json_to_message(hb_ao:get(<<"body">>, Res, Opts), Opts), + ?assertEqual(<<"test-data">>, hb_ao:get(<<"Data">>, Decoded, Opts)). \ No newline at end of file diff --git a/src/dev_lua.erl b/src/dev_lua.erl new file mode 100644 index 000000000..25920793d --- /dev/null +++ b/src/dev_lua.erl @@ -0,0 +1,888 @@ +%%% @doc A device that calls a Lua module upon a request and returns the result. +-module(dev_lua). +-export([info/1, init/3, snapshot/3, normalize/3, functions/3]). +%%% Public Utilities +-export([encode/2, decode/2]). +-export([pure_lua_process_benchmark/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The set of functions that will be sandboxed by default if `sandbox' is set +%%% to only `true'. Setting `sandbox' to a map allows the invoker to specify +%%% which functions should be sandboxed and what to return instead. Providing +%%% a list instead of a map will result in all functions being sandboxed and +%%% returning `sandboxed'. +-define(DEFAULT_SANDBOX, [ + {['_G', io], <<"sandboxed">>}, + {['_G', file], <<"sandboxed">>}, + {['_G', os, execute], <<"sandboxed">>}, + {['_G', os, exit], <<"sandboxed">>}, + {['_G', os, getenv], <<"sandboxed">>}, + {['_G', os, remove], <<"sandboxed">>}, + {['_G', os, rename], <<"sandboxed">>}, + {['_G', os, tmpname], <<"sandboxed">>}, + {['_G', package], <<"sandboxed">>}, + {['_G', loadfile], <<"sandboxed">>}, + {['_G', require], <<"sandboxed">>}, + {['_G', dofile], <<"sandboxed">>}, + {['_G', load], <<"sandboxed">>}, + {['_G', loadfile], <<"sandboxed">>}, + {['_G', loadstring], <<"sandboxed">>} +]). + +%% @doc All keys that are not directly available in the base message are +%% resolved by calling the Lua function in the module of the same name. +%% Additionally, we exclude the `keys', `set', `encode' and `decode' functions +%% which are `message@1.0' core functions, and Lua public utility functions. +info(Base) -> + #{ + default => fun compute/4, + excludes => + [<<"keys">>, <<"set">>, <<"encode">>, <<"decode">>] + ++ maps:keys(Base) + }. + +%% @doc Initialize the device state, loading the script into memory if it is +%% a reference. +init(Base, Req, Opts) -> + ensure_initialized(Base, Req, Opts). + +%% @doc Initialize the Lua VM if it is not already initialized. Optionally takes +%% the script as a Binary string. If not provided, the module will be loaded +%% from the base message. +ensure_initialized(Base, _Req, Opts) -> + case hb_private:from_message(Base) of + #{<<"state">> := _} -> + ?event(debug_lua, lua_state_already_initialized), + {ok, Base}; + _ -> + ?event(debug_lua, initializing_lua_state), + case find_modules(Base, Opts) of + {ok, Modules} -> + initialize(Base, Modules, Opts); + Error -> + Error + end + end. + +%% @doc Find the script in the base message, either by ID or by string. +find_modules(Base, Opts) -> + case hb_ao:get(<<"module">>, {as, <<"message@1.0">>, Base}, Opts) of + not_found -> + {error, <<"no-modules-found">>}; + Module when is_binary(Module) -> + find_modules(Base#{ <<"module">> => [Module] }, Opts); + Module when is_map(Module) -> + % If the module is a map, check its content type to see if it is + % a literal Lua module, or a map of modules with content types. + case hb_ao:get(<<"content-type">>, Module, Opts) of + CT when CT == <<"application/lua">> orelse CT == <<"text/x-lua">> -> + find_modules(Base#{ <<"module">> => [Module] }, Opts); + _ -> + % If the script is not a literal Lua script, assume it is a + % map of scripts with content types, and recurse. + find_modules(Base#{ <<"module">> => maps:values(Module) }, Opts) + end; + Modules when is_list(Modules) -> + % We have found a list of scripts, load them. + load_modules(Modules, Opts) + end. + +%% @doc Load a list of modules for installation into the Lua VM. +load_modules(Modules, Opts) -> load_modules(Modules, Opts, []). +load_modules([], _Opts, Acc) -> + {ok, lists:reverse(Acc)}; +load_modules([ModuleID | Rest], Opts, Acc) when ?IS_ID(ModuleID) -> + case hb_cache:read(ModuleID, Opts) of + {ok, Module} when is_binary(Module) -> + % The ID referred to a binary module item, so we add it to the list + % as-is. + load_modules(Rest, Opts, [{ModuleID, Module}|Acc]); + {ok, ModuleMsg} when is_map(ModuleMsg) -> + % We read a message from the store, so we recurse upon the output, + % as if the module message had beeen given directly. + load_modules([ModuleMsg|Rest], Opts, Acc); + not_found -> + {error, #{ + <<"status">> => 404, + <<"body">> => <<"Lua module '", ModuleID/binary, "' not found.">> + }} + end; +load_modules([Module | Rest], Opts, Acc) when is_map(Module) -> + % We have found a message with a Lua module inside. Search for the binary + % of the program in the body and the data. + ModuleBin = + hb_ao:get_first( + [ + {Module, <<"body">>}, + {Module, <<"data">>} + ], + Module, + Opts + ), + case ModuleBin of + not_found -> + {error, #{ + <<"status">> => 404, + <<"body">> => + << + """ + Lua module not loadable. Lua modules must have a + `body' element set to a binary of the code to load. + """ + >>, + <<"module">> => Module + }}; + ModuleBin -> + % Get the `name' key from the script message if it exists, or + % return the module ID as the module name. + Name = + hb_ao:get_first( + [ + {Module, <<"name">>}, + {Module, <<"id">>} + ], + Module, + Opts + ), + % Load the module into the Lua state. + load_modules(Rest, Opts, [{Name, ModuleBin}|Acc]) + end. + +%% @doc Initialize a new Lua state with a given base message and module. +initialize(Base, Modules, Opts) -> + State0 = luerl:init(), + % Load each script into the Lua state. + State1 = + lists:foldl( + fun({ModuleID, ModuleBin}, StateIn) -> + {ok, _, StateOut} = + luerl:do_dec( + ModuleBin, + [ + {name, hb_util:list(ModuleID)}, + {file, hb_util:list(ModuleID)} + ], + StateIn + ), + StateOut + end, + State0, + Modules + ), + % Apply any sandboxing rules to the state. + State2 = + case hb_ao:get(<<"sandbox">>, {as, <<"message@1.0">>, Base}, false, Opts) of + false -> State1; + true -> sandbox(State1, ?DEFAULT_SANDBOX, Opts); + Spec -> sandbox(State1, Spec, Opts) + end, + % Install the AO-Core Lua library into the state. + {ok, State3} = dev_lua_lib:install(Base, State2, Opts), + % Return the base message with the state added to it. + {ok, hb_private:set(Base, <<"state">>, State3, Opts)}. + +%%% @doc Return a list of all functions in the Lua environment. +functions(Base, _Req, Opts) -> + case hb_private:get(<<"state">>, Base, Opts) of + not_found -> + {error, not_found}; + State -> + {ok, [Res], _S2} = + luerl:do_dec( + << + """ + local __tests = {} + for k, v in pairs(_G) do + if type(v) == "function" then + table.insert(__tests, k) + end + end + return __tests + """ + >>, + State + ), + {ok, hb_util:message_to_ordered_list(decode(Res, Opts))} + end. + +%% @doc Sandbox (render inoperable) a set of Lua functions. Each function is +%% referred to as if it is a path in AO-Core, with its value being what to +%% return to the caller. For example, 'os.exit' would be referred to as +%% referred to as `os/exit'. If preferred, a list rather than a map may be +%% provided, in which case the functions all return `sandboxed'. +sandbox(State, Map, Opts) when is_map(Map) -> + sandbox(State, maps:to_list(Map), Opts); +sandbox(State, [], _Opts) -> + State; +sandbox(State, [{Path, Value} | Rest], Opts) -> + {ok, NextState} = luerl:set_table_keys_dec(Path, Value, State), + sandbox(NextState, Rest, Opts); +sandbox(State, [Path | Rest], Opts) -> + {ok, NextState} = luerl:set_table_keys_dec(Path, <<"sandboxed">>, State), + sandbox(NextState, Rest, Opts). + +%% @doc Call the Lua script with the given arguments. +compute(Key, RawBase, Req, Opts) -> + ?event(debug_lua, compute_called), + {ok, Base} = ensure_initialized(RawBase, Req, Opts), + ?event(debug_lua, ensure_initialized_done), + % Get the state from the base message's private element. + OldPriv = #{ <<"state">> := State } = hb_private:from_message(Base), + % TODO: looks like the script is injected in multiple places, does the + % script need to be passed? + % Get the Lua function to call from the base message. + Function = + hb_ao:get_first( + [ + {Req, <<"body/function">>}, + {Req, <<"function">>}, + {{as, <<"message@1.0">>, Base}, <<"function">>} + ], + Key, + Opts#{ hashpath => ignore } + ), + ?event(debug_lua, function_found), + Params = + hb_ao:get_first( + [ + {Req, <<"body/parameters">>}, + {Req, <<"parameters">>}, + {{as, <<"message@1.0">>, Base}, <<"parameters">>} + ], + [ + hb_private:reset(Base), + Req, + #{} + ], + Opts#{ hashpath => ignore } + ), + ?event(debug_lua, parameters_found), + % Resolve all hyperstate links + ResolvedParams = hb_cache:ensure_all_loaded(Params, Opts), + % Call the VM function with the given arguments. + ?event(lua, + {calling_lua_func, + {function, Function}, + {args, ResolvedParams}, + {req, Req} + } + ), + process_response( + try luerl:call_function_dec( + [Function], + encode(ResolvedParams, Opts), + State + ) + catch + _:Reason:Stacktrace -> {error, Reason, Stacktrace} + end, + OldPriv, + Opts + ). + +%% @doc Process a response to a Luerl invocation. Returns the typical AO-Core +%% HyperBEAM response format. +process_response({ok, [Result], NewState}, Priv, Opts) -> + process_response({ok, [<<"ok">>, Result], NewState}, Priv, Opts); +process_response({ok, [Status, MsgResult], NewState}, Priv, Opts) -> + % If the result is a HyperBEAM device return (`{Status, Msg}'), decode it + % and add the previous `priv' element back into the resulting message. + case decode(MsgResult, Opts) of + Msg when is_map(Msg) -> + ?event(lua, {response, {status, Status}, {msg, Msg}}), + {hb_util:atom(Status), Msg#{ + <<"priv">> => Priv#{ + <<"state">> => NewState + } + }}; + NonMsgRes -> {hb_util:atom(Status), NonMsgRes} + end; +process_response({lua_error, RawError, State}, _Priv, Opts) -> + % An error occurred while calling the Lua function. Parse the stack trace + % and return it. + Error = try decode(luerl:decode(RawError, State), Opts) catch _:_ -> RawError end, + StackTrace = decode_stacktrace(luerl:get_stacktrace(State), State, Opts), + ?event(lua_error, {lua_error, Error, {stacktrace, StackTrace}}), + {error, #{ + <<"status">> => 500, + <<"body">> => Error, + <<"trace">> => hb_ao:normalize_keys(StackTrace, Opts) + }}; +process_response({error, Reason, Trace}, _Priv, _Opts) -> + % An Erlang error occurred while calling the Lua function. Return it. + ?event(lua_error, {trace, Trace}), + TraceBin = iolist_to_binary(hb_format:trace(Trace)), + ?event(lua_error, {formatted, {string, TraceBin}}), + ReasonBin = iolist_to_binary(io_lib:format("~p", [Reason])), + {error, #{ + <<"status">> => 500, + <<"body">> => + << "Erlang error while running Lua: ", ReasonBin/binary >>, + <<"trace">> => TraceBin + }}. + +%% @doc Snapshot the Lua state from a live computation. Normalizes its `priv' +%% state element, then serializes the state to a binary. +snapshot(Base, _Req, Opts) -> + case hb_private:get(<<"state">>, Base, Opts) of + not_found -> + {error, <<"Cannot snapshot Lua state: state not initialized.">>}; + State -> + {ok, #{ <<"body">> => term_to_binary(luerl:externalize(State)) }} + end. + +%% @doc Restore the Lua state from a snapshot, if it exists. +normalize(Base, _Req, RawOpts) -> + Opts = RawOpts#{ hashpath => ignore }, + case hb_private:get(<<"state">>, Base, Opts) of + not_found -> + DeviceKey = + case hb_ao:get(<<"device-key">>, {as, <<"message@1.0">>, Base}, Opts) of + not_found -> []; + Key -> [Key] + end, + ?event(snapshot, + {attempting_to_restore_lua_state, + {msg1, Base}, {device_key, DeviceKey} + } + ), + SerializedState = + hb_ao:get( + [<<"snapshot">>] ++ DeviceKey ++ [<<"body">>], + {as, dev_message, Base}, + Opts + ), + case SerializedState of + not_found -> throw({error, no_lua_state_snapshot_found}); + State -> + ExternalizedState = binary_to_term(State), + InternalizedState = luerl:internalize(ExternalizedState), + ?event(snapshot, loaded_state_from_snapshot), + {ok, hb_private:set(Base, <<"state">>, InternalizedState, Opts)} + end; + _ -> + ?event(snapshot, state_already_initialized), + {ok, Base} + end. + +%% @doc Decode a Lua result into a HyperBEAM `structured@1.0' message. +decode(EncMsg, _Opts) when is_list(EncMsg) andalso length(EncMsg) == 0 -> + % The value is an empty table, so we assume it is a message rather than + % a list. + #{}; +decode(EncMsg = [{_K, _V} | _], Opts) when is_list(EncMsg) -> + decode( + maps:map( + fun(_, V) -> decode(V, Opts) end, + maps:from_list(EncMsg) + ), + Opts + ); +decode(Msg, Opts) when is_map(Msg) -> + % If the message is an ordered list encoded as a map, decode it to a list. + case hb_util:is_ordered_list(Msg, Opts) of + true -> + lists:map( + fun(V) -> decode(V, Opts) end, + hb_util:message_to_ordered_list(Msg) + ); + false -> + Msg + end; +decode(Other, _Opts) -> + Other. + +%% @doc Encode a HyperBEAM `structured@1.0' message into a Lua term. +encode(Map, Opts) when is_map(Map) -> + hb_cache:ensure_all_loaded( + case hb_util:is_ordered_list(Map, Opts) of + true -> encode(hb_util:message_to_ordered_list(Map), Opts); + false -> maps:to_list(maps:map(fun(_, V) -> encode(V, Opts) end, Map)) + end, + Opts + ); +encode(List, Opts) when is_list(List) -> + hb_cache:ensure_all_loaded( + lists:map(fun(V) -> encode(V, Opts) end, List), + Opts + ); +encode(Atom, _Opts) when is_atom(Atom) and (Atom /= false) and (Atom /= true)-> + hb_util:bin(Atom); +encode(Other, _Opts) -> + Other. + +%% @doc Parse a Lua stack trace into a list of messages. +decode_stacktrace(StackTrace, State0, Opts) -> + decode_stacktrace(StackTrace, State0, [], Opts). +decode_stacktrace([], _State, Acc, _Opts) -> + lists:reverse(Acc); +decode_stacktrace([{FuncBin, ParamRefs, FileInfo} | Rest], State0, Acc, Opts) -> + %% Decode all the Lua table refs into Erlang terms + DecodedParams = decode_params(ParamRefs, State0, Opts), + %% Pull out the line number + Line = proplists:get_value(line, FileInfo), + File = proplists:get_value(file, FileInfo, undefined), + ?event(debug_lua_stack, {stack_file, FileInfo}), + %% Build our message‐map + Entry = #{ + <<"function">> => FuncBin, + <<"parameters">> => hb_util:list_to_numbered_message(DecodedParams) + }, + MaybeLine = + if is_binary(File) andalso is_integer(Line) -> + #{ + <<"line">> => + iolist_to_binary( + io_lib:format("~s:~p", [File, Line]) + ) + }; + is_integer(Line) -> + #{ <<"line">> => Line }; + true -> + #{} + end, + decode_stacktrace(Rest, State0, [maps:merge(Entry, MaybeLine)|Acc], Opts). + +%% @doc Decode a list of Lua references, as found in a stack trace, into a +%% list of Erlang terms. +decode_params([], _State, _Opts) -> []; +decode_params([Tref|Rest], State, Opts) -> + Decoded = decode(luerl:decode(Tref, State), Opts), + [Decoded|decode_params(Rest, State, Opts)]. + +%%% Tests +simple_invocation_test() -> + {ok, Script} = file:read_file("test/test.lua"), + Base = #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Script + }, + <<"parameters">> => [] + }, + ?assertEqual(2, hb_ao:get(<<"assoctable/b">>, Base, #{})). + +load_modules_by_id_test_() -> + {timeout, 30, fun load_modules_by_id/0}. +load_modules_by_id() -> + % Start a node to ensure the HTTP services are available. + _Node = hb_http_server:start_node(#{}), + Module = <<"DosEHUAqhl_O5FH3vDqPlgGsG92Guxcm6nrwqnjsDKg">>, + {ok, Acc} = load_modules([Module], #{}), + [{_,Code}|_] = Acc, + <> = Code, + ?assertEqual(<<"function">>, Prefix). + +multiple_modules_test() -> + {ok, Module} = file:read_file("test/test.lua"), + Module2 = + << + """ + function test_second_script() + return 4 + end + """ + >>, + Base = #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => [ + #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + }, + #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module2 + } + ], + <<"parameters">> => [] + }, + ?assertEqual(2, hb_ao:get(<<"assoctable/b">>, Base, #{})), + ?assertEqual(4, hb_ao:get(<<"test_second_script">>, Base, #{})). + +error_response_test() -> + {ok, Module} = file:read_file("test/test.lua"), + Base = #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + }, + <<"parameters">> => [] + }, + ?assertEqual( + {error, <<"Very bad, but Lua caught it.">>}, + hb_ao:resolve(Base, <<"error_response">>, #{}) + ). + +sandboxed_failure_test() -> + {ok, Module} = file:read_file("test/test.lua"), + Base = #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + }, + <<"parameters">> => [], + <<"sandbox">> => true + }, + ?assertMatch({error, _}, hb_ao:resolve(Base, <<"sandboxed_fail">>, #{})). + +%% @doc Run an AO-Core resolution from the Lua environment. +ao_core_sandbox_test() -> + {ok, Module} = file:read_file("test/test.lua"), + Base = #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + }, + <<"parameters">> => [], + <<"device-sandbox">> => [<<"message@1.0">>] + }, + ?assertMatch({error, _}, hb_ao:resolve(Base, <<"ao_relay">>, #{})), + ?assertMatch({ok, _}, hb_ao:resolve(Base, <<"ao_resolve">>, #{})). + +%% @doc Run an AO-Core resolution from the Lua environment. +ao_core_resolution_from_lua_test() -> + {ok, Module} = file:read_file("test/test.lua"), + Base = #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + }, + <<"parameters">> => [] + }, + {ok, Res} = hb_ao:resolve(Base, <<"ao_resolve">>, #{}), + ?assertEqual(<<"Hello, AO world!">>, Res). + +%% @doc Benchmark the performance of Lua executions. +direct_benchmark_test() -> + BenchTime = 3, + {ok, Module} = file:read_file("test/test.lua"), + Base = #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + }, + <<"parameters">> => [] + }, + Iterations = hb_test_utils:benchmark( + fun(X) -> + {ok, _} = hb_ao:resolve(Base, <<"assoctable">>, #{}), + ?event({iteration, X}) + end, + BenchTime + ), + ?event({iterations, Iterations}), + hb_test_utils:benchmark_print( + <<"Direct Lua:">>, + <<"executions">>, + Iterations, + BenchTime + ), + ?assert(Iterations > 10). + +%% @doc Call a non-compute key on a Lua device message and ensure that the +%% function of the same name in the script is called. +invoke_non_compute_key_test() -> + {ok, Module} = file:read_file("test/test.lua"), + Base = #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + }, + <<"test-value">> => 42 + }, + {ok, Result1} = hb_ao:resolve(Base, <<"hello">>, #{}), + ?event({result1, Result1}), + ?assertEqual(42, hb_ao:get(<<"test-value">>, Result1, #{})), + ?assertEqual(<<"world">>, hb_ao:get(<<"hello">>, Result1, #{})), + {ok, Result2} = + hb_ao:resolve( + Base, + #{<<"path">> => <<"hello">>, <<"name">> => <<"Alice">>}, + #{} + ), + ?event({result2, Result2}), + ?assertEqual(<<"Alice">>, hb_ao:get(<<"hello">>, Result2, #{})). + +%% @doc Use a Lua module as a hook on the HTTP server via `~meta@1.0'. +lua_http_hook_test() -> + {ok, Module} = file:read_file("test/test.lua"), + Node = hb_http_server:start_node( + #{ + priv_wallet => ar_wallet:new(), + on => #{ + <<"request">> => + #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + } + } + } + }), + {ok, Res} = hb_http:get(Node, <<"/hello?hello=world">>, #{}), + ?assertMatch(#{ <<"body">> := <<"i like turtles">> }, Res). + +%% @doc Call a process whose `execution-device' is set to `lua@5.3a'. +pure_lua_process_test() -> + Process = generate_lua_process("test/test.lua", #{}), + {ok, _} = hb_cache:write(Process, #{}), + Message = generate_test_message(Process, #{}), + {ok, _} = hb_ao:resolve(Process, Message, #{ hashpath => ignore }), + {ok, Results} = hb_ao:resolve(Process, <<"now">>, #{}), + ?assertEqual(42, hb_ao:get(<<"results/output/body">>, Results, #{})). + +%% @doc Call a process whose `execution-device' is set to `lua@5.3a'. +pure_lua_restore_test() -> + Opts = #{ process_cache_frequency => 1 }, + Process = generate_lua_process("test/test.lua", Opts), + {ok, _} = hb_cache:write(Process, Opts), + Message = generate_test_message(Process, Opts, #{ <<"path">> => <<"inc">>}), + {ok, _} = hb_ao:resolve(Process, Message, Opts#{ hashpath => ignore }), + {ok, Count1} = hb_ao:resolve(Process, <<"now/count">>, Opts), + ?assertEqual(1, Count1), + hb_ao:resolve( + Process, + generate_test_message(Process, #{}, #{ <<"path">> => <<"inc">>}), + Opts + ), + {ok, Count2} = hb_ao:resolve(Process, <<"now/count">>, Opts), + ?assertEqual(2, Count2). + +pure_lua_process_benchmark_test_() -> + {timeout, + 30, + fun() -> + pure_lua_process_benchmark(#{ + process_snapshot_slots => 50 + }) + end}. +pure_lua_process_benchmark(Opts) -> + BenchMsgs = 50, + hb:init(), + Process = generate_lua_process("test/test.lua", Opts), + {ok, _} = hb_cache:write(Process, Opts), + Message = generate_test_message(Process, Opts), + lists:foreach( + fun(X) -> + hb_ao:resolve(Process, Message, Opts#{ hashpath => ignore }), + ?event(debug_lua, {scheduled, X}) + end, + lists:seq(1, BenchMsgs) + ), + ?event(debug_lua, {executing, BenchMsgs}), + BeforeExec = os:system_time(millisecond), + {ok, _} = hb_ao:resolve(Process, <<"now">>, Opts), + AfterExec = os:system_time(millisecond), + hb_test_utils:benchmark_print( + <<"Pure Lua process: Computed">>, + <<"slots">>, + BenchMsgs, + (AfterExec - BeforeExec) / 1000 + ). + +invoke_aos_test() -> + Opts = #{ priv_wallet => hb:wallet() }, + Process = generate_lua_process("test/hyper-aos.lua", Opts), + {ok, _Proc} = hb_cache:write(Process, Opts), + Message = generate_test_message(Process, Opts), + {ok, _Assignment} = hb_ao:resolve(Process, Message, Opts#{ hashpath => ignore }), + {ok, Results} = hb_ao:resolve(Process, <<"now/results/output">>, Opts), + ?assertEqual(<<"1">>, hb_ao:get(<<"data">>, Results, #{})), + ?assertEqual(<<"aos> ">>, hb_ao:get(<<"prompt">>, Results, #{})). + +aos_authority_not_trusted_test() -> + Opts = #{ priv_wallet => ar_wallet:new() }, + Process = generate_lua_process("test/hyper-aos.lua", Opts), + ProcID = hb_message:id(Process, all), + {ok, _} = hb_cache:write(Process, Opts), + Message = hb_message:commit( + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + #{ + <<"target">> => ProcID, + <<"type">> => <<"Message">>, + <<"data">> => <<"1 + 1">>, + <<"random-seed">> => rand:uniform(1337), + <<"action">> => <<"Eval">>, + <<"from-process">> => <<"1234">> + }, + Opts + ) + }, + Opts + ), + ?event({message, Message}), + {ok, _} = hb_ao:resolve(Process, Message, Opts#{ hashpath => ignore }), + {ok, Results} = hb_ao:resolve(Process, <<"now/results/output/data">>, Opts), + ?assertEqual(<<"Message is not trusted.">>, Results). + +%% @doc Benchmark the performance of Lua executions. +aos_process_benchmark_test_() -> + {timeout, 30, fun() -> + BenchMsgs = 10, + Opts = #{ + process_async_cache => true, + hashpath => ignore, + process_snapshot_slots => 50 + }, + Process = generate_lua_process("test/hyper-aos.lua", Opts), + Message = generate_test_message(Process, Opts), + lists:foreach( + fun(X) -> + hb_ao:resolve(Process, Message, Opts), + ?event(debug_lua, {scheduled, X}) + end, + lists:seq(1, BenchMsgs) + ), + ?event(debug_lua, {executing, BenchMsgs}), + BeforeExec = os:system_time(millisecond), + {ok, _} = hb_ao:resolve( + Process, + <<"now">>, + Opts + ), + AfterExec = os:system_time(millisecond), + hb_test_utils:benchmark_print( + <<"HyperAOS process: Computed">>, + <<"slots">>, + BenchMsgs, + (AfterExec - BeforeExec) / 1000 + ) + end}. + +%%% Test helpers + +%% @doc Generate a Lua process message. +generate_lua_process(File, Opts) -> + NormOpts = Opts#{ priv_wallet => hb_opts:get(priv_wallet, hb:wallet(), Opts) }, + Wallet = hb_opts:get(priv_wallet, hb:wallet(), NormOpts), + Address = hb_util:human_id(ar_wallet:to_address(Wallet)), + {ok, Module} = file:read_file(File), + hb_message:commit( + #{ + <<"device">> => <<"process@1.0">>, + <<"type">> => <<"Process">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"execution-device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + }, + <<"authority">> => [ + Address, + <<"E3FJ53E6xtAzcftBpaw2E1H4ZM9h6qy6xz9NXh5lhEQ">> + ], + <<"scheduler-location">> => + hb_util:human_id(ar_wallet:to_address(Wallet)), + <<"test-random-seed">> => rand:uniform(1337) + }, + NormOpts + ). + +%% @doc Generate a test message for a Lua process. +generate_test_message(Process, Opts) -> + generate_test_message( + Process, + Opts, + <<""" + Count = 0 + function add() + Send({Target = 'Foo', Data = 'Bar' }); + Count = Count + 1 + end + add() + return Count + """>> + ). +generate_test_message(Process, Opts, ToEval) when is_binary(ToEval) -> + generate_test_message( + Process, + Opts, + #{ + <<"action">> => <<"Eval">>, + <<"body">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => hb_util:bin(ToEval) + } + } + ); +generate_test_message(Process, Opts, MsgBase) -> + ProcID = hb_message:id(Process, all), + NormOpts = Opts#{ priv_wallet => hb_opts:get(priv_wallet, hb:wallet(), Opts) }, + hb_message:commit(#{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + MsgBase#{ + <<"target">> => ProcID, + <<"type">> => <<"Message">>, + <<"random-seed">> => rand:uniform(1337) + }, + NormOpts + ) + }, + NormOpts + ). + +%% @doc Generate a stack message for the Lua process. +generate_stack(File) -> + Wallet = hb:wallet(), + {ok, Module} = file:read_file(File), + Msg1 = #{ + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => + [ + <<"json-iface@1.0">>, + <<"lua@5.3a">>, + <<"multipass@1.0">> + ], + <<"function">> => <<"json_result">>, + <<"passes">> => 2, + <<"stack-keys">> => [<<"init">>, <<"compute">>], + <<"module">> => Module, + <<"process">> => + hb_message:commit(#{ + <<"type">> => <<"Process">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Module + }, + <<"scheduler">> => hb:address(), + <<"authority">> => hb:address() + }, Wallet) + }, + {ok, Msg2} = hb_ao:resolve(Msg1, <<"init">>, #{}), + Msg2. + +% execute_aos_call(Base) -> +% Req = +% hb_message:commit(#{ +% <<"action">> => <<"Eval">>, +% <<"function">> => <<"json_result">>, +% <<"data">> => <<"return 2">> +% }, +% hb:wallet() +% ), +% execute_aos_call(Base, Req). +% execute_aos_call(Base, Req) -> +% hb_ao:resolve(Base, +% #{ +% <<"path">> => <<"compute">>, +% <<"body">> => Req +% }, +% #{} +% ). \ No newline at end of file diff --git a/src/dev_lua_lib.erl b/src/dev_lua_lib.erl new file mode 100644 index 000000000..ab7bbcc08 --- /dev/null +++ b/src/dev_lua_lib.erl @@ -0,0 +1,203 @@ +%%% @doc A module for providing AO library functions to the Lua environment. +%%% This module contains the implementation of the functions, each by the name +%%% that should be used in the `ao' table in the Lua environment. Every export +%%% is imported into the Lua environment. +%%% +%%% Each function adheres closely to the Luerl calling convention, adding the +%%% appropriate node message as a third argument: +%%% +%%% fun(Args, State, NodeMsg) -> {ResultTerms, NewState} +%%% +%%% As Lua allows for multiple return values, each function returns a list of +%%% terms to grant to the caller. Matching the tuple convention used by AO-Core, +%%% the first term is typically the status, and the second term is the result. +-module(dev_lua_lib). +%%% Library functions. Each exported function is _automatically_ added to the +%%% Lua environment, except for the `install/3' function, which is used to +%%% install the library in the first place. +-export([get/3, resolve/3, set/3, event/3, install/3]). +-include("include/hb.hrl"). + +%%% The set of devices that must be included in the device sandbox for an +%%% execution that is able to perform AO-Core resolutions. Without the following +%%% devices, all resolutions will fail. +-define(MINIMAL_AO_CORE_DEVICES, [<<"structured@1.0">>]). + +%% @doc Install the library into the given Lua environment. +install(Base, State, Opts) -> + % Calculate and set the new `preloaded_devices' option. + AllDevs = hb_opts:get(preloaded_devices, Opts), + DevSandboxDef = + hb_ao:get( + <<"device-sandbox">>, + {as, <<"message@1.0">>, Base}, + false, + Opts + ), + AdmissibleDevs = + case DevSandboxDef of + false -> AllDevs; + DevNames -> + lists:map( + fun(Name) -> + [Dev] = + lists:filter( + fun(X) -> + hb_ao:get(<<"name">>, X, Opts) == Name + end, + AllDevs + ), + Dev + end, + hb_util:message_to_ordered_list( + hb_util:unique(DevNames ++ ?MINIMAL_AO_CORE_DEVICES) + ) + ) + end, + ?event({adding_ao_core_resolver, {device_sandbox, AdmissibleDevs}}), + ExecOpts = + Opts#{ + preloaded_devices => AdmissibleDevs, + hashpath => ignore + }, + % Initialize the AO-Core resolver. + BaseAOTable = + case luerl:get_table_keys_dec([ao], State) of + {ok, nil, _} -> + ?event(no_ao_table), + #{}; + {ok, ExistingTable, _} -> + ?event({existing_ao_table, ExistingTable}), + dev_lua:decode(ExistingTable, Opts) + end, + ?event({base_ao_table, BaseAOTable}), + {ok, State2} = + luerl:set_table_keys_dec( + [ao], + dev_lua:encode(BaseAOTable, Opts), + State + ), + { + ok, + lists:foldl( + fun(FuncName, StateIn) -> + {ok, StateOut} = + luerl:set_table_keys_dec( + [ao, FuncName], + fun(RawArgs, ImportState) -> + ?event(lua_import, {calling_import, {func, FuncName}}), + % Decode the arguments from the Lua environment. + Args = + lists:map( + fun(Arg) -> + dev_lua:decode( + luerl:decode(Arg, ImportState), + Opts + ) + end, + RawArgs + ), + % Call the function with the decoded arguments. + {Res, ResState} = + ?MODULE:FuncName(Args, ImportState, ExecOpts), + % Encode the response for return to Lua + return(Res, ResState, Opts) + end, + StateIn + ), + StateOut + end, + State2, + [ + FuncName + || + {FuncName, _} <- dev_lua_lib:module_info(exports), + FuncName /= module_info, + FuncName /= ?FUNCTION_NAME + ] + ) + }. + +%% @doc Helper function for returning a result from a Lua function. +return(Result, ExecState, Opts) -> + ?event(lua_import, {import_returning, {result, Result}}), + TableEncoded = dev_lua:encode(hb_cache:ensure_all_loaded(Result, Opts), Opts), + {ReturnParams, ResultingState} = + lists:foldr( + fun(LuaEncoded, {Params, StateIn}) -> + {NewParam, NewState} = luerl:encode(LuaEncoded, StateIn), + {[NewParam | Params], NewState} + end, + {[], ExecState}, + TableEncoded + ), + ?event({lua_encoded, ReturnParams}), + {ReturnParams, ResultingState}. + +%% @doc A wrapper function for performing AO-Core resolutions. Offers both the +%% single-message (using `hb_singleton:from/1' to parse) and multiple-message +%% (using `hb_ao:resolve_many/2') variants. +resolve([SingletonMsg], ExecState, ExecOpts) -> + ?event({ao_core_resolver, {msg, SingletonMsg}}), + ParsedMsgs = hb_singleton:from(SingletonMsg, ExecOpts), + ?event({parsed_msgs_to_resolve, ParsedMsgs}), + resolve({many, ParsedMsgs}, ExecState, ExecOpts); +resolve([Base, Path], ExecState, ExecOpts) when is_binary(Path) -> + PathParts = hb_path:term_to_path_parts(Path, ExecOpts), + resolve({many, [Base] ++ PathParts}, ExecState, ExecOpts); +resolve(Msgs, ExecState, ExecOpts) when is_list(Msgs) -> + resolve({many, Msgs}, ExecState, ExecOpts); +resolve({many, Msgs}, ExecState, ExecOpts) -> + MaybeAsMsgs = lists:map(fun convert_as/1, Msgs), + try hb_ao:resolve_many(MaybeAsMsgs, ExecOpts) of + {Status, Res} -> + ?event({resolved_msgs, {status, Status}, {res, Res}, {exec_opts, ExecOpts}}), + {[Status, Res], ExecState} + catch + Error -> + ?event(lua_error, {ao_core_resolver_error, Error}), + {[<<"error">>, Error], ExecState} + end. + +%% @doc A wrapper for `hb_ao''s `get' functionality. +get([Key, Base], ExecState, ExecOpts) -> + ?event({ao_core_get, {base, Base}, {key, Key}}), + NewRes = hb_ao:get(convert_as(Key), convert_as(Base), ExecOpts), + ?event({ao_core_get_result, {result, NewRes}}), + {[NewRes], ExecState}. + +%% @doc Converts any `as' terms from Lua to their HyperBEAM equivalents. +convert_as([<<"as">>, Device, RawMsg]) -> + {as, Device, RawMsg}; +convert_as(Other) -> + Other. + +%% @doc Wrapper for `hb_ao''s `set' functionality. +set([Base, Key, Value], ExecState, ExecOpts) -> + ?event({ao_core_set, {base, Base}, {key, Key}, {value, Value}}), + NewRes = hb_ao:set(Base, Key, Value, ExecOpts), + ?event({ao_core_set_result, {result, NewRes}}), + {[NewRes], ExecState}; +set([Base, NewValues], ExecState, ExecOpts) -> + ?event({ao_core_set, {base, Base}, {new_values, NewValues}}), + NewRes = hb_ao:set(Base, NewValues, ExecOpts), + ?event({ao_core_set_result, {result, NewRes}}), + {[NewRes], ExecState}. + +%% @doc Allows Lua scripts to signal events using the HyperBEAM hosts internal +%% event system. +event([Event], ExecState, Opts) -> + ?event({recalling_event, Event}), + event([global, Event], ExecState, Opts); +event([Group, Event], State, Opts) when is_list(Event) -> + event([Group, list_to_tuple(Event)], State, Opts); +event([Group, Event], ExecState, Opts) -> + ?event( + lua_event, + {event, + {group, Group}, + {event, Event} + } + ), + ?event(Group, Event), + {[<<"ok">>], ExecState}. \ No newline at end of file diff --git a/src/dev_lua_test.erl b/src/dev_lua_test.erl new file mode 100644 index 000000000..0d945934e --- /dev/null +++ b/src/dev_lua_test.erl @@ -0,0 +1,169 @@ +%%% A wrapper module for generating and executing EUnit tests for all Lua modules. +%%% When executed with `rebar3 lua-test`, this module will be invoked and scan the +%%% `scripts' directory for all Lua files, and generate an EUnit test suite for +%%% each one. By default, an individual test is generated for each function in +%%% the global `_G' table that ends in `_test'. +%%% +%%% In order to specify other tests to run instead, the user may employ the +%%% `LUA_TESTS' and `LUA_SCRIPTS' environment variables. The syntax for these +%%% variables is described in the function documentation for `parse_spec'. +%%% +-module(dev_lua_test). +-export([parse_spec/1]). +-include_lib("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Parse a string representation of test descriptions received from the +%% command line via the `LUA_TESTS' environment variable. +%% +%% Supported syntax in loose BNF/RegEx: +%% +%% Definitions := (ModDef,)+ +%% ModDef := ModName(TestDefs)? +%% ModName := ModuleInLUA_SCRIPTS|(FileName[.lua])? +%% TestDefs := (:TestDef)+ +%% TestDef := TestName +%% +%% File names ending in `.lua' are assumed to be relative paths from the current +%% working directory. Module names lacking the `.lua' extension are assumed to +%% be modules found in the `LUA_SCRIPTS' environment variable (defaulting to +%% `scripts/'). +%% +%% For example, to run a single test one could call the following: +%% +%% LUA_TESTS=~/src/LuaScripts/test.yourTest rebar3 lua-tests +%% +%% To specify that one would like to run all of the tests in the +%% `scripts/test.lua' file and two tests from the `scripts/test2.lua' file, the +%% user could provide the following test definition: +%% +%% LUA_TESTS="test,scripts/test2.userTest1|userTest2" rebar3 lua-tests +%% +parse_spec(Str) when is_list(Str) -> + parse_spec(hb_util:bin(Str)); +parse_spec(tests) -> + % The user has not given a test spec, so we default to running all tests in + % the `LUA_SCRIPTS' directory (defaulting to `scripts/'). + Files = + case file:list_dir(ScriptDir = hb_opts:get(lua_scripts)) of + {ok, FileList} -> FileList; + {error, enoent} -> [] + end, + RelevantFiles = + lists:filter( + fun(File) -> + terminates_with(File, <<"lua">>) + end, + Files + ), + ?event({loading_scripts, RelevantFiles}), + [ + { + << + (hb_util:bin(ScriptDir))/binary, + "/", + (hb_util:bin(File))/binary + >>, + tests + } + || + File <- RelevantFiles + ]; +parse_spec(Str) -> + lists:map( + fun(ModDef) -> + [ModName|TestDefs] = binary:split(ModDef, <<":">>, [global, trim_all]), + ScriptDir = hb_util:bin(hb_opts:get(lua_scripts)), + File = + case terminates_with(ModName, <<".lua">>) of + true -> ModName; + false -> << ScriptDir/binary, "/", ModName/binary, ".lua" >> + end, + Tests = + case TestDefs of + [] -> tests; + TestDefs -> TestDefs + end, + {File, Tests} + end, + binary:split(Str, <<",">>, [global, trim_all]) + ). + +%% @doc Main entrypoint for Lua tests. +exec_test_() -> + ScriptDefs = hb_opts:get(lua_tests), + lists:map( + fun({File, Funcs}) -> suite(File, Funcs) end, + ScriptDefs + ). + +%% @doc Generate an EUnit test suite for a given Lua script. If the `Funcs' is +%% the atom `tests' we find all of the global functions in the script, then +%% filter for those ending in `_test' in a similar fashion to Eunit. +suite(File, Funcs) -> + {ok, State} = new_state(File), + {foreach, + fun() -> ok end, + fun(_) -> ok end, + lists:map( + fun(FuncName) -> + { + hb_util:list(File) ++ ":" ++ hb_util:list(FuncName), + fun() -> exec_test(State, FuncName) end + } + end, + case Funcs of + tests -> + lists:filter( + fun(FuncName) -> + terminates_with(FuncName, <<"_test">>) + end, + hb_ao:get(<<"functions">>, State, #{}) + ); + FuncNames -> FuncNames + end + ) + }. + +%% @doc Create a new Lua environment for a given script. +new_state(File) -> + ?event(debug_lua_test, {generating_state_for, File}), + {ok, Module} = file:read_file(hb_util:list(File)), + {ok, _} = + hb_ao:resolve( + #{ + <<"device">> => <<"lua@5.3a">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"name">> => File, + <<"body">> => Module + } + }, + <<"init">>, + #{} + ). + +%% @doc Generate an EUnit test for a given function. +exec_test(State, Function) -> + {Status, Result} = + hb_ao:resolve( + State, + #{ <<"path">> => Function, <<"parameters">> => [] }, + #{} + ), + case Status of + ok -> ok; + error -> + hb_format:print(Result, <<"Lua">>, Function, 1), + ?assertEqual( + ok, + Status + ) + end. + +%%% Utility functions. + +%% @doc Check if a string terminates with a given suffix. +terminates_with(String, Suffix) -> + binary:longest_common_suffix(lists:map(fun hb_util:bin/1, [String, Suffix])) + == byte_size(Suffix). \ No newline at end of file diff --git a/src/dev_lua_test_ledgers.erl b/src/dev_lua_test_ledgers.erl new file mode 100644 index 000000000..33e5215d7 --- /dev/null +++ b/src/dev_lua_test_ledgers.erl @@ -0,0 +1,953 @@ +%%% A collection of Eunit tests for the `lua@5.3a` device, and the +%%% `hyper-token.lua` script. These tests are designed to validate the +%%% functionality of both of these components, and to provide examples +%%% of how to use the `lua@5.3a` device. +%%% +%%% The module is split into four components: +%%% 1. A simple ledger client library. +%%% 2. Assertion functions that verify specific invariants about the state +%%% of ledgers in a test environment. +%%% 3. Utility functions for normalizing the state of a test environment. +%%% 4. Test cases that generate and manipulate ledger networks in test +%%% environments. +%%% +%%% Many client and utility functions in this module handle the conversion of +%%% wallet IDs to human-readable addresses when found in transfers, balances, +%%% and other fields. This is done to make the test cases more readable and +%%% easier to understand -- be careful if following their patterns in other +%%% contexts to either mimic a similar pattern, or to ensure you pass addresses +%%% in these contexts rather that full wallet objects. +-module(dev_lua_test_ledgers). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("include/hb.hrl"). + +%%% Ledger client library. +%%% +%%% A simple, thin library for generating ledgers and interacting with +%%% `hyper-token.lua` processes. + +%% @doc Generate a Lua process definition message. +ledger(Script, Opts) -> + ledger(Script, #{}, Opts). +ledger(Script, Extra, Opts) -> + % If the `balance' key is set in the `Extra' map, ensure that any wallets + % given as keys in the message are converted to human-readable addresses. + HostWallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + ModExtra = + case maps:get(<<"balance">>, Extra, undefined) of + undefined -> Extra; + RawBalance -> + Extra#{ + <<"balance">> => + maps:from_list( + lists:map( + fun({ID, Amount}) when ?IS_ID(ID) -> + {hb_util:human_id(ID), Amount}; + ({Wallet, Amount}) when is_tuple(Wallet) -> + { + hb_util:human_id( + ar_wallet:to_address(Wallet) + ), + Amount + } + end, + maps:to_list(RawBalance) + ) + ) + } + end, + Proc = + hb_message:commit( + maps:merge( + #{ + <<"device">> => <<"process@1.0">>, + <<"type">> => <<"Process">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"scheduler">> => hb_util:human_id(HostWallet), + <<"execution-device">> => <<"lua@5.3a">>, + <<"authority">> => hb_util:human_id(HostWallet), + <<"module">> => lua_script(Script) + }, + ModExtra + ), + Opts#{ priv_wallet => HostWallet } + ), + hb_cache:write(Proc, Opts), + Proc. + +%% @doc Generate a Lua `script' key from a file or list of files. +lua_script(Files) when is_list(Files) -> + [ + #{ + <<"content-type">> => <<"application/lua">>, + <<"module">> => File, + <<"body">> => + hb_util:ok( + file:read_file( + if is_binary(File) -> binary_to_list(File); + true -> File + end + ) + ) + } + || + File <- Files + ]; +lua_script(File) when is_binary(File) -> + hd(lua_script([File])). + +%% @doc Generate a test sub-ledger process definition message. +subledger(Root, Opts) -> + subledger(Root, #{}, Opts). +subledger(Root, Extra, Opts) -> + BareRoot = + maps:without( + [<<"token">>, <<"balance">>], + hb_message:uncommitted(Root) + ), + Proc = + hb_message:commit( + maps:merge( + BareRoot#{ + <<"token">> => hb_message:id(Root, all) + }, + Extra + ), + hb_opts:get(priv_wallet, hb:wallet(), Opts) + ), + hb_cache:write(Proc, Opts), + Proc. + +%% @doc Generate a test transfer message. +transfer(ProcMsg, Sender, Recipient, Quantity, Opts) -> + transfer(ProcMsg, Sender, Recipient, Quantity, undefined, Opts). +transfer(ProcMsg, Sender, Recipient, Quantity, Route, Opts) -> + MaybeRoute = + if Route == undefined -> #{}; + true -> + #{ + <<"route">> => + if is_map(Route) -> hb_message:id(Route, all); + true -> Route + end + } + end, + Xfer = + hb_message:commit(#{ + <<"path">> => <<"push">>, + <<"body">> => + hb_message:commit(MaybeRoute#{ + <<"action">> => <<"Transfer">>, + <<"target">> => hb_message:id(ProcMsg, all), + <<"recipient">> => hb_util:human_id(Recipient), + <<"quantity">> => Quantity + }, + Sender + ) + }, + Sender + ), + hb_ao:resolve( + ProcMsg, + Xfer, + Opts#{ priv_wallet => hb_opts:get(priv_wallet, hb:wallet(), Opts) } + ). + +%% @doc Request that a peer register with a without sub-ledger. +register(ProcMsg, Peer, Opts) when is_map(Peer) -> + register(ProcMsg, hb_message:id(Peer, all), Opts); +register(ProcMsg, PeerID, RawOpts) -> + Opts = + RawOpts#{ + priv_wallet => hb_opts:get(priv_wallet, hb:wallet(), RawOpts) + }, + Reg = + hb_message:commit( + #{ + <<"path">> => <<"push">>, + <<"body">> => + hb_message:commit( + #{ + <<"action">> => <<"register-remote">>, + <<"target">> => hb_message:id(ProcMsg, all), + <<"peer">> => PeerID + }, + Opts + ) + }, + Opts + ), + hb_ao:resolve( + ProcMsg, + Reg, + Opts + ). + +%% @doc Retreive a single balance from the ledger. +balance(ProcMsg, User, Opts) when not ?IS_ID(User) -> + balance(ProcMsg, hb_util:human_id(ar_wallet:to_address(User)), Opts); +balance(ProcMsg, ID, Opts) -> + hb_ao:get(<<"now/balance/", ID/binary>>, ProcMsg, 0, Opts). + +%% @doc Get the total balance for an ID across all ledgers in a set. +balance_total(Procs, ID, Opts) -> + lists:sum( + lists:map( + fun(Proc) -> balance(Proc, ID, Opts) end, + maps:values(normalize_env(Procs)) + ) + ). + +%% @doc Get the balances of a ledger. +balances(ProcMsg, Opts) -> + balances(now, ProcMsg, Opts). +balances(initial, ProcMsg, Opts) -> + balances(<<"">>, ProcMsg, Opts); +balances(Mode, ProcMsg, Opts) when is_atom(Mode) -> + balances(hb_util:bin(Mode), ProcMsg, Opts); +balances(Prefix, ProcMsg, Opts) -> + Balances = hb_ao:get(<>, ProcMsg, #{}, Opts), + hb_private:reset(hb_cache:ensure_all_loaded(Balances, Opts)). + +%% @doc Get the supply of a ledger, either `now` or `initial`. +supply(ProcMsg, Opts) -> + supply(now, ProcMsg, Opts). +supply(Mode, ProcMsg, Opts) -> + lists:sum(maps:values(balances(Mode, ProcMsg, Opts))). + +%% @doc Calculate the supply of tokens in all sub-ledgers, from the balances of +%% the root ledger. +subledger_supply(RootProc, AllProcs, Opts) -> + supply(now, RootProc, Opts) - user_supply(RootProc, AllProcs, Opts). + +%% @doc Calculate the supply of tokens held by users on a ledger, excluding +%% those held in sub-ledgers. +user_supply(Proc, AllProcs, Opts) -> + NormProcs = normalize_without_root(Proc, AllProcs), + SubledgerIDs = maps:keys(NormProcs), + lists:sum( + maps:values( + maps:without( + SubledgerIDs, + balances(now, Proc, Opts) + ) + ) + ). + +%% @doc Get the local expectation of a ledger's balances with peer ledgers. +ledgers(ProcMsg, Opts) -> + case hb_cache:ensure_all_loaded( + hb_ao:get(<<"now/ledgers">>, ProcMsg, #{}, Opts), + Opts + ) of + Msg when is_map(Msg) -> hb_private:reset(Msg); + [] -> #{} + end. + +%% @doc Generate a complete overview of the test environment's balances and +%% ledgers. Optionally, a map of environment names can be provided to make the +%% output more readable. +map(Procs, Opts) -> + NormProcs = normalize_env(Procs), + maps:merge_with( + fun(Key, Balances, Ledgers) -> + MaybeRoot = + case maps:get(Key, NormProcs, #{}) of + #{ <<"token">> := _ } -> #{}; + _ -> #{ root => true } + end, + MaybeRoot#{ + balances => Balances, + ledgers => Ledgers + } + end, + maps:map(fun(_, Proc) -> balances(Proc, Opts) end, NormProcs), + maps:map(fun(_, Proc) -> ledgers(Proc, Opts) end, NormProcs) + ). +map(Procs, EnvNames, Opts) -> + apply_names(map(Procs, Opts), EnvNames, Opts). + +%% @doc Apply a map of environment names to elements in either a map or list. +%% Expects a map of `ID or ProcMsg or Wallet => Name' as the `EnvNames' argument, +%% and a potentially deep map or list of elements to apply the names to. +apply_names(Map, EnvNames, Opts) -> + IDs = + maps:from_list( + lists:filtermap( + fun({Key, V}) -> + try {true, {hb_util:human_id(Key), V}} + catch _:_ -> + try {true, {hb_message:id(Key, all), V}} + catch _:_ -> false + end + end + end, + maps:to_list(EnvNames) + ) + ), + do_apply_names(Map, maps:merge(IDs, EnvNames), Opts). +do_apply_names(Map, EnvNames, Opts) when is_map(Map) -> + maps:from_list( + lists:map( + fun({Key, Proc}) -> + { + apply_names(Key, EnvNames, Opts), + apply_names(Proc, EnvNames, Opts) + } + end, + maps:to_list(Map) + ) + ); +do_apply_names(List, EnvNames, Opts) when is_list(List) -> + lists:map( + fun(Proc) -> + apply_names(Proc, EnvNames, Opts) + end, + List + ); +do_apply_names(Item, Names, _Opts) when is_map_key(Item, Names) -> + maps:get(Item, Names); +do_apply_names(Item, Names, _Opts) -> + try maps:get(hb_util:human_id(Item), Names, Item) + catch _:_ -> Item + end. + +%%% Test ledger network invariants. +%%% +%%% Complex assertions that verify specific invariants about the state of +%%% ledgers in a test environment. These are used to validate the correctness +%%% of the `hyper-token.lua` script. Tested invariants are listed below. +%%% +%%% For every timestep `t_n`, the following invariants must hold: +%%% 1. The root ledger supply at `t_0` must match the current supply. +%%% 2. For every sub-ledger `l`, each expected balance held in `l/now/ledgers` +%%% must equal the balance found at `peer/now/balance/l`. +%%% 3. The sum of all values in `/now/balance` across all sub-ledgers must +%%% equal the root ledger's supply. + +%% @doc Execute all invariant checks for a pair of root ledger and sub-ledgers. +verify_net(RootProc, AllProcs, Opts) -> + verify_net_supply(RootProc, AllProcs, Opts), + verify_net_peer_balances(AllProcs, Opts). + +%% @doc Verify that the initial supply of tokens on the root ledger is the same +%% as the current supply. This invariant will not hold for sub-ledgers, as they +%% 'mint' tokens in their local supply when they receive them from other ledgers. +verify_root_supply(RootProc, Opts) -> + ?assert( + supply(initial, RootProc, Opts) == + supply(now, RootProc, Opts) + + lists:sum(maps:values(ledgers(RootProc, Opts))) + ). + +%% @doc Verify that the sum of all spendable balances held by ledgers in a +%% test network is equal to the initial supply of tokens. +verify_net_supply(RootProc, AllProcs, Opts) -> + verify_root_supply(RootProc, Opts), + StartingRootSupply = supply(initial, RootProc, Opts), + NormProcsWithoutRoot = normalize_without_root(RootProc, AllProcs), + SubledgerIDs = maps:keys(NormProcsWithoutRoot), + RootUserSupply = user_supply(RootProc, NormProcsWithoutRoot, Opts), + SubledgerSupply = subledger_supply(RootProc, AllProcs, Opts), + ?event({verify_net_supply, {root, RootUserSupply}, {subledger, SubledgerSupply}}), + ?assert( + StartingRootSupply == + RootUserSupply + SubledgerSupply + ). + +%% @doc Verify the consistency of all expected ledger balances with their peer +%% ledgers and the actual balances held. +verify_net_peer_balances(AllProcs, Opts) -> + NormProcs = normalize_env(AllProcs), + maps:map( + fun(ValidateProc, _) -> + verify_peer_balances(ValidateProc, NormProcs, Opts) + end, + NormProcs + ). + +%% @doc Verify that a ledger's expectation of its balances with peer ledgers +%% is consistent with the actual balances held. +verify_peer_balances(ValidateProc, AllProcs, Opts) -> + Ledgers = ledgers(ValidateProc, Opts), + NormProcs = normalize_env(AllProcs), + maps:map( + fun(PeerID, ExpectedBalance) -> + ?assertEqual( + ExpectedBalance, + balance(ValidateProc, + maps:get(PeerID, NormProcs), + Opts + ) + ) + end, + Ledgers + ). + +%%% Test utilities. + +%% @doc Normalize a set of processes, representing ledgers in a test environment, +%% to a canonical form: A map of `ID => Proc`. +normalize_env(Procs) when is_map(Procs) -> + normalize_env(maps:values(Procs)); +normalize_env(Procs) when is_list(Procs) -> + maps:from_list( + lists:map( + fun(Proc) -> + {hb_message:id(Proc, all), Proc} + end, + Procs + ) + ). + +%% @doc Return the normalized environment without the root ledger. +normalize_without_root(RootProc, Procs) -> + maps:without([hb_message:id(RootProc, all)], normalize_env(Procs)). + +%% @doc Create a node message for the test that avoids looking up unknown +%% recipients via remote stores. This improves test performance. +test_opts() -> + hb:init(), + #{}. + +%%% Test cases. + +%% @doc Test the `transfer` function. +%% 1. Alice has 100 tokens on a root ledger. +%% 2. Alice sends 1 token to Bob. +%% 3. Alice has 99 tokens, and Bob has 1 token. +transfer_test_() -> {timeout, 30, fun transfer/0}. +transfer() -> + Opts = test_opts(), + Alice = ar_wallet:new(), + Bob = ar_wallet:new(), + Proc = + ledger( + <<"scripts/hyper-token.lua">>, + #{ <<"balance">> => #{ Alice => 100 } }, + Opts + ), + ?assertEqual(100, supply(Proc, Opts)), + transfer(Proc, Alice, Bob, 1, Opts), + ?assertEqual(99, balance(Proc, Alice, Opts)), + ?assertEqual(1, balance(Proc, Bob, Opts)), + ?assertEqual(100, supply(Proc, Opts)). + +%% @doc User's must not be able to send tokens they do not own. We test three +%% cases: +%% 1. Transferring a token when the sender has no tokens. +%% 2. Transferring a token when the sender has less tokens than the amount +%% being transferred. +%% 3. Transferring a binary-encoded amount of tokens that exceed the quantity +%% of tokens the sender has available. +transfer_unauthorized_test_() -> {timeout, 30, fun transfer_unauthorized/0}. +transfer_unauthorized() -> + Opts = test_opts(), + Alice = ar_wallet:new(), + Bob = ar_wallet:new(), + Proc = + ledger( + <<"scripts/hyper-token.lua">>, + #{ <<"balance">> => #{ Alice => 100 } }, + Opts + ), + % 1. Transferring a token when the sender has no tokens. + Result = transfer(Proc, Bob, Alice, 1, Opts), + ?event({unauthorized_transfer, {result, Result}}), + % 2. Transferring a token when the sender has less tokens than the amount + % being transferred. + transfer(Proc, Alice, Bob, 101, Opts), + ?event({unauthorized_transfer, {result, Result}}), + receive after 1000 -> ok end, + ?event({env, map([Proc], #{ Alice => alice, Bob => bob }, Opts)}), + ?assertEqual(100, balance(Proc, Alice, Opts)), + ?assertEqual(0, balance(Proc, Bob, Opts)), + % 3. Transferring a binary-encoded amount of tokens that exceed the quantity + % of tokens the sender has available. + transfer(Proc, Alice, Bob, <<"101">>, Opts), + ?assertEqual(100, balance(Proc, Alice, Opts)), + ?assertEqual(0, balance(Proc, Bob, Opts)), + % Validate the final supply of tokens. + ?assertEqual(100, supply(Proc, Opts)). + +%% @doc Verify that a user can deposit tokens into a sub-ledger. +subledger_deposit_test_() -> {timeout, 30, fun subledger_deposit/0}. +subledger_deposit() -> + Opts = test_opts(), + Alice = ar_wallet:new(), + Proc = + ledger( + <<"scripts/hyper-token.lua">>, + #{ <<"balance">> => #{ Alice => 100 } }, + Opts + ), + SubLedger = subledger(Proc, Opts), + % 1. Alice has tokens on the root ledger. + ?assertEqual(100, balance(Proc, Alice, Opts)), + % 2. Alice deposits tokens into the sub-ledger. + transfer(Proc, Alice, Alice, 10, SubLedger, Opts), + ?event({after_deposit, {result, map([Proc, SubLedger], Opts)} }), + ?assertEqual(90, balance(Proc, Alice, Opts)), + ?assertEqual(10, balance(SubLedger, Alice, Opts)), + % Verify all invariants. + verify_net(Proc, [SubLedger], Opts). + +%% @doc Simulate inter-ledger payments between users on a single sub-ledger: +%% 1. Alice has tokens on the root ledger. +%% 2. Alice sends tokens to the sub-ledger from the root ledger. +%% 3. Alice sends tokens to Bob on the sub-ledger. +%% 4. Bob sends tokens to Alice on the root ledger. +subledger_transfer_test_() -> {timeout, 10, fun subledger_transfer/0}. +subledger_transfer() -> + Opts = test_opts(), + Alice = ar_wallet:new(), + Bob = ar_wallet:new(), + RootLedger = + ledger( + <<"scripts/hyper-token.lua">>, + #{ <<"balance">> => #{ Alice => 100 } }, + Opts + ), + SubLedger = subledger(RootLedger, Opts), + EnvNames = #{ + Alice => alice, + Bob => bob, + RootLedger => root, + SubLedger => subledger + }, + % 1. Alice has tokens on the root ledger. + ?assertEqual(100, balance(RootLedger, Alice, Opts)), + ?event(token_log, {map, map([RootLedger], EnvNames, Opts)}), + % 2. Alice sends tokens to the sub-ledger from the root ledger. + transfer(RootLedger, Alice, Alice, 10, SubLedger, Opts), + ?assertEqual(90, balance(RootLedger, Alice, Opts)), + ?assertEqual(10, balance(SubLedger, Alice, Opts)), + % 3. Alice sends tokens to Bob on the sub-ledger. + transfer(SubLedger, Alice, Bob, 8, Opts), + ?event(token_log, + {state_after_subledger_user_xfer, + {names, map([RootLedger, SubLedger], EnvNames, Opts)}, + {ids, map([RootLedger, SubLedger], Opts)} + }), + % 4. Bob sends tokens to Alice on the root ledger. + transfer(SubLedger, Bob, Bob, 7, RootLedger, Opts), + % Validate the balances of the root and sub-ledgers. + Map = map([RootLedger, SubLedger], EnvNames, Opts), + ?event(token_log, {map, map([RootLedger, SubLedger], Opts)}), + ?assertEqual( + #{ + root => #{ + balances => #{ alice => 90, bob => 7.0, subledger => 3.0 }, + ledgers => #{}, + root => true + }, + subledger => #{ + balances => #{ alice => 2, bob => 1 }, + ledgers => #{} + } + }, + Map + ), + % Validate all invariants. + verify_net(RootLedger, [SubLedger], Opts). + +%% @doc Verify that peer ledgers on the same token are able to register mutually +%% to establish a peer-to-peer connection. +%% +%% Disabled as explicit peer registration is not required for `hyper-token.lua' +%% to function. +subledger_registration_test_disabled() -> + Opts = test_opts(), + Alice = ar_wallet:new(), + RootLedger = + ledger( + <<"scripts/hyper-token.lua">>, + #{ <<"balance">> => #{ Alice => 100 } }, + Opts + ), + SubLedger1 = subledger(RootLedger, Opts), + SubLedger2 = subledger(RootLedger, Opts), + Names = #{ + SubLedger1 => subledger1, + SubLedger2 => subledger2 + }, + ?event(debug, + {subledger, + {sl1, hb_message:id(SubLedger1, none)}, + {sl2, hb_message:id(SubLedger2, none)} + } + ), + % There are no registered peers on either sub-ledger. + ?assertEqual(0, map_size(ledgers(SubLedger1, Opts))), + ?assertEqual(0, map_size(ledgers(SubLedger2, Opts))), + % Alice registers with SubLedger1. + register(SubLedger1, SubLedger2, Opts), + ?event({map, map([SubLedger1, SubLedger2], Names, Opts)}), + ?event({sl1_ledgers, ledgers(SubLedger1, Opts)}), + ?event({sl2_ledgers, ledgers(SubLedger2, Opts)}), + % SubLedger1 and SubLedger2 are now aware of each other. + ?assertEqual(1, map_size(ledgers(SubLedger1, Opts))), + ?assertEqual(1, map_size(ledgers(SubLedger2, Opts))), + % Alice can send tokens to Bob on SubLedger2. + verify_net(RootLedger, [SubLedger1, SubLedger2], Opts). + +single_subledger_to_subledger_test_() -> {timeout, 30, fun single_subledger_to_subledger/0}. +single_subledger_to_subledger() -> + Opts = test_opts(), + Alice = ar_wallet:new(), + Bob = ar_wallet:new(), + RootLedger = + ledger( + <<"scripts/hyper-token.lua">>, + #{ <<"balance">> => #{ Alice => 100 } }, + Opts + ), + SubLedger1 = subledger(RootLedger, Opts), + SL1ID = hb_message:id(SubLedger1, signed, Opts), + ?event({sl1ID, SL1ID}), + SubLedger2 = subledger(RootLedger, Opts), + SL2ID = hb_message:id(SubLedger2, signed, Opts), + ?event({sl2ID, SL2ID}), + Names = #{ + Alice => alice, + Bob => bob, + RootLedger => root, + SubLedger1 => subledger1, + SubLedger2 => subledger2 + }, + ?event({root_ledger, RootLedger}), + ?event({sl1, SubLedger1}), + ?event({sl2, SubLedger2}), + ?assertEqual(100, balance(RootLedger, Alice, Opts)), + % 2. Alice sends 90 tokens to herself on SubLedger1. + ?event({transfer_1}), + transfer(RootLedger, Alice, Alice, 90, SubLedger1, Opts), + ?assertEqual(10, balance(RootLedger, Alice, Opts)), + ?assertEqual(90, balance(SubLedger1, Alice, Opts)), + ?event({transfer_2}), + PushRes = transfer(SubLedger1, Alice, Alice, 80, SubLedger2, Opts), + ?event({push_res, PushRes}), + ?event({map, map([RootLedger, SubLedger1, SubLedger2], Opts)}), + ?assertEqual(80, balance(SubLedger2, Alice, Opts)), + ?assertEqual(10, balance(SubLedger1, Alice, Opts)). + +%% @doc Verify that registered sub-ledgers are able to send tokens to each other +%% without the need for messages on the root ledger. +subledger_to_subledger_test_() -> {timeout, 30, fun subledger_to_subledger/0}. +subledger_to_subledger() -> + Opts = test_opts(), + Alice = ar_wallet:new(), + Bob = ar_wallet:new(), + RootLedger = + ledger( + <<"scripts/hyper-token.lua">>, + #{ <<"balance">> => #{ Alice => 100 } }, + Opts + ), + SubLedger1 = subledger(RootLedger, Opts), + SubLedger2 = subledger(RootLedger, Opts), + Names = #{ + Alice => alice, + Bob => bob, + RootLedger => root, + SubLedger1 => subledger1, + SubLedger2 => subledger2 + }, + % 1. Alice has tokens on the root ledger. + ?assertEqual(100, balance(RootLedger, Alice, Opts)), + % 2. Alice sends 90 tokens to herself on SubLedger1. + transfer(RootLedger, Alice, Alice, 90, SubLedger1, Opts), + % 3. Alice sends 10 tokens to Bob on SubLedger2. + transfer(SubLedger1, Alice, Bob, 10, SubLedger2, Opts), + ?event({map, map([RootLedger, SubLedger1, SubLedger2], Names, Opts)}), + ?assertEqual(10, balance(RootLedger, Alice, Opts)), + ?assertEqual(80, balance(SubLedger1, Alice, Opts)), + ?assertEqual(10, balance(SubLedger2, Bob, Opts)), + verify_net(RootLedger, [SubLedger1, SubLedger2], Opts), + % 5. Bob sends 5 tokens to himself on SubLedger1. + transfer(SubLedger2, Bob, Bob, 5, SubLedger1, Opts), + transfer(SubLedger2, Bob, Alice, 4, SubLedger1, Opts), + ?event({map, map([RootLedger, SubLedger1, SubLedger2], Names, Opts)}), + ?assertEqual(10, balance(RootLedger, Alice, Opts)), + ?assertEqual(5, balance(SubLedger1, Bob, Opts)), + ?assertEqual(84, balance(SubLedger1, Alice, Opts)), + ?assertEqual(1, balance(SubLedger2, Bob, Opts)), + verify_net(RootLedger, [SubLedger1, SubLedger2], Opts). + +%% @doc Verify that a ledger can send tokens to a peer ledger that is not +%% registered with it yet. Each peer ledger must have precisely the same process +%% base message, granting transitive security properties: If a peer trusts its +%% own compute and assignment mechanism, then it can trust messages from exact +%% duplicates of itself. In order for this to be safe, the peer ledger network's +%% base process message must implement sufficicient rollback protections and +%% compute correctness guarantees. +unregistered_peer_transfer_test_() -> {timeout, 30, fun unregistered_peer_transfer/0}. +unregistered_peer_transfer() -> + Opts = #{}, + Alice = ar_wallet:new(), + Bob = ar_wallet:new(), + RootLedger = + ledger( + <<"scripts/hyper-token.lua">>, + #{ <<"balance">> => #{ Alice => 100 } }, + Opts + ), + SubLedgers = [ subledger(RootLedger, Opts) || _ <- lists:seq(1, 3) ], + SubLedger1 = lists:nth(1, SubLedgers), + SubLedger2 = lists:nth(2, SubLedgers), + SubLedger3 = lists:nth(3, SubLedgers), + Names = #{ + Alice => alice, + Bob => bob, + RootLedger => root, + SubLedger1 => subledger1, + SubLedger2 => subledger2, + SubLedger3 => subledger3 + }, + % 1. Alice has tokens on the root ledger. + ?assertEqual(100, balance(RootLedger, Alice, Opts)), + transfer(RootLedger, Alice, Alice, 90, SubLedger1, Opts), + % Verify the state before the multi-hop transfer. + ?assertEqual(10, balance(RootLedger, Alice, Opts)), + ?assertEqual(90, balance(SubLedger1, Alice, Opts)), + % 4. Alice sends 10 tokens to Bob on SubLedger3, via SubLedger2. + transfer(RootLedger, Alice, Bob, 10, SubLedger2, Opts), + ?assertEqual(0, balance(RootLedger, Alice, Opts)), + ?assertEqual(90, balance(SubLedger1, Alice, Opts)), + ?assertEqual(10, balance(SubLedger2, Bob, Opts)), + % 5. Bob sends 10 tokens to himself on SubLedger3. + transfer(SubLedger1, Alice, Bob, 50, SubLedger3, Opts), + % Verify the final state of all ledgers. + ?event(debug, + {map, + map( + [RootLedger, SubLedger1, SubLedger2, SubLedger3], + Names, + Opts + ) + } + ), + ?assertEqual(0, balance(RootLedger, Alice, Opts)), + ?assertEqual(40, balance(SubLedger1, Alice, Opts)), + ?assertEqual(10, balance(SubLedger2, Bob, Opts)), + ?assertEqual(50, balance(SubLedger3, Bob, Opts)), + verify_net(RootLedger, SubLedgers, Opts). + +%% @doc Verify that sub-ledgers can request and enforce multiple scheduler +%% commitments. `hyper-token' always validates that peer `base' processes +%% (the uncommitted process ID without its `scheduler' and `authority' fields) +%% match. It allows us to specify additional constraints on the `scheduler' and +%% `authority' fields while matching against the local ledger's base process +%% message. This test validates the correctness of these constraints. +%% +%% The grammar supported by `hyper-token.lua' allows for the following, where +%% `X = scheduler | authority`: +%% - `X`: A list of `X`s that must (by default) be present in the +%% peer ledger's `X' field. +%% - `X-match`: A count of the number of `X`s that must be present in the +%% peer ledger's `X' field. +%% - `X-required`: A list of `X`s that always must be present in the +%% peer ledger's `X' field. +multischeduler_test_disabled() -> {timeout, 30, fun multischeduler/0}. +multischeduler() -> + BaseOpts = test_opts(), + NodeWallet = ar_wallet:new(), + Scheduler2 = ar_wallet:new(), + Scheduler3 = ar_wallet:new(), + Opts = BaseOpts#{ + priv_wallet => NodeWallet, + identities => #{ + <<"extra-scheduler">> => #{ + priv_wallet => Scheduler2 + } + } + }, + Alice = ar_wallet:new(), + Bob = ar_wallet:new(), + RootLedger = + ledger( + <<"scripts/hyper-token.lua">>, + ProcExtra = + #{ + <<"balance">> => #{ Alice => 100 }, + <<"scheduler">> => + [ + hb_util:human_id(NodeWallet), + hb_util:human_id(Scheduler2) + ], + <<"scheduler-required">> => + [ + hb_util:human_id(NodeWallet) + ] + }, + Opts + ), + % Alice has tokens on the root ledger. She moves them to Bob. + transfer(RootLedger, Alice, Bob, 100, Opts), + ?assertEqual(100, balance(RootLedger, Bob, Opts)), + % Create a new process with with the same schedulers, but do not provide + % the extra scheduler in the `identities' map. + OptsWithoutHostWallet = maps:remove(priv_wallet, Opts), + RootLedger2 = + ledger( + <<"scripts/hyper-token.lua">>, + ProcExtra, + OptsWithoutHostWallet + ), + % Alice has tokens on the root ledger. She tries to move them to Bob. + transfer(RootLedger2, Alice, Bob, 100, OptsWithoutHostWallet), + % The transfer should fail because only one signature will be provided on + % the assignment. + ?assertEqual(0, balance(RootLedger2, Bob, OptsWithoutHostWallet)), + % The transfer should succeed if: + % - Set the `authority-required' field to contain the host wallet, while + % - Setting the `authority-match' field to 1. + OptsWithoutExtraScheduler = #{ priv_wallet => NodeWallet }, + RootLedger3 = + ledger( + <<"scripts/hyper-token.lua">>, + ProcExtra#{ + <<"scheduler-match">> => 1 + }, + OptsWithoutExtraScheduler + ), + transfer(RootLedger3, Alice, Bob, 100, OptsWithoutExtraScheduler), + ?assertEqual(100, balance(RootLedger3, Bob, OptsWithoutExtraScheduler)), + % Ensure that another subledger can be registered to this process with the + % the necessary scheduler shared, but an additional scheduler not shared. + % Further, we ensure that the `scheduler-required' field is satisfied by + % creating a subledger that has two different schedulers, excluding the + % host wallet. + OptsWithSchedulers = OptsWithoutExtraScheduler#{ + identities => #{ + <<"scheduler-1">> => #{ + priv_wallet => Scheduler3 + }, + <<"scheduler-2">> => #{ + priv_wallet => Scheduler2 + }, + <<"scheduler-3">> => #{ + priv_wallet => Scheduler3 + } + } + }, + % Create 3 subledgers with the same process, but different schedulers. Two + % that are valid (containing the `scheduler-required' field), and one that + % is invalid (does not contain the scheduler from `scheduler-required'). + Subledger1 = + subledger( + RootLedger3, + #{ + <<"scheduler">> => + [ + hb_util:human_id(NodeWallet), + hb_util:human_id(Scheduler2) + ], + <<"scheduler-required">> => + [ + hb_util:human_id(NodeWallet) + ] + }, + OptsWithSchedulers + ), + Subledger2 = + subledger( + RootLedger3, + #{ + <<"scheduler">> => + [ + hb_util:human_id(NodeWallet), + hb_util:human_id(Scheduler3) + ], + <<"scheduler-required">> => + [hb_util:human_id(NodeWallet)] + }, + OptsWithSchedulers + ), + Subledger3 = + subledger( + RootLedger3, + #{ + <<"scheduler-required">> => [hb_util:human_id(NodeWallet)], + <<"scheduler">> => + [ + hb_util:human_id(Scheduler2), + hb_util:human_id(Scheduler3) + ] + }, + OptsWithSchedulers + ), + % Create a map of names for the ledgers for use in logging. + Names = #{ + Alice => alice, + Bob => bob, + RootLedger3 => root, + Subledger1 => subledger1, + Subledger2 => subledger2, + Subledger3 => subledger3 + }, + % Bob has tokens on the root ledger. He moves them to Alice on Subledger1. + transfer(RootLedger3, Bob, Alice, 100, Subledger1, OptsWithSchedulers), + transfer(Subledger1, Alice, Bob, 100, Subledger2, OptsWithSchedulers), + % Validate the balance has been transferred to Alice on Subledger2. + ?assertEqual(100, balance(Subledger2, Bob, OptsWithSchedulers)), + % Alice cannot move tokens to Bob on Subledger3, because the + % `scheduler-required' field is not satisfied by the subledger. + ?event(debug_base, + {map, + map( + [RootLedger3, Subledger1, Subledger2, Subledger3], + Names, + OptsWithSchedulers + ) + } + ), + transfer(Subledger2, Bob, Alice, 50, Subledger3, OptsWithSchedulers), + % Validate the balance has not been transferred to Bob on Subledger3. + ?assertEqual(0, balance(Subledger3, Alice, OptsWithSchedulers)), + transfer(Subledger2, Bob, Alice, 50, Subledger1, OptsWithSchedulers), + % Validate that the remaining balance has been transferred to Alice on + % Subledger1. + ?assertEqual(50, balance(Subledger1, Alice, OptsWithSchedulers)), + transfer(Subledger1, Alice, Bob, 50, RootLedger3, OptsWithSchedulers), + % Validate that the balance has been transferred to Bob on the root ledger. + ?assertEqual(50, balance(RootLedger3, Bob, OptsWithSchedulers)). + +%% @doc Ensure that the `hyper-token.lua' script can parse comma-separated +%% IDs in the `scheduler' field of a message. +comma_separated_scheduler_list_test() -> + NodeWallet = hb:wallet(), + Scheduler2 = ar_wallet:new(), + Alice = ar_wallet:new(), + Bob = ar_wallet:new(), + Opts = (test_opts())#{ priv_wallet => NodeWallet, identities => #{ + <<"extra-scheduler">> => #{ + priv_wallet => Scheduler2 + } + } }, + Ledger = + ledger( + <<"scripts/hyper-token.lua">>, + ProcExtra = + #{ + <<"balance">> => #{ Alice => 100 }, + <<"scheduler">> => + iolist_to_binary( + [ + <<"\"">>, + hb_util:human_id(NodeWallet), + <<"\",\"">>, + hb_util:human_id(Scheduler2), + <<"\"">> + ] + ), + <<"scheduler-required">> => + [ + hb_util:human_id(NodeWallet) + ] + }, + Opts + ), + % Alice has tokens on the root ledger. She moves them to Bob. + transfer(Ledger, Alice, Bob, 100, Opts), + ?assertEqual(100, balance(Ledger, Bob, Opts)). diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl new file mode 100644 index 000000000..0713dfa03 --- /dev/null +++ b/src/dev_manifest.erl @@ -0,0 +1,145 @@ +%%% @doc An Arweave path manifest resolution device. Follows the v1 schema: +%%% https://specs.ar.io/?tx=lXLd0OPwo-dJLB_Amz5jgIeDhiOkjXuM3-r0H_aiNj0 +-module(dev_manifest). +-export([index/3, info/0]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Use the `route/4' function as the handler for all requests, aside +%% from `keys' and `set', which are handled by the default resolver. +info() -> + #{ + default => fun route/4, + excludes => [keys, set, committers] + }. + +%% @doc Return the fallback index page when the manifest itself is requested. +index(M1, M2, Opts) -> + ?event({manifest_index_request, M1, M2}), + case route(<<"index">>, M1, M2, Opts) of + {ok, Index} -> + ?event({manifest_index_returned, Index}), + {ok, Index}; + {error, not_found} -> + {error, not_found} + end. + +%% @doc Route a request to the associated data via its manifest. +route(<<"index">>, M1, M2, Opts) -> + ?event({manifest_index, M1, M2}), + case manifest(M1, M2, Opts) of + {ok, JSONStruct} -> + ?event({manifest_json_struct, JSONStruct}), + % Get the path to the index page from the manifest. We make + % sure to use `hb_maps:get/4' to ensure that we do not recurse + % on the `index' key with an `ao' resolve. + Index = + hb_maps:get( + <<"index">>, + JSONStruct, + #{}, + Opts + ), + ?event({manifest_index_found, Index}), + Path = hb_maps:get(<<"path">>, Index, Opts), + case Path of + not_found -> + ?event({manifest_path_not_found, <<"index/path">>}), + {error, not_found}; + _ -> + ?event({manifest_path, Path}), + route(Path, M1, M2, Opts) + end; + {error, not_found} -> + ?event(manifest_not_parsed), + {error, not_found} + end; +route(Key, M1, M2, Opts) -> + ?event({manifest_lookup, Key}), + {ok, Manifest} = manifest(M1, M2, Opts), + {ok, + hb_ao:get( + <<"paths/", Key/binary>>, + {as, <<"message@1.0">>, Manifest}, + Opts + ) + }. + +%% @doc Find and deserialize a manifest from the given base. +manifest(Base, _Req, Opts) -> + JSON = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, Base}, [<<"data">>]}, + {{as, <<"message@1.0">>, Base}, [<<"body">>]} + ], + Opts + ), + ?event({manifest_json, JSON}), + Structured = + hb_cache:ensure_all_loaded( + hb_message:convert(JSON, <<"structured@1.0">>, <<"json@1.0">>, Opts), + Opts + ), + ?event({manifest_structured, {explicit, Structured}}), + Linkified = linkify(Structured, Opts), + ?event({manifest_linkified, {explicit, Linkified}}), + {ok, Linkified}. + +%% @doc Generate a nested message of links to content from a parsed (and +%% structured) manifest. +linkify(#{ <<"id">> := ID }, Opts) -> + LinkOptsBase = (maps:with([store], Opts))#{ scope => [local, remote]}, + {link, ID, LinkOptsBase#{ <<"type">> => <<"link">>, <<"lazy">> => false }}; +linkify(Manifest, Opts) when is_map(Manifest) -> + hb_maps:map( + fun(_Key, Val) -> linkify(Val, Opts) end, + Manifest, + Opts + ); +linkify(Manifest, Opts) when is_list(Manifest) -> + lists:map( + fun(Item) -> linkify(Item, Opts) end, + Manifest + ); +linkify(Manifest, _Opts) -> + Manifest. + +%%% Tests + +resolve_test() -> + Opts = #{ store => hb_opts:get(store, no_viable_store, #{}) }, + IndexPage = #{ + <<"content-type">> => <<"text/html">>, + <<"body">> => <<"Page 1">> + }, + {ok, IndexID} = hb_cache:write(IndexPage, Opts), + Page2 = #{ + <<"content-type">> => <<"text/html">>, + <<"body">> => <<"Page 2">> + }, + {ok, Page2ID} = hb_cache:write(Page2, Opts), + Manifest = #{ + <<"paths">> => #{ + <<"nested">> => #{ <<"page2">> => #{ <<"id">> => Page2ID } }, + <<"page1">> => #{ <<"id">> => IndexID } + }, + <<"index">> => #{ <<"path">> => <<"page1">> } + }, + JSON = hb_message:convert(Manifest, <<"json@1.0">>, <<"structured@1.0">>, Opts), + ManifestMsg = + #{ + <<"device">> => <<"manifest@1.0">>, + <<"body">> => JSON + }, + {ok, ManifestID} = hb_cache:write(ManifestMsg, Opts), + ?event({manifest_id, ManifestID}), + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, #{ <<"body">> := <<"Page 1">> }}, + hb_http:get(Node, << ManifestID/binary, "/index" >>, Opts) + ), + {ok, Res} = hb_http:get(Node, << ManifestID/binary, "/nested/page2" >>, Opts), + ?event({manifest_resolve_test, Res}), + ?assertEqual(<<"Page 2">>, hb_maps:get(<<"body">>, Res, <<"NO BODY">>, Opts)), + ok. \ No newline at end of file diff --git a/src/dev_message.erl b/src/dev_message.erl index 5170055e8..c2e9fab88 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -1,262 +1,900 @@ -%%% @doc The identity device: Simply return a key from the message as it is found -%%% in the message's underlying Erlang map. Private keys (`priv[.*]') are -%%% not included. +%%% @doc The identity device: For non-reserved keys, it simply returns a key +%%% from the message as it is found in the message's underlying Erlang map. +%%% Private keys (`priv[.*]') are not included. +%%% Reserved keys are: `id', `commitments', `committers', `keys', `path', +%%% `set', `remove', `get', and `verify'. Their function comments describe the +%%% behaviour of the device when these keys are set. -module(dev_message). --export([info/0, keys/1, id/1, unsigned_id/1, signed_id/1, signers/1]). --export([set/3, remove/2, get/2, get/3]). +%%% Base AO-Core reserved keys: +-export([info/0, keys/1, keys/2]). +-export([set/3, set_path/3, remove/2, remove/3, get/3, get/4]). +%%% Commitment-specific keys: +-export([id/1, id/2, id/3]). +-export([commit/3, committed/3, committers/1, committers/2, committers/3, verify/3]). +%%% Non-protocol enforced keys: +-export([index/3]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). +-define(DEFAULT_ID_DEVICE, <<"httpsig@1.0">>). +-define(DEFAULT_ATT_DEVICE, <<"httpsig@1.0">>). %% The list of keys that are exported by this device. -define(DEVICE_KEYS, [ - path, - id, - unsigned_id, - signed_id, - signers, - keys, get, - set, - remove + <<"id">>, + <<"commitments">>, + <<"committers">>, + <<"keys">>, + <<"path">>, + <<"set">>, + <<"remove">>, + <<"verify">> ]). %% @doc Return the info for the identity device. info() -> #{ - default => fun get/3 - %exports => ?DEVICE_KEYS + default => fun dev_message:get/4 }. -%% @doc Return the ID of a message. If the message already has an ID, return -%% that. Otherwise, return the signed ID. -id(M) -> - ID = - case get(signature, M) of - {error, not_found} -> raw_id(M, unsigned); - {ok, ?DEFAULT_SIG} -> raw_id(M, unsigned); - _ -> raw_id(M, signed) +%% @doc Generate an index page for a message, in the event that the `body' and +%% `content-type' of a message returned to the client are both empty. We do this +%% as follows: +%% 1. Find the `default_index' key of the node message. If it is a binary, +%% it is assumed to be the name of a device, and we execute the resolution +%% `as` that ID. +%% 2. Merge the base message with the default index message, favoring the default +%% index message's keys over those in the base message, unless the default +%% was a device name. +%% 3. Execute the `default_index_path` (base: `index') upon the message, +%% giving the rest of the request unchanged. +index(Msg, Req, Opts) -> + case hb_opts:get(default_index, not_found, Opts) of + not_found -> + {error, <<"No default index message set.">>}; + DefaultIndex -> + hb_ao:resolve( + case is_map(DefaultIndex) of + true -> maps:merge(Msg, DefaultIndex); + false -> {as, DefaultIndex, Msg} + end, + Req#{ + <<"path">> => + case hb_maps:find(<<"path">>, DefaultIndex, Opts) of + {ok, Path} -> Path; + _ -> + hb_opts:get(default_index_path, <<"index">>, Opts) + end + }, + Opts + ) + end. + +%% @doc Return the ID of a message, using the `committers' list if it exists. +%% If the `committers' key is `all', return the ID including all known +%% commitments -- `none' yields the ID without any commitments. If the +%% `committers' key is a list/map, return the ID including only the specified +%% commitments. +%% +%% The `id-device' key in the message can be used to specify the device that +%% should be used to calculate the ID. If it is not set, the default device +%% (`httpsig@1.0') is used. +%% +%% Note: This function _does not_ use AO-Core's `get/3' function, as it +%% as it would require significant computation. We may want to change this +%% if/when non-map message structures are created. +id(Base) -> id(Base, #{}). +id(Base, Req) -> id(Base, Req, #{}). +id(Base, _, NodeOpts) when is_binary(Base) -> + % Return the hashpath of the message in native format, to match the native + % format of the message ID return. + {ok, hb_util:human_id(hb_path:hashpath(Base, NodeOpts))}; +id(RawBase, Req, NodeOpts) -> + % Ensure that the base message is a normalized before proceeding. + IDOpts = NodeOpts#{ linkify_mode => discard }, + Base = + ensure_commitments_loaded( + hb_message:convert(RawBase, tabm, IDOpts), + NodeOpts + ), + % Remove the commitments from the base message if there are none, after + % filtering for the committers specified in the request. + ModBase = #{ <<"commitments">> := Commitments } + = with_relevant_commitments(Base, Req, IDOpts), + ?event(debug_commitments, + {generating_ids, + {selected_commitments, Commitments}, + {req, Req}, + {msg, Base} + } + ), + case hb_maps:keys(Commitments) of + [] -> + % If there are no commitments, we must (re)calculate the ID. + ?event(id, no_commitments_found_in_id_call), + calculate_id(hb_maps:without([<<"commitments">>], ModBase), Req, IDOpts); + IDs -> + % Accumulate the relevant IDs into a single value. This is performed + % by module arithmetic of each of the IDs. The effect of this is that: + % 1. New IDs can be added to the combined ID without requiring any + % recalculation of other IDs. + % 2. New IDs can be added in any order, and will compare to the same + % value as if they were added in other orders. + % 3. Subsequently, combined IDs cannot be used to express ordering of + % the underlying commitments. + % This works for single IDs as well as lists of IDs, because the + % accumulation function starts with a buffer of zero encoded as a + % 256-bit binary. Subsequently, a single ID on its own 'accumulates' + % to itself. + ?event(id, {accumulating_existing_ids, IDs}), + {ok, + hb_util:human_id( + hb_crypto:accumulate( + lists:map(fun hb_util:native_id/1, IDs) + ) + ) + } + end. + +calculate_id(Base, Req, NodeOpts) -> + % Find the ID device for the message. + ?event(linkify, {calculate_ids, {base, Base}}), + IDMod = + case id_device(Base, NodeOpts) of + {ok, IDDev} -> IDDev; + {error, Error} -> throw({id, Error}) + end, + ?event(linkify, {generating_id, {idmod, IDMod}, {base, Base}}), + % Get the device module from the message, or use the default if it is not + % set. We can tell if the device is not set (or is the default) by checking + % whether the device module is the same as this module. + DevMod = + case hb_ao:message_to_device(#{ <<"device">> => IDMod }, NodeOpts) of + ?MODULE -> + hb_ao:message_to_device( + #{ <<"device">> => ?DEFAULT_ID_DEVICE }, + NodeOpts + ); + Module -> Module + end, + % Apply the function's default `commit' function with the appropriate arguments. + % If it doesn't exist, error. + case hb_ao:find_exported_function(Base, DevMod, commit, 3, NodeOpts) of + {ok, Fun} -> + ?event(id, {called_id_device, IDMod}, NodeOpts), + {ok, #{ <<"commitments">> := Comms} } = + apply( + Fun, + hb_ao:truncate_args( + Fun, + [Base, Req#{ <<"type">> => <<"unsigned">> }, NodeOpts] + ) + ), + ?event(id, {generated_id, {type, unsigned}, {commitments, maps:keys(Comms)}}), + {ok, hd(maps:keys(Comms))}; + not_found -> throw({id, id_resolver_not_found_for_device, DevMod}) + end. + +%% @doc Locate the ID device of a message. The ID device is determined the +%% `device' set in _all_ of the commitments. If no commitments are present, +%% the default device (`httpsig@1.0') is used. +id_device(#{ <<"commitments">> := Commitments }, Opts) -> + % Get the device from the first commitment. + UnfilteredDevs = + hb_maps:map( + fun(_, #{ <<"commitment-device">> := CommitmentDev }) -> + CommitmentDev; + (_, _) -> undefined + end, + Commitments, + Opts + ), + % Filter out the undefined devices. + Devs = + lists:filter( + fun(Dev) -> Dev =/= undefined end, + hb_maps:values(UnfilteredDevs, Opts) + ), + % If there are no devices, return the default. + case Devs of + [] -> {ok, ?DEFAULT_ID_DEVICE}; + [Dev] -> {ok, Dev}; + [FirstDev|Rest] -> + % If there are multiple devices amongst the set, err. + MultiDeviceMessage = lists:all(fun(Dev) -> Dev =:= FirstDev end, Rest), + case MultiDeviceMessage of + false -> {error, {multiple_id_devices, Devs}}; + true -> {ok, FirstDev} + end + end; +id_device(_, _) -> + {ok, ?DEFAULT_ID_DEVICE}. + +%% @doc Return the committers of a message that are present in the given request. +committers(Base) -> committers(Base, #{}). +committers(Base, Req) -> committers(Base, Req, #{}). +committers(#{ <<"commitments">> := Commitments }, _, NodeOpts) -> + ?event(debug_commitments, {calculating_committers, {commitments, Commitments}}), + {ok, + hb_maps:values( + hb_maps:filtermap( + fun(_ID, Commitment) -> + Committer = maps:get(<<"committer">>, Commitment, undefined), + ?event(debug_commitments, {committers, {committer, Committer}}), + case Committer of + undefined -> false; + Committer -> {true, Committer} + end + end, + Commitments, + NodeOpts + ), + NodeOpts + ) + }; +committers(_, _, _) -> + {ok, []}. + +%% @doc Commit to a message, using the `commitment-device' key to specify the +%% device that should be used to commit to the message. If the key is not set, +%% the default device (`httpsig@1.0') is used. +commit(Self, Req, Opts) -> + {ok, Base} = hb_message:find_target(Self, Req, Opts), + AttDev = + case hb_maps:get(<<"commitment-device">>, Req, not_specified, Opts) of + not_specified -> + hb_opts:get(commitment_device, no_viable_commitment_device, Opts); + Dev -> Dev + end, + % We _do not_ set the `device' key in the message, as the device will be + % part of the commitment. Instead, we find the device module's `commit' + % function and apply it. + CommitOpts = Opts#{ linkify_mode => offload }, + AttMod = hb_ao:message_to_device(#{ <<"device">> => AttDev }, CommitOpts), + {ok, AttFun} = hb_ao:find_exported_function(Base, AttMod, commit, 3, CommitOpts), + % Encode to a TABM + Loaded = + ensure_commitments_loaded( + hb_message:convert(Base, tabm, CommitOpts), + Opts + ), + {ok, Committed} = + apply( + AttFun, + hb_ao:truncate_args( + AttFun, + [ + Loaded, + Req#{ <<"type">> => maps:get(<<"type">>, Req, <<"signed">>) }, + CommitOpts + ] + ) + ), + {ok, hb_message:convert(Committed, <<"structured@1.0">>, tabm, CommitOpts)}. + +%% @doc Verify a message. By default, all commitments are verified. The +%% `committers' key in the request can be used to specify that only the +%% commitments from specific committers should be verified. Similarly, specific +%% commitments can be specified using the `commitments' key. +verify(Self, Req, Opts) -> + % Get the target message of the verification request. + {ok, RawBase} = hb_message:find_target(Self, Req, Opts), + Base = + hb_message:convert( + ensure_commitments_loaded( + RawBase, + Opts + ), + tabm, + Opts + ), + ?event(verify, {verify, {base_found, Base}}), + Commitments = maps:get(<<"commitments">>, Base, #{}), + IDsToVerify = commitment_ids_from_request(Base, Req, Opts), + % Generate the new commitment request base messsage by removing the keys + % used by this function (path, committers, commitments) and returning the + % remaining keys. This message will then be merged with each commitment + % message to generate the final request, allowing the caller to pass + % additional keys to the commitment device. + ReqBase = + maps:without( + [<<"path">>, <<"committers">>, <<"commitments">>], + Req + ), + % Verify the commitments. Stop execution if any fail. + Res = + lists:all( + fun(CommitmentID) -> + {ok, Res} = + verify_commitment( + Base, + maps:merge( + ReqBase, + maps:get(CommitmentID, Commitments) + ), + Opts + ), + ?event(verify, + {verify_commitment_res, + {commitment_id, CommitmentID}, + {res, Res} + }), + Res + end, + IDsToVerify + ), + ?event(verify, {verify, {res, Res}}), + {ok, Res}. + +%% @doc Execute a function for a single commitment in the context of its +%% parent message. +%% Note: Assumes that the `commitments' key has already been removed from the +%% message if applicable. +verify_commitment(Base, Commitment, Opts) -> + ?event(verify, {verifying_commitment, {commitment, Commitment}, {msg, Base}}), + AttDev = + hb_maps:get( + <<"commitment-device">>, + Commitment, + ?DEFAULT_ATT_DEVICE, + Opts + ), + AttMod = + hb_ao:message_to_device( + #{ <<"device">> => AttDev }, + Opts + ), + {ok, AttFun} = + hb_ao:find_exported_function( + Base, + AttMod, + verify, + 3, + Opts + ), + apply(AttFun, [Base, Commitment, Opts]). + +%% @doc Return the list of committed keys from a message. +committed(Self, Req, Opts) -> + % Get the target message of the verification request and ensure its + % commitments are loaded. + {ok, RawBase} = + hb_message:find_target( + Self, + Req, + Opts + ), + Base = ensure_commitments_loaded(RawBase, Opts), + CommitmentIDs = commitment_ids_from_request(Base, Req, Opts), + ?event(debug_commitments, + {calculating_committed, + {commitment_ids, CommitmentIDs}, + {req, Req} + } + ), + Commitments = maps:get(<<"commitments">>, Base, #{}), + % Get the list of committed keys from each committer. + CommitmentKeys = + lists:map( + fun(CommitmentID) -> + Commitment = maps:get(CommitmentID, Commitments), + % The committed keys will be a TABM encoded numbered map + % so we must decode it to its underlying list of normalized keys + % for comparison purposes. + hb_util:message_to_ordered_list( + maps:get(<<"committed">>, Commitment), + Opts + ) + end, + CommitmentIDs + ), + % Remove commitments that are not in *every* committer's list. + % To start, we need to create the super-set of committed keys. + AllCommittedKeys = + lists:foldr( + fun(Key, Acc) -> + case lists:member(Key, Acc) of + true -> Acc; + false -> [Key | Acc] + end + end, + [], + lists:flatten(CommitmentKeys) + ), + % Next, we filter the list of all committed keys to only include those that + % are present in every committer's list. + OnlyCommittedKeys = + lists:filter( + fun(Key) -> + lists:all( + fun(CommittedKeys) -> lists:member(Key, CommittedKeys) end, + CommitmentKeys + ) + end, + AllCommittedKeys + ), + % Remove any `+link` suffixes from TABM-form committed keys if the `raw` flag + % is not set. This means that callers to `committed/3' will receive a list of + % keys that they can match against the 'normal' representation of the message + % in devices, etc., without exposure to TABM-specifics. If `raw' is set, the + % recipient receives the `committed` list in its unprocessed form. + CommittedNormalizedKeys = + case maps:get(<<"raw">>, Req, false) of + true -> OnlyCommittedKeys; + false -> + lists:map( + fun hb_link:remove_link_specifier/1, + OnlyCommittedKeys + ) + end, + ?event({only_committed_keys, CommittedNormalizedKeys}), + {ok, CommittedNormalizedKeys}. + +%% @doc Return a message with only the relevant commitments for a given request. +%% See `commitment_ids_from_request/3' for more information on the request format. +with_relevant_commitments(Base, Req, Opts) -> + Commitments = maps:get(<<"commitments">>, Base, #{}), + CommitmentIDs = commitment_ids_from_request(Base, Req, Opts), + Base#{ <<"commitments">> => maps:with(CommitmentIDs, Commitments) }. + +%% @doc Implements a standardized form of specifying commitment IDs for a +%% message request. The caller may specify a list of committers (by address) +%% or a list of commitment IDs directly. They may specify both, in which case +%% the returned list will be the union of the two lists. In each case, they +%% may specify `all' or `none' for each group. If no specifiers are provided, +%% the default is `all' for commitments -- also implying `all' for committers. +commitment_ids_from_request(Base, Req, Opts) -> + Commitments = maps:get(<<"commitments">>, Base, #{}), + ReqCommitters = + case maps:get(<<"committers">>, Req, <<"none">>) of + X when is_list(X) -> X; + CommitterDescriptor -> hb_ao:normalize_key(CommitterDescriptor) + end, + RawReqCommitments = maps:get(<<"commitments">>, Req, <<"none">>), + ReqCommitments = + case RawReqCommitments of + X2 when is_list(X2) -> X2; + CommitmentDescriptor -> hb_ao:normalize_key(CommitmentDescriptor) + end, + ?event(debug_commitments, + {commitment_ids_from_request, + {req_commitments, ReqCommitments}, + {req_committers, ReqCommitters}} + ), + % Get the commitments to verify. + FromCommitmentIDs = + case ReqCommitments of + <<"none">> -> []; + <<"all">> -> hb_maps:keys(Commitments, Opts); + CommitmentIDs -> + if is_list(CommitmentIDs) -> CommitmentIDs; + true -> [CommitmentIDs] + end + end, + FromCommitterAddrs = + case ReqCommitters of + <<"none">> -> + ?event(no_commitment_ids_for_committers), + []; + <<"all">> -> + ?event(debug_commitments, {getting_commitment_ids_for_all_committers}), + {ok, Committers} = committers(Base, Req, Opts), + ?event(debug_commitments, {commitment_ids_from_committers, Committers}), + commitment_ids_from_committers(Committers, Commitments, Opts); + RawCommitterAddrs -> + ?event({getting_commitment_ids_for_committers, RawCommitterAddrs}), + CommitterAddrs = + if is_list(RawCommitterAddrs) -> RawCommitterAddrs; + true -> [RawCommitterAddrs] + end, + commitment_ids_from_committers(CommitterAddrs, Commitments, Opts) + end, + Res = + case FromCommitterAddrs ++ FromCommitmentIDs of + [] -> + % The request is for no committers, and no explicit commitments. + % Subsequently, we return the commitment using the default + % commitment device, if it exists. + lists:filter( + fun(CommitmentID) -> + Comm = maps:get(CommitmentID, Commitments), + Dev = maps:get(<<"commitment-device">>, Comm, undefined), + case Dev of + ?DEFAULT_ATT_DEVICE -> + not hb_maps:is_key(<<"committer">>, Comm); + _ -> false + end + end, + maps:keys(Commitments) + ); + FinalCommitmentIDs -> FinalCommitmentIDs end, - {ok, ID}. - -%% @doc Wrap a call to the `hb_util:id/2' function, which returns the -%% unsigned ID of a message. -unsigned_id(M) -> - {ok, raw_id(M, unsigned)}. - -%% @doc Return the signed ID of a message. -signed_id(M) -> - try - {ok, raw_id(M, signed)} - catch - _:_ -> {error, not_signed} + ?event({commitment_ids_from_request, {base, Base}, {req, Req}, {res, Res}}), + Res. + +%% @doc Ensure that the `commitments` submessage of a base message is fully +%% loaded into local memory. +ensure_commitments_loaded(NonRelevant, _Opts) when not is_map(NonRelevant) -> + NonRelevant; +ensure_commitments_loaded(M = #{ <<"commitments">> := Link}, Opts) when ?IS_LINK(Link) -> + M#{ + <<"commitments">> => hb_cache:ensure_all_loaded(Link, Opts) + }; +ensure_commitments_loaded(M, _Opts) -> + M. + +%% @doc Returns a list of commitment IDs in a commitments map that are relevant +%% for a list of given committer addresses. +commitment_ids_from_committers(CommitterAddrs, Commitments, Opts) -> + % Get the IDs of all commitments for each committer. + Comms = + lists:map( + fun(RawCommitterAddr) -> + CommitterAddr = hb_cache:ensure_loaded(RawCommitterAddr, Opts), + % For each committer, filter the commitments to only + % include those with the matching committer address. + IDs = + maps:values(maps:filtermap( + fun(ID, Msg) -> + % If the committer address matches, return + % the ID. If not, ignore the commitment. + case hb_maps:get(<<"committer">>, Msg, undefined) of + CommitterAddr -> {true, ID}; + _ -> false + end + end, + Commitments + ) + ), + {CommitterAddr, IDs} + end, + CommitterAddrs + ), + % Check that each committer has at least one commitment. + EachCommitterHasCommitment = + lists:all(fun({_, IDs}) -> IDs =/= [] end, Comms), + % If all committers have at least one commitment, return the + % IDs of all commitments. If any committer does not have a + % commitment, error. + case EachCommitterHasCommitment of + true -> lists:flatten([ IDs || {_, IDs} <- Comms ]); + false -> + % Get the list of committers that do not have a + % commitment. + MissingCommitters = + [ + MissingCommitter + || + {MissingCommitter, []} <- Comms + ], + throw( + {verify, + {requested_committers_not_found, + {missing_commitments, MissingCommitters} + } + } + ) end. -%% @doc Encode an ID in any format to a normalized, b64u 43 character binary. -raw_id(Item) -> raw_id(Item, unsigned). -raw_id(TX, Type) when is_record(TX, tx) -> - hb_util:encode(ar_bundles:id(TX, Type)); -raw_id(Map, Type) when is_map(Map) -> - TX = hb_message:convert(Map, tx, converge, #{}), - hb_util:encode(ar_bundles:id(TX, Type)); -raw_id(Bin, _) when is_binary(Bin) andalso byte_size(Bin) == 43 -> - Bin; -raw_id(Bin, _) when is_binary(Bin) andalso byte_size(Bin) == 32 -> - hb_util:encode(Bin); -raw_id(Data, _) when is_list(Data) -> - raw_id(list_to_binary(Data)). - -%% @doc Return the signers of a message. -signers(M) -> - {ok, hb_util:list_to_numbered_map(hb_message:signers(M))}. - -%% @doc Set keys in a message. Takes a map of key-value pairs and sets them in -%% the message, overwriting any existing values. +%% @doc Deep merge keys in a message. Takes a map of key-value pairs and sets +%% them in the message, overwriting any existing values. set(Message1, NewValuesMsg, Opts) -> + OriginalPriv = hb_private:from_message(Message1), % Filter keys that are in the default device (this one). + {ok, NewValuesKeys} = keys(NewValuesMsg, Opts), KeysToSet = lists:filter( fun(Key) -> - not lists:member(Key, ?DEVICE_KEYS) andalso - (maps:get(Key, NewValuesMsg, undefined) =/= undefined) + not lists:member(Key, ?DEVICE_KEYS ++ [<<"set-mode">>]) andalso + (hb_maps:get(Key, NewValuesMsg, undefined, Opts) =/= undefined) end, - hb_converge:keys(NewValuesMsg, Opts#{ topic => ?MODULE }) + NewValuesKeys ), % Find keys in the message that are already set (case-insensitive), and % note them for removal. - NormalizedKeysToSet = lists:map(fun hb_converge:to_key/1, KeysToSet), ConflictingKeys = lists:filter( - fun(Key) -> - lists:member(hb_converge:to_key(Key), NormalizedKeysToSet) - end, - maps:keys(Message1) + fun(Key) -> lists:member(Key, KeysToSet) end, + hb_maps:keys(Message1, Opts) ), UnsetKeys = lists:filter( fun(Key) -> - case maps:get(Key, NewValuesMsg, not_found) of + case hb_maps:get(Key, NewValuesMsg, not_found, Opts) of unset -> true; _ -> false end end, - maps:keys(Message1) + hb_maps:keys(Message1, Opts) ), - { - ok, - maps:merge( - maps:without(ConflictingKeys ++ UnsetKeys, Message1), - maps:from_list( - lists:filtermap( - fun(Key) -> - case maps:get(Key, NewValuesMsg, undefined) of - undefined -> false; - unset -> false; - Value -> {true, {Key, Value}} - end - end, - KeysToSet - ) - ) - ) - }. - -%% @doc Remove a key or keys from a message. -remove(Message1, #{ item := Key }) -> - remove(Message1, #{ items => [hb_converge:to_key(Key)] }); -remove(Message1, #{ items := Keys }) -> - NormalizedKeysToRemove = lists:map(fun hb_converge:to_key/1, Keys), - { - ok, - maps:filtermap( - fun(KeyN, Val) -> - NormalizedKeyN = hb_converge:to_key(KeyN), - case lists:member(NormalizedKeyN, NormalizedKeysToRemove) of - true -> false; - false -> {true, Val} + % Base message with keys-to-unset removed + BaseValues = hb_maps:without(UnsetKeys, Message1, Opts), + ?event(message_set, + {performing_set, + {conflicting_keys, ConflictingKeys}, + {keys_to_unset, UnsetKeys}, + {new_values, NewValuesMsg}, + {original_message, Message1} + } + ), + % Create the map of new values + NewValues = hb_maps:from_list( + lists:filtermap( + fun(Key) -> + case hb_maps:get(Key, NewValuesMsg, undefined, Opts) of + undefined -> false; + unset -> false; + Value -> {true, {Key, Value}} end end, - Message1 + KeysToSet ) - }. + ), + % Caclulate if the keys to be set conflict with any committed keys. + {ok, CommittedKeys} = + committed( + Message1, + #{ + <<"committers">> => <<"all">> + }, + Opts + ), + ?event(message_set, + {setting, + {committed_keys, CommittedKeys}, + {keys_to_set, KeysToSet}, + {message, Message1} + } + ), + OverwrittenCommittedKeys = + lists:filtermap( + fun(Key) -> + NormKey = hb_ao:normalize_key(Key), + ?event({checking_committed_key, {key, Key}, {norm_key, NormKey}}), + Res = case lists:member(NormKey, KeysToSet) of + true -> {true, NormKey}; + false -> false + end, + Res + end, + CommittedKeys + ), + ?event({setting, {overwritten_committed_keys, OverwrittenCommittedKeys}}), + % Combine with deep merge or if `set-mode` is `explicit' then just merge. + Merged = + hb_private:set_priv( + case maps:get(<<"set-mode">>, NewValuesMsg, <<"deep">>) of + <<"explicit">> -> maps:merge(BaseValues, NewValues); + _ -> hb_util:deep_merge(BaseValues, NewValues, Opts) + end, + OriginalPriv + ), + case OverwrittenCommittedKeys of + [] -> + ?event(message_set, {no_overwritten_committed_keys, {merged, Merged}}), + {ok, Merged}; + _ -> + % We did overwrite some keys, but do their values match the original? + % If not, we must remove the commitments. + case hb_message:match(Merged, Message1, Opts) of + true -> + ?event(message_set, {set_keys_matched, {merged, Merged}}), + {ok, Merged}; + _ -> + ?event( + message_set, + {set_conflict_removing_commitments, {merged, Merged}} + ), + {ok, hb_maps:without([<<"commitments">>], Merged, Opts)} + end + end. + +%% @doc Special case of `set/3' for setting the `path' key. This cannot be set +%% using the normal `set' function, as the `path' is a reserved key, used to +%% transmit the present key that is being executed. Subsequently, to call `path' +%% we would need to set `path' to `set', removing the ability to specify its +%% new value. +set_path(Base, #{ <<"value">> := Value }, Opts) -> + set_path(Base, Value, Opts); +set_path(Base, Value, Opts) when not is_map(Value) -> + % Determine whether the `path' key is committed. If it is, we remove the + % commitment if the new value is different. We try to minimize work by + % doing the `hb_maps:get` first, as it is far cheaper than calculating + % the committed keys. + BaseWithCorrectedComms = + case hb_maps:get(<<"path">>, Base, undefined, Opts) of + Value -> Base; + _ -> + % The new value is different, but is it committed? If so, we + % must remove the commitments. + case hb_message:is_signed_key(<<"path">>, Base, Opts) of + true -> hb_message:uncommitted(Base, Opts); + false -> Base + end + end, + case Value of + unset -> + {ok, hb_maps:without([<<"path">>], BaseWithCorrectedComms, Opts)}; + _ -> + BaseWithCorrectedComms#{ <<"path">> => Value } + end. + +%% @doc Remove a key or keys from a message. +remove(Message1, Key) -> + remove(Message1, Key, #{}). + +remove(Message1, #{ <<"item">> := Key }, Opts) -> + remove(Message1, #{ <<"items">> => [Key] }, Opts); +remove(Message1, #{ <<"items">> := Keys }, Opts) -> + { ok, hb_maps:without(Keys, Message1, Opts) }. %% @doc Get the public keys of a message. -keys(Msg) when not is_map(Msg) -> - case hb_converge:ensure_message(Msg) of - NormMsg when is_map(NormMsg) -> keys(NormMsg); +keys(Msg) -> + keys(Msg, #{}). + +keys(Msg, Opts) when not is_map(Msg) -> + case hb_ao:normalize_keys(Msg, Opts) of + NormMsg when is_map(NormMsg) -> keys(NormMsg, Opts); _ -> throw(badarg) end; -keys(Msg) -> +keys(Msg, Opts) -> { ok, lists:filter( fun(Key) -> not hb_private:is_private(Key) end, - maps:keys(Msg) + hb_maps:keys(Msg, Opts) ) }. %% @doc Return the value associated with the key as it exists in the message's %% underlying Erlang map. First check the public keys, then check case- %% insensitively if the key is a binary. -get(Key, Msg) -> get(Key, Msg, #{ path => get }). -get(Key, Msg, _Msg2) -> - ?event({getting_key, {key, Key}, {msg, Msg}}), - {ok, PublicKeys} = keys(Msg), - case lists:member(Key, PublicKeys) of - true -> {ok, maps:get(Key, Msg)}; - false -> case_insensitive_get(Key, Msg) - end. +get(Key, Msg, Opts) -> get(Key, Msg, #{ <<"path">> => <<"get">> }, Opts). +get(Key, Msg, _Msg2, Opts) -> + case hb_private:is_private(Key) of + true -> {error, not_found}; + false -> + case hb_maps:get(Key, Msg, not_found, Opts) of + not_found -> case_insensitive_get(Key, Msg, Opts); + Value -> {ok, Value} + end + end. %% @doc Key matching should be case insensitive, following RFC-9110, so we %% implement a case-insensitive key lookup rather than delegating to -%% `maps:get/2'. Encode the key to a binary if it is not already. -case_insensitive_get(Key, Msg) -> - {ok, Keys} = keys(Msg), - %?event({case_insensitive_get, {key, Key}, {keys, Keys}}), - case_insensitive_get(Key, Msg, Keys). -case_insensitive_get(Key, Msg, Keys) when byte_size(Key) > 43 -> - do_case_insensitive_get(Key, Msg, Keys); -case_insensitive_get(Key, Msg, Keys) -> - do_case_insensitive_get(hb_converge:to_key(Key), Msg, Keys). -do_case_insensitive_get(_Key, _Msg, []) -> {error, not_found}; -do_case_insensitive_get(Key, Msg, [CurrKey | Keys]) -> - case hb_converge:to_key(CurrKey) of - Key -> {ok, maps:get(CurrKey, Msg)}; - _ -> do_case_insensitive_get(Key, Msg, Keys) - end. +%% `hb_maps:get/2'. Encode the key to a binary if it is not already. +case_insensitive_get(Key, Msg, Opts) -> + NormKey = hb_util:to_lower(hb_util:bin(Key)), + NormMsg = hb_ao:normalize_keys(Msg, Opts), + case hb_maps:get(NormKey, NormMsg, not_found, Opts) of + not_found -> {error, not_found}; + Value -> {ok, Value} + end. %%% Tests %%% Internal module functionality tests: get_keys_mod_test() -> - ?assertEqual([a], maps:keys(#{a => 1})). + ?assertEqual([a], hb_maps:keys(#{a => 1}, #{})). is_private_mod_test() -> - ?assertEqual(true, hb_private:is_private(private)), ?assertEqual(true, hb_private:is_private(<<"private">>)), ?assertEqual(true, hb_private:is_private(<<"private.foo">>)), - ?assertEqual(false, hb_private:is_private(a)), - % Generate a long list of characters and check it does not match. - ?assertEqual(false, hb_private:is_private([ C || C <- lists:seq($a, $z) ])). + ?assertEqual(false, hb_private:is_private(<<"a">>)). %%% Device functionality tests: keys_from_device_test() -> - ?assertEqual({ok, [a]}, hb_converge:resolve(#{a => 1}, keys, #{})). + ?assertEqual({ok, [<<"a">>]}, hb_ao:resolve(#{ <<"a">> => 1 }, keys, #{})). case_insensitive_get_test() -> - ?assertEqual({ok, 1}, case_insensitive_get(a, #{a => 1})), - ?assertEqual({ok, 1}, case_insensitive_get(a, #{ <<"A">> => 1 })), - ?assertEqual({ok, 1}, case_insensitive_get(a, #{ <<"a">> => 1 })), - ?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{a => 1})), - ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{a => 1})), - ?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{ <<"a">> => 1 })), - ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{ <<"A">> => 1 })). + ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{ <<"a">> => 1 }, #{})), +% ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{ <<"A">> => 1 }, #{})), + ?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{ <<"a">> => 1 }, #{})). + %?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{ <<"A">> => 1 }, #{})). private_keys_are_filtered_test() -> ?assertEqual( - {ok, [a]}, - hb_converge:resolve(#{a => 1, private => 2}, keys, #{}) + {ok, [<<"a">>]}, + hb_ao:resolve(#{ <<"a">> => 1, <<"private">> => 2 }, keys, #{}) ), ?assertEqual( - {ok, [a]}, - hb_converge:resolve(#{a => 1, "priv_foo" => 4}, keys, #{}) + {ok, [<<"a">>]}, + hb_ao:resolve(#{ <<"a">> => 1, <<"priv_foo">> => 4 }, keys, #{}) ). cannot_get_private_keys_test() -> ?assertEqual( {error, not_found}, - hb_converge:resolve(#{ a => 1, private_key => 2 }, private_key, #{}) + hb_ao:resolve( + #{ <<"a">> => 1, <<"private_key">> => 2 }, + <<"private_key">>, + #{ hashpath => ignore } + ) ). key_from_device_test() -> - ?assertEqual({ok, 1}, hb_converge:resolve(#{a => 1}, a, #{})). + ?assertEqual({ok, 1}, hb_ao:resolve(#{ <<"a">> => 1 }, <<"a">>, #{})). remove_test() -> - Msg = #{ <<"Key1">> => <<"Value1">>, <<"Key2">> => <<"Value2">> }, - ?assertMatch({ok, #{ <<"Key2">> := <<"Value2">> }}, - hb_converge:resolve(Msg, #{ path => remove, item => <<"Key1">> }, #{})), + Msg = #{ <<"key1">> => <<"Value1">>, <<"key2">> => <<"Value2">> }, + ?assertMatch({ok, #{ <<"key2">> := <<"Value2">> }}, + hb_ao:resolve( + Msg, + #{ <<"path">> => <<"remove">>, <<"item">> => <<"key1">> }, + #{ hashpath => ignore } + ) + ), ?assertMatch({ok, #{}}, - hb_converge:resolve( + hb_ao:resolve( Msg, - #{ path => remove, items => [<<"Key1">>, <<"Key2">>] }, - #{} + #{ <<"path">> => <<"remove">>, <<"items">> => [<<"key1">>, <<"key2">>] }, + #{ hashpath => ignore } ) ). set_conflicting_keys_test() -> - Msg1 = #{ <<"Dangerous">> => <<"Value1">> }, - Msg2 = #{ path => set, dangerous => <<"Value2">> }, - ?assertMatch({ok, #{ dangerous := <<"Value2">> }}, - hb_converge:resolve(Msg1, Msg2, #{})). + Msg1 = #{ <<"dangerous">> => <<"Value1">> }, + Msg2 = #{ <<"path">> => <<"set">>, <<"dangerous">> => <<"Value2">> }, + ?assertMatch({ok, #{ <<"dangerous">> := <<"Value2">> }}, + hb_ao:resolve(Msg1, Msg2, #{})). unset_with_set_test() -> - Msg1 = #{ <<"Dangerous">> => <<"Value1">> }, - Msg2 = #{ path => set, dangerous => unset }, - ?assertMatch({ok, Msg3} when map_size(Msg3) == 0, - hb_converge:resolve(Msg1, Msg2, #{ hashpath => ignore })). + Msg1 = #{ <<"dangerous">> => <<"Value1">> }, + Msg2 = #{ <<"path">> => <<"set">>, <<"dangerous">> => unset }, + ?assertMatch({ok, Msg3} when ?IS_EMPTY_MESSAGE(Msg3), + hb_ao:resolve(Msg1, Msg2, #{ hashpath => ignore })). + +deep_unset_test() -> + Opts = #{ hashpath => ignore }, + Msg1 = #{ + <<"test-key1">> => <<"Value1">>, + <<"deep">> => #{ + <<"test-key2">> => <<"Value2">>, + <<"test-key3">> => <<"Value3">> + } + }, + Msg2 = hb_ao:set(Msg1, #{ <<"deep/test-key2">> => unset }, Opts), + ?assertEqual(#{ + <<"test-key1">> => <<"Value1">>, + <<"deep">> => #{ <<"test-key3">> => <<"Value3">> } + }, + Msg2 + ), + Msg3 = hb_ao:set(Msg2, <<"deep/test-key3">>, unset, Opts), + ?assertEqual(#{ + <<"test-key1">> => <<"Value1">>, + <<"deep">> => #{} + }, + Msg3 + ), + Msg4 = hb_ao:set(Msg3, #{ <<"deep">> => unset }, Opts), + ?assertEqual(#{ <<"test-key1">> => <<"Value1">> }, Msg4). set_ignore_undefined_test() -> - Msg1 = #{ <<"Test-Key">> => <<"Value1">> }, - Msg2 = #{ path => set, <<"Test-Key">> => undefined }, - ?assertEqual({ok, #{ <<"Test-Key">> => <<"Value1">> }}, - set(Msg1, Msg2, #{ hashpath => ignore })). + Msg1 = #{ <<"test-key">> => <<"Value1">> }, + Msg2 = #{ <<"path">> => <<"set">>, <<"test-key">> => undefined }, + ?assertEqual(#{ <<"test-key">> => <<"Value1">> }, + hb_private:reset(hb_util:ok(set(Msg1, Msg2, #{ hashpath => ignore })))). +verify_test() -> + Unsigned = #{ <<"a">> => <<"b">> }, + Signed = hb_message:commit(Unsigned, hb:wallet()), + ?event({signed, Signed}), + BadSigned = Signed#{ <<"a">> => <<"c">> }, + ?event({bad_signed, BadSigned}), + ?assertEqual(false, hb_message:verify(BadSigned)), + ?assertEqual({ok, true}, + hb_ao:resolve( + #{ <<"device">> => <<"message@1.0">> }, + #{ <<"path">> => <<"verify">>, <<"body">> => Signed }, + #{ hashpath => ignore } + ) + ), + % Test that we can verify a message without specifying the device explicitly. + ?assertEqual({ok, true}, + hb_ao:resolve( + #{}, + #{ <<"path">> => <<"verify">>, <<"body">> => Signed }, + #{ hashpath => ignore } + ) + ). \ No newline at end of file diff --git a/src/dev_messenger.erl b/src/dev_messenger.erl deleted file mode 100644 index 4bf479394..000000000 --- a/src/dev_messenger.erl +++ /dev/null @@ -1,121 +0,0 @@ -%%% @doc A device that relays messages to their `Target` destination, -%%% optionally waiting for a resulting message upon which further -%%% computation can be performed. -%%% -%%% This device is typically employed in two modes: -%%% -%%% 1. Invoked in order to force calculation of a hashpath, and to -%%% `push` sub-messages from the result to a further hashpath (for -%%% another computation), or to any traditional HTTP/1.1-3 URL. -%%% 2. Added to a stack of devices, resulting in a 'real-time' `push` -%%% of messages as a result of a computation. -%%% -%%% The `Messenger` device honors the following paramaters: -%%% ``` -%%% Recursive: Determines whether the device should seek to evaluate -%%% pushed messages, and push messages that result from -%%% those. This mode allows interactions that require many -%%% interactions with paralell computations to be -%%% orchestrated on behalf of a user. Default: True. -%%% -%%% Monitor: The device should periodically execute a hashpath, every -%%% `Value` time period, pushing the resulting messages as -%%% in other modes. Default: Not set. -%%% -%%% Allow-URLs: Whether messages for which the `Target` is a centralized -%%% web location (a fully-qualified IP/FDQN address) should be -%%% honored (`True`) or ignored (`False`) by the messenger. -%%% If set to `True`, the device will relay messages as given -%%% in the `Base` to the URL specified by `Target`, adding the -%%% response message received from the remote HTTP server to -%%% its own output message. -%%% Messenger-Keys: A list of the keys for which the messenger device should -%%% be active. Default: [<<"Push">>]. -%%% Report: Determines whether the results of the message pushing -%%% should be reported in the resulting output, and if set, -%%% to which subpath. -%%% ''' -%%% Each parameter is expected to be located in the `BaseMessage`. --module(dev_messenger). --export([push/3]). --include("include/hb.hrl"). - -%% @doc Search for messages to `push` in the base message. -push(M1, M2, Opts) -> - Prefix = - hb_converge:get( - <<"Input-Prefix">>, - {as, dev_message, M1}, - <<>>, - Opts - ), - ShouldReport = - hb_converge:get(<< Prefix/binary, "/Report">>, M1, false, Opts), - case ShouldReport of - false -> - % We do not need to report our progress, so we can spawn a new - % Erlang process to handle pushing for us. - spawn(fun() -> dispatch(M1, M2, Opts) end), - {ok, M1}; - Subpath -> - % Synchronously push messages, taking the returned value and - % placing it into the BaseMsg at `Subpath`. - hb_converge:set( - M1, - #{ Subpath => dispatch(M1, M2, Opts) }, - Opts - ) - end. - -dispatch(Msg1, Msg2, Opts) -> - case hb_converge:get(<<"Target">>, Msg2, Opts) of - not_found -> - ?event({uploading_to_arweave_as_no_target_set, Msg2}), - hb_client:upload(Msg2); - Target = << "http", _/binary >> -> - ?event({posting_to_http, Target, Msg2}), - M1Allowed = hb_converge:get(<<"Allow-URLs">>, Msg1, Opts), - OptsAllowed = hb_opts:get(allow_urls, false, Opts), - case {M1Allowed, OptsAllowed} of - {A1, A2} when (not A1) or (A2 == false) -> - throw(settings_disallow_http_post); - {true, RawAllowed} -> - URLSpecs = - case RawAllowed of - true -> []; - _ -> RawAllowed - end, - Admissable = - (RawAllowed == true) orelse - lists:any( - fun(Spec) -> - case Target of - <> -> true; - _ -> false - end - end, - URLSpecs - ), - ?event({sending_to_http, Target, Msg2}), - case Admissable of - true -> - case hb_converge:get(<<"Method">>, Msg2, Opts) of - <<"POST">> -> hb_http:post(Target, Msg2); - _ -> hb_http:get(Target, Msg2) - end; - false -> throw(settings_disallow_http_post) - end - end; - Target -> - ?event( - {pushing_to_target, Target, hb_path:from_message(hashpath, Msg1)} - ), - {ok, Downstream} = hb_converge:resolve( - #{ path => <> }, - Opts - ), - #{ - <<"Target">> => Target, - <<"Resulted-In">> => Downstream - } - end. \ No newline at end of file diff --git a/src/dev_meta.erl b/src/dev_meta.erl index 3cda8e2df..a89551919 100644 --- a/src/dev_meta.erl +++ b/src/dev_meta.erl @@ -1,69 +1,762 @@ %%% @doc The hyperbeam meta device, which is the default entry point %%% for all messages processed by the machine. This device executes a -%%% Converge singleton request, after first applying the node's -%%% pre-processor, if set. +%%% AO-Core singleton request, after first applying the node's +%%% pre-processor, if set. The pre-processor can halt the request by +%%% returning an error, or return a modified version if it deems necessary -- +%%% the result of the pre-processor is used as the request for the AO-Core +%%% resolver. Additionally, a post-processor can be set, which is executed after +%%% the AO-Core resolver has returned a result. -module(dev_meta). --export([handle/2]). +-export([info/1, info/3, build/3, handle/2, adopt_node_message/2, is/2, is/3]). +-export([is_operator/2]). -include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). +%%% Include the auto-generated build info header file. +-include_lib("../_build/hb_buildinfo.hrl"). + +%% @doc Ensure that the helper function `adopt_node_message/2' is not exported. +%% The naming of this method carefully avoids a clash with the exported `info/3' +%% function. We would like the node information to be easily accessible via the +%% `info' endpoint, but AO-Core also uses `info' as the name of the function +%% that grants device information. The device call takes two or fewer arguments, +%% so we are safe to use the name for both purposes in this case, as the user +%% info call will match the three-argument version of the function. If in the +%% future the `request' is added as an argument to AO-Core's internal `info' +%% function, we will need to find a different approach. +info(_) -> #{ exports => [info, build] }. + +%% @doc Utility function for determining if a request is from the `operator' of +%% the node. +is_operator(Request, NodeMsg) -> + RequestSigners = hb_message:signers(Request, NodeMsg), + Operator = + hb_opts:get( + operator, + case hb_opts:get(priv_wallet, no_viable_wallet, NodeMsg) of + no_viable_wallet -> unclaimed; + Wallet -> ar_wallet:to_address(Wallet) + end, + NodeMsg + ), + EncOperator = + case Operator of + unclaimed -> unclaimed; + NativeAddress -> hb_util:human_id(NativeAddress) + end, + EncOperator == unclaimed orelse lists:member(EncOperator, RequestSigners). +%% @doc Emits the version number and commit hash of the HyperBEAM node source, +%% if available. +%% +%% We include the short hash separately, as the length of this hash may change in +%% the future, depending on the git version/config used to build the node. +%% Subsequently, rather than embedding the `git-short-hash-length', for the +%% avoidance of doubt, we include the short hash separately, as well as its long +%% hash. +build(_, _, _NodeMsg) -> + {ok, + #{ + <<"node">> => <<"HyperBEAM">>, + <<"version">> => ?HYPERBEAM_VERSION, + <<"source">> => ?HB_BUILD_SOURCE, + <<"source-short">> => ?HB_BUILD_SOURCE_SHORT, + <<"build-time">> => ?HB_BUILD_TIME + } + }. %% @doc Normalize and route messages downstream based on their path. Messages -%% with a `Meta` key are routed to the `handle_meta/2` function, while all -%% other messages are routed to the `handle_converge/2` function. +%% with a `Meta' key are routed to the `handle_meta/2' function, while all +%% other messages are routed to the `handle_resolve/2' function. handle(NodeMsg, RawRequest) -> - ?event(debug, {request, RawRequest}), - NormRequest = hb_singleton:from(RawRequest), - ?event(debug, {norm_request, NormRequest}), - case is_meta_request(NormRequest) of - true -> handle_meta(NormRequest, NodeMsg); - false -> handle_converge(NormRequest, NodeMsg) + ?event({singleton_tabm_request, RawRequest}), + NormRequest = hb_singleton:from(RawRequest, NodeMsg), + ?event( + http, + {request, + hb_cache:ensure_all_loaded( + hb_ao:normalize_keys(NormRequest, NodeMsg), + NodeMsg + ) + } + ), + case hb_opts:get(initialized, false, NodeMsg) of + false -> + Res = + embed_status( + hb_ao:force_message( + handle_initialize(NormRequest, NodeMsg), + NodeMsg + ), + NodeMsg + ), + Res; + _ -> handle_resolve(RawRequest, NormRequest, NodeMsg) end. -%% @doc Handle a potential list of messages, checking if the first message -%% has a path of `Meta`. -is_meta_request([PrimaryMsg | _]) -> hb_path:hd(PrimaryMsg, #{}) == <<"Meta">>; -is_meta_request(_) -> false. - -%% @doc Get/set the node message based on the request method. If the request -%% is a `POST`, we check that the request is signed by the owner of the node. -%% If not, we return the node message as-is, aside all keys that are -%% private (according to `hb_private`). -handle_meta([Request|_], NodeMsg) -> - case hb_converge:get(<<"method">>, Request, NodeMsg) of - <<"GET">> -> - {ok, hb_private:reset(Request, NodeMsg)}; +handle_initialize([Base = #{ <<"device">> := Dev}, Req = #{ <<"path">> := Path }|_], NodeMsg) -> + ?event({got, {device, Dev}, {path, Path}}), + case {Dev, Path} of + {<<"meta@1.0">>, <<"info">>} -> info(Base, Req, NodeMsg); + _ -> {error, <<"Node must be initialized before use.">>} + end; +handle_initialize([{as, <<"meta@1.0">>, _}|Rest], NodeMsg) -> + handle_initialize([#{ <<"device">> => <<"meta@1.0">>}|Rest], NodeMsg); +handle_initialize([_|Rest], NodeMsg) -> + handle_initialize(Rest, NodeMsg); +handle_initialize([], _NodeMsg) -> + {error, <<"Node must be initialized before use.">>}. + +%% @doc Get/set the node message. If the request is a `POST', we check that the +%% request is signed by the owner of the node. If not, we return the node message +%% as-is, aside all keys that are private (according to `hb_private'). +info(_, Request, NodeMsg) -> + case hb_ao:get(<<"method">>, Request, NodeMsg) of <<"POST">> -> - ReqSigners = hb_message:signers(Request, NodeMsg), - Owner = hb_opts:get(owner, no_owner_set, NodeMsg), - case lists:member(Owner, ReqSigners) of - false -> - {error, {unauthorized, Request}}; - true -> - hb_http_server:setops(NodeMsg), - {ok, <<"OK">>} + case hb_ao:get(<<"initialized">>, NodeMsg, not_found, NodeMsg) of + permanent -> + embed_status( + {error, + <<"The node message of this machine is already " + "permanent. It cannot be changed.">> + }, + NodeMsg + ); + _ -> + update_node_message(Request, NodeMsg) end; - _ -> {error, {unsupported_method, Request}} + _ -> + ?event({get_config_req, Request, NodeMsg}), + DynamicKeys = add_dynamic_keys(NodeMsg), + embed_status({ok, filter_node_msg(DynamicKeys, NodeMsg)}, NodeMsg) end. -%% @doc Handle a Converge request, which is a list of messages. We apply +%% @doc Remove items from the node message that are not encodable into a +%% message. +filter_node_msg(Msg, NodeMsg) when is_map(Msg) -> + hb_maps:map(fun(_, Value) -> filter_node_msg(Value, NodeMsg) end, hb_private:reset(Msg), NodeMsg); +filter_node_msg(Msg, NodeMsg) when is_list(Msg) -> + lists:map(fun(Item) -> filter_node_msg(Item, NodeMsg) end, Msg); +filter_node_msg(Tuple, _NodeMsg) when is_tuple(Tuple) -> + <<"Unencodable value.">>; +filter_node_msg(Other, _NodeMsg) -> + Other. + +%% @doc Add dynamic keys to the node message. +add_dynamic_keys(NodeMsg) -> + UpdatedNodeMsg = + case hb_opts:get(priv_wallet, no_viable_wallet, NodeMsg) of + no_viable_wallet -> + NodeMsg; + Wallet -> + %% Create a new map with address and merge it (overwriting existing) + Address = hb_util:id(ar_wallet:to_address(Wallet)), + NodeMsg#{ address => Address, <<"address">> => Address } + end, + add_identity_addresses(UpdatedNodeMsg). + +add_identity_addresses(NodeMsg) -> + Identities = hb_opts:get(identities, #{}, NodeMsg), + NewIdentities = maps:map(fun(_, Identity) -> + Identity#{ + <<"address">> => hb_util:human_id( + hb_opts:get(priv_wallet, hb:wallet(), Identity) + ) + } + end, Identities), + NodeMsg#{ <<"identities">> => NewIdentities }. + +%% @doc Validate that the request is signed by the operator of the node, then +%% allow them to update the node message. +update_node_message(Request, NodeMsg) -> + case is(admin, Request, NodeMsg) of + false -> + ?event({set_node_message_fail, Request}), + embed_status({error, <<"Unauthorized">>}, NodeMsg); + true -> + case adopt_node_message(Request, NodeMsg) of + {ok, NewNodeMsg} -> + NewH = hb_opts:get(node_history, [], NewNodeMsg), + embed_status( + {ok, + #{ + <<"body">> => + iolist_to_binary( + io_lib:format( + "Node message updated. History: ~p" + "updates.", + [length(NewH)] + ) + ), + <<"history-length">> => length(NewH) + } + }, + NodeMsg + ); + {error, Reason} -> + ?event({set_node_message_fail, Request, Reason}), + embed_status({error, Reason}, NodeMsg) + end + end. + +%% @doc Attempt to adopt changes to a node message. +adopt_node_message(Request, NodeMsg) -> + ?event({set_node_message_success, Request}), + % Ensure that the node history is updated and the http_server ID is + % not overridden. + case hb_opts:get(initialized, permanent, NodeMsg) of + permanent -> + {error, <<"Node message is already permanent.">>}; + _ -> + hb_http_server:set_opts(Request, NodeMsg) + end. + +%% @doc Handle an AO-Core request, which is a list of messages. We apply %% the node's pre-processor to the request first, and then resolve the request -%% using the node's Converge implementation if its response was `ok`. -%% After execution, we run the node's `postprocessor` message on the result of +%% using the node's AO-Core implementation if its response was `ok'. +%% After execution, we run the node's `response' hook on the result of %% the request before returning the result it grants back to the user. -handle_converge(Request, NodeMsg) -> - case resolve_processor(preprocessor, Request, NodeMsg) of - {ok, PreProcMsg} -> - case hb_converge:resolve(PreProcMsg, NodeMsg) of - {ok, ConvergedMsg} -> - resolve_processor(postprocessor, ConvergedMsg, NodeMsg); - {error, Error} -> {error, Error} +handle_resolve(Req, Msgs, NodeMsg) -> + TracePID = hb_opts:get(trace, no_tracer_set, NodeMsg), + % Apply the pre-processor to the request. + ?event(http_request, + {resolve_hook, + {raw_request, Req}, + {parsed_request_sequence, Msgs} + } + ), + LoadedMsgs = hb_cache:ensure_all_loaded(Msgs, NodeMsg), + case resolve_hook(<<"request">>, Req, LoadedMsgs, NodeMsg) of + {ok, PreProcessedMsg} -> + ?event(http_request, {request_after_preprocessing, PreProcessedMsg}), + AfterPreprocOpts = hb_http_server:get_opts(NodeMsg), + % Resolve the request message. + HTTPOpts = hb_maps:merge( + AfterPreprocOpts, + hb_opts:get(http_extra_opts, #{}, NodeMsg), + NodeMsg + ), + Res = + hb_ao:resolve_many( + PreProcessedMsg, + HTTPOpts#{ force_message => true, trace => TracePID } + ), + {ok, StatusEmbeddedRes} = embed_status(Res, NodeMsg), + AfterResolveOpts = hb_http_server:get_opts(NodeMsg), + % Apply the post-processor to the result. + Output = maybe_sign( + embed_status( + resolve_hook( + <<"response">>, + Req, + StatusEmbeddedRes, + AfterResolveOpts + ), + NodeMsg + ), + NodeMsg + ), + ?event(http_request, + {http_request, + {request, Req}, + {result, Output} + } + ), + Output; + Res -> embed_status(hb_ao:force_message(Res, NodeMsg), NodeMsg) + end. + +%% @doc Execute a hook from the node message upon the user's request. The +%% invocation of the hook provides a request of the following form: +%% 
+%%      /path => request | response
+%%      /request => the original request singleton
+%%      /body => parsed sequence of messages to process | the execution result
+%%
+resolve_hook(HookName, InitiatingRequest, Body, NodeMsg) -> + HookReq = + #{ + <<"request">> => InitiatingRequest, + <<"body">> => Body + }, + ?event(hook, {resolve_hook, HookName, HookReq}), + case dev_hook:on(HookName, HookReq, NodeMsg) of + {ok, #{ <<"body">> := ResponseBody }} -> + ?event(hook, + {resolve_hook_success, + {name, HookName}, + {response_body, ResponseBody} + } + ), + {ok, ResponseBody}; + {error, _} = Error -> + ?event(hook, + {resolve_hook_error, + {name, HookName}, + {error, Error} + } + ), + Error; + Other -> + {error, Other} + end. + +%% @doc Wrap the result of a device call in a status. +embed_status({ErlStatus, Res}, NodeMsg) when is_map(Res) -> + case lists:member(<<"status">>, hb_message:committed(Res, all, NodeMsg)) of + false -> + HTTPCode = status_code({ErlStatus, Res}, NodeMsg), + {ok, Res#{ <<"status">> => HTTPCode }}; + true -> + {ok, Res} + end; +embed_status({ErlStatus, Res}, NodeMsg) -> + HTTPCode = status_code({ErlStatus, Res}, NodeMsg), + {ok, #{ <<"status">> => HTTPCode, <<"body">> => Res }}. + +%% @doc Calculate the appropriate HTTP status code for an AO-Core result. +%% The order of precedence is: +%% 1. The status code from the message. +%% 2. The HTTP representation of the status code. +%% 3. The default status code. +status_code({ErlStatus, Msg}, NodeMsg) -> + case message_to_status(Msg, NodeMsg) of + default -> status_code(ErlStatus, NodeMsg); + RawStatus -> RawStatus + end; +status_code(ok, _NodeMsg) -> 200; +status_code(error, _NodeMsg) -> 400; +status_code(created, _NodeMsg) -> 201; +status_code(not_found, _NodeMsg) -> 404; +status_code(failure, _NodeMsg) -> 500; +status_code(unavailable, _NodeMsg) -> 503; +status_code(unauthorized, _NodeMsg) -> 401; +status_code(forbidden, _NodeMsg) -> 403; +status_code(_, _NodeMsg) -> 200. + +%% @doc Get the HTTP status code from a transaction (if it exists). +message_to_status(#{ <<"body">> := Status }, NodeMsg) when is_atom(Status) -> + status_code(Status, NodeMsg); +message_to_status(Item, NodeMsg) when is_map(Item) -> + % Note: We use `dev_message' directly here, such that we do not cause + % additional AO-Core calls for every request. This is particularly important + % if a remote server is being used for all AO-Core requests by a node. + case dev_message:get(<<"status">>, Item, NodeMsg) of + {ok, RawStatus} when is_integer(RawStatus) -> RawStatus; + {ok, RawStatus} when is_atom(RawStatus) -> + status_code(RawStatus, NodeMsg); + {ok, RawStatus} -> + % If we can convert the status to an integer, do so. + try binary_to_integer(RawStatus) + catch + error:badarg -> + % We can't convert the status to an integer, but we may be + % able to convert it to an existing atom status code. + try + status_code( + binary_to_existing_atom(RawStatus, latin1), + NodeMsg + ) + catch + error:badarg -> + % We can't convert the status to an integer or atom, + % so we return the default status code. + default + end + end; + _ -> default + end; +message_to_status(Item, NodeMsg) when is_atom(Item) -> + status_code(Item, NodeMsg); +message_to_status(_Item, _NodeMsg) -> + default. + +%% @doc Sign the result of a device call if the node is configured to do so. +maybe_sign({Status, Res}, NodeMsg) -> + {Status, maybe_sign(Res, NodeMsg)}; +maybe_sign(Res, NodeMsg) -> + ?event({maybe_sign, Res}), + case hb_opts:get(force_signed, false, NodeMsg) of + true -> + case hb_message:signers(Res, NodeMsg) of + [] -> hb_message:commit(Res, NodeMsg); + _ -> Res end; - {error, Error} -> {error, Error} + false -> Res end. -%% @doc execute a message from the node message upon the user's request. -resolve_processor(Processor, Request, NodeMsg) -> - case hb_opts:get(Processor, undefined, NodeMsg) of - undefined -> {ok, Request}; - ProcessorMsg -> - hb_converge:resolve(ProcessorMsg, Request, NodeMsg) +%% @doc Check if the request in question is signed by a given `role' on the node. +%% The `role' can be one of `operator' or `initiator'. +is(Request, NodeMsg) -> + is(operator, Request, NodeMsg). +is(admin, Request, NodeMsg) -> + % Does the caller have the right to change the node message? + RequestSigners = hb_message:signers(Request, NodeMsg), + ValidOperator = + hb_util:bin( + hb_opts:get( + operator, + case hb_opts:get(priv_wallet, no_viable_wallet, NodeMsg) of + no_viable_wallet -> unclaimed; + Wallet -> ar_wallet:to_address(Wallet) + end, + NodeMsg + ) + ), + EncOperator = + case ValidOperator of + <<"unclaimed">> -> unclaimed; + NativeAddress -> hb_util:human_id(NativeAddress) + end, + ?event({is, + {operator, + {valid_operator, ValidOperator}, + {encoded_operator, EncOperator}, + {request_signers, RequestSigners} + } + }), + EncOperator == unclaimed orelse lists:member(EncOperator, RequestSigners); +is(operator, Req, NodeMsg) -> + % Is the caller explicitly set to be the operator? + % Get the operator from the node message + Operator = hb_opts:get(operator, unclaimed, NodeMsg), + % Get the request signers + RequestSigners = hb_message:signers(Req, NodeMsg), + % Ensure the operator is present in the request + lists:member(Operator, RequestSigners); +is(initiator, Request, NodeMsg) -> + % Is the caller the first identity that configured the node message? + NodeHistory = hb_opts:get(node_history, [], NodeMsg), + % Check if node_history exists and is not empty + case NodeHistory of + [] -> + ?event(green_zone, {init, node_history, empty}), + false; + [InitializationRequest | _] -> + % Extract signature from first entry + InitializationRequestSigners = hb_message:signers(InitializationRequest, NodeMsg), + % Get request signers + RequestSigners = hb_message:signers(Request, NodeMsg), + % Ensure all signers of the initalization request are present in the + % request. + AllSignersPresent = + lists:all( + fun(Signer) -> lists:member(Signer, RequestSigners) end, + InitializationRequestSigners + ), + case AllSignersPresent of + true -> + {ok, true}; + false -> + {error, #{ + <<"status">> => 401, + <<"message">> => <<"Invalid request signature.">> + }} + end end. + +%%% Tests + +%% @doc Test that we can get the node message. +config_test() -> + StoreOpts = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + }, + Node = hb_http_server:start_node(Opts = #{ test_config_item => <<"test">>, store => StoreOpts }), + {ok, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, Opts), + ?assertEqual(<<"test">>, hb_ao:get(<<"test_config_item">>, Res, Opts)). + +%% @doc Test that we can't get the node message if the requested key is private. +priv_inaccessible_test() -> + Node = hb_http_server:start_node( + #{ + test_config_item => <<"test">>, + priv_key => <<"BAD">> + } + ), + {ok, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, #{}), + ?event({res, Res}), + ?assertEqual(<<"test">>, hb_ao:get(<<"test_config_item">>, Res, #{})), + ?assertEqual(not_found, hb_ao:get(<<"priv_key">>, Res, #{})). + +%% @doc Test that we can't set the node message if the request is not signed by +%% the owner of the node. +unauthorized_set_node_msg_fails_test() -> + StoreOpts = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + }, + Node = hb_http_server:start_node(Opts = #{ store => StoreOpts, priv_wallet => ar_wallet:new() }), + {error, _} = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/~meta@1.0/info">>, + <<"evil_config_item">> => <<"BAD">> + }, + Opts#{ priv_wallet => ar_wallet:new() } + ), + #{} + ), + {ok, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, Opts), + ?assertEqual(not_found, hb_ao:get(<<"evil_config_item">>, Res, Opts)), + ?assertEqual(0, length(hb_ao:get(<<"node_history">>, Res, [], Opts))). + +%% @doc Test that we can set the node message if the request is signed by the +%% owner of the node. +authorized_set_node_msg_succeeds_test() -> + StoreOpts = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + }, + Owner = ar_wallet:new(), + Node = hb_http_server:start_node( + Opts = #{ + operator => hb_util:human_id(ar_wallet:to_address(Owner)), + test_config_item => <<"test">>, + store => StoreOpts + } + ), + {ok, SetRes} = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/~meta@1.0/info">>, + <<"test_config_item">> => <<"test2">> + }, + Opts#{ priv_wallet => Owner } + ), + Opts + ), + ?event({res, SetRes}), + {ok, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, Opts), + ?event({res, Res}), + ?assertEqual(<<"test2">>, hb_ao:get(<<"test_config_item">>, Res, Opts)), + ?assertEqual(1, length(hb_ao:get(<<"node_history">>, Res, [], Opts))). + +%% @doc Test that an uninitialized node will not run computation. +uninitialized_node_test() -> + Node = hb_http_server:start_node(#{ initialized => false }), + {error, Res} = hb_http:get(Node, <<"/key1?1.key1=value1">>, #{}), + ?event({res, Res}), + ?assertEqual(<<"Node must be initialized before use.">>, Res). + +%% @doc Test that a permanent node message cannot be changed. +permanent_node_message_test() -> + StoreOpts = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + }, + Owner = ar_wallet:new(), + Node = hb_http_server:start_node( + Opts =#{ + operator => <<"unclaimed">>, + initialized => false, + test_config_item => <<"test">>, + store => StoreOpts + } + ), + {ok, SetRes1} = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/~meta@1.0/info">>, + <<"test_config_item">> => <<"test2">>, + initialized => <<"permanent">> + }, + Opts#{ priv_wallet => Owner } + ), + Opts + ), + ?event({set_res, SetRes1}), + {ok, Res} = hb_http:get(Node, #{ <<"path">> => <<"/~meta@1.0/info">> }, Opts), + ?event({get_res, Res}), + ?assertEqual(<<"test2">>, hb_ao:get(<<"test_config_item">>, Res, Opts)), + {error, SetRes2} = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/~meta@1.0/info">>, + <<"test_config_item">> => <<"bad_value">> + }, + Opts#{ priv_wallet => Owner } + ), + Opts + ), + ?event({set_res, SetRes2}), + {ok, Res2} = hb_http:get(Node, #{ <<"path">> => <<"/~meta@1.0/info">> }, Opts), + ?event({get_res, Res2}), + ?assertEqual(<<"test2">>, hb_ao:get(<<"test_config_item">>, Res2, Opts)), + ?assertEqual(1, length(hb_ao:get(<<"node_history">>, Res2, [], Opts))). + +%% @doc Test that we can claim the node correctly and set the node message after. +claim_node_test() -> + StoreOpts = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + }, + Owner = ar_wallet:new(), + Address = ar_wallet:to_address(Owner), + Node = hb_http_server:start_node( + Opts = #{ + operator => unclaimed, + test_config_item => <<"test">>, + store => StoreOpts + } + ), + {ok, SetRes} = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/~meta@1.0/info">>, + <<"operator">> => hb_util:human_id(Address) + }, + Opts#{ priv_wallet => Owner} + ), + Opts + ), + ?event({res, SetRes}), + {ok, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, Opts), + ?event({res, Res}), + ?assertEqual(hb_util:human_id(Address), hb_ao:get(<<"operator">>, Res, Opts)), + {ok, SetRes2} = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/~meta@1.0/info">>, + <<"test_config_item">> => <<"test2">> + }, + Opts#{ priv_wallet => Owner } + ), + Opts + ), + ?event({res, SetRes2}), + {ok, Res2} = hb_http:get(Node, <<"/~meta@1.0/info">>, Opts), + ?event({res, Res2}), + ?assertEqual(<<"test2">>, hb_ao:get(<<"test_config_item">>, Res2, Opts)), + ?assertEqual(2, length(hb_ao:get(<<"node_history">>, Res2, [], Opts))). + +%% Test that we can use a hook upon a request. +request_response_hooks_test() -> + Parent = self(), + Node = hb_http_server:start_node( + #{ + on => + #{ + <<"request">> => + #{ + <<"device">> => #{ + <<"request">> => + fun(_, #{ <<"body">> := Msgs }, _) -> + Parent ! {hook, request}, + {ok, #{ <<"body">> => Msgs} } + end + } + }, + <<"response">> => + #{ + <<"device">> => #{ + <<"response">> => + fun(_, #{ <<"body">> := Msgs }, _) -> + Parent ! {hook, response}, + {ok, #{ <<"body">> => Msgs} } + end + } + } + }, + http_extra_opts => #{ + <<"cache-control">> => [<<"no-store">>, <<"no-cache">>] + } + }), + hb_http:get(Node, <<"/~meta@1.0/info">>, #{}), + % Receive both of the responses from the hooks, if possible. + Res = + receive + {hook, request} -> + receive {hook, response} -> true after 100 -> false end + after 100 -> + false + end, + ?assert(Res). + +%% @doc Test that we can halt a request if the hook returns an error. +halt_request_test() -> + Node = hb_http_server:start_node( + #{ + on => + #{ + <<"request">> => + #{ + <<"device">> => #{ + <<"request">> => + fun(_, _, _) -> + {error, <<"Bad">>} + end + } + } + } + }), + {error, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, #{}), + ?assertEqual(<<"Bad">>, Res). + +%% @doc Test that a hook can modify a request. +modify_request_test() -> + Node = hb_http_server:start_node( + #{ + on => + #{ + <<"request">> => + #{ + <<"device">> => #{ + <<"request">> => + fun(_, #{ <<"body">> := [M|Ms] }, _) -> + { + ok, + #{ + <<"body">> => + [ + M#{ + <<"added">> => + <<"value">> + } + | + Ms + ] + } + } + end + } + } + } + }), + {ok, Res} = hb_http:get(Node, <<"/added">>, #{}), + ?assertEqual(<<"value">>, Res). + +%% @doc Test that version information is available and returned correctly. +buildinfo_test() -> + Node = hb_http_server:start_node(#{}), + ?assertEqual( + {ok, <<"HyperBEAM">>}, + hb_http:get(Node, <<"/~meta@1.0/build/node">>, #{}) + ), + ?assertEqual( + {ok, ?HYPERBEAM_VERSION}, + hb_http:get(Node, <<"/~meta@1.0/build/version">>, #{}) + ), + ?assertEqual( + {ok, ?HB_BUILD_SOURCE}, + hb_http:get(Node, <<"/~meta@1.0/build/source">>, #{}) + ), + ?assertEqual( + {ok, ?HB_BUILD_SOURCE_SHORT}, + hb_http:get(Node, <<"/~meta@1.0/build/source-short">>, #{}) + ), + ?assertEqual( + {ok, ?HB_BUILD_TIME}, + hb_http:get(Node, <<"/~meta@1.0/build/build-time">>, #{}) + ). diff --git a/src/dev_monitor.erl b/src/dev_monitor.erl index 2114729eb..86d3b1e5b 100644 --- a/src/dev_monitor.erl +++ b/src/dev_monitor.erl @@ -9,18 +9,18 @@ %%% functions must not mutate state. init(State, _, InitState) -> - {ok, State#{ monitors => InitState }}. + {ok, State#{ <<"monitors">> => InitState }}. -execute(Message, State = #{ pass := Pass, passes := Passes }) when Pass == Passes -> +execute(Message, State = #{ <<"pass">> := Pass, <<"passes">> := Passes }) when Pass == Passes -> signal(State, {message, Message}); execute(_, S) -> {ok, S}. -add_monitor(Mon, State = #{ monitors := Monitors }) -> - {ok, State#{ monitors => [Mon | Monitors] }}. +add_monitor(Mon, State = #{ <<"monitors">> := Monitors }) -> + {ok, State#{ <<"monitors">> => [Mon | Monitors] }}. end_of_schedule(State) -> signal(State, end_of_schedule). -signal(State = #{ monitors := StartingMonitors }, Signal) -> +signal(State = #{ <<"monitors">> := StartingMonitors }, Signal) -> RemainingMonitors = lists:filter( fun(Mon) -> @@ -32,6 +32,6 @@ signal(State = #{ monitors := StartingMonitors }, Signal) -> StartingMonitors ), ?event({remaining_monitors, length(RemainingMonitors)}), - {ok, State#{ monitors := RemainingMonitors }}. + {ok, State#{ <<"monitors">> := RemainingMonitors }}. uses() -> all. \ No newline at end of file diff --git a/src/dev_mu.erl b/src/dev_mu.erl deleted file mode 100644 index 8f435e1c1..000000000 --- a/src/dev_mu.erl +++ /dev/null @@ -1,128 +0,0 @@ --module(dev_mu). --export([push/2]). --include("include/hb.hrl"). - -%%% The main pushing logic for messages around the system.s - -%% We should run the following device stack on the message: -%% dev_scheduler -> dev_cu -> dev_poda -%% After execution we take the result and fork again based on it. --define(PUSH_DEV_STACK, [dev_scheduler, dev_cu, dev_poda]). - -%% @doc The main entry point for pushing a message. Assumes the message is -%% a carrier message, and will extract the carried message to push from it. -push(CarrierMsg, State) -> - % First pass: We need to verify the message and start the logger. - Msg = - case CarrierMsg#tx.data of - #{ <<"1">> := CarriedMsg } -> - CarriedMsg; - _ -> CarrierMsg - end, - ?event({starting_push_for, - {unsigned, hb_util:id(Msg, unsigned)}, - {signed, hb_util:id(Msg, signed)}, - {target, hb_util:id(Msg#tx.target)} - }), - ?no_prod(fix_mu_push_validation), - case ar_bundles:verify_item(Msg) of - _ -> - Logger = - case maps:get(logger, State, undefined) of - undefined -> hb_logger:start(); - X -> X - end, - hb_logger:register(Logger), - fork( - #result { - messages = [Msg] - }, - State#{ - depth => 0, - store => maps:get(store, State, hb_opts:get(store)), - logger => Logger, - wallet => maps:get(wallet, State, hb:wallet()) - } - ), - % TODO: Implement trace waiting. - ResTX = ar_bundles:sign_item( - #tx{ tags = [{<<"Status">>, <<"200">>}]}, - hb:wallet()), - {ok, #{ results => ResTX }} - %false -> - % {error, cannot_push_invalid_message} - end. - -%% Take a computation result and fork each message/spawn/... into its own worker. -fork(Res, Opts) -> - push_messages(upload, Res#result.spawns, Opts), - push_messages(upload, Res#result.messages, Opts), - push_messages(attest, Res#result.assignments, Opts). - -push_messages(upload, Messages, Opts) -> - lists:foreach( - fun(Message) -> - spawn( - fun() -> - ?event( - {mu_forking_for, - {unsigned, hb_util:id(Message, unsigned)}, - {signed, hb_util:id(Message, signed)}, - {target, hb_util:id(Message#tx.target)}, - {logger, maps:get(logger, Opts, undefined)} - } - ), - Stack = dev_stack:create(?PUSH_DEV_STACK), - {ok, Results} = hb_converge:resolve( - {dev_stack, execute}, - push, - [ - #{ - devices => Stack, - message => Message, - logger => maps:get(logger, Opts, undefined), - store => maps:get(store, Opts, hb_opts:get(store)), - wallet => maps:get(wallet, Opts, hb:wallet()) - } - ] - ), - ?event({pushing_result_for_computed_message, - {unsigned, hb_util:id(Message, unsigned)}, - {signed, hb_util:id(Message, signed)}, - {target, hb_util:id(Message#tx.target)} - }), - handle_push_result(Results, Opts) - end - ) - end, - maybe_to_list(Messages) - ); -push_messages(attest, Assignments, #{ logger := Logger }) -> - lists:foreach( - fun(Assignment) -> - hb_logger:log(Logger, {ok, "Assigning ", ar_bundles:id(Assignment, signed)}), - hb_client:assign(Assignment), - ?no_prod("After assigning, don't we want to push the message?") - end, - maybe_to_list(Assignments) - ). - -handle_push_result(Results, Opts = #{ depth := Depth }) -> - % Second pass: We have the results, so we can fork the messages/spawns/... - Res = #result{ - messages = maps:get(<<"/Outbox">>, Results, #{}), - assignments = maps:get(<<"/Assignment">>, Results, #{}), - spawns = maps:get(<<"/Spawn">>, Results, #{}) - }, - ?event({push_recursing, - {depth, Depth}, - {messages, maps:size(Res#result.messages)}, - {assignments, maps:size(Res#result.assignments)}, - {spawns, maps:size(Res#result.spawns)} - }), - fork(Res, Opts#{ depth => Depth + 1 }). - -maybe_to_list(Map) when is_map(Map) -> [V || {_K, V} <- maps:to_list(Map)]; -maybe_to_list(undefined) -> []; -maybe_to_list(Else) when not is_list(Else) -> [Else]; -maybe_to_list(Else) -> Else. \ No newline at end of file diff --git a/src/dev_multipass.erl b/src/dev_multipass.erl index 57512e526..e3ab47b7b 100644 --- a/src/dev_multipass.erl +++ b/src/dev_multipass.erl @@ -3,6 +3,7 @@ %%% execution passes to be completed in sequence across devices. -module(dev_multipass). -export([info/1]). +-include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). info(_M1) -> @@ -12,13 +13,13 @@ info(_M1) -> %% @doc Forward the keys function to the message device, handle all others %% with deduplication. We only act on the first pass. -handle(keys, M1, _M2, _Opts) -> - dev_message:keys(M1); -handle(set, M1, M2, Opts) -> +handle(<<"keys">>, M1, _M2, Opts) -> + dev_message:keys(M1, Opts); +handle(<<"set">>, M1, M2, Opts) -> dev_message:set(M1, M2, Opts); handle(_Key, M1, _M2, Opts) -> - Passes = hb_converge:get(<<"Passes">>, {as, dev_message, M1}, 1, Opts), - Pass = hb_converge:get(<<"Pass">>, {as, dev_message, M1}, 1, Opts), + Passes = hb_ao:get(<<"passes">>, {as, dev_message, M1}, 1, Opts), + Pass = hb_ao:get(<<"pass">>, {as, dev_message, M1}, 1, Opts), case Pass < Passes of true -> {pass, M1}; false -> {ok, M1} @@ -29,10 +30,11 @@ handle(_Key, M1, _M2, Opts) -> basic_multipass_test() -> Msg1 = #{ - <<"device">> => <<"Multipass/1.0">>, - <<"Passes">> => 2, - <<"Pass">> => 1 + <<"device">> => <<"multipass@1.0">>, + <<"passes">> => 2, + <<"pass">> => 1 }, - Msg2 = Msg1#{ <<"Pass">> => 2 }, - ?assertMatch({pass, _}, hb_converge:resolve(Msg1, <<"Compute">>, #{})), - ?assertMatch({ok, _}, hb_converge:resolve(Msg2, <<"Compute">>, #{})). \ No newline at end of file + Msg2 = Msg1#{ <<"pass">> => 2 }, + ?assertMatch({pass, _}, hb_ao:resolve(Msg1, <<"Compute">>, #{})), + ?event(alive), + ?assertMatch({ok, _}, hb_ao:resolve(Msg2, <<"Compute">>, #{})). \ No newline at end of file diff --git a/src/dev_name.erl b/src/dev_name.erl new file mode 100644 index 000000000..04d018550 --- /dev/null +++ b/src/dev_name.erl @@ -0,0 +1,149 @@ +%%% @doc A device for resolving names to their corresponding values, through the +%%% use of a `resolver' interface. Each `resolver' is a message that can be +%%% given a `key' and returns an associated value. The device will attempt to +%%% match the key against each resolver in turn, and return the value of the +%%% first resolver that matches. +-module(dev_name). +-export([info/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Configure the `default' key to proxy to the `resolver/4' function. +%% Exclude the `keys' and `set' keys from being processed by this device, as +%% these are needed to modify the base message itself. +info(_) -> + #{ + default => fun resolve/4, + excludes => [<<"keys">>, <<"set">>] + }. + +%% @doc Resolve a name to its corresponding value. The name is given by the key +%% called. For example, `GET /~name@1.0/hello&load=false' grants the value of +%% `hello'. If the `load' key is set to `true', the value is treated as a +%% pointer and its contents is loaded from the cache. For example, +%% `GET /~name@1.0/reference' yields the message at the path specified by the +%% `reference' key. +resolve(Key, _, Req, Opts) -> + Resolvers = hb_opts:get(name_resolvers, [], Opts), + ?event({resolvers, Resolvers}), + case match_resolver(Key, Resolvers, Opts) of + {ok, Resolved} -> + case hb_util:atom(hb_ao:get(<<"load">>, Req, true, Opts)) of + false -> + {ok, Resolved}; + true -> + hb_cache:read(Resolved, Opts) + end; + not_found -> + not_found + end. + +%% @doc Find the first resolver that matches the key and return its value. +match_resolver(_Key, [], _Opts) -> + not_found; +match_resolver(Key, [Resolver | Resolvers], Opts) -> + case execute_resolver(Key, Resolver, Opts) of + {ok, Value} -> + ?event({resolver_found, {key, Key}, {value, Value}}), + {ok, Value}; + _ -> + match_resolver(Key, Resolvers, Opts) + end. + +%% @doc Execute a resolver with the given key and return its value. +execute_resolver(Key, Resolver, Opts) -> + ?event({executing, {key, Key}, {resolver, Resolver}}), + hb_ao:resolve( + Resolver, + #{ <<"path">> => <<"lookup">>, <<"key">> => Key }, + Opts + ). + +%%% Tests. + +no_resolvers_test() -> + ?assertEqual( + not_found, + resolve(<<"hello">>, #{}, #{}, #{ only => local }) + ). + +message_lookup_device_resolver(Msg) -> + #{ + <<"device">> => #{ + <<"lookup">> => fun(_, Req, Opts) -> + Key = hb_ao:get(<<"key">>, Req, Opts), + ?event({test_resolver_executing, {key, Key}, {req, Req}, {msg, Msg}}), + case maps:get(Key, Msg, not_found) of + not_found -> + ?event({test_resolver_not_found, {key, Key}, {msg, Msg}}), + {error, not_found}; + Value -> + ?event({test_resolver_found, {key, Key}, {value, Value}}), + {ok, Value} + end + end + } + }. + +single_resolver_test() -> + ?assertEqual( + {ok, <<"world">>}, + resolve( + <<"hello">>, + #{}, + #{ <<"load">> => false }, + #{ + name_resolvers => [ + message_lookup_device_resolver( + #{<<"hello">> => <<"world">>} + ) + ] + } + ) + ). + +multiple_resolvers_test() -> + ?assertEqual( + {ok, <<"bigger-world">>}, + resolve( + <<"hello">>, + #{}, + #{ <<"load">> => false }, + #{ + name_resolvers => [ + message_lookup_device_resolver( + #{<<"irrelevant">> => <<"world">>} + ), + message_lookup_device_resolver( + #{<<"hello">> => <<"bigger-world">>} + ) + ] + } + ) + ). + +%% @doc Test that we can resolve messages from a name loaded with the device. +load_and_execute_test() -> + TestKey = <<"test-key", (hb_util:bin(erlang:system_time(millisecond)))/binary>>, + {ok, ID} = hb_cache:write( + #{ + <<"deep">> => <<"PING">> + }, + #{} + ), + ?assertEqual( + {ok, <<"PING">>}, + hb_ao:resolve_many( + [ + #{ <<"device">> => <<"name@1.0">> }, + #{ <<"path">> => TestKey }, + #{ <<"path">> => <<"deep">> } + ], + #{ + name_resolvers => [ + message_lookup_device_resolver(#{ <<"irrelevant">> => ID }), + message_lookup_device_resolver(#{ TestKey => ID }) + ] + } + ) + ). \ No newline at end of file diff --git a/src/dev_node_process.erl b/src/dev_node_process.erl new file mode 100644 index 000000000..5366b6793 --- /dev/null +++ b/src/dev_node_process.erl @@ -0,0 +1,208 @@ +%%% @doc A device that implements the singleton pattern for processes specific +%%% to an individual node. This device uses the `local-name@1.0' device to +%%% register processes with names locally, persistenting them across reboots. +%%% +%%% Definitions of singleton processes are expected to be found with their +%%% names in the `node_processes' section of the node message. +-module(dev_node_process). +-export([info/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Register a default handler for the device. Inherits `keys' and `set' +%% from the default device. +info(_Opts) -> + #{ + default => fun lookup/4, + excludes => [<<"set">>, <<"keys">>] + }. + +%% @doc Lookup a process by name. +lookup(Name, _Base, Req, Opts) -> + ?event(node_process, {lookup, {name, Name}}), + LookupRes = + hb_ao:resolve( + #{ <<"device">> => <<"local-name@1.0">> }, + #{ <<"path">> => <<"lookup">>, <<"key">> => Name, <<"load">> => true }, + Opts + ), + case LookupRes of + {ok, ProcessID} -> + hb_cache:read(ProcessID, Opts); + {error, not_found} -> + case hb_ao:get(<<"spawn">>, Req, true, Opts) of + true -> + spawn_register(Name, Opts); + false -> + {error, not_found} + end + end. + +%% @doc Spawn a new process according to the process definition found in the +%% node message, and register it with the given name. +spawn_register(Name, Opts) -> + case hb_opts:get(node_processes, #{}, Opts) of + #{ Name := BaseDef } -> + % We have found the base process definition. Augment it with the + % node's address as necessary, then commit to the result. + ?event(node_process, {registering, {name, Name}, {base_def, BaseDef}}), + Signed = + hb_message:commit( + augment_definition(BaseDef, Opts), + Opts, + hb_opts:get(node_process_spawn_codec, <<"httpsig@1.0">>, Opts) + ), + ?event(node_process, {signed, {name, Name}, {signed, Signed}}), + ID = hb_message:id(Signed, signed, Opts), + ?event(node_process, {spawned, {name, Name}, {process, Signed}}), + % `POST' to the schedule device for the process to start its sequence. + {ok, Assignment} = + hb_ao:resolve( + Signed, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => Signed + }, + Opts + ), + ?event(node_process, {initialized, {name, Name}, {assignment, Assignment}}), + RegResult = + dev_local_name:direct_register( + #{ <<"key">> => Name, <<"value">> => ID }, + Opts + ), + ?event(node_process, {registered, {name, Name}, {process_id, ID}}), + case RegResult of + {ok, _} -> + {ok, Signed}; + {error, Err} -> + {error, #{ + <<"status">> => 500, + <<"body">> => <<"Failed to register process.">>, + <<"details">> => Err + }} + end; + _ -> + % We could not find the base process definition for the given name + % in the node message. + {error, not_found} + end. + +%% @doc Augment the given process definition with the node's address. +augment_definition(BaseDef, Opts) -> + Address = + hb_util:human_id( + ar_wallet:to_address( + hb_opts:get(priv_wallet, no_viable_wallet, Opts) + ) + ), + SchedulersFromBase = + hb_util:binary_to_addresses( + hb_ao:get(<<"scheduler">>, BaseDef, <<>>, Opts) + ), + AuthoritiesFromBase = + hb_util:binary_to_addresses( + hb_ao:get(<<"authority">>, BaseDef, <<>>, Opts) + ), + Schedulers = (SchedulersFromBase -- [Address]) ++ [Address], + Authorities = (AuthoritiesFromBase -- [Address]) ++ [Address], + % Normalize the scheduler and authority lists to binary strings. + hb_ao:set( + #{ + <<"scheduler">> => Schedulers, + <<"authority">> => Authorities + }, + BaseDef, + Opts + ). + +%%% Tests + +%%% The name that should be used for the singleton process during tests. +-define(TEST_NAME, <<"test-node-process">>). + +%% @doc Helper function to generate a test environment and its options. +generate_test_opts() -> + {ok, Module} = file:read_file(<<"test/test.lua">>), + generate_test_opts(#{ + ?TEST_NAME => #{ + <<"device">> => <<"process@1.0">>, + <<"execution-device">> => <<"lua@5.3a">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"module">> => #{ + <<"content-type">> => <<"text/x-lua">>, + <<"body">> => Module + } + } + }). +generate_test_opts(Defs) -> + #{ + node_processes => Defs, + priv_wallet => ar_wallet:new() + }. + +lookup_no_spawn_test() -> + Opts = generate_test_opts(), + ?assertEqual( + {error, not_found}, + lookup(<<"name1">>, #{}, #{}, Opts) + ). + +lookup_spawn_test() -> + Opts = generate_test_opts(), + Res1 = {_, Process1} = + hb_ao:resolve( + #{ <<"device">> => <<"node-process@1.0">> }, + ?TEST_NAME, + Opts + ), + ?assertMatch( + {ok, #{ <<"device">> := <<"process@1.0">> }}, + Res1 + ), + {ok, Process2} = hb_ao:resolve( + #{ <<"device">> => <<"node-process@1.0">> }, + ?TEST_NAME, + Opts + ), + ?assertEqual( + hb_cache:ensure_all_loaded(Process1, Opts), + hb_cache:ensure_all_loaded(Process2, Opts) + ). + +%% @doc Test that a process can be spawned, executed upon, and its result retrieved. +lookup_execute_test() -> + Opts = generate_test_opts(), + Res1 = + hb_ao:resolve_many( + [ + #{ <<"device">> => <<"node-process@1.0">> }, + ?TEST_NAME, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + #{ + <<"path">> => <<"compute">>, + <<"test-key">> => <<"test-value">> + }, + Opts + ) + } + ], + Opts + ), + ?assertMatch( + {ok, #{ <<"slot">> := 1 }}, + Res1 + ), + ?assertMatch( + 42, + hb_ao:get( + << ?TEST_NAME/binary, "/now/results/output/body" >>, + #{ <<"device">> => <<"node-process@1.0">> }, + Opts + ) + ). \ No newline at end of file diff --git a/src/dev_p4.erl b/src/dev_p4.erl index f94088a30..b072ae1bd 100644 --- a/src/dev_p4.erl +++ b/src/dev_p4.erl @@ -1,6 +1,537 @@ +%%% @doc The HyperBEAM core payment ledger. This module allows the operator to +%%% specify another device that can act as a pricing mechanism for transactions +%%% on the node, as well as orchestrating a payment ledger to calculate whether +%%% the node should fulfil services for users. +%%% +%%% The device requires the following node message settings in order to function: +%%% +%%% - `p4_pricing-device': The device that will estimate the cost of a request. +%%% - `p4_ledger-device': The device that will act as a payment ledger. +%%% +%%% The pricing device should implement the following keys: +%%% 
+%%%             `GET /estimate?type=pre|post&body=[...]&request=RequestMessage'
+%%%             `GET /price?type=pre|post&body=[...]&request=RequestMessage'
+%%%
+%%% +%%% The `body' key is used to pass either the request or response messages to the +%%% device. The `type' key is used to specify whether the inquiry is for a request +%%% (pre) or a response (post) object. Requests carry lists of messages that will +%%% be executed, while responses carry the results of the execution. The `price' +%%% key may return `infinity' if the node will not serve a user under any +%%% circumstances. Else, the value returned by the `price' key will be passed to +%%% the ledger device as the `amount' key. +%%% +%%% A ledger device should implement the following keys: +%%% 
+%%%             `POST /credit?message=PaymentMessage&request=RequestMessage'
+%%%             `POST /charge?amount=PriceMessage&request=RequestMessage'
+%%%             `GET /balance?request=RequestMessage'
+%%%
+%%% +%%% The `type' key is optional and defaults to `pre'. If `type' is set to `post', +%%% the charge must be applied to the ledger, whereas the `pre' type is used to +%%% check whether the charge would succeed before execution. -module(dev_p4). --export([push/2]). +-export([request/3, response/3, balance/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). -push(_Item, S) -> - % TODO: Check payment. - {ok, S}. \ No newline at end of file +%%% The default list of routes that should not be charged for. +-define(DEFAULT_NON_CHARGABLE_ROUTES, [ + #{ <<"template">> => <<"/~p4@1.0/balance">> }, + #{ <<"template">> => <<"/~p4@1.0/topup">> }, + #{ <<"template">> => <<"/~meta@1.0/*">> } +]). + +%% @doc Estimate the cost of a transaction and decide whether to proceed with +%% a request. The default behavior if `pricing-device' or `p4_balances' are +%% not set is to proceed, so it is important that a user initialize them. +request(State, Raw, NodeMsg) -> + PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), + LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), + Messages = hb_ao:get(<<"body">>, Raw, NodeMsg#{ hashpath => ignore }), + Request = hb_ao:get(<<"request">>, Raw, NodeMsg), + IsChargable = is_chargable_req(Request, NodeMsg), + ?event(payment, + {preprocess_with_devices, + PricingDevice, + LedgerDevice, + {chargable, IsChargable} + } + ), + case {IsChargable, (PricingDevice =/= false) and (LedgerDevice =/= false)} of + {false, _} -> + ?event(payment, non_chargable_route), + {ok, #{ <<"body">> => Messages }}; + {true, false} -> + ?event(payment, {p4_pre_pricing_response, {error, <<"infinity">>}}), + {ok, #{ <<"body">> => Messages }}; + {true, true} -> + PricingMsg = State#{ <<"device">> => PricingDevice }, + LedgerMsg = State#{ <<"device">> => LedgerDevice }, + PricingReq = #{ + <<"path">> => <<"estimate">>, + <<"request">> => Request, + <<"body">> => Messages + }, + ?event({p4_pricing_request, {devmsg, PricingMsg}, {req, PricingReq}}), + case hb_ao:resolve(PricingMsg, PricingReq, NodeMsg) of + {ok, <<"infinity">>} -> + % The device states that under no circumstances should we + % proceed with the request. + ?event(payment, {p4_pre_pricing_response, {error, <<"infinity">>}}), + {error, + <<"Node will not service this request " + "under any circumstances.">>}; + {ok, 0} -> + % The device has estimated the cost of the request to be + % zero, so we proceed. + {ok, #{ <<"body">> => Messages }}; + {ok, Price} -> + % The device has estimated the cost of the request. We check + % the user's balance to see if they have enough funds to + % service the request. + LedgerReq = #{ + <<"path">> => <<"balance">>, + <<"target">> => + case hb_message:signers(Request, NodeMsg) of + [Signer] -> Signer; + [] -> <<"unknown">>; + Multiple -> Multiple + end, + <<"request">> => Request + }, + ?event(payment, {p4_pre_pricing_estimate, Price}), + case hb_ao:resolve(LedgerMsg, LedgerReq, NodeMsg) of + {ok, Sufficient} when + Sufficient =:= true orelse + Sufficient =:= <<"infinity">> -> + % The ledger device has confirmed that the user has + % enough funds for the request, so we proceed. + ?event(payment, + {p4_pre_ledger_response, + {balance_check, guaranteed} + } + ), + {ok, #{ <<"body">> => Messages }}; + {ok, Balance} when Balance >= Price -> + % The user has enough funds to service the request, + % so we proceed. + ?event(payment, + {p4_pre_ledger_response, + {balance_check, sufficient} + } + ), + {ok, #{ <<"body">> => Messages }}; + {ok, Balance} -> + % The user does not have enough funds to service + % the request, so we don't proceed. + ?event(payment, + {insufficient_funds, + {balance, Balance}, + {price, Price} + } + ), + {error, #{ + <<"status">> => 402, + <<"body">> => <<"Insufficient funds">>, + <<"price">> => Price, + <<"balance">> => Balance + }}; + {error, Error} -> + % The ledger device is unable to process the request, + % so we don't proceed. + ?event(payment, + {pre_ledger_validation, + {error, Error}, + {base, LedgerMsg}, + {req, LedgerReq} + } + ), + {error, #{ + <<"status">> => 500, + <<"body">> => <<"Error checking ledger balance.">> + }} + end; + {ErrType, Err} -> + % The device is unable to estimate the cost of the request, + % so we don't proceed. + ?event({p4_pricing_error, {type, ErrType}}), + {error, + #{ + <<"type">> => ErrType, + <<"body">> => + <<"Could not estimate price of request.">> + } + } + end + end. + +%% @doc Postprocess the request after it has been fulfilled. +response(State, RawResponse, NodeMsg) -> + PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), + LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), + Response = + hb_ao:get( + <<"body">>, + RawResponse, + NodeMsg#{ hashpath => ignore } + ), + Request = hb_ao:get(<<"request">>, RawResponse, NodeMsg), + ?event(payment, {post_processing_with_devices, PricingDevice, LedgerDevice}), + ?event({response_hook, {request, Request}, {response, Response}}), + case ((PricingDevice =/= false) and (LedgerDevice =/= false)) andalso + is_chargable_req(Request, NodeMsg) of + false -> + {ok, #{ <<"body">> => Response }}; + true -> + PricingMsg = State#{ <<"device">> => PricingDevice }, + LedgerMsg = State#{ <<"device">> => LedgerDevice }, + PricingReq = #{ + <<"path">> => <<"price">>, + <<"request">> => Request, + <<"body">> => Response + }, + ?event({post_pricing_request, PricingReq}), + PricingRes = + case hb_ao:resolve(PricingMsg, PricingReq, NodeMsg) of + {error, _Error} -> + % The pricing device is unable to give us a cost for + % the request, so we try to estimate it instead. + EstimateReq = PricingReq#{ <<"path">> => <<"estimate">> }, + hb_ao:resolve(PricingMsg, EstimateReq, NodeMsg); + {ok, P} -> {ok, P} + end, + ?event(payment, {p4_post_pricing_response, PricingRes}), + case PricingRes of + {ok, 0} -> + % The pricing device has estimated the cost of the request + % to be zero, so we proceed. + {ok, #{ <<"body">> => Response }}; + {ok, Price} -> + % We have successfully determined the cost of the request, + % so we proceed to charge the user's account. We sign the + % request with the node's private key, as it is the node + % that is performing the charge, not the user. + LedgerReq = + hb_message:commit( + #{ + <<"path">> => <<"charge">>, + <<"quantity">> => Price, + <<"account">> => + case hb_message:signers(Request, NodeMsg) of + [Signer] -> Signer; + Multiple -> Multiple + end, + <<"recipient">> => + case hb_opts:get(p4_recipient, undefined, NodeMsg) of + Addr when ?IS_ID(Addr) -> + hb_util:human_id(Addr); + _ -> + case hb_opts:get(operator, undefined, NodeMsg) of + undefined -> + <<"unknown">>; + Operator-> + hb_util:human_id(Operator) + end + end, + <<"request">> => Request + }, + NodeMsg + ), + ?event(payment, + {post_charge, + {msg, LedgerMsg}, + {req, LedgerReq} + } + ), + case hb_ao:resolve(LedgerMsg, LedgerReq, NodeMsg) of + {ok, _} -> + ?event(payment, {p4_post_ledger_response, {ok, Price}}), + % Return the original response. + {ok, #{ <<"body">> => Response }}; + {error, Error} -> + ?event(payment, {p4_post_ledger_response, {error, Error}}), + % The charge failed, so we return the error from the + % ledger device. + {error, Error} + end; + {error, PricingError} -> + % The pricing device is unable to process the request, + % so we don't proceed. + {error, PricingError} + end + end. + +%% @doc Get the balance of a user in the ledger. +balance(_, Req, NodeMsg) -> + case dev_hook:find(<<"request">>, NodeMsg) of + [] -> + {error, <<"No request hook found.">>}; + [Handler] -> + LedgerDevice = + hb_ao:get(<<"ledger-device">>, Handler, false, NodeMsg), + LedgerMsg = Handler#{ <<"device">> => LedgerDevice }, + LedgerReq = #{ + <<"path">> => <<"balance">>, + <<"request">> => Req + }, + ?event({ledger_message, {ledger_msg, LedgerMsg}}), + case hb_ao:resolve(LedgerMsg, LedgerReq, NodeMsg) of + {ok, Balance} -> + {ok, Balance}; + {error, Error} -> + {error, Error} + end + end. + +%% @doc The node operator may elect to make certain routes non-chargable, using +%% the `routes' syntax also used to declare routes in `router@1.0'. +is_chargable_req(Req, NodeMsg) -> + NonChargableRoutes = + hb_opts:get( + p4_non_chargable_routes, + ?DEFAULT_NON_CHARGABLE_ROUTES, + NodeMsg + ), + Matches = + dev_router:match( + #{ <<"routes">> => NonChargableRoutes }, + Req, + NodeMsg + ), + ?event( + { + is_chargable, + {non_chargable_routes, NonChargableRoutes}, + {req, Req}, + {matches, Matches} + } + ), + case Matches of + {error, no_matching_route} -> true; + _ -> false + end. + +%%% Tests + +test_opts(Opts) -> + test_opts(Opts, <<"faff@1.0">>). +test_opts(Opts, PricingDev) -> + test_opts(Opts, PricingDev, <<"faff@1.0">>). +test_opts(Opts, PricingDev, LedgerDev) -> + ProcessorMsg = + #{ + <<"device">> => <<"p4@1.0">>, + <<"pricing-device">> => PricingDev, + <<"ledger-device">> => LedgerDev + }, + Opts#{ + on => #{ + <<"request">> => ProcessorMsg, + <<"response">> => ProcessorMsg + } + }. + +%% @doc Simple test of p4's capabilities with the `faff@1.0' device. +faff_test() -> + GoodWallet = ar_wallet:new(), + BadWallet = ar_wallet:new(), + Node = hb_http_server:start_node( + test_opts( + #{ + faff_allow_list => + [hb_util:human_id(ar_wallet:to_address(GoodWallet))] + } + ) + ), + Req = #{ + <<"path">> => <<"/greeting">>, + <<"greeting">> => <<"Hello, world!">> + }, + GoodSignedReq = hb_message:commit(Req, GoodWallet), + ?event({req, GoodSignedReq}), + BadSignedReq = hb_message:commit(Req, BadWallet), + ?event({req, BadSignedReq}), + {ok, Res} = hb_http:get(Node, GoodSignedReq, #{}), + ?event(payment, {res, Res}), + ?assertEqual(<<"Hello, world!">>, Res), + ?assertMatch({error, _}, hb_http:get(Node, BadSignedReq, #{})). + +%% @doc Test that a non-chargable route is not charged for. +non_chargable_route_test() -> + Wallet = ar_wallet:new(), + Processor = + #{ + <<"device">> => <<"p4@1.0">>, + <<"ledger-device">> => <<"simple-pay@1.0">>, + <<"pricing-device">> => <<"simple-pay@1.0">> + }, + Node = hb_http_server:start_node( + #{ + p4_non_chargable_routes => + [ + #{ <<"template">> => <<"/~p4@1.0/balance">> }, + #{ <<"template">> => <<"/~meta@1.0/*/*">> } + ], + on => #{ + <<"request">> => Processor, + <<"response">> => Processor + }, + operator => hb:address() + } + ), + Req = #{ + <<"path">> => <<"/~p4@1.0/balance">> + }, + GoodSignedReq = hb_message:commit(Req, Wallet), + Res = hb_http:get(Node, GoodSignedReq, #{}), + ?event({res1, Res}), + ?assertMatch({ok, 0}, Res), + Req2 = #{ <<"path">> => <<"/~meta@1.0/info/operator">> }, + GoodSignedReq2 = hb_message:commit(Req2, Wallet), + Res2 = hb_http:get(Node, GoodSignedReq2, #{}), + ?event({res2, Res2}), + OperatorAddress = hb_util:human_id(hb:address()), + ?assertEqual({ok, OperatorAddress}, Res2), + Req3 = #{ <<"path">> => <<"/~scheduler@1.0">> }, + BadSignedReq3 = hb_message:commit(Req3, Wallet), + Res3 = hb_http:get(Node, BadSignedReq3, #{}), + ?event({res3, Res3}), + ?assertMatch({error, _}, Res3). + +%% @doc Ensure that Lua scripts can be used as pricing and ledger devices. Our +%% scripts come in two components: +%% 1. A `process' script which is executed as a persistent `local-process' on the +%% node, and which maintains the state of the ledger. This process runs +%% `hyper-token.lua' as its base, then adds the logic of `hyper-token-p4.lua' +%% to it. This secondary script implements the `charge' function that `p4@1.0' +%% will call to charge a user's account. +%% 2. A `client' script, which is executed as a `p4@1.0' ledger device, which +%% uses `~push@1.0' to send requests to the ledger `process'. +hyper_token_ledger_test_() -> + {timeout, 60, fun hyper_token_ledger/0}. +hyper_token_ledger() -> + % Create the wallets necessary and read the files containing the scripts. + HostWallet = ar_wallet:new(), + HostAddress = hb_util:human_id(HostWallet), + OperatorWallet = ar_wallet:new(), + OperatorAddress = hb_util:human_id(OperatorWallet), + AliceWallet = ar_wallet:new(), + AliceAddress = hb_util:human_id(AliceWallet), + BobWallet = ar_wallet:new(), + BobAddress = hb_util:human_id(BobWallet), + {ok, TokenScript} = file:read_file("scripts/hyper-token.lua"), + {ok, ProcessScript} = file:read_file("scripts/hyper-token-p4.lua"), + {ok, ClientScript} = file:read_file("scripts/hyper-token-p4-client.lua"), + % Create the processor device, contains component (1): The script that + % pushes requests to the ledger `process'. + Processor = + #{ + <<"device">> => <<"p4@1.0">>, + <<"ledger-device">> => <<"lua@5.3a">>, + <<"pricing-device">> => <<"simple-pay@1.0">>, + <<"module">> => #{ + <<"content-type">> => <<"text/x-lua">>, + <<"name">> => <<"scripts/hyper-token-p4-client.lua">>, + <<"body">> => ClientScript + }, + <<"ledger-path">> => <<"/ledger~node-process@1.0">> + }, + % Start the node with the processor and the `local-process' ledger + % (component 2) running the `hyper-token.lua' and `hyper-token-p4.lua' + % scripts. `hyper-token.lua' implements the core token ledger, while + % `hyper-token-p4.lua' implements the `charge' function that `p4@1.0' will + % call to charge a user's account upon charges. We initialize the ledger + % with 100 tokens for Alice. + Node = + hb_http_server:start_node( + #{ + store => [hb_test_utils:test_store()], + priv_wallet => HostWallet, + p4_non_chargable_routes => + [ + #{ + <<"template">> => <<"/*~node-process@1.0/*">> + } + ], + on => #{ + <<"request">> => Processor, + <<"response">> => Processor + }, + operator => OperatorAddress, + node_processes => #{ + <<"ledger">> => #{ + <<"device">> => <<"process@1.0">>, + <<"execution-device">> => <<"lua@5.3a">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"module">> => [ + #{ + <<"content-type">> => <<"text/x-lua">>, + <<"name">> => <<"scripts/hyper-token.lua">>, + <<"body">> => TokenScript + }, + #{ + <<"content-type">> => <<"text/x-lua">>, + <<"name">> => <<"scripts/hyper-token-p4.lua">>, + <<"body">> => ProcessScript + } + ], + <<"balance">> => #{ AliceAddress => 100 }, + <<"admin">> => HostAddress + % <<"operator">> => + % hb_util:human_id(ar_wallet:to_address(HostWallet)) + } + } + } + ), + % To start, we attempt a request from Bob, which should fail because he + % has no tokens. + Req = #{ + <<"path">> => <<"/greeting">>, + <<"greeting">> => <<"Hello, world!">> + }, + SignedReq = hb_message:commit(Req, BobWallet), + Res = hb_http:get(Node, SignedReq, #{}), + ?event({expected_failure, Res}), + ?assertMatch({error, _}, Res), + % We then move 50 tokens from Alice to Bob. + {ok, TopupRes} = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/ledger~node-process@1.0/schedule">>, + <<"body">> => + hb_message:commit( + #{ + <<"path">> => <<"transfer">>, + <<"quantity">> => 50, + <<"recipient">> => BobAddress + }, + AliceWallet + ) + }, + HostWallet + ), + #{} + ), + % We now attempt Bob's request again, which should succeed. + ?event({topup_res, TopupRes}), + ResAfterTopup = hb_http:get(Node, SignedReq, #{}), + ?event({res_after_topup, ResAfterTopup}), + ?assertMatch({ok, <<"Hello, world!">>}, ResAfterTopup), + % We now check the balance of Bob. It should have been charged 2 tokens from + % the 50 Alice sent him. + {ok, Balances} = + hb_http:get( + Node, + <<"/ledger~node-process@1.0/now/balance">>, + #{} + ), + ?event(debug_charge, {balances, Balances}), + ?assertMatch(48, hb_ao:get(BobAddress, Balances, #{})), + % Finally, we check the balance of the operator. It should be 2 tokens, + % the amount that was charged from Alice. + ?assertMatch(2, hb_ao:get(OperatorAddress, Balances, #{})). \ No newline at end of file diff --git a/src/dev_patch.erl b/src/dev_patch.erl new file mode 100644 index 000000000..f7ef33e26 --- /dev/null +++ b/src/dev_patch.erl @@ -0,0 +1,344 @@ +%%% @doc A device that can be used to reorganize a message: Moving data from +%%% one path inside it to another. This device's function runs in two modes: +%%% +%%% 1. When using `all' to move all data at the path given in `from' to the +%%% path given in `to'. +%%% 2. When using `patches' to move all submessages in the source to the target, +%%% _if_ they have a `method' key of `PATCH' or a `device' key of `patch@1.0'. +%%% +%%% Source and destination paths may be prepended by `base:` or `req:` keys to +%%% indicate that they are relative to either of the message's that the +%%% computation is being performed on. +%%% +%%% The search order for finding the source and destination keys is as follows, +%%% where `X` is either `from' or `to`: +%%% +%%% 1. The `patch-X' key of the execution message. +%%% 2. The `X' key of the execution message. +%%% 3. The `patch-X' key of the request message. +%%% 4. The `X' key of the request message. +%%% +%%% Additionally, this device implements the standard computation device keys, +%%% allowing it to be used as an element of an execution stack pipeline, etc. +-module(dev_patch). +-export([all/3, patches/3]). +%%% `execution-device` standard hooks: +-export([init/3, compute/3, normalize/3, snapshot/3]). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("include/hb.hrl"). + +%% @doc Necessary hooks for compliance with the `execution-device' standard. +init(Msg1, _Msg2, _Opts) -> {ok, Msg1}. +normalize(Msg1, _Msg2, _Opts) -> {ok, Msg1}. +snapshot(Msg1, _Msg2, _Opts) -> {ok, Msg1}. +compute(Msg1, Msg2, Opts) -> patches(Msg1, Msg2, Opts). + +%% @doc Get the value found at the `patch-from' key of the message, or the +%% `from' key if the former is not present. Remove it from the message and set +%% the new source to the value found. +all(Msg1, Msg2, Opts) -> + move(all, Msg1, Msg2, Opts). + +%% @doc Find relevant `PATCH' messages in the given source key of the execution +%% and request messages, and apply them to the given destination key of the +%% request. +patches(Msg1, Msg2, Opts) -> + move(patches, Msg1, Msg2, Opts). + +%% @doc Unified executor for the `all' and `patches' modes. +move(Mode, Msg1, Msg2, Opts) -> + maybe + % Find the input paths. + % For `from' we parse the path to see if it is relative to the request + % or the base message. This is not needed for `to' because it is + % always relative to the request. + RawPatchFrom = + hb_ao:get_first( + [ + {Msg2, <<"patch-from">>}, + {Msg1, <<"patch-from">>}, + {Msg2, <<"from">>}, + {Msg1, <<"from">>} + ], + <<"/">>, + Opts + ), + {FromMsg, PatchFromParts} = + case hb_path:term_to_path_parts(RawPatchFrom) of + [BinKey|RestKeys] -> + case binary:split(BinKey, <<":">>) of + [<<"base">>, RestKey] -> + {Msg1, [RestKey|RestKeys]}; + [<<"req">>, RestKey] -> + {Msg2, [RestKey|RestKeys]}; + _ -> + {Msg1, RawPatchFrom} + end; + _ -> + {Msg1, RawPatchFrom} + end, + ?event({patch_from_parts, {explicit, PatchFromParts}}), + PatchFrom = + case hb_path:to_binary(PatchFromParts) of + <<"">> -> <<"/">>; + Path -> Path + end, + ?event({patch_from, PatchFrom}), + PatchTo = + hb_ao:get_first( + [ + {Msg2, <<"patch-to">>}, + {Msg1, <<"patch-to">>}, + {Msg2, <<"to">>}, + {Msg1, <<"to">>} + ], + <<"/">>, + Opts + ), + ?event({patch_from, PatchFrom}), + ?event({patch_to, PatchTo}), + % Get the source of the patches from the message. Makes the `maybe' + % statement return `{error, not_found}' if the source is not found. + {ok, Source} ?= hb_ao:resolve(FromMsg, PatchFrom, Opts), + % Find all messages with the PATCH request. + {ToWrite, NewSourceValue} = + case Mode of + patches -> + maps:fold( + fun(Key, Msg, {PatchAcc, NewSourceAcc}) -> + Method = hb_ao:get(<<"method">>, Msg, Opts) + == <<"PATCH">>, + Device = hb_ao:get(<<"device">>, Msg, Opts) + == <<"patch@1.0">>, + if Method orelse Device -> + {PatchAcc#{Key => Msg}, NewSourceAcc}; + true -> + {PatchAcc, NewSourceAcc#{ Key => Msg }} + end + end, + {#{}, #{}}, + Source + ); + all -> + {Source, unset} + end, + ?event({source_data, ToWrite}), + ?event({new_data_for_source_path, NewSourceValue}), + % Remove the source from the message and set the new source. + FromMsgWithoutSource = + hb_ao:set( + FromMsg, + PatchFrom, + <<"patch-error">>, + Opts + ), + FromMsgWithNewSource = + hb_ao:set( + FromMsgWithoutSource, + #{ PatchFrom => NewSourceValue }, + Opts + ), + % If the `mode` is `patches`, we need to remove the `method` key from + % them, if present. + ToWriteMod = + case Mode of + all -> ToWrite; + patches -> + maps:fold( + fun(_, Patch, MsgN) -> + ?event({patching, {patch, Patch}, {before, MsgN}}), + Res = + hb_ao:set( + MsgN, + maps:without([<<"method">>], Patch), + Opts + ), + ?event({patched, {'after', Res}}), + Res + end, + #{}, + ToWrite + ) + end, + % Find the target to apply the patches to, and apply them. + PatchedResult = + hb_ao:set( + FromMsgWithNewSource, + PatchTo, + ToWriteMod, + Opts + ), + % Return the patched message and the source, less the patches. + ?event({patch_result, PatchedResult}), + {ok, PatchedResult} + end. + +%%% Tests + +uninitialized_patch_test() -> + InitState = #{ + <<"device">> => <<"patch@1.0">>, + <<"results">> => #{ + <<"outbox">> => #{ + <<"1">> => #{ + <<"method">> => <<"PATCH">>, + <<"prices">> => #{ + <<"apple">> => 100, + <<"banana">> => 200 + } + }, + <<"2">> => #{ + <<"method">> => <<"GET">>, + <<"prices">> => #{ + <<"apple">> => 1000 + } + } + } + }, + <<"other-message">> => <<"other-value">>, + <<"patch-to">> => <<"/">>, + <<"patch-from">> => <<"/results/outbox">> + }, + {ok, ResolvedState} = + hb_ao:resolve( + InitState, + <<"compute">>, + #{} + ), + ?event({resolved_state, ResolvedState}), + ?assertEqual( + 100, + hb_ao:get(<<"prices/apple">>, ResolvedState, #{}) + ), + ?assertMatch( + not_found, + hb_ao:get(<<"results/outbox/1">>, ResolvedState, #{}) + ). + +patch_to_submessage_test() -> + InitState = #{ + <<"device">> => <<"patch@1.0">>, + <<"results">> => #{ + <<"outbox">> => #{ + <<"1">> => + hb_message:commit(#{ + <<"method">> => <<"PATCH">>, + <<"prices">> => #{ + <<"apple">> => 100, + <<"banana">> => 200 + } + }, + hb:wallet() + ) + } + }, + <<"state">> => #{ + <<"prices">> => #{ + <<"apple">> => 1000 + } + }, + <<"other-message">> => <<"other-value">>, + <<"patch-to">> => <<"/state">>, + <<"patch-from">> => <<"/results/outbox">> + }, + {ok, ResolvedState} = + hb_ao:resolve( + InitState, + <<"compute">>, + #{} + ), + ?event({resolved_state, ResolvedState}), + ?assertEqual( + 100, + hb_ao:get(<<"state/prices/apple">>, ResolvedState, #{}) + ). + +all_mode_test() -> + InitState = #{ + <<"device">> => <<"patch@1.0">>, + <<"input">> => #{ + <<"zones">> => #{ + <<"1">> => #{ + <<"method">> => <<"PATCH">>, + <<"prices">> => #{ + <<"apple">> => 100, + <<"banana">> => 200 + } + }, + <<"2">> => #{ + <<"method">> => <<"GET">>, + <<"prices">> => #{ + <<"orange">> => 300 + } + } + } + }, + <<"state">> => #{ + <<"prices">> => #{ + <<"apple">> => 1000 + } + } + }, + {ok, ResolvedState} = + hb_ao:resolve( + InitState, + #{ + <<"path">> => <<"all">>, + <<"patch-to">> => <<"/state">>, + <<"patch-from">> => <<"/input/zones">> + }, + #{} + ), + ?event({resolved_state, ResolvedState}), + ?assertEqual( + 100, + hb_ao:get(<<"state/1/prices/apple">>, ResolvedState, #{}) + ), + ?assertEqual( + 300, + hb_ao:get(<<"state/2/prices/orange">>, ResolvedState, #{}) + ), + ?assertEqual( + not_found, + hb_ao:get(<<"input/zones">>, ResolvedState, #{}) + ). + +req_prefix_test() -> + BaseMsg = #{ + <<"device">> => <<"patch@1.0">>, + <<"state">> => #{ + <<"prices">> => #{ + <<"apple">> => 1000 + } + } + }, + ReqMsg = #{ + <<"path">> => <<"all">>, + <<"patch-from">> => <<"req:/results/outbox/1">>, + <<"patch-to">> => <<"/state">>, + <<"results">> => #{ + <<"outbox">> => #{ + <<"1">> => #{ + <<"method">> => <<"PATCH">>, + <<"prices">> => #{ + <<"apple">> => 100, + <<"banana">> => 200 + } + } + } + } + }, + {ok, ResolvedState} = hb_ao:resolve(BaseMsg, ReqMsg, #{}), + ?event({resolved_state, ResolvedState}), + ?assertEqual( + 100, + hb_ao:get(<<"state/prices/apple">>, ResolvedState, #{}) + ), + ?assertEqual( + 200, + hb_ao:get(<<"state/prices/banana">>, ResolvedState, #{}) + ), + ?assertEqual( + not_found, + hb_ao:get(<<"results/outbox/1">>, ResolvedState, #{}) + ). \ No newline at end of file diff --git a/src/dev_poda.erl b/src/dev_poda.erl index d6c6e5620..ab9ede9fb 100644 --- a/src/dev_poda.erl +++ b/src/dev_poda.erl @@ -1,10 +1,4 @@ --module(dev_poda). --export([init/2, execute/3]). --export([is_user_signed/1]). --export([push/2]). --include("include/hb.hrl"). --hb_debug(print). - +%%% @doc A simple exemplar decentralized proof of authority consensus algorithm %%% A simple exemplar decentralized proof of authority consensus algorithm %%% for AO processes. This device is split into two flows, spanning three %%% actions. @@ -12,8 +6,14 @@ %%% Execution flow: %%% 1. Initialization. %%% 2. Validation of incoming messages before execution. -%%% Attestation flow: -%%% 1. Adding attestations to results, either on a CU or MU. +%%% Commitment flow: +%%% 1. Adding commitments to results, either on a CU or MU. +-module(dev_poda). +-export([init/2, execute/3]). +-export([is_user_signed/1]). +-export([push/3]). +-include("include/hb.hrl"). +-hb_debug(print). %%% Execution flow: Initialization. @@ -23,11 +23,11 @@ init(S, Params) -> extract_opts(Params) -> Authorities = lists:filtermap( - fun({<<"Authority">>, Addr}) -> {true, Addr}; + fun({<<"authority">>, Addr}) -> {true, Addr}; (_) -> false end, Params ), - {_, RawQuorum} = lists:keyfind(<<"Quorum">>, 1, Params), + {_, RawQuorum} = lists:keyfind(<<"quorum">>, 1, Params), Quorum = binary_to_integer(RawQuorum), ?event({poda_authorities, Authorities}), #{ @@ -37,46 +37,47 @@ extract_opts(Params) -> %%% Execution flow: Pre-execution validation. -execute(Outer = #tx { data = #{ <<"Message">> := Msg } }, S = #{ pass := 1 }, Opts) -> +execute(Outer = #tx { data = #{ <<"body">> := Msg } }, S = #{ <<"pass">> := 1 }, Opts) -> case is_user_signed(Msg) of true -> {ok, S}; false -> - % For now, the message itself will be at `/Message/Message`. case validate(Msg, Opts) of true -> ?event({poda_validated, ok}), % Add the validations to the VFS. - Atts = - maps:to_list( + Comms = + hb_maps:to_list( case Msg of - #tx { data = #{ <<"Attestations">> := #tx { data = X } }} -> X; - #tx { data = #{ <<"Attestations">> := X }} -> X; - #{ <<"Attestations">> := X } -> X - end + #tx { data = #{ <<"commitments">> := #tx { data = X } }} -> X; + #tx { data = #{ <<"commitments">> := X }} -> X; + #{ <<"commitments">> := X } -> X + end, + Opts ), VFS1 = lists:foldl( - fun({_, Attestation}, Acc) -> - Id = ar_bundles:signer(Attestation), + fun({_, Commitment}, Acc) -> + Id = ar_bundles:signer(Commitment), Encoded = hb_util:encode(Id), - maps:put( - <<"/Attestations/", Encoded/binary>>, - Attestation#tx.data, - Acc + hb_maps:put( + <<"/commitments/", Encoded/binary>>, + Commitment#tx.data, + Acc, + Opts ) end, - maps:get(vfs, S, #{}), - Atts + hb_maps:get(vfs, S, #{}, Opts), + Comms ), % Update the arg prefix to include the unwrapped message. - {ok, S#{ vfs => VFS1, arg_prefix => + {ok, S#{ <<"vfs">> => VFS1, <<"arg_prefix">> => [ - % Traverse two layers of `/Message/Message` to get - % the actual message, then replace `/Message` with it. + % Traverse two layers of `/Message/Message' to get + % the actual message, then replace `/Message' with it. Outer#tx{ data = (Outer#tx.data)#{ - <<"Message">> => maps:get(<<"Message">>, Msg#tx.data) + <<"body">> => hb_maps:get(<<"body">>, Msg#tx.data, Opts) } } ] @@ -84,7 +85,7 @@ execute(Outer = #tx { data = #{ <<"Message">> := Msg } }, S = #{ pass := 1 }, Op {false, Reason} -> return_error(S, Reason) end end; -execute(_M, S = #{ pass := 3, results := _Results }, _Opts) -> +execute(_M, S = #{ <<"pass">> := 3, <<"results">> := _Results }, _Opts) -> {ok, S}; execute(_M, S, _Opts) -> {ok, S}. @@ -94,31 +95,31 @@ validate(Msg, Opts) -> validate_stage(1, Msg, Opts) when is_record(Msg, tx) -> validate_stage(1, Msg#tx.data, Opts); -validate_stage(1, #{ <<"Attestations">> := Attestations, <<"Message">> := Content }, Opts) -> - validate_stage(2, Attestations, Content, Opts); +validate_stage(1, #{ <<"commitments">> := Commitments, <<"body">> := Content }, Opts) -> + validate_stage(2, Commitments, Content, Opts); validate_stage(1, _M, _Opts) -> {false, <<"Required PoDA messages missing">>}. -validate_stage(2, #tx { data = Attestations }, Content, Opts) -> - validate_stage(2, Attestations, Content, Opts); -validate_stage(2, Attestations, Content, Opts) -> - % Ensure that all attestations are valid and signed by a +validate_stage(2, #tx { data = Commitments }, Content, Opts) -> + validate_stage(2, Commitments, Content, Opts); +validate_stage(2, Commitments, Content, Opts) -> + % Ensure that all commitments are valid and signed by a % trusted authority. case lists:all( - fun({_, Att}) -> - ar_bundles:verify_item(Att) + fun({_, Comm}) -> + ar_bundles:verify_item(Comm) end, - maps:to_list(Attestations) + hb_maps:to_list(Commitments, Opts) ) of - true -> validate_stage(3, Content, Attestations, Opts); - false -> {false, <<"Invalid attestations">>} + true -> validate_stage(3, Content, Commitments, Opts); + false -> {false, <<"Invalid commitments">>} end; -validate_stage(3, Content, Attestations, Opts = #{ quorum := Quorum }) -> +validate_stage(3, Content, Commitments, Opts = #{ <<"quorum">> := Quorum }) -> Validations = lists:filter( - fun({_, Att}) -> validate_attestation(Content, Att, Opts) end, - maps:to_list(Attestations) + fun({_, Comm}) -> validate_commitment(Content, Comm, Opts) end, + hb_maps:to_list(Commitments, Opts) ), ?event({poda_validations, length(Validations)}), case length(Validations) >= Quorum of @@ -128,21 +129,20 @@ validate_stage(3, Content, Attestations, Opts = #{ quorum := Quorum }) -> false -> {false, <<"Not enough validations">>} end. -validate_attestation(Msg, Att, Opts) -> +validate_commitment(Msg, Comm, Opts) -> MsgID = hb_util:encode(ar_bundles:id(Msg, unsigned)), - AttSigner = hb_util:encode(ar_bundles:signer(Att)), - ?event({poda_attestation, {signer, AttSigner, maps:get(authorities, Opts)}, {msg_id, MsgID}}), - ValidSigner = lists:member(AttSigner, maps:get(authorities, Opts)), - ?no_prod(use_real_signature_verification), - ValidSignature = ar_bundles:verify_item(Att), - RelevantMsg = ar_bundles:id(Att, unsigned) == MsgID orelse - (lists:keyfind(<<"Attestation-For">>, 1, Att#tx.tags) - == {<<"Attestation-For">>, MsgID}) orelse - ar_bundles:member(ar_bundles:id(Msg, unsigned), Att), + AttSigner = hb_util:encode(ar_bundles:signer(Comm)), + ?event({poda_commitment, {signer, AttSigner, hb_maps:get(authorities, Opts, undefined, Opts)}, {msg_id, MsgID}}), + ValidSigner = lists:member(AttSigner, hb_maps:get(authorities, Opts, undefined, Opts)), + ValidSignature = ar_bundles:verify_item(Comm), + RelevantMsg = ar_bundles:id(Comm, unsigned) == MsgID orelse + (lists:keyfind(<<"commitment-for">>, 1, Comm#tx.tags) + == {<<"commitment-for">>, MsgID}) orelse + ar_bundles:member(ar_bundles:id(Msg, unsigned), Comm), case ValidSigner and ValidSignature and RelevantMsg of false -> - ?event({poda_attestation_invalid, - {attestation, ar_bundles:id(Att, signed)}, + ?event({poda_commitment_invalid, + {commitment, ar_bundles:id(Comm, signed)}, {signer, AttSigner}, {valid_signer, ValidSigner}, {valid_signature, ValidSignature}, @@ -154,84 +154,87 @@ validate_attestation(Msg, Att, Opts) -> %%% Execution flow: Error handling. %%% Skip execution of this message, instead returning an error message. -return_error(S = #{ wallet := Wallet }, Reason) -> +return_error(S = #{ <<"wallet">> := Wallet }, Reason) -> ?event({poda_return_error, Reason}), ?debug_wait(10000), {skip, S#{ results => #{ - <<"/Outbox">> => + <<"/outbox">> => ar_bundles:sign_item( #tx{ data = Reason, - tags = [{<<"Error">>, <<"PoDA">>}] + tags = [{<<"error">>, <<"PoDA">>}] }, Wallet ) } }}. -is_user_signed(#tx { data = #{ <<"Message">> := Msg } }) -> - ?no_prod(use_real_attestation_detection), - lists:keyfind(<<"From-Process">>, 1, Msg#tx.tags) == false; +%%% @doc Determines if a user committed +is_user_signed(#tx { data = #{ <<"body">> := Msg } }) -> + ?no_prod(use_real_commitment_detection), + lists:keyfind(<<"from-process">>, 1, Msg#tx.tags) == false; is_user_signed(_) -> true. -%%% Attestation flow: Adding attestations to results. +%%% Commitment flow: Adding commitments to results. -%% @doc Hook used by the MU pathway (currently) to add attestations to an +%% @doc Hook used by the MU pathway (currently) to add commitments to an %% outbound message if the computation requests it. -push(_Item, S = #{ results := ResultsMsg }) -> - NewRes = attest_to_results(ResultsMsg, S), - {ok, S#{ results => NewRes }}. +push(_Item, S = #{ <<"results">> := ResultsMsg }, Opts) -> + NewRes = commit_to_results(ResultsMsg, S, Opts), + {ok, S#{ <<"results">> => NewRes }}. -attest_to_results(Msg, S) -> +commit_to_results(Msg, S, Opts) -> case is_map(Msg#tx.data) of true -> - % Add attestations to the outbox and spawn items. - maps:map( + % Add commitments to the outbox and spawn items. + hb_maps:map( fun(Key, IndexMsg) -> - ?no_prod("Currently we only attest to the outbox and spawn items." + ?no_prod("Currently we only commit to the outbox and spawn items." "Make it general?"), - case lists:member(Key, [<<"/Outbox">>, <<"/Spawn">>]) of + case lists:member(Key, [<<"/outbox">>, <<"/spawn">>]) of true -> - ?event({poda_starting_to_attest_to_result, Key}), - maps:map( - fun(_, DeepMsg) -> add_attestations(DeepMsg, S) end, - IndexMsg#tx.data + ?event({poda_starting_to_commit_to_result, Key}), + hb_maps:map( + fun(_, DeepMsg) -> add_commitments(DeepMsg, S, Opts) end, + IndexMsg#tx.data, + Opts ); false -> IndexMsg end end, - Msg#tx.data + Msg#tx.data, + Opts ); false -> Msg end. -add_attestations(NewMsg, S = #{ assignment := Assignment, store := _Store, logger := _Logger, wallet := Wallet }) -> +add_commitments(NewMsg, S = #{ <<"assignment">> := Assignment, <<"store">> := _Store, <<"logger">> := _Logger, <<"wallet">> := Wallet }, Opts) -> Process = find_process(NewMsg, S), - case is_record(Process, tx) andalso lists:member({<<"Device">>, <<"PODA">>}, Process#tx.tags) of + case is_record(Process, tx) andalso lists:member({<<"device">>, <<"PODA">>}, Process#tx.tags) of true -> - #{ authorities := InitAuthorities, quorum := Quorum } = + #{ <<"authorities">> := InitAuthorities, <<"quorum">> := Quorum } = extract_opts(Process#tx.tags), ?event({poda_push, InitAuthorities, Quorum}), % Aggregate validations from other nodes. - % TODO: Filter out attestations from the current node. + % TODO: Filter out commitments from the current node. MsgID = hb_util:encode(ar_bundles:id(NewMsg, unsigned)), - ?event({poda_add_attestations_from, InitAuthorities, {self,hb:address()}}), - Attestations = pfiltermap( + ?event({poda_add_commitments_from, InitAuthorities, {self,hb:address()}}), + Commitments = pfiltermap( fun(Address) -> - case hb_router:find(compute, ar_bundles:id(Process, unsigned), Address) of + case hb_router:find(compute, ar_bundles:id(Process, unsigned), Address, Opts) of {ok, ComputeNode} -> - ?event({poda_asking_peer_for_attestation, ComputeNode, <<"Attest-To">>, MsgID}), - Res = hb_client:compute( + ?event({poda_asking_peer_for_commitment, ComputeNode, <<"commit-to">>, MsgID}), + Res = hb_client:resolve( ComputeNode, ar_bundles:id(Process, signed), ar_bundles:id(Assignment, signed), - #{ <<"Attest-To">> => MsgID } + #{ <<"commit-to">> => MsgID } ), case Res of - {ok, Att} -> - ?event({poda_got_attestation_from_peer, ComputeNode}), - {true, Att}; + {ok, Comm} -> + ?event({poda_got_commitment_from_peer, ComputeNode}), + {true, Comm}; _ -> false end; _ -> false @@ -239,44 +242,44 @@ add_attestations(NewMsg, S = #{ assignment := Assignment, store := _Store, logge end, ?event(InitAuthorities -- [hb:address()]) ), - LocalAttestation = ar_bundles:sign_item( - #tx{ tags = [{<<"Attestation-For">>, MsgID}], data = <<>> }, + LocalCommitment = ar_bundles:sign_item( + #tx{ tags = [{<<"commitment-for">>, MsgID}], data = <<>> }, Wallet ), - CompleteAttestations = + CompleteCommitments = ar_bundles:sign_item( ar_bundles:normalize( #tx { data = - maps:from_list( + hb_maps:from_list( lists:zipwith( - fun(Index, Att) -> {integer_to_binary(Index), Att} end, - lists:seq(1, length([LocalAttestation | Attestations])), - AttList = [LocalAttestation | Attestations] + fun(Index, Comm) -> {integer_to_binary(Index), Comm} end, + lists:seq(1, length([LocalCommitment | Commitments])), + AttList = [LocalCommitment | Commitments] ) ) } ), Wallet ), - AttestationBundle = ar_bundles:sign_item( + CommitmentBundle = ar_bundles:sign_item( ar_bundles:normalize( #tx{ target = NewMsg#tx.target, data = #{ - <<"Attestations">> => CompleteAttestations, - <<"Message">> => NewMsg + <<"commitments">> => CompleteCommitments, + <<"body">> => NewMsg } } ), Wallet ), - ?event({poda_attestation_bundle_signed, {attestations, length(AttList)}}), - AttestationBundle; + ?event({poda_commitment_bundle_signed, {commitments, length(AttList)}}), + CommitmentBundle; false -> NewMsg end. -%% @doc Helper function for parallel execution of attestation +%% @doc Helper function for parallel execution of commitment %% gathering. pfiltermap(Pred, List) -> Parent = self(), @@ -309,16 +312,16 @@ pfiltermap(Pred, List) -> ]. %% @doc Find the process that this message is targeting, in order to -%% determine which attestations to add. -find_process(Item, #{ logger := _Logger, store := Store }) -> +%% determine which commitments to add. +find_process(Item, #{ <<"logger">> := _Logger, <<"store">> := Store }) -> case Item#tx.target of X when X =/= <<>> -> ?event({poda_find_process, hb_util:id(Item#tx.target)}), - {ok, Proc} = hb_cache:read_message(Store, hb_util:id(Item#tx.target)), + {ok, Proc} = hb_cache:read(Store, hb_util:id(Item#tx.target)), Proc; _ -> - case lists:keyfind(<<"Type">>, 1, Item#tx.tags) of - {<<"Type">>, <<"Process">>} -> Item; + case lists:keyfind(<<"type">>, 1, Item#tx.tags) of + {<<"type">>, <<"process">>} -> Item; _ -> process_not_specified end end. \ No newline at end of file diff --git a/src/dev_process.erl b/src/dev_process.erl index f364185e1..abeb76bd8 100644 --- a/src/dev_process.erl +++ b/src/dev_process.erl @@ -1,5 +1,5 @@ %%% @doc This module contains the device implementation of AO processes -%%% in Converge. The core functionality of the module is in 'routing' requests +%%% in AO-Core. The core functionality of the module is in 'routing' requests %%% for different functionality (scheduling, computing, and pushing messages) %%% to the appropriate device. This is achieved by swapping out the device %%% of the process message with the necessary component in order to run the @@ -14,7 +14,7 @@ %%% `dev_process_cache' for details. %%% %%% The external API of the device is as follows: -%%% ``` +%%% 
 %%% GET /ID/Schedule:                Returns the messages in the schedule
 %%% POST /ID/Schedule:               Adds a message to the schedule
 %%% 
@@ -22,10 +22,10 @@
 %%%                                  applying a message
 %%% GET /ID/Now:                     Returns the `/Results' key of the latest 
 %%%                                  computed message
-%%% '''
+%%%
%%% %%% An example process definition will look like this: -%%% ``` +%%% 
 %%%     Device: Process/1.0
 %%%     Scheduler-Device: Scheduler/1.0
 %%%     Execution-Device: Stack/1.0
@@ -38,7 +38,7 @@
 %%%         Authority: B
 %%%         Authority: C
 %%%         Quorum: 2
-%%% '''
+%%%
%%% %%% Runtime options: %%% Cache-Frequency: The number of assignments that will be computed @@ -47,299 +47,622 @@ %%% assignments, in addition to `/Results'. -module(dev_process). %%% Public API --export([info/1, compute/3, schedule/3, slot/3, now/3, push/3, snapshot/3]). +-export([info/1, as/3, compute/3, schedule/3, slot/3, now/3, push/3, snapshot/3]). -export([ensure_process_key/2]). +%%% Public utilities +-export([as_process/2, process_id/3]). %%% Test helpers --export([test_aos_process/0, dev_test_process/0, test_wasm_process/1]). +-export([test_aos_process/0, test_aos_process/1, dev_test_process/0, test_wasm_process/1]). +-export([schedule_aos_call/2, schedule_aos_call/3, init/0]). %%% Tests -export([do_test_restore/0]). -include_lib("eunit/include/eunit.hrl"). -include_lib("include/hb.hrl"). %% The frequency at which the process state should be cached. Can be overridden -%% with the `cache_frequency` option. --define(DEFAULT_CACHE_FREQ, 10). +%% with the `process_snapshot_slots' or `process_snapshot_time' options. +-if(TEST == true). +-define(DEFAULT_SNAPSHOT_SLOTS, 1). +-define(DEFAULT_SNAPSHOT_TIME, undefined). +-else. +-define(DEFAULT_SNAPSHOT_SLOTS, undefined). +-define(DEFAULT_SNAPSHOT_TIME, 60). +-endif. %% @doc When the info key is called, we should return the process exports. info(_Msg1) -> #{ worker => fun dev_process_worker:server/3, - grouper => fun dev_process_worker:group/3 + grouper => fun dev_process_worker:group/3, + await => fun dev_process_worker:await/5, + excludes => [ + <<"test">>, + <<"init">>, + <<"ping_ping_script">>, + <<"schedule_aos_call">>, + <<"test_aos_process">>, + <<"dev_test_process">>, + <<"test_wasm_process">> + ] }. +%% @doc Return the process state with the device swapped out for the device +%% of the given key. +as(RawMsg1, Msg2, Opts) -> + {ok, Msg1} = ensure_loaded(RawMsg1, Msg2, Opts), + Key = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, Msg2}, <<"as">>}, + {{as, <<"message@1.0">>, Msg2}, <<"as-device">>} + ], + <<"execution">>, + Opts + ), + {ok, + hb_util:deep_merge( + ensure_process_key(Msg1, Opts), + #{ + <<"device">> => + hb_maps:get( + << Key/binary, "-device">>, + Msg1, + default_device(Msg1, Key, Opts), + Opts + ), + % Configure input prefix for proper message routing within the + % device + <<"input-prefix">> => + case hb_maps:get(<<"input-prefix">>, Msg1, not_found, Opts) of + not_found -> <<"process">>; + Prefix -> Prefix + end, + % Configure output prefixes for result organization + <<"output-prefixes">> => + hb_maps:get( + <>, + Msg1, + undefined, % Undefined in set will be ignored. + Opts + ) + }, + Opts + ) + }. + +%% @doc Returns the default device for a given piece of functionality. Expects +%% the `process/variant' key to be set in the message. The `execution-device' +%% _must_ be set in all processes aside those marked with `ao.TN.1' variant. +%% This is in order to ensure that post-mainnet processes do not default to +%% using infrastructure that should not be present on nodes in the future. +default_device(Msg1, Key, Opts) -> + NormKey = hb_ao:normalize_key(Key), + case {NormKey, hb_util:deep_get(<<"process/variant">>, Msg1, Opts)} of + {<<"execution">>, <<"ao.TN.1">>} -> <<"genesis-wasm@1.0">>; + _ -> default_device_index(NormKey) + end. +default_device_index(<<"scheduler">>) -> <<"scheduler@1.0">>; +default_device_index(<<"execution">>) -> <<"genesis-wasm@1.0">>; +default_device_index(<<"push">>) -> <<"push@1.0">>. %% @doc Wraps functions in the Scheduler device. schedule(Msg1, Msg2, Opts) -> - run_as(<<"Scheduler">>, Msg1, Msg2, Opts). + run_as(<<"scheduler">>, Msg1, Msg2, Opts). slot(Msg1, Msg2, Opts) -> - ?event({slot_called, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), - run_as(<<"Scheduler">>, Msg1, Msg2, Opts). + ?event({slot_called, {msg1, Msg1}, {msg2, Msg2}}), + run_as(<<"scheduler">>, Msg1, Msg2, Opts). next(Msg1, _Msg2, Opts) -> - run_as(<<"Scheduler">>, Msg1, next, Opts). + run_as(<<"scheduler">>, Msg1, next, Opts). snapshot(RawMsg1, _Msg2, Opts) -> Msg1 = ensure_process_key(RawMsg1, Opts), {ok, SnapshotMsg} = run_as( - <<"Execution">>, + <<"execution">>, Msg1, - #{ path => <<"Snapshot">>, <<"Mode">> => <<"Map">> }, - Opts#{ cache_control => [] } + #{ <<"path">> => <<"snapshot">>, <<"mode">> => <<"Map">> }, + Opts#{ + cache_control => [<<"no-cache">>, <<"no-store">>], + hashpath => ignore + } ), - ProcID = hb_converge:get(<<"id">>, Msg1, Opts), - Slot = hb_converge:get(<<"Current-Slot">>, Msg1, Opts), + ProcID = hb_message:id(Msg1, all, Opts), + Slot = hb_ao:get(<<"at-slot">>, {as, <<"message@1.0">>, Msg1}, Opts), {ok, hb_private:set( - hb_converge:set( - SnapshotMsg, - #{ <<"Cache-Control">> => [<<"store">>] }, - Opts - ), - #{ <<"priv/Additional-Hashpaths">> => + SnapshotMsg#{ <<"cache-control">> => [<<"store">>] }, + #{ <<"priv/additional-hashpaths">> => [ - hb_path:to_binary([ProcID, <<"Snapshot">>, Slot]) + hb_path:to_binary([ProcID, <<"snapshot">>, Slot]) ] }, Opts ) }. +%% @doc Returns the process ID of the current process. +process_id(Msg1, Msg2, Opts) -> + case hb_ao:get(<<"process">>, Msg1, Opts#{ hashpath => ignore }) of + not_found -> + process_id(ensure_process_key(Msg1, Opts), Msg2, Opts); + Process -> + hb_message:id( + Process, + hb_util:atom(maps:get(<<"commitments">>, Msg2, <<"all">>)), + Opts + ) + end. %% @doc Before computation begins, a boot phase is required. This phase %% allows devices on the execution stack to initialize themselves. We set the %% `Initialized' key to `True' to indicate that the process has been %% initialized. -init(Msg1, _Msg2, Opts) -> - ?event({init_called, {msg1, Msg1}, {opts, Opts}}), +init(Msg1, Msg2, Opts) -> + ?event({init_called, {msg1, Msg1}, {msg2, Msg2}}), {ok, Initialized} = - run_as(<<"Execution">>, Msg1, #{ path => init }, Opts), + run_as(<<"execution">>, Msg1, #{ <<"path">> => init }, Opts), { ok, - hb_converge:set( + hb_ao:set( Initialized, #{ - <<"Initialized">> => <<"True">>, - <<"Current-Slot">> => -1 + <<"initialized">> => <<"true">>, + <<"at-slot">> => -1 }, Opts ) }. -%% @doc Compute the result of an assignment applied to the process state, if it -%% is the next message. +%% @doc Compute the result of an assignment applied to the process state. +%% This function serves as the main entry point for compute operations and routes +%% between two distinct execution paths: +%% +%% - GET method: Normal compute execution that applies messages to process state +%% and advances the state permanently. Used for regular process execution. +%% +%% - POST method: Dryrun compute execution that simulates message processing +%% without permanently modifying process state. Used for testing message +%% handlers and previewing results. The POST method is the key entry point +%% for the dryrun functionality that allows external clients to test +%% message processing without side effects. compute(Msg1, Msg2, Opts) -> - % If we do not have a live state, restore or initialize one. - {ok, Loaded} = - ensure_loaded( - ensure_process_key(Msg1, Opts), - Msg2, - Opts - ), - {ok, Normalized} = - run_as( - <<"Execution">>, - Loaded, - normalize, + ProcBase = ensure_process_key(Msg1, Opts), + ProcID = process_id(ProcBase, #{}, Opts), + TargetSlot = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, Msg2}, <<"compute">>}, + {{as, <<"message@1.0">>, Msg2}, <<"slot">>} + ], Opts ), - ProcID = hb_converge:get(<<"Process/id">>, Loaded, Opts), - ?event({compute_called, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), - do_compute( - ProcID, - Normalized, - Msg2, - hb_converge:get(<<"Slot">>, Msg2, Opts), - Opts - ). + case TargetSlot of + not_found -> + % The slot is not set, so we need to serve the latest known state. + % We do this by setting the `process_now_from_cache' option to `true'. + now(Msg1, Msg2, Opts#{ process_now_from_cache => true }); + RawSlot -> + Slot = hb_util:int(RawSlot), + case dev_process_cache:read(ProcID, Slot, Opts) of + {ok, Result} -> + % The result is already cached, so we can return it. + ?event( + {compute_result_cached, + {proc_id, ProcID}, + {slot, Slot}, + {result, Result} + } + ), + {ok, without_snapshot(Result, Opts)}; + not_found -> + {ok, Loaded} = ensure_loaded(ProcBase, Msg2, Opts), + ?event(compute, + {computing, {process_id, ProcID}, + {to_slot, Slot}}, + Opts + ), + compute_to_slot( + ProcID, + Loaded, + Msg2, + Slot, + Opts + ) + end + end. %% @doc Continually get and apply the next assignment from the scheduler until %% we reach the target slot that the user has requested. -do_compute(ProcID, Msg1, Msg2, TargetSlot, Opts) -> - ?event({do_compute_called, {target_slot, TargetSlot}, {msg1, Msg1}}), - case hb_converge:get(<<"Current-Slot">>, Msg1, Opts) of +compute_to_slot(ProcID, Msg1, Msg2, TargetSlot, Opts) -> + CurrentSlot = hb_ao:get(<<"at-slot">>, Msg1, Opts#{ hashpath => ignore }), + ?event(compute_short, + {starting_compute, + {proc_id, ProcID}, + {current, CurrentSlot}, + {target, TargetSlot} + } + ), + case CurrentSlot of CurrentSlot when CurrentSlot > TargetSlot -> - throw({error, {already_calculated_slot, TargetSlot}}); - CurrentSlot when CurrentSlot == TargetSlot -> - % We reached the target height so we return. - ?event({reached_target_slot_returning_state, TargetSlot}), - {ok, as_process(Msg1, Opts)}; - CurrentSlot -> - % Get the next input from the scheduler device. - {ok, #{ <<"Message">> := ToProcess, <<"State">> := State }} = - next(Msg1, Msg2, Opts), - ?event(process_compute, - { - executing, - {msg1, Msg1}, - {msg2, ToProcess} + % The cache should already have the result, so we should never end up + % here. Depending on the type of process, 'rewinding' may require + % re-computing from a significantly earlier checkpoint, so for now + % we throw an error. + ?event(compute, {error_already_calculated_slot, {target, TargetSlot}, {current, CurrentSlot}}), + throw( + {error, + {already_calculated_slot, + {target, TargetSlot}, + {current, CurrentSlot} + } } + ); + CurrentSlot when CurrentSlot == TargetSlot -> + % We reached the target height so we force a snapshot and return. + ?event(compute, {reached_target_slot_returning_state, TargetSlot}), + store_result( + true, + ProcID, + TargetSlot, + Msg1, + Msg2, + Opts ), - {ok, Msg3} = - run_as( - <<"Execution">>, - State, - ToProcess, - Opts - ), - % Cache the `Memory` key every `Cache-Frequency` slots. - Freq = - hb_opts:get( - process_cache_frequency, - ?DEFAULT_CACHE_FREQ, - Opts - ), - case CurrentSlot rem Freq of - 0 -> - case snapshot(Msg3, Msg2, Opts) of - {ok, Snapshot} -> - ?event(snapshot, - {got_snapshot, - {storing_as_slot, CurrentSlot} - } - ), - dev_process_cache:write( + {ok, without_snapshot(as_process(Msg1, Opts), Opts)}; + CurrentSlot -> + % Compute the next state transition. + NextSlot = CurrentSlot + 1, + % Get the next input message from the scheduler device. + case next(Msg1, Msg2, Opts) of + {error, Res} -> + % If the scheduler device cannot provide a next message, + % we return its error details, along with the current slot. + ?event(compute, + {error_getting_schedule, + {error, Res}, + {phase, <<"get-schedule">>}, + {attempted_slot, NextSlot} + } + ), + {error, Res#{ + <<"phase">> => <<"get-schedule">>, + <<"attempted-slot">> => NextSlot + }}; + {ok, #{ <<"body">> := SlotMsg, <<"state">> := State }} -> + % Compute the next single state transition. + case compute_slot(ProcID, State, SlotMsg, Msg2, Opts) of + {ok, NewState} -> + % Continue computing to the target slot. + compute_to_slot( ProcID, - CurrentSlot, - Snapshot, + NewState, + Msg2, + TargetSlot, Opts ); - not_found -> - ?event(no_result_for_snapshot), - nothing_to_store - end; - _ -> nothing_to_do - end, - ?event({do_compute_result, {msg3, Msg3}}), - do_compute( - ProcID, - hb_converge:set( - Msg3, - #{ <<"Current-Slot">> => CurrentSlot + 1 }, + {error, Error} -> + % If the compute_slot function returns an error, + % we return the error details, along with the current + % slot. + ErrMsg = + if is_map(Error) -> + Error; + true -> #{ <<"error">> => Error } + end, + ?event(compute, + {error_computing_slot, + {error, ErrMsg}, + {phase, <<"compute">>}, + {attempted_slot, NextSlot} + } + ), + {error, + ErrMsg#{ + <<"phase">> => <<"compute">>, + <<"attempted-slot">> => NextSlot + } + } + end + end + end. + +%% @doc Compute a single slot for a process, given an initialized state. +compute_slot(ProcID, State, RawInputMsg, ReqMsg, Opts) -> + % Ensure that the next slot is the slot that we are expecting, just + % in case there is a scheduler device error. + NextSlot = hb_util:int(hb_ao:get(<<"slot">>, RawInputMsg, Opts)), + % If the input message does not have a path, set it to `compute'. + InputMsg = + case hb_path:from_message(request, RawInputMsg, Opts) of + undefined -> RawInputMsg#{ <<"path">> => <<"compute">> }; + _ -> RawInputMsg + end, + ?event(compute,{input_msg, InputMsg}), + ?event(compute, {executing, {proc_id, ProcID}, {slot, NextSlot}}, Opts), + % Unset the previous results. + UnsetResults = hb_ao:set(State, #{ <<"results">> => unset }, Opts), + Res = run_as(<<"execution">>, UnsetResults, InputMsg, Opts), + case Res of + {ok, NewProcStateMsg} -> + % We have now transformed slot n -> n + 1. Increment the current slot. + NewProcStateMsgWithSlot = + hb_ao:set( + NewProcStateMsg, + #{ <<"device">> => <<"process@1.0">>, <<"at-slot">> => NextSlot }, Opts ), - Msg2, - TargetSlot, + % Notify any waiters that the result for a slot is now available. + dev_process_worker:notify_compute( + ProcID, + NextSlot, + {ok, NewProcStateMsgWithSlot}, Opts - ) + ), + ProcStateWithSnapshot = + store_result( + false, + ProcID, + NextSlot, + NewProcStateMsgWithSlot, + ReqMsg, + Opts + ), + {ok, ProcStateWithSnapshot}; + {error, Error} -> + {error, Error} end. -%% @doc Returns the `/Results' key of the latest computed message. -now(RawMsg1, _Msg2, Opts) -> +%% @doc Store the resulting state in the cache, potentially with the snapshot +%% key. +store_result(ForceSnapshot, ProcID, Slot, Msg3, Msg2, Opts) -> + % Cache the `Snapshot' key as frequently as the node is configured to. + Msg3MaybeWithSnapshot = + case ForceSnapshot orelse should_snapshot(Slot, Msg3, Opts) of + false -> Msg3; + true -> + ?event(compute_debug, + {snapshotting, {proc_id, ProcID}, {slot, Slot}}, Opts), + {ok, Snapshot} = snapshot(Msg3, Msg2, Opts), + ?event(snapshot, + {got_snapshot, + {storing_as_slot, Slot}, + {snapshot, Snapshot} + } + ), + ?event(snapshot, + {snapshot_generated, + {proc_id, ProcID}, + {slot, Slot}, + {snapshot, Snapshot} + }, + Opts + ), + WithLastSnapshot = + hb_private:set( + Msg3#{ <<"snapshot">> => Snapshot }, + <<"last-snapshot">>, + os:system_time(second), + Opts + ), + ?event(debug_interval, + {snapshot_with_last_snapshot, + {proc_id, ProcID}, + {slot, Slot}, + {snapshot, WithLastSnapshot} + } + ), + hb_cache:ensure_all_loaded(WithLastSnapshot, Opts) + end, + ?event(compute, {caching_result, {proc_id, ProcID}, {slot, Slot}}, Opts), + Writer = + fun() -> + dev_process_cache:write(ProcID, Slot, Msg3MaybeWithSnapshot, Opts) + end, + case hb_opts:get(process_async_cache, true, Opts) of + true -> + spawn(Writer), + ?event(compute, {caching_delegated, {proc_id, ProcID}, {slot, Slot}}, Opts); + false -> + Writer(), + ?event(compute, {caching_completed, {proc_id, ProcID}, {slot, Slot}}, Opts) + end, + hb_maps:without([<<"snapshot">>], Msg3MaybeWithSnapshot, Opts). + +%% @doc Should we snapshot a new full state result? First, we check if the +%% `process_snapshot_time' option is set. If it is, we check if the elapsed time +%% since the last snapshot is greater than the value. We also check the +%% `process_snapshot_slots' option. If it is set, we check if the slot is +%% a multiple of the interval. If either are true, we must snapshot. +should_snapshot(Slot, Msg3, Opts) -> + should_snapshot_slots(Slot, Opts) + orelse should_snapshot_time(Msg3, Opts). + +%% @doc Calculate if we should snapshot based on the number of slots. +should_snapshot_slots(Slot, Opts) -> + case hb_opts:get(process_snapshot_slots, ?DEFAULT_SNAPSHOT_SLOTS, Opts) of + Undef when (Undef == undefined) or (Undef == <<"false">>) -> + false; + RawSnapshotSlots -> + SnapshotSlots = hb_util:int(RawSnapshotSlots), + Slot rem SnapshotSlots == 0 + end. + +%% @doc Calculate if we should snapshot based on the elapsed time since the last +%% snapshot. +should_snapshot_time(Msg3, Opts) -> + case hb_opts:get(process_snapshot_time, ?DEFAULT_SNAPSHOT_TIME, Opts) of + Undef when (Undef == undefined) or (Undef == <<"false">>) -> + false; + RawSecs -> + Secs = hb_util:int(RawSecs), + case hb_private:get(<<"last-snapshot">>, Msg3, undefined, Opts) of + undefined -> + ?event( + debug_interval, + {no_last_snapshot, + {interval, Secs}, + {msg, Msg3} + } + ), + true; + OldTimestamp -> + ?event( + debug_interval, + {calculating, + {secs, Secs}, + {timestamp, OldTimestamp}, + {now, os:system_time(second)} + } + ), + os:system_time(second) > OldTimestamp + hb_util:int(Secs) + end + end. + +%% @doc Returns the known state of the process at either the current slot, or +%% the latest slot in the cache depending on the `process_now_from_cache' option. +now(RawMsg1, Msg2, Opts) -> Msg1 = ensure_process_key(RawMsg1, Opts), - {ok, CurrentSlot} = hb_converge:resolve(Msg1, #{ path => <<"Slot/Current-Slot">> }, Opts), - ProcessID = hb_converge:get(<<"Process/id">>, Msg1, Opts), - ?event({now_called, {process, ProcessID}, {slot, CurrentSlot}}), - hb_converge:resolve( - Msg1, - #{ path => <<"Compute/Results">>, <<"Slot">> => CurrentSlot }, - Opts - ). + ProcessID = process_id(Msg1, #{}, Opts), + case hb_opts:get(process_now_from_cache, false, Opts) of + false -> + {ok, CurrentSlot} = + hb_ao:resolve( + Msg1, + #{ <<"path">> => <<"slot/current">> }, + Opts + ), + ?event({now_called, {process, ProcessID}, {slot, CurrentSlot}}), + hb_ao:resolve( + Msg1, + #{ <<"path">> => <<"compute">>, <<"slot">> => CurrentSlot }, + Opts + ); + CacheParam -> + % We are serving the latest known state from the cache, rather + % than computing it. + LatestKnown = dev_process_cache:latest(ProcessID, [], Opts), + case LatestKnown of + {ok, LatestSlot, RawLatestMsg} -> + LatestMsg = without_snapshot(RawLatestMsg, Opts), + ?event(compute_short, + {serving_latest_cached_state, + {proc_id, ProcessID}, + {slot, LatestSlot} + }, + Opts + ), + ?event( + {serving_from_cache, + {proc_id, ProcessID}, + {slot, LatestSlot}, + {msg, LatestMsg} + }), + dev_process_worker:notify_compute( + ProcessID, + LatestSlot, + {ok, LatestMsg}, + Opts + ), + {ok, LatestMsg}; + _ -> + if CacheParam =/= always -> + % The node is configured to use the cache if possible, + % but forcing computation is also admissible. Subsequently, + % as no other option is available, we compute the state. + now(Msg1, Msg2, Opts#{ process_now_from_cache => false }); + true -> + % The node is configured to only serve the latest known + % state from the cache, so we return the latest slot. + {failure, <<"No cached state available.">>} + end + end + end. %% @doc Recursively push messages to the scheduler until we find a message %% that does not lead to any further messages being scheduled. push(Msg1, Msg2, Opts) -> - Wallet = hb:wallet(), - PushMsgSlot = hb_converge:get(<<"Slot">>, Msg2, Opts), - {ok, Outbox} = hb_converge:resolve( - Msg1, - #{ path => <<"Compute/Results/Outbox">>, <<"Slot">> => PushMsgSlot }, - Opts#{ spawn_worker => true } - ), - case ?IS_EMPTY_MESSAGE(Outbox) of - true -> - {ok, #{}}; - false -> - {ok, maps:map( - fun(Key, MsgToPush) -> - case hb_converge:get(<<"Target">>, MsgToPush, Opts) of - not_found -> - ?event({skip_no_target, {key, Key}, MsgToPush}), - {ok, <<"No Target. Did not push.">>}; - Target -> - ?event( - {pushing_child, - {originates_from_slot, PushMsgSlot}, - {outbox_key, Key} - } - ), - {ok, NextSlotOnProc} = hb_converge:resolve( - Msg1, - #{ - method => <<"POST">>, - path => <<"Schedule/Slot">>, - <<"Message">> => - PushedMsg = hb_message:sign( - MsgToPush, - Wallet - ) - }, - Opts - ), - PushedMsgID = hb_converge:get(<<"id">>, PushedMsg, Opts), - ?event( - {push_scheduled, - {assigned_slot, NextSlotOnProc}, - {target, Target} - }), - {ok, Downstream} = hb_converge:resolve( - Msg1, - #{ path => <<"Push">>, <<"Slot">> => NextSlotOnProc }, - Opts - ), - #{ - <<"id">> => PushedMsgID, - <<"Target">> => Target, - <<"Slot">> => NextSlotOnProc, - <<"Resulted-In">> => Downstream - } - end - end, - maps:without([hashpath], Outbox) - )} - end. + ProcBase = ensure_process_key(Msg1, Opts), + run_as(<<"push">>, ProcBase, Msg2, Opts). %% @doc Ensure that the process message we have in memory is live and %% up-to-date. ensure_loaded(Msg1, Msg2, Opts) -> % Get the nonce we are currently on and the inbound nonce. - TargetSlot = hb_converge:get(<<"Slot">>, Msg2, undefined, Opts), - ProcID = - case hb_converge:get(<<"Process/id">>, {as, dev_message, Msg1}, Opts) of - not_found -> - hb_converge:get(<<"id">>, Msg1, Opts); - P -> P - end, - case hb_converge:get(<<"Initialized">>, Msg1, <<"False">>, Opts) of - <<"False">> -> + TargetSlot = hb_ao:get(<<"slot">>, Msg2, undefined, Opts), + ProcID = process_id(Msg1, #{}, Opts), + ?event({ensure_loaded, {msg1, Msg1}, {msg2, Msg2}}), + case hb_ao:get(<<"initialized">>, Msg1, Opts) of + <<"true">> -> + ?event(already_initialized), + {ok, Msg1}; + _ -> + ?event(not_initialized), % Try to load the latest complete state from disk. LoadRes = dev_process_cache:latest( ProcID, - [], + [<<"snapshot+link">>], TargetSlot, Opts ), - ?event({snapshot_load_res, {proc_id, ProcID}, {res, LoadRes}, {target, TargetSlot}}), + ?event(compute, + {snapshot_load_res, + {proc_id, ProcID}, + {res, LoadRes}, + {target, TargetSlot} + } + ), case LoadRes of - {ok, LoadedSlot, SnapshotMsg} -> + {ok, MaybeLoadedSlot, MaybeLoadedSnapshotMsg} -> % Restore the devices in the executor stack with the % loaded state. This allows the devices to load any % necessary 'shadow' state (state not represented in % the public component of a message) into memory. - % Do not update the hashpath while we do this. - ?event(snapshot, {loaded_state_checkpoint, ProcID, LoadedSlot}), - {ok, - hb_converge:set( - Msg1, - #{ - <<"Initialized">> => <<"True">>, - <<"Current-Slot">> => LoadedSlot, - <<"Snapshot">> => SnapshotMsg - }, + % Do not update the hashpath while we do this, and remove + % the snapshot key after we have normalized the message. + LoadedSnapshotMsg = + hb_cache:ensure_all_loaded( + MaybeLoadedSnapshotMsg, + Opts + ), + Process = hb_maps:get(<<"process">>, LoadedSnapshotMsg, Opts), + #{ <<"commitments">> := HmacCommits} = + hb_message:with_commitments( + #{ <<"type">> => <<"hmac-sha256">>}, + Process, + Opts), + #{ <<"commitments">> := SignCommits } = + hb_message:with_commitments(ProcID, Process, Opts), + UpdateProcess = hb_maps:put( + <<"commitments">>, + hb_maps:merge(HmacCommits, SignCommits), + Process, + Opts + ), + LoadedSnapshotMsg2 = + LoadedSnapshotMsg#{ + <<"process">> => UpdateProcess, + <<"initialized">> => <<"true">> + }, + LoadedSlot = hb_cache:ensure_all_loaded(MaybeLoadedSlot, Opts), + ?event(compute, {found_state_checkpoint, ProcID, LoadedSnapshotMsg2}), + {ok, Normalized} = + run_as( + <<"execution">>, + LoadedSnapshotMsg2, + normalize, Opts#{ hashpath => ignore } - ) - }; + ), + NormalizedWithoutSnapshot = without_snapshot(Normalized, Opts), + ?event(snapshot, + {loaded_state_checkpoint_result, + {proc_id, ProcID}, + {slot, LoadedSlot}, + {after_normalization, NormalizedWithoutSnapshot} + } + ), + {ok, NormalizedWithoutSnapshot}; not_found -> % If we do not have a checkpoint, initialize the % process from scratch. @@ -350,66 +673,123 @@ ensure_loaded(Msg1, Msg2, Opts) -> } ), init(Msg1, Msg2, Opts) - end; - <<"True">> -> {ok, Msg1} + end end. +%% @doc Remove the `snapshot' key from a message and return it. +without_snapshot(Msg, Opts) -> + hb_maps:remove(<<"snapshot">>, Msg, Opts). + %% @doc Run a message against Msg1, with the device being swapped out for %% the device found at `Key'. After execution, the device is swapped back %% to the original device if the device is the same as we left it. +run_as(Key, Msg1, Path, Opts) when not is_map(Path) -> + run_as(Key, Msg1, #{ <<"path">> => Path }, Opts); run_as(Key, Msg1, Msg2, Opts) -> - BaseDevice = hb_converge:get(<<"Device">>, {as, dev_message, Msg1}, Opts), - %?event({running_as, {key, Key}, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), - {ok, PreparedMsg} = - dev_message:set( + % Store the original device so we can restore it after execution + BaseDevice = hb_maps:get(<<"device">>, Msg1, not_found, Opts), + ?event({running_as, {key, {explicit, Key}}, {req, Msg2}}), + % Prepare the message with the specialized device configuration. + % This sets up the device context for the specific operation type. + PreparedMsg = + hb_util:deep_merge( ensure_process_key(Msg1, Opts), #{ - device => - DeviceSet = hb_converge:get( - << Key/binary, "-Device">>, - {as, dev_message, Msg1}, - Opts - ), - <<"Input-Prefix">> => - case hb_converge:get(<<"Input-Prefix">>, Msg1, Opts) of - not_found -> <<"Process">>; + <<"device">> => + DeviceSet = + hb_maps:get( + << Key/binary, "-device">>, + Msg1, + default_device(Msg1, Key, Opts), + Opts + ), + % Configure input prefix for proper message routing within the device + <<"input-prefix">> => + case hb_maps:get(<<"input-prefix">>, Msg1, not_found, Opts) of + not_found -> <<"process">>; Prefix -> Prefix end, - <<"Output-Prefixes">> => - hb_converge:get( - <>, - {as, dev_message, Msg1}, undefined, Opts) + % Configure output prefixes for result organization + <<"output-prefixes">> => + hb_maps:get( + <>, + Msg1, + undefined, % Undefined in set will be ignored. + Opts + ) }, Opts ), - %?event({resolving_proc, {msg1, PreparedMsg}, {msg2, Msg2}, {opts, Opts}}), - {ok, BaseResult} = - hb_converge:resolve( + ?event(debug_prefix, + {input_prefix, hb_maps:get(<<"output-prefixes">>, PreparedMsg, not_found, Opts) + }), + % Execute the message through the specialized device. + {Status, BaseResult} = + hb_ao:resolve( PreparedMsg, Msg2, Opts ), - case BaseResult of - #{ device := DeviceSet } -> - {ok, hb_converge:set(BaseResult, #{ device => BaseDevice })}; + % Restore the original device context after execution. + % This ensures the process maintains its identity after device delegation. + case {Status, BaseResult} of + {ok, #{ <<"device">> := DeviceSet }} -> + {ok, hb_ao:set(BaseResult, #{ <<"device">> => BaseDevice }, Opts)}; _ -> ?event({returning_base_result, BaseResult}), - {ok, BaseResult} + {Status, BaseResult} end. %% @doc Change the message to for that has the device set as this module. -%% In situations where the key that is `run_as` returns a message with a +%% In situations where the key that is `run_as' returns a message with a %% transformed device, this is useful. as_process(Msg1, Opts) -> - {ok, Proc} = dev_message:set(Msg1, #{ device => <<"Process/1.0">> }, Opts), + {ok, Proc} = dev_message:set(Msg1, #{ <<"device">> => <<"process@1.0">> }, Opts), Proc. -%% @doc Helper function to store a copy of the `process` key in the message. +%% @doc Helper function to store a copy of the `process' key in the message. ensure_process_key(Msg1, Opts) -> - case hb_converge:get(<<"Process">>, {as, dev_message, Msg1}, Opts) of + case hb_maps:get(<<"process">>, Msg1, not_found, Opts) of not_found -> - hb_converge:set( - Msg1, #{ <<"Process">> => Msg1 }, Opts#{ hashpath => ignore }); + % If the message has lost its signers, we need to re-read it from + % the cache. This can happen if the message was 'cast' to a different + % device, leading the signers to be unset. + ProcessMsg = + case hb_message:signers(Msg1, Opts) of + [] -> + ?event({process_key_not_found_no_signers, {msg1, Msg1}}), + case hb_cache:read(hb_message:id(Msg1, all, Opts), Opts) of + {ok, Proc} -> Proc; + not_found -> + % Fallback to the original message if we cannot + % read it from the cache. + Msg1 + end; + Signers -> + ?event( + {process_key_not_found_but_signers_present, + {signers, Signers}, + {msg1, Msg1} + } + ), + Msg1 + end, + {ok, Committed} = hb_message:with_only_committed(ProcessMsg, Opts), + ?event( + {process_key_before_set, + {msg1, Msg1}, + {process_msg, {explicit, ProcessMsg}}, + {committed, Committed} + } + ), + Res = + hb_ao:set( + hb_message:uncommitted(Msg1, Opts), + #{ <<"process">> => Committed }, + Opts#{ hashpath => ignore } + ), + ?event({set_process_key_res, {msg1, Msg1}, {process_msg, ProcessMsg}, {res, Res}}), + Res; _ -> Msg1 end. @@ -422,140 +802,191 @@ init() -> %% @doc Generate a process message with a random number, and no %% executor. test_base_process() -> - Wallet = hb:wallet(), + test_base_process(#{}). +test_base_process(Opts) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), Address = hb_util:human_id(ar_wallet:to_address(Wallet)), - #{ - device => <<"Process/1.0">>, - <<"Scheduler-Device">> => <<"Scheduler/1.0">>, - <<"Scheduler-Location">> => Address, - <<"Type">> => <<"Process">>, - <<"Test-Random-Seed">> => rand:uniform(1337) - }. + hb_message:commit(#{ + <<"device">> => <<"process@1.0">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"scheduler-location">> => hb_opts:get(scheduler, Address, Opts), + <<"type">> => <<"Process">>, + <<"test-random-seed">> => rand:uniform(1337) + }, Wallet). test_wasm_process(WASMImage) -> - #{ image := WASMImageID } = dev_wasm:cache_wasm_image(WASMImage), - maps:merge(test_base_process(), #{ - <<"Execution-Device">> => <<"Stack/1.0">>, - <<"Device-Stack">> => [<<"WASM-64/1.0">>], - <<"Image">> => WASMImageID - }). + test_wasm_process(WASMImage, #{}). +test_wasm_process(WASMImage, Opts) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + #{ <<"image">> := WASMImageID } = dev_wasm:cache_wasm_image(WASMImage, Opts), + hb_message:commit( + hb_maps:merge( + hb_message:uncommitted(test_base_process(Opts), Opts), + #{ + <<"execution-device">> => <<"stack@1.0">>, + <<"device-stack">> => [<<"wasm-64@1.0">>], + <<"image">> => WASMImageID + }, + Opts + ), + Opts#{ priv_wallet => Wallet} + ). %% @doc Generate a process message with a random number, and the %% `dev_wasm' device for execution. test_aos_process() -> - Wallet = hb:wallet(), - WASMProc = test_wasm_process(<<"test/aos-2-pure-xs.wasm">>), - hb_message:sign(maps:merge(WASMProc, #{ - <<"Device-Stack">> => - [ - <<"WASI/1.0">>, - <<"JSON-Iface/1.0">>, - <<"WASM-64/1.0">>, - <<"Multipass/1.0">> - ], - <<"Output-Prefix">> => <<"WASM">>, - <<"Passes">> => 2, - <<"Stack-Keys">> => - [ - <<"Init">>, - <<"Compute">>, - <<"Snapshot">>, - <<"Normalize">> - ], - <<"Scheduler">> => hb:address(), - <<"Authority">> => hb:address() - }), Wallet). + test_aos_process(#{}). +test_aos_process(Opts) -> + test_aos_process(Opts, [ + <<"wasi@1.0">>, + <<"json-iface@1.0">>, + <<"wasm-64@1.0">>, + <<"multipass@1.0">> + ]). +test_aos_process(Opts, Stack) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + Address = hb_util:human_id(ar_wallet:to_address(Wallet)), + WASMProc = test_wasm_process(<<"test/aos-2-pure-xs.wasm">>, Opts), + hb_message:commit( + hb_maps:merge( + hb_message:uncommitted(WASMProc, Opts), + #{ + <<"device-stack">> => Stack, + <<"execution-device">> => <<"stack@1.0">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"output-prefix">> => <<"wasm">>, + <<"patch-from">> => <<"/results/outbox">>, + <<"passes">> => 2, + <<"stack-keys">> => + [ + <<"init">>, + <<"compute">>, + <<"snapshot">>, + <<"normalize">> + ], + <<"scheduler">> => + hb_opts:get(scheduler, Address, Opts), + <<"authority">> => + hb_opts:get(authority, Address, Opts) + }, Opts), + Opts#{ priv_wallet => Wallet} + ). %% @doc Generate a device that has a stack of two `dev_test's for %% execution. This should generate a message state has doubled %% `Already-Seen' elements for each assigned slot. dev_test_process() -> - maps:merge(test_base_process(), #{ - <<"Execution-Device">> => <<"Stack/1.0">>, - <<"Device-Stack">> => [<<"Test-Device/1.0">>, <<"Test-Device/1.0">>] - }). - -schedule_test_message(Msg1, Text) -> - schedule_test_message(Msg1, Text, #{}). -schedule_test_message(Msg1, Text, MsgBase) -> Wallet = hb:wallet(), - Msg2 = hb_message:sign(#{ - path => <<"Schedule">>, - <<"Method">> => <<"POST">>, - <<"Message">> => - MsgBase#{ - <<"Type">> => <<"Message">>, - <<"Test-Label">> => Text - } - }, Wallet), - {ok, _} = hb_converge:resolve(Msg1, Msg2, #{}). + hb_message:commit( + hb_maps:merge(test_base_process(), #{ + <<"execution-device">> => <<"stack@1.0">>, + <<"device-stack">> => [<<"test-device@1.0">>, <<"test-device@1.0">>] + }, #{}), + Wallet + ). -schedule_aos_call(Msg1, Code) -> +schedule_test_message(Msg1, Text, Opts) -> + schedule_test_message(Msg1, Text, #{}, Opts). +schedule_test_message(Msg1, Text, MsgBase, Opts) -> Wallet = hb:wallet(), - ProcID = hb_converge:get(id, Msg1, #{}), - Msg2 = hb_message:sign(#{ - <<"Action">> => <<"Eval">>, - data => Code, - target => ProcID - }, Wallet), - schedule_test_message(Msg1, <<"TEST MSG">>, Msg2). + UncommittedBase = hb_message:uncommitted(MsgBase, Opts), + Msg2 = + hb_message:commit(#{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + UncommittedBase#{ + <<"type">> => <<"Message">>, + <<"test-label">> => Text + }, + Opts#{ priv_wallet => Wallet} + ) + }, + Opts#{ priv_wallet => Wallet} + ), + {ok, _} = hb_ao:resolve(Msg1, Msg2, Opts). -schedule_wasm_call(Msg1, FuncName, Params) -> - Msg2 = #{ - path => <<"Schedule">>, - <<"Method">> => <<"POST">>, - <<"Message">> => +schedule_aos_call(Msg1, Code) -> + schedule_aos_call(Msg1, Code, #{}). +schedule_aos_call(Msg1, Code, Opts) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + ProcID = hb_message:id(Msg1, all), + Msg2 = + hb_message:commit( #{ - <<"Type">> => <<"Message">>, - <<"WASM-Function">> => FuncName, - <<"WASM-Params">> => Params - } - }, - ?assertMatch({ok, _}, hb_converge:resolve(Msg1, Msg2, #{})). + <<"action">> => <<"Eval">>, + <<"data">> => Code, + <<"target">> => ProcID + }, + Opts#{priv_wallet => Wallet} + ), + schedule_test_message(Msg1, <<"TEST MSG">>, Msg2, Opts). -schedule_on_process_test() -> - init(), - Msg1 = test_aos_process(), - schedule_test_message(Msg1, <<"TEST TEXT 1">>), - schedule_test_message(Msg1, <<"TEST TEXT 2">>), - ?event(messages_scheduled), - {ok, SchedulerRes} = - hb_converge:resolve(Msg1, #{ - <<"Method">> => <<"GET">>, - path => <<"Schedule">> - }, #{}), - ?assertMatch( - <<"TEST TEXT 1">>, - hb_converge:get(<<"Assignments/0/Message/Test-Label">>, SchedulerRes) - ), - ?assertMatch( - <<"TEST TEXT 2">>, - hb_converge:get(<<"Assignments/1/Message/Test-Label">>, SchedulerRes) - ). +schedule_wasm_call(Msg1, FuncName, Params) -> + schedule_wasm_call(Msg1, FuncName, Params, #{}). +schedule_wasm_call(Msg1, FuncName, Params, Opts) -> + Wallet = hb:wallet(), + Msg2 = hb_message:commit(#{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + #{ + <<"type">> => <<"Message">>, + <<"function">> => FuncName, + <<"parameters">> => Params + }, + Opts#{ priv_wallet => Wallet} + ) + }, Opts#{ priv_wallet => Wallet}), + ?assertMatch({ok, _}, hb_ao:resolve(Msg1, Msg2, Opts)). + +schedule_on_process_test_() -> + {timeout, 30, fun()-> + init(), + Msg1 = test_aos_process(), + schedule_test_message(Msg1, <<"TEST TEXT 1">>, #{}), + schedule_test_message(Msg1, <<"TEST TEXT 2">>, #{}), + ?event(messages_scheduled), + {ok, SchedulerRes} = + hb_ao:resolve(Msg1, #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"schedule">> + }, #{}), + ?assertMatch( + <<"TEST TEXT 1">>, + hb_ao:get(<<"assignments/0/body/test-label">>, SchedulerRes) + ), + ?assertMatch( + <<"TEST TEXT 2">>, + hb_ao:get(<<"assignments/1/body/test-label">>, SchedulerRes) + ) + end}. get_scheduler_slot_test() -> init(), Msg1 = test_base_process(), - schedule_test_message(Msg1, <<"TEST TEXT 1">>), - schedule_test_message(Msg1, <<"TEST TEXT 2">>), + schedule_test_message(Msg1, <<"TEST TEXT 1">>, #{}), + schedule_test_message(Msg1, <<"TEST TEXT 2">>, #{}), Msg2 = #{ - path => <<"Slot">>, - <<"Method">> => <<"GET">> + <<"path">> => <<"slot">>, + <<"method">> => <<"GET">> }, ?assertMatch( - {ok, #{ <<"Current-Slot">> := CurrentSlot }} when CurrentSlot > 0, - hb_converge:resolve(Msg1, Msg2, #{}) + {ok, #{ <<"current">> := CurrentSlot }} when CurrentSlot > 0, + hb_ao:resolve(Msg1, Msg2, #{}) ). recursive_path_resolution_test() -> init(), Msg1 = test_base_process(), - schedule_test_message(Msg1, <<"TEST TEXT 1">>), + schedule_test_message(Msg1, <<"TEST TEXT 1">>, #{}), CurrentSlot = - hb_converge:resolve( + hb_ao:resolve( Msg1, - #{ path => <<"Slot/Current-Slot">> }, - #{ hashpath => ignore } + #{ <<"path">> => <<"slot/current">> }, + #{ <<"hashpath">> => ignore } ), ?event({resolved_current_slot, CurrentSlot}), ?assertMatch( @@ -567,21 +998,21 @@ recursive_path_resolution_test() -> test_device_compute_test() -> init(), Msg1 = dev_test_process(), - schedule_test_message(Msg1, <<"TEST TEXT 1">>), - schedule_test_message(Msg1, <<"TEST TEXT 2">>), + schedule_test_message(Msg1, <<"TEST TEXT 1">>, #{}), + schedule_test_message(Msg1, <<"TEST TEXT 2">>, #{}), ?assertMatch( {ok, <<"TEST TEXT 2">>}, - hb_converge:resolve( + hb_ao:resolve( Msg1, - <<"Schedule/Assignments/1/Message/Test-Label">>, - #{ hashpath => ignore } + <<"schedule/assignments/1/body/test-label">>, + #{ <<"hashpath">> => ignore } ) ), - Msg2 = #{ path => <<"Compute">>, <<"Slot">> => 1 }, - {ok, Msg3} = hb_converge:resolve(Msg1, Msg2, #{}), + Msg2 = #{ <<"path">> => <<"compute">>, <<"slot">> => 1 }, + {ok, Msg3} = hb_ao:resolve(Msg1, Msg2, #{}), ?event({computed_message, {msg3, Msg3}}), - ?assertEqual(1, hb_converge:get(<<"Results/Assignment-Slot">>, Msg3, #{})), - ?assertEqual([1,1,0,0], hb_converge:get(<<"Already-Seen">>, Msg3, #{})). + ?assertEqual(1, hb_ao:get(<<"results/assignment-slot">>, Msg3, #{})), + ?assertEqual([1,1,0,0], hb_ao:get(<<"already-seen">>, Msg3, #{})). wasm_compute_test() -> init(), @@ -589,21 +1020,80 @@ wasm_compute_test() -> schedule_wasm_call(Msg1, <<"fac">>, [5.0]), schedule_wasm_call(Msg1, <<"fac">>, [6.0]), {ok, Msg3} = - hb_converge:resolve( + hb_ao:resolve( Msg1, - #{ path => <<"Compute">>, <<"Slot">> => 0 }, - #{ hashpath => ignore } + #{ <<"path">> => <<"compute">>, <<"slot">> => 0 }, + #{ <<"hashpath">> => ignore } ), ?event({computed_message, {msg3, Msg3}}), - ?assertEqual([120.0], hb_converge:get(<<"Results/Output">>, Msg3, #{})), + ?assertEqual([120.0], hb_ao:get(<<"results/output">>, Msg3, #{})), {ok, Msg4} = - hb_converge:resolve( + hb_ao:resolve( Msg1, - #{ path => <<"Compute">>, <<"Slot">> => 1 }, - #{ hashpath => ignore } + #{ <<"path">> => <<"compute">>, <<"slot">> => 1 }, + #{ <<"hashpath">> => ignore } ), ?event({computed_message, {msg4, Msg4}}), - ?assertEqual([720.0], hb_converge:get(<<"Results/Output">>, Msg4, #{})). + ?assertEqual([720.0], hb_ao:get(<<"results/output">>, Msg4, #{})). + +wasm_compute_from_id_test() -> + init(), + Opts = #{ cache_control => <<"always">> }, + Msg1 = test_wasm_process(<<"test/test-64.wasm">>), + schedule_wasm_call(Msg1, <<"fac">>, [5.0], Opts), + Msg1ID = hb_message:id(Msg1, all), + Msg2 = #{ <<"path">> => <<"compute">>, <<"slot">> => 0 }, + {ok, Msg3} = hb_ao:resolve(Msg1ID, Msg2, Opts), + ?event(process_compute, {computed_message, {msg3, Msg3}}), + ?assertEqual([120.0], hb_ao:get(<<"results/output">>, Msg3, Opts)). + +http_wasm_process_by_id_test() -> + rand:seed(default), + SchedWallet = ar_wallet:new(), + Node = hb_http_server:start_node(Opts = #{ + port => 10000 + rand:uniform(10000), + priv_wallet => SchedWallet, + cache_control => <<"always">>, + process_async_cache => false, + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-mainnet">> + } + }), + Wallet = ar_wallet:new(), + Proc = test_wasm_process(<<"test/test-64.wasm">>, Opts), + hb_cache:write(Proc, Opts), + ProcID = hb_util:human_id(hb_message:id(Proc, all)), + InitRes = + hb_http:post( + Node, + << "/schedule" >>, + Proc, + #{} + ), + ?event({schedule_proc_res, InitRes}), + ExecMsg = + hb_message:commit(#{ + <<"target">> => ProcID, + <<"type">> => <<"Message">>, + <<"function">> => <<"fac">>, + <<"parameters">> => [5.0] + }, + Wallet + ), + {ok, Msg3} = hb_http:post(Node, << ProcID/binary, "/schedule">>, ExecMsg, #{}), + ?event({schedule_msg_res, {msg3, Msg3}}), + {ok, Msg4} = + hb_http:get( + Node, + #{ + <<"path">> => << ProcID/binary, "/compute">>, + <<"slot">> => 1 + }, + #{} + ), + ?event({compute_msg_res, {msg4, Msg4}}), + ?assertEqual([120.0], hb_ao:get(<<"results/output">>, Msg4, #{})). aos_compute_test_() -> {timeout, 30, fun() -> @@ -611,19 +1101,131 @@ aos_compute_test_() -> Msg1 = test_aos_process(), schedule_aos_call(Msg1, <<"return 1+1">>), schedule_aos_call(Msg1, <<"return 2+2">>), - Msg2 = #{ path => <<"Compute">>, <<"Slot">> => 0 }, - {ok, Msg3} = hb_converge:resolve(Msg1, Msg2, #{}), - {ok, Res} = hb_converge:resolve(Msg3, <<"Results">>, #{}), + Msg2 = #{ <<"path">> => <<"compute">>, <<"slot">> => 0 }, + {ok, Msg3} = hb_ao:resolve(Msg1, Msg2, #{}), + {ok, Res} = hb_ao:resolve(Msg3, <<"results">>, #{}), ?event({computed_message, {msg3, Res}}), - {ok, Data} = hb_converge:resolve(Res, <<"Data">>, #{}), + {ok, Data} = hb_ao:resolve(Res, <<"data">>, #{}), ?event({computed_data, Data}), ?assertEqual(<<"2">>, Data), - Msg4 = #{ path => <<"Compute">>, <<"Slot">> => 1 }, - {ok, Msg5} = hb_converge:resolve(Msg1, Msg4, #{}), - ?assertEqual(<<"4">>, hb_converge:get(<<"Results/Data">>, Msg5, #{})), + Msg4 = #{ <<"path">> => <<"compute">>, <<"slot">> => 1 }, + {ok, Msg5} = hb_ao:resolve(Msg1, Msg4, #{}), + ?assertEqual(<<"4">>, hb_ao:get(<<"results/data">>, Msg5, #{})), {ok, Msg5} end}. +aos_browsable_state_test_() -> + {timeout, 30, fun() -> + init(), + Msg1 = test_aos_process(), + schedule_aos_call(Msg1, + <<"table.insert(ao.outbox.Messages, { target = ao.id, ", + "action = \"State\", ", + "data = { deep = 4, bool = true } })">> + ), + Msg2 = #{ <<"path">> => <<"compute">>, <<"slot">> => 0 }, + {ok, Msg3} = + hb_ao:resolve_many( + [Msg1, Msg2, <<"results">>, <<"outbox">>, 1, <<"data">>, <<"deep">>], + #{ cache_control => <<"always">> } + ), + ID = hb_message:id(Msg1), + ?event({computed_message, {id, {explicit, ID}}}), + ?assertEqual(4, Msg3) + end}. + +aos_state_access_via_http_test_() -> + {timeout, 60, fun() -> + rand:seed(default), + Wallet = ar_wallet:new(), + Node = hb_http_server:start_node(Opts = #{ + port => 10000 + rand:uniform(10000), + priv_wallet => Wallet, + cache_control => <<"always">>, + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-mainnet">> + }, + force_signed_requests => true + }), + Proc = test_aos_process(Opts), + ProcID = hb_util:human_id(hb_message:id(Proc, all)), + {ok, _InitRes} = hb_http:post(Node, <<"/schedule">>, Proc, Opts), + Msg2 = hb_message:commit(#{ + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"type">> => <<"Message">>, + <<"action">> => <<"Eval">>, + <<"data">> => + <<"table.insert(ao.outbox.Messages, { target = ao.id,", + " action = \"State\", data = { ", + "[\"content-type\"] = \"text/html\", ", + "[\"body\"] = \"

Hello, world!

\"", + "}})">>, + <<"target">> => ProcID + }, Wallet), + {ok, Msg3} = hb_http:post(Node, << ProcID/binary, "/schedule">>, Msg2, Opts), + ?event({schedule_msg_res, {msg3, Msg3}}), + {ok, Msg4} = + hb_http:get( + Node, + #{ + <<"path">> => << ProcID/binary, "/compute/results/outbox/1/data" >>, + <<"slot">> => 1 + }, + Opts + ), + ?event({compute_msg_res, {msg4, Msg4}}), + ?event( + {try_yourself, + {explicit, + << + Node/binary, + "/", + ProcID/binary, + "/compute&slot=1/results/outbox/1/data" + >> + } + } + ), + ?assertMatch(#{ <<"body">> := <<"

Hello, world!

">> }, Msg4), + ok + end}. + +aos_state_patch_test_() -> + {timeout, 30, fun() -> + Wallet = hb:wallet(), + init(), + Msg1Raw = test_aos_process(#{}, [ + <<"wasi@1.0">>, + <<"json-iface@1.0">>, + <<"wasm-64@1.0">>, + <<"patch@1.0">>, + <<"multipass@1.0">> + ]), + {ok, Msg1} = hb_message:with_only_committed(Msg1Raw, #{}), + ProcID = hb_message:id(Msg1, all), + Msg2 = (hb_message:commit(#{ + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"target">> => ProcID, + <<"type">> => <<"Message">>, + <<"action">> => <<"Eval">>, + <<"data">> => + << + "table.insert(ao.outbox.Messages, " + "{ method = \"PATCH\", x = \"banana\" })" + >> + }, Wallet))#{ <<"path">> => <<"schedule">>, <<"method">> => <<"POST">> }, + {ok, _} = hb_ao:resolve(Msg1, Msg2, #{}), + Msg3 = #{ <<"path">> => <<"compute">>, <<"slot">> => 0 }, + {ok, Msg4} = hb_ao:resolve(Msg1, Msg3, #{}), + ?event({computed_message, {msg3, Msg4}}), + {ok, Data} = hb_ao:resolve(Msg4, <<"x">>, #{}), + ?event({computed_data, Data}), + ?assertEqual(<<"banana">>, Data) + end}. + %% @doc Manually test state restoration without using the cache. restore_test_() -> {timeout, 30, fun do_test_restore/0}. @@ -632,58 +1234,64 @@ do_test_restore() -> % 1. Set variables in Lua. % 2. Return the variable. % Execute the first computation, then the second as a disconnected process. - Opts = #{ process_cache_frequency => 1 }, + Opts = #{ process_cache_frequency => 1, process_async_cache => false }, init(), - Msg1 = test_aos_process(), - schedule_aos_call(Msg1, <<"X = 42">>), - schedule_aos_call(Msg1, <<"X = 1337">>), - schedule_aos_call(Msg1, <<"return X">>), + Store = hb_opts:get(store, no_viable_store, Opts), + ResetRes = hb_store:reset(Store), + ?event({reset_store, {result, ResetRes}, {store, Store}}), + Msg1 = test_aos_process(Opts), + schedule_aos_call(Msg1, <<"X = 42">>, Opts), + schedule_aos_call(Msg1, <<"X = 1337">>, Opts), + schedule_aos_call(Msg1, <<"return X">>, Opts), % Compute the first message. {ok, _} = - hb_converge:resolve( + hb_ao:resolve( Msg1, - #{ path => <<"Compute">>, <<"Slot">> => 1 }, + #{ <<"path">> => <<"compute">>, <<"slot">> => 1 }, Opts ), {ok, ResultB} = - hb_converge:resolve( + hb_ao:resolve( Msg1, - #{ path => <<"Compute">>, <<"Slot">> => 2 }, + #{ <<"path">> => <<"compute">>, <<"slot">> => 2 }, Opts ), ?event({result_b, ResultB}), - ?assertEqual(<<"1337">>, hb_converge:get(<<"Results/Data">>, ResultB, #{})). + ?assertEqual(<<"1337">>, hb_ao:get(<<"results/data">>, ResultB, #{})). -now_results_test() -> +now_results_test_() -> {timeout, 30, fun() -> init(), Msg1 = test_aos_process(), schedule_aos_call(Msg1, <<"return 1+1">>), schedule_aos_call(Msg1, <<"return 2+2">>), - ?assertEqual({ok, <<"4">>}, hb_converge:resolve(Msg1, <<"Now/Data">>, #{})) + ?assertEqual({ok, <<"4">>}, hb_ao:resolve(Msg1, <<"now/results/data">>, #{})) end}. -full_push_test_() -> - {timeout, 30, fun() -> - init(), - Msg1 = test_aos_process(), - Script = ping_ping_script(3), - ?event({script, Script}), - {ok, Msg2} = schedule_aos_call(Msg1, Script), - ?event({init_sched_result, Msg2}), - {ok, StartingMsgSlot} = - hb_converge:resolve(Msg2, #{ path => <<"Slot">> }, #{}), - Msg3 = - #{ - path => <<"Push">>, - <<"Slot">> => StartingMsgSlot - }, - {ok, _} = hb_converge:resolve(Msg1, Msg3, #{}), - ?assertEqual( - {ok, <<"Done.">>}, - hb_converge:resolve(Msg1, <<"Now/Data">>, #{}) - ) - end}. +prior_results_accessible_test_() -> + {timeout, 30, fun() -> + init(), + Opts = #{ + process_async_cache => false + }, + Msg1 = test_aos_process(), + schedule_aos_call(Msg1, <<"return 1+1">>), + schedule_aos_call(Msg1, <<"return 2+2">>), + ?assertEqual( + {ok, <<"4">>}, + hb_ao:resolve(Msg1, <<"now/results/data">>, Opts) + ), + {ok, Results} = + hb_ao:resolve( + Msg1, + #{ <<"path">> => <<"compute">>, <<"slot">> => 1 }, + Opts + ), + ?assertMatch( + #{ <<"results">> := #{ <<"data">> := <<"4">> } }, + hb_cache:ensure_all_loaded(Results, Opts) + ) + end}. persistent_process_test() -> {timeout, 30, fun() -> @@ -694,19 +1302,19 @@ persistent_process_test() -> schedule_aos_call(Msg1, <<"return X">>), T0 = hb:now(), FirstSlotMsg2 = #{ - path => <<"Compute">>, - <<"Slot">> => 0 + <<"path">> => <<"compute">>, + <<"slot">> => 0 }, ?assertMatch( {ok, _}, - hb_converge:resolve(Msg1, FirstSlotMsg2, #{ spawn_worker => true }) + hb_ao:resolve(Msg1, FirstSlotMsg2, #{ spawn_worker => true }) ), T1 = hb:now(), ThirdSlotMsg2 = #{ - path => <<"Compute">>, - <<"Slot">> => 2 + <<"path">> => <<"compute">>, + <<"slot">> => 2 }, - Res = hb_converge:resolve(Msg1, ThirdSlotMsg2, #{}), + Res = hb_ao:resolve(Msg1, ThirdSlotMsg2, #{}), ?event({computed_message, {msg3, Res}}), ?assertMatch( {ok, _}, @@ -726,12 +1334,12 @@ simple_wasm_persistent_worker_benchmark_test() -> schedule_wasm_call(Msg1, <<"fac">>, [5.0]), schedule_wasm_call(Msg1, <<"fac">>, [6.0]), {ok, Initialized} = - hb_converge:resolve( + hb_ao:resolve( Msg1, - #{ path => <<"Compute">>, <<"Slot">> => 1 }, - #{ spawn_worker => true } + #{ <<"path">> => <<"compute">>, <<"slot">> => 1 }, + #{ spawn_worker => true, process_workers => true } ), - Iterations = hb:benchmark( + Iterations = hb_test_utils:benchmark( fun(Iteration) -> schedule_wasm_call( Initialized, @@ -740,9 +1348,9 @@ simple_wasm_persistent_worker_benchmark_test() -> ), ?assertMatch( {ok, _}, - hb_converge:resolve( + hb_ao:resolve( Initialized, - #{ path => <<"Compute">>, <<"Slot">> => Iteration + 1 }, + #{ <<"path">> => <<"compute">>, <<"slot">> => Iteration + 1 }, #{} ) ) @@ -750,28 +1358,28 @@ simple_wasm_persistent_worker_benchmark_test() -> BenchTime ), ?event(benchmark, {scheduled, Iterations}), - hb_util:eunit_print( - "Scheduled and evaluated ~p simple wasm process messages in ~p s (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] + hb_format:eunit_print( + "Scheduled and evaluated ~p simple wasm process messages in ~p s (~s msg/s)", + [Iterations, BenchTime, hb_util:human_int(Iterations / BenchTime)] ), - ?assert(Iterations > 2), + ?assert(Iterations >= 2), ok. aos_persistent_worker_benchmark_test_() -> {timeout, 30, fun() -> - BenchTime = 4, + BenchTime = 5, init(), Msg1 = test_aos_process(), schedule_aos_call(Msg1, <<"X=1337">>), FirstSlotMsg2 = #{ - path => <<"Compute">>, - <<"Slot">> => 0 + <<"path">> => <<"compute">>, + <<"slot">> => 0 }, ?assertMatch( {ok, _}, - hb_converge:resolve(Msg1, FirstSlotMsg2, #{ spawn_worker => true }) + hb_ao:resolve(Msg1, FirstSlotMsg2, #{ spawn_worker => true }) ), - Iterations = hb:benchmark( + Iterations = hb_test_utils:benchmark( fun(Iteration) -> schedule_aos_call( Msg1, @@ -779,9 +1387,9 @@ aos_persistent_worker_benchmark_test_() -> ), ?assertMatch( {ok, _}, - hb_converge:resolve( + hb_ao:resolve( Msg1, - #{ path => <<"Compute">>, <<"Slot">> => Iteration }, + #{ <<"path">> => <<"compute">>, <<"slot">> => Iteration }, #{} ) ) @@ -789,28 +1397,10 @@ aos_persistent_worker_benchmark_test_() -> BenchTime ), ?event(benchmark, {scheduled, Iterations}), - hb_util:eunit_print( - "Scheduled and evaluated ~p AOS process messages in ~p s (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] + hb_format:eunit_print( + "Scheduled and evaluated ~p AOS process messages in ~p s (~s msg/s)", + [Iterations, BenchTime, hb_util:human_int(Iterations / BenchTime)] ), ?assert(Iterations >= 2), ok - end}. - -%%% Test helpers - -ping_ping_script(Limit) -> - << - "Handlers.add(\"Ping\",\n" - " function(m)\n" - " C = tonumber(m.Count)\n" - " if C <= ", (integer_to_binary(Limit))/binary, " then\n" - " Send({ Target = ao.id, Action = \"Ping\", Count = C + 1 })\n" - " print(\"Ping\", C + 1)\n" - " else\n" - " print(\"Done.\")\n" - " end\n" - " end\n" - ")\n" - "Send({ Target = ao.id, Action = \"Ping\", Count = 1 })\n" - >>. + end}. \ No newline at end of file diff --git a/src/dev_process_cache.erl b/src/dev_process_cache.erl index 6d883885f..df3b32f07 100644 --- a/src/dev_process_cache.erl +++ b/src/dev_process_cache.erl @@ -26,10 +26,17 @@ write(ProcID, Slot, Msg, Opts) -> MsgIDPath = path( ProcID, - ID = hb_util:human_id(hb_converge:get(id, Msg)), + ID = hb_util:human_id(hb_ao:get(id, Msg, Opts)), Opts ), - ?event({linking_id, {proc_id, ProcID}, {id, ID}, {path, MsgIDPath}}), + ?event( + {linking_id, + {proc_id, ProcID}, + {slot, Slot}, + {id, ID}, + {path, MsgIDPath} + } + ), hb_cache:link(Root, MsgIDPath, Opts), % Return the slot number path. {ok, SlotNumPath}. @@ -42,13 +49,13 @@ path(ProcID, Ref, PathSuffix, Opts) -> hb_store:path( Store, [ - <<"Computed">>, + <<"computed">>, hb_util:human_id(ProcID) ] ++ case Ref of - Int when is_integer(Int) -> ["Slot", integer_to_binary(Int)]; + Int when is_integer(Int) -> ["slot", integer_to_binary(Int)]; root -> []; - slot_root -> ["Slot"]; + slot_root -> ["slot"]; _ -> [Ref] end ++ PathSuffix ). @@ -60,6 +67,13 @@ latest(ProcID, Opts) -> latest(ProcID, [], Opts). latest(ProcID, RequiredPath, Opts) -> latest(ProcID, RequiredPath, undefined, Opts). latest(ProcID, RawRequiredPath, Limit, Opts) -> + ?event( + {latest_called, + {proc_id, ProcID}, + {required_path, RawRequiredPath}, + {limit, Limit} + } + ), % Convert the required path to a list of _binary_ keys. RequiredPath = case RawRequiredPath of @@ -68,11 +82,13 @@ latest(ProcID, RawRequiredPath, Limit, Opts) -> _ -> hb_path:term_to_path_parts( RawRequiredPath, - Opts#{ atom_keys => false } + Opts ) end, + ?event({required_path_converted, {proc_id, ProcID}, {required_path, RequiredPath}}), Path = path(ProcID, slot_root, Opts), AllSlots = hb_cache:list_numbered(Path, Opts), + ?event({all_slots, {proc_id, ProcID}, {slots, AllSlots}}), CappedSlots = case Limit of undefined -> AllSlots; @@ -120,7 +136,7 @@ first_with_path(ProcID, RequiredPath, [Slot | Rest], Opts, Store) -> ResolvedPath = hb_store:resolve(Store, RawPath), ?event({trying_slot, {slot, Slot}, {path, RawPath}, {resolved_path, ResolvedPath}}), case hb_store:type(Store, ResolvedPath) of - no_viable_store -> + not_found -> first_with_path(ProcID, RequiredPath, Rest, Opts, Store); _ -> Slot @@ -137,8 +153,7 @@ process_cache_suite_test_() -> [ {Name, Opts} || - {Name, Opts} <- hb_store:test_stores(), - Name =/= hb_store_rocksdb % Disable rocksdb for now + {Name, Opts} <- hb_store:test_stores() ] ). @@ -147,7 +162,7 @@ process_cache_suite_test_() -> test_write_and_read_output(Opts) -> Proc = hb_cache:test_signed( #{ <<"test-item">> => hb_cache:test_unsigned(<<"test-body-data">>) }), - ProcID = hb_util:human_id(hb_converge:get(id, Proc)), + ProcID = hb_util:human_id(hb_ao:get(id, Proc)), Item1 = hb_cache:test_signed(<<"Simple signed output #1">>), Item2 = hb_cache:test_unsigned(<<"Simple unsigned output #2">>), {ok, Path0} = write(ProcID, 0, Item1, Opts), @@ -161,10 +176,10 @@ test_write_and_read_output(Opts) -> {ok, ReadItem2BySlotNum} = read(ProcID, 1, Opts), ?assert(hb_message:match(Item2, ReadItem2BySlotNum)), {ok, ReadItem1ByID} = - read(ProcID, hb_util:human_id(hb_converge:get(id, Item1)), Opts), + read(ProcID, hb_util:human_id(hb_ao:get(id, Item1)), Opts), ?assert(hb_message:match(Item1, ReadItem1ByID)), {ok, ReadItem2ByID} = - read(ProcID, hb_util:human_id(hb_converge:get(unsigned_id, Item2)), Opts), + read(ProcID, hb_util:human_id(hb_message:id(Item2, all)), Opts), ?assert(hb_message:match(Item2, ReadItem2ByID)). %% @doc Test for retrieving the latest computed output for a process. @@ -174,9 +189,9 @@ find_latest_outputs(Opts) -> ResetRes = hb_store:reset(Store), ?event({reset_store, {result, ResetRes}, {store, Store}}), Proc1 = dev_process:test_aos_process(), - ProcID = hb_util:human_id(hb_converge:get(id, Proc1)), + ProcID = hb_util:human_id(hb_ao:get(id, Proc1, Opts)), % Create messages for the slots, with only the middle slot having a - % `/Process` field, while the top slot has a `/Deep/Process` field. + % `/Process' field, while the top slot has a `/Deep/Process' field. Msg0 = #{ <<"Results">> => #{ <<"Result-Number">> => 0 } }, Msg1 = #{ @@ -205,5 +220,5 @@ find_latest_outputs(Opts) -> {ok, 2, ReadMsg2Required} = latest(ProcID, <<"Deep/Process">>, Opts), ?assert(hb_message:match(Msg2, ReadMsg2Required)), ?event(read_latest_slot_with_deep_key), - {ok, 1, ReadMsg1} = latest(ProcID, <<"">>, 1, Opts), + {ok, 1, ReadMsg1} = latest(ProcID, [], 1, Opts), ?assert(hb_message:match(Msg1, ReadMsg1)). diff --git a/src/dev_process_worker.erl b/src/dev_process_worker.erl index 747b0b16d..bdc71ee2a 100644 --- a/src/dev_process_worker.erl +++ b/src/dev_process_worker.erl @@ -1,9 +1,9 @@ %%% @doc A long-lived process worker that keeps state in memory between -%%% calls. Implements the interface of `hb_converge' to receive and respond +%%% calls. Implements the interface of `hb_ao' to receive and respond %%% to computation requests regarding a process as a singleton. -module(dev_process_worker). --export([server/3, stop/1, group/3]). +-export([server/3, stop/1, group/3, await/5, notify_compute/4]). -include_lib("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -13,30 +13,150 @@ group(Msg1, undefined, Opts) -> hb_persistent:default_grouper(Msg1, undefined, Opts); group(Msg1, Msg2, Opts) -> - case hb_path:matches(<<"Compute">>, hb_path:hd(Msg2, Opts)) of + case hb_opts:get(process_workers, false, Opts) of + false -> + hb_persistent:default_grouper(Msg1, Msg2, Opts); true -> - process_to_group_name(Msg1, Opts); - _ -> - hb_persistent:default_grouper(Msg1, Msg2, Opts) + case Msg2 of + undefined -> + hb_persistent:default_grouper(Msg1, undefined, Opts); + _ -> + case hb_path:matches(<<"compute">>, hb_path:hd(Msg2, Opts)) of + true -> + process_to_group_name(Msg1, Opts); + _ -> + hb_persistent:default_grouper(Msg1, Msg2, Opts) + end + end end. process_to_group_name(Msg1, Opts) -> - hb_util:human_id( - hb_converge:get( - <<"Process/id">>, - {as, - dev_message, - dev_process:ensure_process_key(Msg1, Opts) - }, - Opts#{ hashpath => ignore } - ) - ). + Initialized = dev_process:ensure_process_key(Msg1, Opts), + ProcMsg = hb_ao:get(<<"process">>, Initialized, Opts#{ hashpath => ignore }), + ID = hb_message:id(ProcMsg, all), + ?event({process_to_group_name, {id, ID}, {msg1, Msg1}}), + hb_util:human_id(ID). %% @doc Spawn a new worker process. This is called after the end of the first -%% execution of `hb_converge:resolve/3', so the state we are given is the +%% execution of `hb_ao:resolve/3', so the state we are given is the %% already current. server(GroupName, Msg1, Opts) -> - hb_persistent:default_worker(GroupName, Msg1, Opts#{ static_worker => true }). + ServerOpts = Opts#{ + await_inprogress => false, + spawn_worker => false, + process_workers => false + }, + % The maximum amount of time the worker will wait for a request before + % checking the cache for a snapshot. Default: 5 minutes. + Timeout = hb_opts:get(process_worker_max_idle, 300_000, Opts), + ?event(worker, {waiting_for_req, {group, GroupName}}), + receive + {resolve, Listener, GroupName, Msg2, ListenerOpts} -> + TargetSlot = hb_ao:get(<<"slot">>, Msg2, Opts), + ?event(worker, + {work_received, + {group, GroupName}, + {slot, TargetSlot}, + {listener, Listener} + } + ), + Res = + hb_ao:resolve( + Msg1, + #{ <<"path">> => <<"compute">>, <<"slot">> => TargetSlot }, + hb_maps:merge(ListenerOpts, ServerOpts, Opts) + ), + ?event(worker, {work_done, {group, GroupName}, {req, Msg2}, {res, Res}}), + send_notification(Listener, GroupName, TargetSlot, Res), + server( + GroupName, + case Res of + {ok, Msg3} -> Msg3; + _ -> Msg1 + end, + Opts + ); + stop -> + ?event(worker, {stopping, {group, GroupName}, {msg1, Msg1}}), + exit(normal) + after Timeout -> + % We have hit the in-memory persistence timeout. Generate a snapshot + % of the current process state and ensure it is cached. + hb_ao:resolve( + Msg1, + <<"snapshot">>, + ServerOpts#{ <<"cache-control">> => [<<"store">>] } + ), + % Return the current process state. + {ok, Msg1} + end. + +%% @doc Await a resolution from a worker executing the `process@1.0' device. +await(Worker, GroupName, Msg1, Msg2, Opts) -> + case hb_path:matches(<<"compute">>, hb_path:hd(Msg2, Opts)) of + false -> + hb_persistent:default_await(Worker, GroupName, Msg1, Msg2, Opts); + true -> + TargetSlot = hb_ao:get(<<"slot">>, Msg2, any, Opts), + ?event({awaiting_compute, + {worker, Worker}, + {group, GroupName}, + {target_slot, TargetSlot} + }), + receive + {resolved, _, GroupName, {slot, RecvdSlot}, Res} + when RecvdSlot == TargetSlot orelse TargetSlot == any -> + ?event(compute_debug, {notified_of_resolution, + {target, TargetSlot}, + {group, GroupName} + }), + Res; + {resolved, _, GroupName, {slot, RecvdSlot}, _Res} -> + ?event(compute_debug, {waiting_again, + {target, TargetSlot}, + {recvd, RecvdSlot}, + {worker, Worker}, + {group, GroupName} + }), + await(Worker, GroupName, Msg1, Msg2, Opts); + {'DOWN', _R, process, Worker, _Reason} -> + ?event(compute_debug, + {leader_died, + {group, GroupName}, + {leader, Worker}, + {target, TargetSlot} + } + ), + {error, leader_died} + end + end. + +%% @doc Notify any waiters for a specific slot of the computed results. +notify_compute(GroupName, SlotToNotify, Msg3, Opts) -> + notify_compute(GroupName, SlotToNotify, Msg3, Opts, 0). +notify_compute(GroupName, SlotToNotify, Msg3, Opts, Count) -> + ?event({notifying_of_computed_slot, {group, GroupName}, {slot, SlotToNotify}}), + receive + {resolve, Listener, GroupName, #{ <<"slot">> := SlotToNotify }, _ListenerOpts} -> + send_notification(Listener, GroupName, SlotToNotify, Msg3), + notify_compute(GroupName, SlotToNotify, Msg3, Opts, Count + 1); + {resolve, Listener, GroupName, Msg, _ListenerOpts} + when is_map(Msg) andalso not is_map_key(<<"slot">>, Msg) -> + send_notification(Listener, GroupName, SlotToNotify, Msg3), + notify_compute(GroupName, SlotToNotify, Msg3, Opts, Count + 1) + after 0 -> + ?event(worker_short, + {finished_notifying, + {group, GroupName}, + {slot, SlotToNotify}, + {listeners, Count} + } + ) + end. + +send_notification(Listener, GroupName, SlotToNotify, Msg3) -> + ?event({sending_notification, {group, GroupName}, {slot, SlotToNotify}}), + Listener ! {resolved, self(), GroupName, {slot, SlotToNotify}, Msg3}. %% @doc Stop a worker process. stop(Worker) -> @@ -51,18 +171,18 @@ test_init() -> info_test() -> test_init(), M1 = dev_process:test_wasm_process(<<"test/aos-2-pure-xs.wasm">>), - Res = hb_converge:info(M1, #{}), - ?assertEqual(fun dev_process_worker:group/3, maps:get(grouper, Res)). + Res = hb_ao:info(M1, #{}), + ?assertEqual(fun dev_process_worker:group/3, hb_maps:get(grouper, Res, undefined, #{})). grouper_test() -> test_init(), M1 = dev_process:test_aos_process(), - M2 = #{ path => <<"Compute">>, v => 1 }, - M3 = #{ path => <<"Compute">>, v => 2 }, - M4 = #{ path => <<"Not-Compute">>, v => 3 }, - G1 = hb_persistent:group(M1, M2, #{}), - G2 = hb_persistent:group(M1, M3, #{}), - G3 = hb_persistent:group(M1, M4, #{}), + M2 = #{ <<"path">> => <<"compute">>, <<"v">> => 1 }, + M3 = #{ <<"path">> => <<"compute">>, <<"v">> => 2 }, + M4 = #{ <<"path">> => <<"not-compute">>, <<"v">> => 3 }, + G1 = hb_persistent:group(M1, M2, #{ process_workers => true }), + G2 = hb_persistent:group(M1, M3, #{ process_workers => true }), + G3 = hb_persistent:group(M1, M4, #{ process_workers => true }), ?event({group_samples, {g1, G1}, {g2, G2}, {g3, G3}}), ?assertEqual(G1, G2), ?assertNotEqual(G1, G3). \ No newline at end of file diff --git a/src/dev_profile.erl b/src/dev_profile.erl new file mode 100644 index 000000000..4883f2026 --- /dev/null +++ b/src/dev_profile.erl @@ -0,0 +1,370 @@ +%%% @doc A module for running different profiling tools upon HyperBEAM executions. +%%% This device allows a variety of profiling tools to be used and for their +%%% outputs to be returned as messages, or displayed locally on the console. +%%% +%%% When called from an AO-Core request, the path at the given key is resolved. +%%% If the `eval' function is instead directly invoked via Erlang, the first +%%% argument may be a function to profile instead. +-module(dev_profile). +-export([info/1, eval/1, eval/2, eval/3, eval/4]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% @doc Default to the `eval' function. +info(_) -> + #{ + excludes => [<<"keys">>, <<"set">>], + default => fun eval/4 + }. + +%% @doc Invoke a profiling tool on a function or an AO-Core resolution. If a +%% message is provided as the first argument, a function to profile is produced +%% from an AO-Core resolution of the path referenced by `path` key. For example, +%% `/~profile@1.0/run?run=/~meta@1.0/build' will resolve to the `build' function +%% in the `~meta@1.0' device. The `request` message (if provided) is passed +%% downstream to the profiling engine as well as the AO-Core resolution. +%% +%% If the `return-mode' option is not set, we default to `console' for Erlang +%% invocations. We determine this by checking if the first argument is a +%% function. This is not possible if the function has been invoked by an +%% AO-Core resolution. +%% +%% When in `return-mode: console' mode, the return format will be +%% `{EngineOut, Res}' where `EngineOut' is the output from the engine, and `Res' +%% is the result of the function or resolution. In `return-mode: message' mode, +%% the return format will be `{ok, EngineMessage}' where `EngineMessage' is the +%% output from the engine formatted as an AO-Core message. +eval(Fun) -> eval(Fun, #{}). +eval(Fun, Opts) -> eval(Fun, #{}, Opts). +eval(Fun, Req, Opts) when is_function(Fun) -> + do_eval( + Fun, + case return_mode(Req, Opts, undefined) of + undefined -> Req#{ <<"return-mode">> => <<"open">> }; + _ -> Req + end, + Opts + ); +eval(Base, Request, Opts) -> + eval(<<"eval">>, Base, Request, Opts). +eval(PathKey, Base, Req, Opts) when not is_function(Base) -> + case hb_ao:get(PathKey, Req, undefined, Opts) of + undefined -> + { + error, + << + "Path key `", + (hb_util:bin(PathKey))/binary, + "` not found in request." + >> + }; + Path -> + do_eval( + fun() -> hb_ao:resolve(Req#{ <<"path">> => Path }, Opts) end, + Req, + Opts + ) + end. + +do_eval(Fun, Req, Opts) -> + % Validate the request and options, then invoke the engine-specific profile + % function. We match the user-requested engine against the supported engines + % on the node. Each engine takes three arguments: + % 1. The function to profile. + % 2. The request message containing user-provided options. + % 3. The node options. + maybe + true ?= validate_enabled(Opts), + true ?= validate_signer(Req, Opts), + true ?= validate_return_mode(Req, Opts), + {ok, ProfileEngine} ?= engine(hb_ao:get(<<"engine">>, Req, default, Opts)), + ProfileEngine(Fun, Req, Opts) + else + {unknown_engine, Unknown} -> + {error, <<"Unsupported engine `", (hb_util:bin(Unknown))/binary, "`">>}; + {validation_error, disabled} -> + {error, <<"Profiling is disabled.">>}; + {validation_error, invalid_request} -> + {error, <<"Invalid request.">>} + end. + +%%% Validation helpers: + +%% @doc Find the profiling options. The supported options for `profiling' in the +%% node message are: +%% - `true' to enable profiling to allow all requests to be profiled. +%% - `false' to disable all profiling. +%% - A list of signers whose requests are allowed to be profiled. +%% If the `profiling' option is not set, check the following other node config +%% options to determine if profiling should be enabled: +%% - Node message key `mode': `prod' => false, others continue; +%% - `TEST` build flag: `true' => true, others => false. +find_profiling_config(Opts) -> + case hb_opts:get(profiling, not_found, Opts) of + not_found -> + case hb_opts:get(mode, prod, Opts) of + prod -> false; + _ -> hb_features:test() + end; + EnableProfiling -> EnableProfiling + end. + +%% @doc Validate that profiling is enabled. +%% +%% Return the calculated value _only_ if it is false. If it is not, return +%% true. This allows the `profiling` option to also be used to set a list +%% of valid signers for profiling requests. +validate_enabled(Opts) -> + case find_profiling_config(Opts) of + false -> {validation_error, disabled}; + _ -> true + end. + +%% @doc Validate that the request return mode is acceptable. We only allow the +%% `open' mode if the node is in `debug' mode. +validate_return_mode(Req, Opts) -> + case return_mode(Req, Opts) of + <<"open">> -> hb_opts:get(mode, prod, Opts) == debug; + _ -> true + end. + +%% @doc Validate that the request is from a valid signer, if set by the node +%% operator. If the `profiling' config option is `true', all requests are +%% allowed. If it is a list, check if the request is from a valid signer. +validate_signer(Req, Opts) -> + case find_profiling_config(Opts) of + ValidSigners when is_list(ValidSigners) -> + lists:any( + fun(Signer) -> lists:member(Signer, ValidSigners) end, + hb_message:signers(Req, Opts) + ); + EnableProfiling -> EnableProfiling + end orelse {validation_error, invalid_signer}. + +%% @doc Return the profiling function for the given engine. +engine(<<"eflame">>) -> {ok, fun eflame_profile/3}; +engine(<<"eprof">>) -> {ok, fun eprof_profile/3}; +engine(<<"event">>) -> {ok, fun event_profile/3}; +engine(default) -> {ok, default()}; +engine(Unknown) -> {unknown_engine, Unknown}. + +%% @doc Return the default profiling engine to use. `eflame' if preferred if +%% available, falling back to `eprof' if not. +default() -> + case hb_features:eflame() of + true -> fun eflame_profile/3; + false -> fun eprof_profile/3 + end. + +%%% Profiling engines. + +-ifdef(ENABLE_EFLAME). +%% @doc Profile a function using the `eflame' tool. This tool is only available +%% if HyperBEAM was built with the `eflame' feature (`rebar3 as eflame ...'). +%% If the return mode is `open` and the `profiler_allow_open` option is `true`, +%% we open the flame graph. The command to open the graph is specified by the +%% `profiler_open_cmd' node option, or `open' if not set. A delay of 500ms is +%% added to allow the file to be opened before it is cleaned up. This delay can +%% be configured by the `profiler_open_delay' node option, or 500ms if not set. +eflame_profile(Fun, Req, Opts) -> + File = temp_file(), + Res = eflame:apply(normal, File, Fun, []), + MergeStacks = hb_maps:get(<<"mode">>, Req, <<"merge">>, Opts), + EflameDir = code:lib_dir(eflame), + % Get the name of the function to profile. If the path in the request is + % set, attempt to find it. If that is not found, we use the bare path. + % This follows the semantics of the request evaluation scheme, in which the + % user's provided path is a pointer to the actual path to resolve. If the + % path is not set, we use Erlang's short string encoding of the function. + Name = + case hb_maps:get(<<"path">>, Req, undefined, Opts) of + undefined -> hb_util:bin(io_lib:format("~p", [Fun])); + Path -> + case hb_maps:get(Path, Req, undefined, Opts) of + undefined -> hb_util:bin(Path); + EvalPath -> hb_util:bin(EvalPath) + end + end, + StackToFlameScript = hb_util:bin(filename:join(EflameDir, "flamegraph.pl")), + FlameArg = + case MergeStacks of + <<"merge">> -> <<"">>; + <<"time">> -> <<"--flamechart">> + end, + PreparedCommand = + hb_util:list( + << + "cat ", (hb_util:bin(File))/binary, + " | uniq -c | awk '{print $2, \" \", $1}' | ", + StackToFlameScript/binary, " ", FlameArg/binary, + " --title=\"", Name/binary, "\"" + >> + ), + Flame = hb_util:bin(os:cmd(PreparedCommand)), + ?event(debug_profile, + {flame_graph, + {name, Name}, + {command, PreparedCommand}, + {flame, Flame} + } + ), + file:delete(File), + case return_mode(Req, Opts) of + <<"open">> -> + % We cannot return a text version of the flame graph, so we open it + % on the local machine. + file:write_file( + SVG = filename:absname(temp_file(<<"svg">>)), + Flame + ), + ?event(debug_profile, {svg, SVG}), + case hb_opts:get(profiler_allow_open, true, Opts) of + true -> + OpenCmd = hb_opts:get(profiler_open_cmd, "open", Opts), + CmdRes = os:cmd(OpenCmd ++ " " ++ hb_util:list(SVG)), + timer:sleep(hb_opts:get(profiler_open_delay, 500, Opts)), + ?event(debug_profile, {open_command_result, CmdRes}), + file:delete(SVG), + {ok, Res}; + _ -> + {SVG, Res} + end; + <<"message">> -> + % We can return the flame graph as an SVG image suitable for output + % to a browser. + {ok, + #{ + <<"content-type">> => <<"image/svg+xml">>, + <<"body">> => Flame + } + } + end. +-else. +eflame_profile(_Fun, _Req, _Opts) -> + {error, <<"eflame is not enabled.">>}. +-endif. + +%% @doc Profile a function using the `eprof' tool. +eprof_profile(Fun, Req, Opts) -> + File = temp_file(), + % Attempt to profile the function, stopping the profiler afterwards. + Res = + try + eprof:start(), + eprof:start_profiling([self()]), + Fun() + catch + _:_ -> {error, <<"Execution of function to profile failed.">>} + after eprof:stop_profiling() + end, + % If we are writing to the console we do not need to write and read the + % file. + case return_mode(Req, Opts) of + <<"message">> -> eprof:log(File); + _ -> do_nothing + end, + eprof:analyze(total), + eprof:stop(), + case return_mode(Req, Opts) of + <<"message">> -> + {ok, Log} = file:read_file(File), + file:delete(File), + { + ok, + #{ + <<"content-type">> => <<"text/plain">>, + <<"body">> => Log + } + }; + _ -> + Res + end. + +%% @doc Profile using HyperBEAM's events. +event_profile(Fun, Req, Opts) -> + Start = hb_event:counters(), + Fun(), + End = hb_event:counters(), + Diff = hb_message:diff(Start, End, Opts), + case return_mode(Req, Opts) of + <<"message">> -> + {ok, Diff}; + <<"console">> -> + hb_format:print(Diff), + {ok, Diff} + end. + +%%% Engine helpers: Generalized tools useful for multiple engines. + +%% @doc Get the return mode of a profiler run. The run mode is set to `console' +%% by the default handler if the profiler is called from Erlang, and `message' +%% if the profiler is called from AO-Core. +return_mode(Req, Opts) -> + return_mode(Req, Opts, <<"message">>). +return_mode(Req, Opts, Default) -> + hb_ao:get(<<"return-mode">>, Req, Default, Opts). + +%% @doc Returns a temporary filename for use in a profiling run. +temp_file() -> temp_file(<<"out">>). +temp_file(Ext) -> + << + "profile-", + (integer_to_binary(os:system_time(microsecond)))/binary, + ".", + Ext/binary + >>. + +%%% Tests. + +eprof_fun_test() -> test_engine(function, <<"eprof">>). +eprof_resolution_test() -> test_engine(resolution, <<"eprof">>). + +-ifdef(ENABLE_EFLAME). +eflame_fun_test() -> test_engine(function, <<"eflame">>). +eflame_resolution_test() -> test_engine(resolution, <<"eflame">>). +-endif. + +%%% Test utilities. + +%% @doc Run a test and validate the output for a given engine. +test_engine(Type, Engine) -> + validate_profiler_output(Engine, test_profiler_exec(Type, Engine)). + +%% @doc Invoke an engine in either a function (as called from Erlang) or +%% resolution context. We get the build information from the node in order to +%% simulate some basic compute that is performant. +test_profiler_exec(function, Engine) -> + eval( + fun() -> dev_meta:build(#{}, #{}, #{}) end, + #{ <<"engine">> => Engine, <<"return-mode">> => <<"message">> }, + #{} + ); +test_profiler_exec(resolution, Engine) -> + hb_ao:resolve( + #{ + <<"path">> => <<"/~profile@1.0/run?run=/~meta@1.0/build">>, + <<"engine">> => Engine, <<"return-mode">> => <<"message">> }, + #{} + ). + +%% @doc Verify the expected type of output from a profiler. +validate_profiler_output(<<"eprof">>, Res) -> + ?assertMatch( + {ok, + #{ + <<"content-type">> := <<"text/plain">>, + <<"body">> := Body + } + } when byte_size(Body) > 100, + Res + ); +validate_profiler_output(<<"eflame">>, Res) -> + ?assertMatch( + {ok, + #{ + <<"content-type">> := <<"image/svg+xml">>, + <<"body">> := Body + } + } when byte_size(Body) > 100, + Res + ). \ No newline at end of file diff --git a/src/dev_push.erl b/src/dev_push.erl new file mode 100644 index 000000000..5408c2416 --- /dev/null +++ b/src/dev_push.erl @@ -0,0 +1,1096 @@ +%%% @doc `push@1.0' takes a message or slot number, evaluates it, and recursively +%%% pushes the resulting messages to other processes. The `push'ing mechanism +%%% continues until the there are no remaining messages to push. +-module(dev_push). +%%% Public API +-export([push/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Push either a message or an assigned slot number. If a `Process' is +%% provided in the `body' of the request, it will be scheduled (initializing +%% it if it does not exist). Otherwise, the message specified by the given +%% `slot' key will be pushed. +%% +%% Optional parameters: +%% `/result-depth': The depth to which the full contents of the result +%% will be included in the response. Default: 1, returning +%% the full result of the first message, but only the 'tree' +%% of downstream messages. +%% `/push-mode': Whether or not the push should be done asynchronously. +%% Default: `sync', pushing synchronously. +push(Base, Req, Opts) -> + Process = dev_process:as_process(Base, Opts), + ?event(push, {push_base, {base, Process}, {req, Req}}, Opts), + case hb_ao:get(<<"slot">>, {as, <<"message@1.0">>, Req}, no_slot, Opts) of + no_slot -> + case schedule_initial_message(Process, Req, Opts) of + {ok, Assignment} -> + case find_type(hb_ao:get(<<"body">>, Assignment, Opts), Opts) of + <<"Process">> -> + ?event(push, + {initializing_process, + {base, Process}, + {assignment, Assignment}}, + Opts + ), + {ok, Assignment}; + _ -> + ?event(push, + {pushing_message, + {base, Process}, + {assignment, Assignment} + }, + Opts + ), + push_with_mode(Process, Assignment, Opts) + end; + {error, Res} -> {error, Res} + end; + _ -> push_with_mode(Process, Req, Opts) + end. + +push_with_mode(Process, Req, Opts) -> + Mode = is_async(Process, Req, Opts), + case Mode of + <<"sync">> -> + do_push(Process, Req, Opts); + <<"async">> -> + spawn(fun() -> do_push(Process, Req, Opts) end) + end. + +%% @doc Determine if the push is asynchronous. +is_async(Process, Req, Opts) -> + hb_ao:get_first( + [ + {Req, <<"push-mode">>}, + {Process, <<"push-mode">>}, + {Process, <<"process/push-mode">>} + ], + <<"sync">>, + Opts + ). + +%% @doc Push a message or slot number, including its downstream results. +do_push(PrimaryProcess, Assignment, Opts) -> + Slot = hb_ao:get(<<"slot">>, Assignment, Opts), + ID = dev_process:process_id(PrimaryProcess, #{}, Opts), + UncommittedID = + dev_process:process_id( + PrimaryProcess, + #{ <<"commitments">> => <<"none">> }, + Opts + ), + BaseID = calculate_base_id(PrimaryProcess, Opts), + ?event(debug, + {push_computing_outbox, + {process_id, ID}, + {base_id, BaseID}, + {slot, Slot} + } + ), + ?event(push, {push_computing_outbox, {process_id, ID}, {slot, Slot}}), + {Status, Result} = hb_ao:resolve( + {as, <<"process@1.0">>, PrimaryProcess}, + #{ <<"path">> => <<"compute/results">>, <<"slot">> => Slot }, + Opts#{ hashpath => ignore } + ), + % Determine if we should include the full compute result in our response. + IncludeDepth = hb_ao:get(<<"result-depth">>, Assignment, 1, Opts), + AdditionalRes = + case IncludeDepth of + X when X > 0 -> Result; + _ -> #{} + end, + ?event(push_depth, {depth, IncludeDepth, {assignment, Assignment}}), + ?event(push, {push_computed, {process, ID}, {slot, Slot}}), + ?event(debug, + {push_computed, + {status, Status}, + {assignment, Assignment}, + {request, hb_maps:get(<<"body">>, Assignment, Assignment, Opts)}, + {result, + if is_list(Result) -> + hb_ao:normalize_keys(Result); + true -> Result + end + } + }), + case {Status, hb_ao:get(<<"outbox">>, Result, #{}, Opts)} of + {ok, NoResults} when ?IS_EMPTY_MESSAGE(NoResults) -> + ?event(push_short, {done, {process, {string, ID}}, {slot, Slot}}), + {ok, AdditionalRes#{ <<"slot">> => Slot, <<"process">> => ID }}; + {ok, Outbox} -> + ?event(push, {push_found_outbox, {outbox, Outbox}}), + Downstream = + hb_maps:map( + fun(Key, RawMsgToPush = #{ <<"target">> := Target }) -> + MsgToPush = + case maybe_evaluate_message(RawMsgToPush, Opts) of + {ok, R} -> R; + Err -> + #{ + <<"resolve">> => <<"error">>, + <<"target">> => ID, + <<"status">> => 400, + <<"outbox-index">> => Key, + <<"reason">> => Err, + <<"source">> => RawMsgToPush + } + end, + case hb_cache:read(Target, Opts) of + {ok, DownstreamProcess} -> + push_result_message( + DownstreamProcess, + MsgToPush, + #{ + <<"process">> => ID, + <<"slot">> => Slot, + <<"outbox-key">> => Key, + <<"result-depth">> => IncludeDepth, + <<"from-base">> => BaseID, + <<"from-uncommitted">> => UncommittedID, + <<"from-scheduler">> => + hb_ao:get( + <<"scheduler">>, + PrimaryProcess, + Opts + ), + <<"from-authority">> => + hb_ao:get( + <<"authority">>, + PrimaryProcess, + Opts + ) + }, + Opts + ); + not_found -> + #{ + <<"response">> => <<"error">>, + <<"status">> => 404, + <<"target">> => Target, + <<"reason">> => + <<"Could not access target process!">> + } + end; + (Key, Msg) -> + #{ + <<"response">> => <<"error">>, + <<"status">> => 404, + <<"outbox-index">> => Key, + <<"reason">> => + <<"Target process not available.">>, + <<"message">> => Msg + } + end, + hb_util:lower_case_key_map( + hb_ao:normalize_keys(hb_private:reset(Outbox)), + Opts + ), + Opts + ), + {ok, maps:merge(Downstream, AdditionalRes#{ + <<"slot">> => Slot, + <<"process">> => ID + })}; + {Err, Error} when Err == error; Err == failure -> + ?event(push, {push_failed_to_find_outbox, {error, Error}}, Opts), + {error, Error} + end. + + +%% @doc If the outbox message has a path we interpret it as a request to perform +%% AO-Core eval and schedule the result. Additionally, we remove the `target` +%% from the base message before execution and re-add it to the result, such that +%% the target to schedule the execution result upon is not confused with +%% functional components of the evaluation. +maybe_evaluate_message(Message, Opts) -> + case hb_ao:get(<<"resolve">>, Message, Opts) of + not_found -> + {ok, Message}; + ResolvePath -> + ReqMsg = + maps:without( + [<<"target">>], + Message + ), + ResolveOpts = Opts#{ force_message => true }, + case hb_ao:resolve(ReqMsg#{ <<"path">> => ResolvePath }, ResolveOpts) of + {ok, EvalRes} -> + { + ok, + EvalRes#{ + <<"target">> => + hb_ao:get( + <<"target">>, + Message, + Opts + ) + } + }; + Err -> Err + end + end. + +%% @doc Push a downstream message result. The `Origin' map contains information +%% about the origin of the message: The process that originated the message, +%% the slot number from which it was sent, and the outbox key of the message, +%% and the depth to which downstream results should be included in the message. +push_result_message(TargetProcess, MsgToPush, Origin, Opts) -> + NormMsgToPush = hb_util:lower_case_key_map(MsgToPush, Opts), + case hb_ao:get(<<"target">>, NormMsgToPush, undefined, Opts) of + undefined -> + ?event(push, + {skip_no_target, {msg, MsgToPush}, {origin, Origin}}, + Opts + ), + #{}; + TargetID -> + ?event(push, + {pushing_child, + {target, TargetID}, + {msg, MsgToPush}, + {origin, Origin} + }, + Opts + ), + case schedule_result(TargetProcess, MsgToPush, Origin, Opts) of + {ok, Assignment} -> + % Analyze the result of the message push. + NextSlotOnProc = hb_ao:get(<<"slot">>, Assignment, Opts), + PushedMsg = hb_ao:get(<<"body">>, Assignment, Opts), + % Get the ID of the message that was pushed. We already have + % the 'origin' message, but we need the signed ID. + PushedMsgID = hb_message:id(PushedMsg, all, Opts), + ?event(push_short, + {pushed_message_to, + {process, TargetID}, + {slot, NextSlotOnProc} + } + ), + {ok, TargetBase} = hb_cache:read(TargetID, Opts), + TargetAsProcess = dev_process:ensure_process_key(TargetBase, Opts), + RecvdID = hb_message:id(TargetBase, all, Opts), + ?event(push, {recvd_id, {id, RecvdID}, {msg, TargetAsProcess}}), + % Push the message downstream. We decrease the result-depth. + Recurse = + hb_ao:resolve( + {as, <<"process@1.0">>, TargetAsProcess}, + #{ + <<"path">> => <<"push">>, + <<"slot">> => NextSlotOnProc, + <<"result-depth">> => + hb_ao:get( + <<"result-depth">>, + Origin, + 1, + Opts + ) - 1 + }, + Opts#{ cache_control => <<"always">> } + ), + case Recurse of + {ok, Downstream} -> + #{ + <<"id">> => PushedMsgID, + <<"target">> => TargetID, + <<"slot">> => NextSlotOnProc, + <<"resulted-in">> => Downstream + }; + {error, Error} -> + ?event(push, {push_failed, {error, Error}}, Opts), + #{ + <<"response">> => <<"error">>, + <<"target">> => TargetID, + <<"reason">> => Error + } + end; + {error, Error} -> + ?event(push, {push_failed, {error, Error}}, Opts), + #{ + <<"response">> => <<"error">>, + <<"target">> => TargetID, + <<"reason">> => Error + } + end + end. + +%% @doc Augment the message with from-* keys, if it doesn't already have them. +normalize_message(MsgToPush, Opts) -> + hb_ao:set( + MsgToPush, + #{ + <<"target">> => target_process(MsgToPush, Opts) + }, + Opts#{ hashpath => ignore } + ). + +%% @doc Find the target process ID for a message to push. +target_process(MsgToPush, Opts) -> + case hb_ao:get(<<"target">>, MsgToPush, Opts) of + not_found -> undefined; + RawTarget -> extract(target, RawTarget) + end. + +%% @doc Return either the `target' or the `hint'. +extract(hint, Raw) -> + {_, Hint} = split_target(Raw), + Hint; +extract(target, Raw) -> + {Target, _} = split_target(Raw), + Target. + +%% @doc Split the target into the process ID and the optional query string. +split_target(RawTarget) -> + case binary:split(RawTarget, [<<"?">>, <<"&">>]) of + [Target, QStr] -> {Target, QStr}; + _ -> {RawTarget, <<>>} + end. + +%% @doc Calculate the base ID for a process. The base ID is not just the +%% uncommitted process ID. It also excludes the `authority' and `scheduler' +%% keys. +calculate_base_id(GivenProcess, Opts) -> + Process = + case hb_ao:get(<<"process">>, GivenProcess, Opts#{ hashpath => ignore }) of + not_found -> GivenProcess; + Proc -> Proc + end, + BaseProcess = maps:without([<<"authority">>, <<"scheduler">>], Process), + {ok, BaseID} = hb_ao:resolve( + BaseProcess, + #{ <<"path">> => <<"id">>, <<"commitments">> => <<"none">> }, + Opts + ), + ?event({push_generated_base, {id, BaseID}, {base, BaseProcess}}), + BaseID. + +%% @doc Add the necessary keys to the message to be scheduled, then schedule it. +%% If the remote scheduler does not support the given codec, it will be +%% downgraded and re-signed. +schedule_result(TargetProcess, MsgToPush, Origin, Opts) -> + schedule_result(TargetProcess, MsgToPush, <<"httpsig@1.0">>, Origin, Opts). +schedule_result(TargetProcess, MsgToPush, Codec, Origin, Opts) -> + Target = hb_ao:get(<<"target">>, MsgToPush, Opts), + ?event(push, + {push_scheduling_result, + {target, {string, Target}}, + {target_process, TargetProcess}, + {msg, MsgToPush}, + {codec, Codec}, + {origin, Origin} + }, + Opts + ), + AugmentedMsg = augment_message(Origin, MsgToPush, Opts), + ?event(push, {prepared_msg, {msg, AugmentedMsg}}, Opts), + % Load the `accept-id`'d wallet into the `Opts` map, if requested. + SignedMsg = apply_security(AugmentedMsg, TargetProcess, Codec, Opts), + ScheduleReq = #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => SignedMsg + }, + ?event(push, {schedule_req, {req, ScheduleReq}}, Opts), + ?event(debug, + {push_scheduling_result, + {signed_req, SignedMsg}, + {verifies, hb_message:verify(SignedMsg, signers, Opts)} + } + ), + {ErlStatus, Res} = + case hb_message:signers(SignedMsg, Opts) of + [] -> + {error, + << + "Application of security policy failed: ", + "No identities matching authority were found." + >> + }; + _Committers -> + hb_ao:resolve( + {as, <<"process@1.0">>, TargetProcess}, + ScheduleReq, + Opts#{ cache_control => <<"always">> } + ) + end, + ?event(push, {push_sched_result, {status, ErlStatus}, {response, Res}}, Opts), + case {ErlStatus, hb_ao:get(<<"status">>, Res, 200, Opts)} of + {ok, 200} -> + {ok, Res}; + {ok, 307} -> + Location = hb_ao:get(<<"location">>, Res, Opts), + ?event(push, {redirect, {location, {explicit, Location}}}), + NormMsg = normalize_message(MsgToPush, Opts), + SignedNormMsg = hb_message:commit(NormMsg, Opts), + remote_schedule_result(Location, SignedNormMsg, Opts); + {error, 422} -> + ?event(push, {wrong_format, {422, Res}, {codec, Codec}}, Opts), + case Codec of + <<"ans104@1.0">> -> + {error, Res}; + <<"httpsig@1.0">> -> + ?event(push, + {downgrading_to_ans104, + {422, Res}, + {codec, Codec}, + {origin, Origin} + }, + Opts + ), + schedule_result( + TargetProcess, + MsgToPush, + <<"ans104@1.0">>, + Origin, + Opts + ) + end; + {error, _} -> + {error, Res} + end. + +%% @doc Set the necessary keys in order for the recipient to know where the +%% message came from. +augment_message(Origin, ToSched, Opts) -> + ?event(push, {adding_keys, {origin, Origin}, {to, ToSched}}, Opts), + hb_message:uncommitted( + hb_ao:set( + ToSched, + #{ + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"type">> => <<"Message">>, + <<"from-process">> => maps:get(<<"process">>, Origin), + <<"from-uncommitted">> => maps:get(<<"from-uncommitted">>, Origin), + <<"from-base">> => maps:get(<<"from-base">>, Origin), + <<"from-scheduler">> => maps:get(<<"from-scheduler">>, Origin), + <<"from-authority">> => maps:get(<<"from-authority">>, Origin) + }, + Opts#{ hashpath => ignore } + ) + ). + +%% @doc Apply the recipient's security policy to the message. Observes the +%% following parameters in order to calculate the appropriate security policy: +%% - `policy': A message that generates a security policy message. +%% - `authority': A single committer, or list of comma separated committers. +%% - (Default: Signs with default wallet) +apply_security(Msg, TargetProcess, Codec, Opts) -> + apply_security(policy, Msg, TargetProcess, Codec, Opts). +apply_security(policy, Msg, TargetProcess, Codec, Opts) -> + case hb_ao:get(<<"policy">>, TargetProcess, not_found, Opts) of + not_found -> apply_security(authority, Msg, TargetProcess, Codec, Opts); + Policy -> + case hb_ao:resolve(Policy, Opts) of + {ok, PolicyOpts} -> + case hb_ao:get(<<"accept-committers">>, PolicyOpts, Opts) of + not_found -> + apply_security( + authority, + Msg, + TargetProcess, + Codec, + Opts + ); + Committers -> + commit_result(Msg, Committers, Codec, Opts) + end; + {error, Error} -> + ?event(push, {policy_error, {error, Error}}, Opts), + apply_security(authority, Msg, TargetProcess, Codec, Opts) + end + end; +apply_security(authority, Msg, TargetProcess, Codec, Opts) -> + case hb_ao:get(<<"authority">>, TargetProcess, Opts) of + not_found -> apply_security(default, Msg, TargetProcess, Codec, Opts); + Authorities when is_list(Authorities) -> + % The `authority` key has already been parsed into a list of + % committers. Sign with all local valid keys. + commit_result(Msg, Authorities, Codec, Opts); + Authority -> + % Parse the authority string into a list of committers. Sign with + % all local valid keys. + ?event(push, {found_authority, {authority, Authority}}, Opts), + commit_result( + Msg, + hb_util:binary_to_addresses(Authority), + Codec, + Opts + ) + end; +apply_security(default, Msg, TargetProcess, Codec, Opts) -> + ?event(push, {default_policy, {target, TargetProcess}}, Opts), + commit_result( + Msg, + [hb_util:human_id(hb_opts:get(priv_wallet, no_viable_wallet, Opts))], + Codec, + Opts + ). + +% @doc Attempt to sign a result message with the given committers. +commit_result(Msg, [], Codec, Opts) -> + case hb_opts:get(push_always_sign, true, Opts) of + true -> hb_message:commit(hb_message:uncommitted(Msg), Opts, Codec); + false -> Msg + end; +commit_result(Msg, Committers, Codec, Opts) -> + Signed = lists:foldl( + fun(Committer, Acc) -> + case hb_opts:as(Committer, Opts) of + {ok, CommitterOpts} -> + ?event(debug_commit, {signing_with_identity, Committer}), + hb_message:commit(Acc, CommitterOpts, Codec); + {error, not_found} -> + ?event(debug_commit, desired_signer_not_available_on_node), + ?event(push, + {policy_warning, + { + unknown_committer, + Committer + } + }, + Opts + ), + Acc + end + end, + hb_message:uncommitted(Msg), + Committers + ), + ?event(debug_commit, + {signed_message_as, {explicit, hb_message:signers(Signed, Opts)}} + ), + case hb_message:signers(Signed, Opts) of + [] -> + ?event(debug_commit, signing_with_default_identity), + commit_result(Msg, [], Codec, Opts); + _FoundSigners -> + Signed + end. + +%% @doc Push a message or a process, prior to pushing the resulting slot number. +schedule_initial_message(Base, Req, Opts) -> + ModReq = Req#{ <<"path">> => <<"schedule">>, <<"method">> => <<"POST">> }, + ?event(push, {initial_push, {base, Base}, {req, ModReq}}, Opts), + case hb_ao:resolve(Base, ModReq, Opts) of + {ok, Res} -> + case hb_ao:get(<<"status">>, Res, 200, Opts) of + 200 -> {ok, Res}; + 307 -> + Location = hb_ao:get(<<"location">>, Res, Opts), + remote_schedule_result(Location, Req, Opts) + end; + {error, Res = #{ <<"status">> := 422 }} -> + ?event(push, {initial_push_wrong_format, {error, Res}}, Opts), + {error, Res}; + {error, Res} -> + ?event(push, {initial_push_error, {error, Res}}, Opts), + {error, Res} + end. + +remote_schedule_result(Location, SignedReq, Opts) -> + ?event(push, {remote_schedule_result, {location, Location}, {req, SignedReq}}, Opts), + {Node, RedirectPath} = parse_redirect(Location, Opts), + Path = + case find_type(SignedReq, Opts) of + <<"Process">> -> <<"/schedule">>; + <<"Message">> -> RedirectPath + end, + % Store a copy of the message for ourselves. + {ok, _} = hb_cache:write(SignedReq, Opts), + ?event(push, {remote_schedule_result, {path, Path}}, Opts), + case hb_http:post(Node, Path, hb_maps:without([<<"path">>], SignedReq, Opts), Opts) of + {ok, Res} -> + ?event(push, {remote_schedule_result, {res, Res}}, Opts), + case hb_ao:get(<<"status">>, Res, 200, Opts) of + 200 -> {ok, Res}; + 307 -> + NewLocation = hb_ao:get(<<"location">>, Res, Opts), + remote_schedule_result(NewLocation, SignedReq, Opts) + end; + {error, Res} -> + {error, Res} + end. + +find_type(Req, Opts) -> + hb_ao:get_first( + [ + {Req, <<"type">>}, + {Req, <<"body/type">>} + ], + Opts + ). + +parse_redirect(Location, Opts) -> + Parsed = uri_string:parse(Location), + Node = + uri_string:recompose( + (hb_maps:remove(query, Parsed, Opts))#{ + path => <<"/schedule">> + } + ), + {Node, hb_maps:get(path, Parsed, undefined, Opts)}. + +%%% Tests + +full_push_test_() -> + {timeout, 30, fun() -> + dev_process:init(), + Opts = #{ + process_async_cache => false, + priv_wallet => hb:wallet(), + cache_control => <<"always">>, + store => [ + #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-TEST">> }, + #{ <<"store-module">> => hb_store_gateway, + <<"store">> => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + } + } + ] + }, + Msg1 = dev_process:test_aos_process(Opts), + hb_cache:write(Msg1, Opts), + {ok, SchedInit} = + hb_ao:resolve(Msg1, #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => Msg1 + }, + Opts + ), + ?event({test_setup, {msg1, Msg1}, {sched_init, SchedInit}}), + Script = ping_pong_script(2), + ?event({script, Script}), + {ok, Msg2} = dev_process:schedule_aos_call(Msg1, Script, Opts), + ?event({msg_sched_result, Msg2}), + {ok, StartingMsgSlot} = + hb_ao:resolve(Msg2, #{ <<"path">> => <<"slot">> }, Opts), + ?event({starting_msg_slot, StartingMsgSlot}), + Msg3 = + #{ + <<"path">> => <<"push">>, + <<"slot">> => StartingMsgSlot + }, + {ok, _} = hb_ao:resolve(Msg1, Msg3, Opts), + ?assertEqual( + {ok, <<"Done.">>}, + hb_ao:resolve(Msg1, <<"now/results/data">>, Opts) + ) + end}. + +push_as_identity_test_() -> + {timeout, 90, fun() -> + dev_process:init(), + % Create a new identity for the scheduler. + DefaultWallet = hb:wallet(), + SchedulingWallet = ar_wallet:new(), + SchedulingID = hb_util:human_id(SchedulingWallet), + ComputeWallet = ar_wallet:new(), + ComputeID = hb_util:human_id(ComputeWallet), + Opts = #{ + priv_wallet => DefaultWallet, + cache_control => <<"always">>, + identities => #{ + SchedulingID => #{ + priv_wallet => SchedulingWallet, + store => [hb_test_utils:test_store()] + }, + ComputeID => #{ + priv_wallet => ComputeWallet + } + } + }, + % Create a new test AOS process, which will use the given identities as + % its authority and scheduler. + Msg1 = + dev_process:test_aos_process( + Opts#{ + authority => ComputeID, + scheduler => [SchedulingID, ComputeID] + } + ), + ?event({msg1, Msg1}), + % Perform the remainder of the test as with `full_push_test_/0'. + hb_cache:write(Msg1, Opts), + {ok, SchedInit} = + hb_ao:resolve(Msg1, #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => Msg1 + }, + Opts + ), + ?event({test_setup, {msg1, Msg1}, {sched_init, SchedInit}}), + Script = ping_pong_script(2), + ?event({script, Script}), + {ok, Msg2} = dev_process:schedule_aos_call(Msg1, Script), + ?event(push, {msg_sched_result, Msg2}), + {ok, StartingMsgSlot} = + hb_ao:resolve(Msg2, #{ <<"path">> => <<"slot">> }, Opts), + ?event({starting_msg_slot, StartingMsgSlot}), + Msg3 = + #{ + <<"path">> => <<"push">>, + <<"slot">> => StartingMsgSlot + }, + {ok, _} = hb_ao:resolve(Msg1, Msg3, Opts), + ?assertEqual( + {ok, <<"Done.">>}, + hb_ao:resolve(Msg1, <<"now/results/data">>, Opts) + ), + % Validate that the scheduler's wallet was used to sign the message. + Committers = + hb_ao:get( + <<"schedule/assignments/2/committers">>, + Msg1, + Opts + ), + ?assert(lists:member(SchedulingID, Committers)), + ?assert(lists:member(ComputeID, Committers)), + % Validate that the compute wallet was used to sign the message. + ?assertEqual( + [ComputeID], + hb_ao:get(<<"schedule/assignments/2/body/committers">>, Msg1, Opts) + ) + end}. + +multi_process_push_test_() -> + {timeout, 30, fun() -> + dev_process:init(), + Opts = #{ + priv_wallet => hb:wallet(), + cache_control => <<"always">> + }, + Proc1 = dev_process:test_aos_process(Opts), + hb_cache:write(Proc1, Opts), + {ok, _SchedInit1} = + hb_ao:resolve(Proc1, #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => Proc1 + }, + Opts + ), + {ok, _} = dev_process:schedule_aos_call(Proc1, reply_script()), + Proc2 = dev_process:test_aos_process(Opts), + hb_cache:write(Proc2, Opts), + {ok, _SchedInit2} = + hb_ao:resolve(Proc2, #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => Proc2 + }, + Opts + ), + ProcID1 = hb_message:id(Proc1, all, Opts), + ProcID2 = hb_message:id(Proc2, all, Opts), + ?event(push, {testing_with, {proc1_id, ProcID1}, {proc2_id, ProcID2}}), + {ok, ToPush} = dev_process:schedule_aos_call( + Proc2, + << + "Handlers.add(\"Pong\",\n" + " function (test) return true end,\n" + " function(m)\n" + " print(\"GOT PONG\")\n" + " end\n" + ")\n" + "Send({ Target = \"", (ProcID1)/binary, "\", Action = \"Ping\" })" + >> + ), + SlotToPush = hb_ao:get(<<"slot">>, ToPush, Opts), + ?event(push, {slot_to_push_proc2, SlotToPush}), + Msg3 = + #{ + <<"path">> => <<"push">>, + <<"slot">> => SlotToPush, + <<"result-depth">> => 1 + }, + {ok, PushResult} = hb_ao:resolve(Proc2, Msg3, Opts), + ?event(push, {push_result_proc2, PushResult}), + AfterPush = hb_ao:resolve(Proc2, <<"now/results/data">>, Opts), + ?event(push, {after_push, AfterPush}), + ?assertEqual({ok, <<"GOT PONG">>}, AfterPush) + end}. + +push_with_redirect_hint_test_disabled() -> + {timeout, 30, fun() -> + dev_process:init(), + Stores = + [ + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + } + ], + ExtOpts = #{ priv_wallet => ar_wallet:new(), store => Stores }, + LocalOpts = #{ priv_wallet => hb:wallet(), store => Stores }, + ExtScheduler = hb_http_server:start_node(ExtOpts), + ?event(push, {external_scheduler, {location, ExtScheduler}}), + % Create the Pong server and client + Client = dev_process:test_aos_process(), + PongServer = dev_process:test_aos_process(ExtOpts), + % Push the new process that runs on the external scheduler + {ok, ServerSchedResp} = + hb_http:post( + ExtScheduler, + <<"/push">>, + PongServer, + ExtOpts + ), + ?event(push, {pong_server_sched_resp, ServerSchedResp}), + % Get the IDs of the server process + PongServerID = + hb_ao:get( + <<"process/id">>, + dev_process:ensure_process_key(PongServer, LocalOpts), + LocalOpts + ), + {ok, ServerScriptSchedResp} = + hb_http:post( + ExtScheduler, + <>, + #{ + <<"body">> => + hb_message:commit( + #{ + <<"target">> => PongServerID, + <<"action">> => <<"Eval">>, + <<"type">> => <<"Message">>, + <<"data">> => reply_script() + }, + ExtOpts + ) + }, + ExtOpts + ), + ?event(push, {pong_server_script_sched_resp, ServerScriptSchedResp}), + {ok, ToPush} = + dev_process:schedule_aos_call( + Client, + << + "Handlers.add(\"Pong\",\n" + " function (test) return true end,\n" + " function(m)\n" + " print(\"GOT PONG\")\n" + " end\n" + ")\n" + "Send({ Target = \"", + (PongServerID)/binary, "?hint=", + (ExtScheduler)/binary, + "\", Action = \"Ping\" })\n" + >>, + LocalOpts + ), + SlotToPush = hb_ao:get(<<"slot">>, ToPush, LocalOpts), + ?event(push, {slot_to_push_client, SlotToPush}), + Msg3 = #{ <<"path">> => <<"push">>, <<"slot">> => SlotToPush }, + {ok, PushResult} = hb_ao:resolve(Client, Msg3, LocalOpts), + ?event(push, {push_result_client, PushResult}), + AfterPush = hb_ao:resolve(Client, <<"now/results/data">>, LocalOpts), + ?event(push, {after_push, AfterPush}), + % Note: This test currently only gets a reply that the message was not + % trusted by the process. To fix this, we would have to add another + % trusted authority to the `test_aos_process' call. For now, this is + % enough to validate that redirects are pushed through correctly. + ?assertEqual({ok, <<"GOT PONG">>}, AfterPush) + end}. + +push_prompts_encoding_change_test_() -> + {timeout, 30, fun push_prompts_encoding_change/0}. +push_prompts_encoding_change() -> + dev_process:init(), + Opts = #{ + priv_wallet => hb:wallet(), + cache_control => <<"always">>, + store => + [ + #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-TEST">> }, + % Include a gateway store so that we can get the legacynet + % process when needed. + #{ <<"store-module">> => hb_store_gateway, + <<"store">> => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + } + } + ] + }, + Msg = hb_message:commit(#{ + <<"path">> => <<"push">>, + <<"method">> => <<"POST">>, + <<"target">> => <<"QQiMcAge5ZtxcUV7ruxpi16KYRE8UBP0GAAqCIJPXz0">>, + <<"action">> => <<"Eval">>, + <<"data">> => <<"print(\"Please ignore!\")">> + }, Opts), + ?event(push, {msg1, Msg}), + Res = + hb_ao:resolve_many( + [ + <<"QQiMcAge5ZtxcUV7ruxpi16KYRE8UBP0GAAqCIJPXz0">>, + {as, <<"process@1.0">>, <<>>}, + Msg + ], + Opts + ), + ?assertMatch({error, #{ <<"status">> := 422 }}, Res). + +oracle_push_test_() -> {timeout, 30, fun oracle_push/0}. +oracle_push() -> + dev_process:init(), + Client = dev_process:test_aos_process(), + {ok, _} = hb_cache:write(Client, #{}), + {ok, _} = dev_process:schedule_aos_call(Client, oracle_script()), + Msg3 = + #{ + <<"path">> => <<"push">>, + <<"slot">> => 0 + }, + {ok, PushResult} = hb_ao:resolve(Client, Msg3, #{ priv_wallet => hb:wallet() }), + ?event({result, PushResult}), + ComputeRes = + hb_ao:resolve( + Client, + <<"now/results/data">>, + #{ priv_wallet => hb:wallet() } + ), + ?event({compute_res, ComputeRes}), + ?assertMatch({ok, _}, ComputeRes). + +-ifdef(ENABLE_GENESIS_WASM). +%% @doc Test that a message that generates another message which resides on an +%% ANS-104 scheduler leads to `~push@1.0` re-signing the message correctly. +%% Requires `ENABLE_GENESIS_WASM' to be enabled. +nested_push_prompts_encoding_change_test_() -> + {timeout, 30, fun nested_push_prompts_encoding_change/0}. +nested_push_prompts_encoding_change() -> + dev_process:init(), + Opts = #{ + priv_wallet => hb:wallet(), + cache_control => <<"always">>, + store => hb_opts:get(store) + }, + ?event(push_debug, {opts, Opts}), + Msg1 = dev_process:test_aos_process(Opts), + hb_cache:write(Msg1, Opts), + {ok, SchedInit} = + hb_ao:resolve(Msg1, #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => Msg1 + }, + Opts + ), + ?event({test_setup, {msg1, Msg1}, {sched_init, SchedInit}}), + Script = message_to_legacynet_scheduler_script(), + ?event({script, Script}), + {ok, Msg2} = dev_process:schedule_aos_call(Msg1, Script), + ?event(push, {msg_sched_result, Msg2}), + {ok, StartingMsgSlot} = + hb_ao:resolve(Msg2, #{ <<"path">> => <<"slot">> }, Opts), + ?event({starting_msg_slot, StartingMsgSlot}), + Msg3 = + #{ + <<"path">> => <<"push">>, + <<"slot">> => StartingMsgSlot + }, + {ok, Res} = hb_ao:resolve(Msg1, Msg3, Opts), + ?event(push, {res, Res}), + Msg = hb_message:commit(#{ + <<"path">> => <<"push">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + #{ + <<"target">> => hb_message:id(Msg1, all, Opts), + <<"action">> => <<"Ping">> + }, + Opts + ) + }, Opts), + ?event(push, {msg1, Msg}), + Res2 = + hb_ao:resolve_many( + [ + hb_message:id(Msg1, all, Opts), + {as, <<"process@1.0">>, <<>>}, + Msg + ], + Opts + ), + ?assertMatch({ok, #{ <<"1">> := #{ <<"resulted-in">> := _ }}}, Res2). +-endif. +%%% Test helpers + +ping_pong_script(Limit) -> + << + "Handlers.add(\"Ping\",\n" + " function (test) return true end,\n" + " function(m)\n" + " C = tonumber(m.Count)\n" + " if C <= ", (integer_to_binary(Limit))/binary, " then\n" + " Send({ Target = ao.id, Action = \"Ping\", Count = C + 1 })\n" + " print(\"Ping\", C + 1)\n" + " else\n" + " print(\"Done.\")\n" + " end\n" + " end\n" + ")\n" + "Send({ Target = ao.id, Action = \"Ping\", Count = 1 })\n" + >>. + +reply_script() -> + << + """ + Handlers.add("Reply", + { Action = "Ping" }, + function(m) + print("Replying to...") + print(m.From) + Send({ Target = m.From, Action = "Reply", Message = "Pong!" }) + print("Done.") + end + ) + """ + >>. + +message_to_legacynet_scheduler_script() -> + << + """ + Handlers.add("Ping", + { Action = "Ping" }, + function(m) + print("Pinging...") + print(m.From) + Send({ + Target = "QQiMcAge5ZtxcUV7ruxpi16KYRE8UBP0GAAqCIJPXz0", + Action = "Ping" + }) + print("Done.") + end + ) + """ + >>. + +oracle_script() -> + << + """ + Handlers.add("Oracle", + function(m) + return true + end, + function(m) + print(m.Body) + end + ) + Send({ + target = ao.id, + resolve = "/~relay@1.0/call", + ["relay-path"] = "https://arweave.net" + }) + + """ + >>. \ No newline at end of file diff --git a/src/dev_query.erl b/src/dev_query.erl new file mode 100644 index 000000000..5416fe390 --- /dev/null +++ b/src/dev_query.erl @@ -0,0 +1,332 @@ +%%% @doc A discovery engine for searching for and returning messages found in +%%% a node's cache, through supported stores. +%%% +%%% This device supports various modes of matching, including: +%%% +%%% - `all' (default): Match all keys in the request message. +%%% - `base': Match all keys in the base message. +%%% - `only': Match only the key(s) specified in the `only' key. +%%% +%%% The `only' key can be a binary, a map, or a list of keys. If it is a binary, +%%% it is split on commas to get a list of keys to search for. If it is a message, +%%% it is used directly as the match spec. If it is a list, it is assumed to be +%%% a list of keys that we should select from the request or base message and +%%% use as the match spec. +%%% +%%% The `return' key can be used to specify the type of data to return. +%%% +%%% - `count': Return the number of matches. +%%% - `paths': Return the paths of the matches in a list. +%%% - `messages': Return the messages associated with each match in a list. +%%% - `first-path': Return the first path of the matches. +%%% - `first-message': Return the first message of the matches. +%%% - `boolean': Return a boolean indicating whether any matches were found. +-module(dev_query). +%%% Message matching API: +-export([info/1, only/3, all/3, base/3]). +%%% GraphQL API: +-export([graphql/3, has_results/3]). +%%% Test setup: +-export([test_setup/0]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%%% Keys that should typically be excluded from searches. +-define( + DEFAULT_EXCLUDES, + [<<"path">>, <<"commitments">>, <<"return">>, <<"exclude">>, <<"only">>] +). + +info(_Opts) -> + #{ + excludes => [<<"keys">>, <<"set">>], + default => fun default/4 + }. + +%% @doc Execute the query via GraphQL. +graphql(Req, Base, Opts) -> + dev_query_graphql:handle(Req, Base, Opts). + +%% @doc Return whether a GraphQL esponse in a message has transaction results. +%% This key is used in HB's gateway client multirequest configuration to +%% determine if the response from the node should be considered admissible. +has_results(Base, Req, Opts) -> + JSON = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, Base}, <<"body">>}, + {{as, <<"message@1.0">>, Req}, <<"body">>} + ], + <<"{}">>, + Opts + ), + Decoded = hb_json:decode(JSON), + ?event(debug_multi, {has_results, {decoded_json, Decoded}}), + case Decoded of + #{ <<"data">> := #{ <<"transactions">> := #{ <<"edges">> := Nodes } } } + when length(Nodes) > 0 -> + true; + _ -> false + end. + +%% @doc Search for the keys specified in the request message. +default(_, Base, Req, Opts) -> + all(Base, Req, Opts). + +%% @doc Search the node's store for all of the keys and values in the request, +%% aside from the `commitments' and `path' keys. +all(Base, Req, Opts) -> + match(Req, Base, Req, Opts). + +%% @doc Search the node's store for all of the keys and values in the base +%% message, aside from the `commitments' and `path' keys. +base(Base, Req, Opts) -> + match(Base, Base, Req, Opts). + +%% @doc Search only for the (list of) key(s) specified in `only' in the request. +%% The `only' key can be a binary, a map, or a list of keys. See the moduledoc +%% for semantics. +only(Base, Req, Opts) -> + case hb_maps:get(<<"only">>, Req, not_found, Opts) of + KeyBin when is_binary(KeyBin) -> + % The descriptor is a binary, so we split it on commas to get a + % list of keys to search for. If there is only one key, we + % return a list with that key. + match(binary:split(KeyBin, <<",">>, [global]), Base, Req, Opts); + Spec when is_map(Spec) -> + % The descriptor is a map, so we use it as the match spec. + match(Spec, Base, Req, Opts); + Keys when is_list(Keys) -> + % The descriptor is a list, so we assume that it is a list of + % keys that we should select from the request and use as the + % match spec. + match(Keys, Base, Req, Opts); + not_found -> + % We cannot find the key to match upon. Return an error. + {error, not_found} + end. + +%% @doc Match the request against the base message, using the keys to select +%% the values from the request and (if not found) the values from the base +%% message. +match(Keys, Base, Req, Opts) when is_list(Keys) -> + UserSpec = + maps:from_list( + lists:filtermap( + fun(Key) -> + % Search for the value in the request. If not found, + % look in the base message. + Value = + hb_maps:get( + Key, + Req, + hb_maps:get(Key, Base, not_found, Opts), + Opts + ), + if Value == not_found -> false; + true -> {true, {Key, Value}} + end + end, + Keys + ) + ), + match(UserSpec, Base, Req, Opts); +match(UserSpec, _Base, Req, Opts) -> + ?event({matching, {spec, UserSpec}}), + FilteredSpec = + hb_maps:without( + hb_maps:get(<<"exclude">>, Req, ?DEFAULT_EXCLUDES, Opts), + UserSpec + ), + ReturnType = hb_maps:get(<<"return">>, Req, <<"paths">>, Opts), + ?event({matching, {spec, FilteredSpec}, {return, ReturnType}}), + case hb_cache:match(FilteredSpec, Opts) of + {ok, Matches} when ReturnType == <<"count">> -> + ?event({matched, {paths, Matches}}), + {ok, length(Matches)}; + {ok, Matches} when ReturnType == <<"paths">> -> + ?event({matched, {paths, Matches}}), + {ok, Matches}; + {ok, Matches} when ReturnType == <<"messages">> -> + ?event({matched, {paths, Matches}}), + Messages = + lists:map( + fun(Path) -> + hb_util:ok(hb_cache:read(Path, Opts)) + end, + Matches + ), + ?event({matched, {messages, Messages}}), + {ok, Messages}; + {ok, Matches} when ReturnType == <<"first-path">> -> + ?event({matched, {paths, Matches}}), + {ok, hd(Matches)}; + {ok, Matches} when ReturnType == <<"first">> + orelse ReturnType == <<"first-message">> -> + ?event({matched, {paths, Matches}}), + {ok, hb_util:ok(hb_cache:read(hd(Matches), Opts))}; + {ok, Matches} when ReturnType == <<"boolean">> -> + ?event({matched, {paths, Matches}}), + {ok, length(Matches) > 0}; + not_found when ReturnType == <<"boolean">> -> + {ok, false}; + not_found -> + {error, not_found} + end. + +%%% Tests + +%% @doc Return test options with a test store. +test_setup() -> + Store = hb_test_utils:test_store(hb_store_lmdb), + Opts = #{ store => Store, priv_wallet => hb:wallet() }, + % Write a simple message. + hb_cache:write( + #{ + <<"basic">> => <<"binary-value">>, + <<"basic-2">> => <<"binary-value-2">> + }, + Opts + ), + % Write a nested and committed message. + hb_cache:write( + hb_message:commit( + #{ + <<"test-key">> => <<"test-value">>, + <<"test-key-2">> => <<"test-value-2">>, + <<"nested">> => Nested = #{ + <<"test-key-3">> => <<"test-value-3">>, + <<"test-key-4">> => <<"test-value-4">> + } + }, + Opts + ), + Opts + ), + % Write a list message with complex keys. + hb_cache:write([<<"a">>, 2, ok], Opts), + {ok, Opts, #{ <<"nested">> => hb_message:id(Nested, all, Opts) }}. + +%% @doc Search for and find a basic test key. +basic_test() -> + {ok, Opts, _} = test_setup(), + {ok, [ID]} = hb_ao:resolve(<<"~query@1.0/all?basic=binary-value">>, Opts), + {ok, Read} = hb_cache:read(ID, Opts), + ?assertEqual(<<"binary-value">>, hb_maps:get(<<"basic">>, Read)), + ?assertEqual(<<"binary-value-2">>, hb_maps:get(<<"basic-2">>, Read)), + {ok, [Msg]} = + hb_ao:resolve( + <<"~query@1.0/all?basic-2=binary-value-2&return=messages">>, + Opts + ), + ?assertEqual(<<"binary-value-2">>, hb_maps:get(<<"basic-2">>, Msg)), + ok. + +%% @doc Ensure that we can search for and match only a single key. +only_test() -> + {ok, Opts, _} = test_setup(), + {ok, [Msg]} = + hb_ao:resolve( + <<"~query@1.0/only=basic&basic=binary-value&wrong=1&return=messages">>, + Opts + ), + ?assertEqual(<<"binary-value">>, hb_maps:get(<<"basic">>, Msg)), + ok. + +%% @doc Ensure that we can specify multiple keys to match. +multiple_test() -> + {ok, Opts, _} = test_setup(), + {ok, [Msg]} = + hb_ao:resolve( + << + "~query@1.0/only=basic,basic-2", + "&basic=binary-value&basic-2=binary-value-2", + "&return=messages" + >>, + Opts + ), + ?assertEqual(<<"binary-value">>, hb_maps:get(<<"basic">>, Msg)), + ?assertEqual(<<"binary-value-2">>, hb_maps:get(<<"basic-2">>, Msg)), + ok. + +%% @doc Search for and find a nested test key. +nested_test() -> + {ok, Opts, _} = test_setup(), + {ok, [MsgWithNested]} = + hb_ao:resolve( + <<"~query@1.0/all?test-key=test-value&return=messages">>, + Opts + ), + ?assert(hb_maps:is_key(<<"nested">>, MsgWithNested, Opts)), + Nested = hb_maps:get(<<"nested">>, MsgWithNested, undefined, Opts), + ?assertEqual(<<"test-value-3">>, hb_maps:get(<<"test-key-3">>, Nested, Opts)), + ?assertEqual(<<"test-value-4">>, hb_maps:get(<<"test-key-4">>, Nested, Opts)), + ok. + +%% @doc Search for and find a list message with typed elements. +list_test() -> + {ok, Opts, _} = test_setup(), + {ok, [Msg]} = + hb_ao:resolve( + <<"~query@1.0/all?2+integer=2&3+atom=ok&return=messages">>, + Opts + ), + ?assertEqual([<<"a">>, 2, ok], Msg), + ok. + +%% @doc Ensure user's can opt not to specify a key to resolve, instead specifying +%% only the matchable keys in the message. +return_key_test() -> + {ok, Opts, _} = test_setup(), + {ok, [ID]} = + hb_ao:resolve( + <<"~query@1.0/basic=binary-value">>, + Opts + ), + {ok, Msg} = hb_cache:read(ID, Opts), + ?assertEqual(<<"binary-value">>, hb_maps:get(<<"basic">>, Msg, Opts)), + ok. + +%% @doc Validate the functioning of various return types. +return_types_test() -> + {ok, Opts, _} = test_setup(), + {ok, [Msg]} = + hb_ao:resolve( + <<"~query@1.0/basic=binary-value&return=messages">>, + Opts + ), + ?assertEqual(<<"binary-value">>, hb_maps:get(<<"basic">>, Msg, Opts)), + ?assertEqual( + {ok, 1}, + hb_ao:resolve( + <<"~query@1.0/basic=binary-value&return=count">>, + Opts + ) + ), + ?assertEqual( + {ok, true}, + hb_ao:resolve( + <<"~query@1.0/basic=binary-value&return=boolean">>, + Opts + ) + ), + ?assertEqual( + {ok, <<"binary-value">>}, + hb_ao:resolve( + <<"~query@1.0/basic=binary-value&return=first-message/basic">>, + Opts + ) + ), + ok. + +http_test() -> + {ok, Opts, _} = test_setup(), + Node = hb_http_server:start_node(Opts), + {ok, Msg} = + hb_http:get( + Node, + <<"~query@1.0/only=basic&basic=binary-value?return=first">>, + Opts + ), + ?assertEqual(<<"binary-value">>, hb_maps:get(<<"basic">>, Msg, Opts)), + ok. \ No newline at end of file diff --git a/src/dev_query_arweave.erl b/src/dev_query_arweave.erl new file mode 100644 index 000000000..ed7e53670 --- /dev/null +++ b/src/dev_query_arweave.erl @@ -0,0 +1,310 @@ +%%% @doc An implementation of the Arweave GraphQL API, inside the `~query@1.0' +%%% device. +-module(dev_query_arweave). +%%% AO-Core API: +-export([query/4]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% @doc The arguments that are supported by the Arweave GraphQL API. +-define(SUPPORTED_QUERY_ARGS, + [ + <<"height">>, + <<"id">>, + <<"ids">>, + <<"tags">>, + <<"owners">>, + <<"recipients">> + ] +). + +%% @doc Handle an Arweave GraphQL query for either transactions or blocks. +query(List, <<"edges">>, _Args, _Opts) -> + {ok, [{ok, Msg} || Msg <- List]}; +query(Msg, <<"node">>, _Args, _Opts) -> + {ok, Msg}; +query(Obj, <<"transaction">>, Args, Opts) -> + case query(Obj, <<"transactions">>, Args, Opts) of + {ok, []} -> {ok, null}; + {ok, [Msg|_]} -> {ok, Msg} + end; +query(Obj, <<"transactions">>, Args, Opts) -> + ?event({transactions_query, + {object, Obj}, + {field, <<"transactions">>}, + {args, Args} + }), + Matches = match_args(Args, Opts), + ?event({transactions_matches, Matches}), + Messages = + lists:filtermap( + fun(Match) -> + case hb_cache:read(Match, Opts) of + {ok, Msg} -> {true, Msg}; + not_found -> false + end + end, + Matches + ), + {ok, Messages}; +query(Obj, <<"block">>, Args, Opts) -> + case query(Obj, <<"blocks">>, Args, Opts) of + {ok, []} -> {ok, null}; + {ok, [Msg|_]} -> {ok, Msg} + end; +query(Obj, <<"blocks">>, Args, Opts) -> + ?event({blocks, + {object, Obj}, + {field, <<"blocks">>}, + {args, Args} + }), + Matches = match_args(Args, Opts), + ?event({blocks_matches, Matches}), + Blocks = + lists:filtermap( + fun(Match) -> + case hb_cache:read(Match, Opts) of + {ok, Msg} -> {true, Msg}; + not_found -> false + end + end, + Matches + ), + % Return the blocks as a list of messages. + % Individual access methods are defined below. + {ok, Blocks}; +query(Block, <<"previous">>, _Args, Opts) -> + {ok, hb_maps:get(<<"previous_block">>, Block, null, Opts)}; +query(Block, <<"height">>, _Args, Opts) -> + {ok, hb_maps:get(<<"height">>, Block, null, Opts)}; +query(Block, <<"timestamp">>, _Args, Opts) -> + {ok, hb_maps:get(<<"timestamp">>, Block, null, Opts)}; +query(Msg, <<"signature">>, _Args, Opts) -> + % Return the signature of the transaction. + % Other TX access methods are defined below. + case hb_maps:get(<<"commitments">>, Msg, not_found, Opts) of + not_found -> {ok, null}; + Commitments -> + case maps:to_list(Commitments) of + [] -> {ok, null}; + [{_CommitmentID, Commitment} | _] -> + {ok, hb_maps:get(<<"signature">>, Commitment, null, Opts)} + end + end; +query(Msg, <<"owner">>, _Args, Opts) -> + ?event({query_owner, Msg}), + case hb_message:commitments(#{ <<"committer">> => '_' }, Msg, Opts) of + not_found -> {ok, null}; + Commitments -> + case hb_maps:keys(Commitments) of + [] -> {ok, null}; + [CommID | _] -> + {ok, Commitment} = hb_maps:find(CommID, Commitments, Opts), + {ok, Address} = hb_maps:find(<<"committer">>, Commitment, Opts), + {ok, KeyID} = hb_maps:find(<<"keyid">>, Commitment, Opts), + Key = dev_codec_httpsig_keyid:remove_scheme_prefix(KeyID), + {ok, #{ + <<"address">> => Address, + <<"key">> => Key + }} + end + end; +query(#{ <<"key">> := Key }, <<"key">>, _Args, _Opts) -> + {ok, Key}; +query(#{ <<"address">> := Address }, <<"address">>, _Args, _Opts) -> + {ok, Address}; +query(Msg, <<"recipient">>, _Args, Opts) -> + case find_field_key(<<"field-target">>, Msg, Opts) of + {ok, null} -> {ok, <<"">>}; + OkRes -> OkRes + end; +query(Msg, <<"anchor">>, _Args, Opts) -> + case find_field_key(<<"field-anchor">>, Msg, Opts) of + {ok, null} -> {ok, <<"">>}; + {ok, Anchor} -> {ok, hb_util:human_id(Anchor)} + end; +query(Msg, <<"data">>, _Args, Opts) -> + Data = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, Msg}, <<"data">>}, + {{as, <<"message@1.0">>, Msg}, <<"body">>} + ], + <<>>, + Opts + ), + Type = hb_maps:get(<<"content-type">>, Msg, null, Opts), + {ok, #{ <<"data">> => Data, <<"type">> => Type }}; +query(#{ <<"data">> := Data }, <<"size">>, _Args, _Opts) -> + {ok, byte_size(Data)}; +query(#{ <<"type">> := Type }, <<"type">>, _Args, _Opts) -> + {ok, Type}; +query(Obj, Field, Args, _Opts) -> + ?event({unimplemented_transactions_query, + {object, Obj}, + {field, Field}, + {args, Args} + }), + {ok, <<"Not implemented.">>}. + +%% @doc Find and return a value from the fields of a message (from its +%% commitments). +find_field_key(Field, Msg, Opts) -> + case hb_message:commitments(#{ Field => '_' }, Msg, Opts) of + not_found -> {ok, null}; + Commitments -> + case hb_maps:keys(Commitments) of + [] -> {ok, null}; + [CommID | _] -> + {ok, Commitment} = hb_maps:find(CommID, Commitments, Opts), + case hb_maps:find(Field, Commitment, Opts) of + {ok, Value} -> {ok, Value}; + error -> {ok, null} + end + end + end. + +%% @doc Progressively generate matches from each argument for a transaction +%% query. +match_args(Args, Opts) when is_map(Args) -> + match_args( + maps:to_list( + maps:with( + ?SUPPORTED_QUERY_ARGS, + Args + ) + ), + [], + Opts + ). +match_args([], Results, Opts) -> + ?event({match_args_results, Results}), + Matches = + lists:foldl( + fun(Result, Acc) -> + hb_util:list_with(resolve_ids(Result, Opts), Acc) + end, + resolve_ids(hd(Results), Opts), + tl(Results) + ), + hb_util:unique( + lists:flatten( + [ + all_ids(ID, Opts) + || + ID <- Matches + ] + ) + ); +match_args([{Field, X} | Rest], Acc, Opts) -> + MatchRes = match(Field, X, Opts), + ?event({match, {field, Field}, {arg, X}, {match_res, MatchRes}}), + case MatchRes of + {ok, Result} -> + match_args(Rest, [Result | Acc], Opts); + _Error -> + match_args(Rest, Acc, Opts) + end. + +%% @doc Generate a match upon `tags' in the arguments, if given. +match(_, null, _) -> ignore; +match(<<"height">>, Heights, Opts) -> + Min = hb_maps:get(<<"min">>, Heights, 0, Opts), + Max = + case hb_maps:find(<<"max">>, Heights, Opts) of + {ok, GivenMax} -> GivenMax; + error -> + {ok, Latest} = dev_arweave_block_cache:latest(Opts), + Latest + end, + #{ store := ScopedStores } = scope(Opts), + {ok, + lists:filtermap( + fun(Height) -> + Path = dev_arweave_block_cache:path(Height, Opts), + case hb_store:type(ScopedStores, Path) of + not_found -> false; + _ -> {true, hb_store:resolve(ScopedStores, Path)} + end + end, + lists:seq(Min, Max) + ) + }; +match(<<"id">>, ID, _Opts) -> + {ok, [ID]}; +match(<<"ids">>, IDs, _Opts) -> + {ok, IDs}; +match(<<"tags">>, Tags, Opts) -> + hb_cache:match(dev_query_graphql:keys_to_template(Tags), Opts); +match(<<"owners">>, Owners, Opts) -> + {ok, matching_commitments(<<"committer">>, Owners, Opts)}; +match(<<"owner">>, Owner, Opts) -> + Res = matching_commitments(<<"committer">>, Owner, Opts), + ?event({match_owner, Owner, Res}), + {ok, Res}; +match(<<"recipients">>, Recipients, Opts) -> + {ok, matching_commitments(<<"field-target">>, Recipients, Opts)}; +match(UnsupportedFilter, _, _) -> + throw({unsupported_query_filter, UnsupportedFilter}). + +%% @doc Return the base IDs for messages that have a matching commitment. +matching_commitments(Field, Values, Opts) when is_list(Values) -> + hb_util:unique(lists:flatten( + lists:map( + fun(Value) -> matching_commitments(Field, Value, Opts) end, + Values + ) + )); +matching_commitments(Field, Value, Opts) when is_binary(Value) -> + case hb_cache:match(#{ Field => Value }, Opts) of + {ok, IDs} -> + ?event( + {found_matching_commitments, + {field, Field}, + {value, Value}, + {ids, IDs} + } + ), + lists:map(fun(ID) -> commitment_id_to_base_id(ID, Opts) end, IDs); + not_found -> not_found + end. + +%% @doc Convert a commitment message's ID to a base ID. +commitment_id_to_base_id(ID, Opts) -> + Store = hb_opts:get(store, no_store, Opts), + ?event({commitment_id_to_base_id, ID}), + case hb_store:read(Store, << ID/binary, "/signature">>) of + {ok, EncSig} -> + Sig = hb_util:decode(EncSig), + ?event({commitment_id_to_base_id_sig, Sig}), + hb_util:encode(hb_crypto:sha256(Sig)); + not_found -> not_found + end. + +%% @doc Find all IDs for a message, by any of its other IDs. +all_ids(ID, Opts) -> + Store = hb_opts:get(store, no_store, Opts), + case hb_store:list(Store, << ID/binary, "/commitments">>) of + {ok, []} -> [ID]; + {ok, CommitmentIDs} -> CommitmentIDs; + _ -> [] + end. + +%% @doc Scope the stores used for block matching. The searched stores can be +%% scoped by setting the `query_arweave_scope' option. +scope(Opts) -> + Scope = hb_opts:get(query_arweave_scope, [local], Opts), + hb_store:scope(Opts, Scope). + +%% @doc Resolve a list of IDs to their store paths, using the stores provided. +resolve_ids(IDs, Opts) -> + Scoped = scope(Opts), + lists:map( + fun(ID) -> + case hb_cache:read(ID, Opts) of + {ok, Msg} -> hb_message:id(Msg, uncommitted, Scoped); + not_found -> ID + end + end, + IDs + ). \ No newline at end of file diff --git a/src/dev_query_graphql.erl b/src/dev_query_graphql.erl new file mode 100644 index 000000000..b3db8d3bc --- /dev/null +++ b/src/dev_query_graphql.erl @@ -0,0 +1,450 @@ +%%% @doc A GraphQL interface for querying a node's cache. Accessible through the +%%% `~query@1.0/graphql' device key. +-module(dev_query_graphql). +%%% AO-Core API: +-export([handle/3]). +%%% GraphQL Callbacks: +-export([execute/4]). +%%% Submodule helpers: +-export([keys_to_template/1, test_query/3, test_query/4]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%%% Constants. +-define(DEFAULT_QUERY_TIMEOUT, 10000). +-define(START_TIMEOUT, 3000). + +%%% `Message' query keys. +-define(MESSAGE_QUERY_KEYS, + [ + <<"id">>, + <<"message">>, + <<"keys">>, + <<"tags">>, + <<"name">>, + <<"value">> + ] +). + +%% @doc Returns the complete GraphQL schema. +schema() -> + hb_util:ok(file:read_file("scripts/schema.gql")). + +%% @doc Ensure that the GraphQL schema and context are initialized. Can be +%% called many times. +ensure_started() -> ensure_started(#{}). +ensure_started(Opts) -> + case hb_name:lookup(graphql_controller) of + PID when is_pid(PID) -> ok; + undefined -> + Parent = self(), + PID = + spawn_link( + fun() -> + init(Opts), + Parent ! {started, self()}, + receive stop -> ok end + end + ), + receive {started, PID} -> ok + after ?START_TIMEOUT -> exit(graphql_start_timeout) + end + end. + +%% @doc Initialize the GraphQL schema and context. Should only be called once. +init(_Opts) -> + ?event(graphql_init_called), + application:ensure_all_started(graphql), + ?event(graphql_application_started), + GraphQLOpts = + #{ + scalars => #{ default => ?MODULE }, + interfaces => #{ default => ?MODULE }, + unions => #{ default => ?MODULE }, + objects => #{ default => ?MODULE }, + enums => #{ default => ?MODULE } + }, + ok = graphql:load_schema(GraphQLOpts, schema()), + ?event(graphql_schema_loaded), + Root = + {root, + #{ + query => 'Query', + interfaces => [] + } + }, + ok = graphql:insert_schema_definition(Root), + ?event(graphql_schema_definition_inserted), + ok = graphql:validate_schema(), + ?event(graphql_schema_validated), + hb_name:register(graphql_controller, self()), + ?event(graphql_controller_registered), + ok. + +handle(_Base, RawReq, Opts) -> + ?event({request, RawReq}), + Req = + case hb_maps:find(<<"query">>, RawReq, Opts) of + {ok, _} -> RawReq; + error -> + % Parse the query, assuming that the request body is a JSON + % object with the necessary fields. + hb_json:decode(hb_maps:get(<<"body">>, RawReq, <<>>, Opts)) + end, + ?event({request, {processed, Req}}), + Query = hb_maps:get(<<"query">>, Req, <<>>, Opts), + OpName = hb_maps:get(<<"operationName">>, Req, undefined, Opts), + Vars = hb_maps:get(<<"variables">>, Req, #{}, Opts), + ?event( + {graphql_run_called, + {query, Query}, + {operation, OpName}, + {variables, Vars} + } + ), + ensure_started(), + case graphql:parse(Query) of + {ok, AST} -> + ?event(graphql_parsed), + try + ?event(graphql_type_checking), + {ok, #{fun_env := FunEnv, ast := AST2 }} = graphql:type_check(AST), + ?event(graphql_type_checked_successfully), + ok = graphql:validate(AST2), + ?event(graphql_validated), + Coerced = graphql:type_check_params(FunEnv, OpName, Vars), + ?event(graphql_type_checked_params), + Ctx = + #{ + params => Coerced, + operation_name => OpName, + default_timeout => + hb_opts:get( + query_timeout, + ?DEFAULT_QUERY_TIMEOUT, + Opts + ), + opts => Opts + }, + ?event(graphql_context_created), + Response = graphql:execute(Ctx, AST2), + ?event(graphql_executed), + JSON = hb_json:encode(Response), + ?event({graphql_response, {bytes, byte_size(JSON)}}), + {ok, + #{ + <<"content-type">> => <<"application/json">>, + <<"body">> => JSON + } + } + catch + throw:Error:Stacktrace -> + ?event({graphql_error, {error, Error}, {trace, Stacktrace}}), + {error, Error} + end + end. + +%% @doc The main entrypoint for resolving GraphQL elements, called by the +%% GraphQL library. We split the resolution flows into two separated functions: +%% `message_query/4' for the HyperBEAM native API, and `dev_query_arweave:query/4' +%% for the Arweave-compatible API. +execute(#{opts := Opts}, Obj, Field, Args) -> + ?event({graphql_query, {object, Obj}, {field, Field}, {args, Args}}), + case lists:member(Field, ?MESSAGE_QUERY_KEYS) of + true -> message_query(Obj, Field, Args, Opts); + false -> dev_query_arweave:query(Obj, Field, Args, Opts) + end. + +%% @doc Handle a HyperBEAM `message' query. +message_query(Obj, <<"message">>, #{<<"keys">> := Keys}, Opts) -> + Template = keys_to_template(Keys), + ?event( + {graphql_execute_called, + {object, Obj}, + {field, <<"message">>}, + {raw_keys, Keys}, + {template, Template} + } + ), + case hb_cache:match(Template, Opts) of + {ok, [ID | _IDs]} -> + ?event({graphql_cache_match_found, ID}), + {ok, Msg} = hb_cache:read(ID, Opts), + ?event({graphql_cache_read, Msg}), + {ok, Msg}; + not_found -> + ?event(graphql_cache_match_not_found), + {ok, #{<<"id">> => <<"not-found">>, <<"keys">> => #{}}} + end; +message_query(Msg, Field, _Args, Opts) when Field =:= <<"keys">>; Field =:= <<"tags">> -> + OnlyKeys = + hb_maps:to_list( + hb_private:reset( + hb_maps:without( + [<<"data">>, <<"body">>], + hb_message:uncommitted(Msg, Opts), + Opts + ) + ), + Opts + ), + ?event({message_query_keys_or_tags, {object, Msg}, {only_keys, OnlyKeys}}), + Res = { + ok, + [ + {ok, + #{ + <<"name">> => Name, + <<"value">> => hb_cache:ensure_loaded(Value, Opts) + } + } + || + {Name, Value} <- OnlyKeys + ] + }, + ?event({message_query_keys_or_tags_result, Res}), + Res; +message_query(Msg, Field, _Args, Opts) + when Field =:= <<"name">> orelse Field =:= <<"value">> -> + ?event({message_query_name_or_value, {object, Msg}, {field, Field}}), + {ok, hb_maps:get(Field, Msg, null, Opts)}; +message_query(Msg = #{ <<"independent_hash">> := _ }, <<"id">>, _Args, Opts) -> + {ok, hb_maps:get(<<"independent_hash">>, Msg, null, Opts)}; +message_query(Msg, <<"id">>, _Args, Opts) -> + ?event({message_query_id, {object, Msg}}), + {ok, hb_message:id(Msg, all, Opts)}; +message_query(_Obj, _Field, _, _) -> + {ok, <<"Not found.">>}. + +keys_to_template(Keys) -> + maps:from_list(lists:foldl( + fun(#{<<"name">> := Name, <<"value">> := Value}, Acc) -> + [{Name, Value} | Acc]; + (#{<<"name">> := Name, <<"values">> := [Value]}, Acc) -> + [{Name, Value} | Acc]; + (#{<<"name">> := Name, <<"values">> := Values}, _Acc) -> + throw( + {multivalue_tag_search_not_supported, #{ + <<"name">> => Name, + <<"values">> => Values + }} + ) + end, + [], + Keys + )). + +%%% Test helpers. + +test_query(Node, Query, Opts) -> + test_query(Node, Query, undefined, Opts). +test_query(Node, Query, Variables, Opts) -> + test_query(Node, Query, Variables, undefined, Opts). +test_query(Node, Query, Variables, OperationName, Opts) -> + UnencodedPayload = + maps:filter( + fun(_, undefined) -> false; + (_, _) -> true + end, + #{ + <<"query">> => Query, + <<"variables">> => Variables, + <<"operationName">> => OperationName + } + ), + ?event({test_query_unencoded_payload, UnencodedPayload}), + {ok, Res} = + hb_http:post( + Node, + #{ + <<"path">> => <<"~query@1.0/graphql">>, + <<"content-type">> => <<"application/json">>, + <<"codec-device">> => <<"json@1.0">>, + <<"body">> => hb_json:encode(UnencodedPayload) + }, + Opts + ), + hb_json:decode(hb_maps:get(<<"body">>, Res, <<>>, Opts)). + +%%% Tests + +lookup_test() -> + {ok, Opts, _} = dev_query:test_setup(), + Node = hb_http_server:start_node(Opts), + Query = + <<""" + query GetMessage { + message( + keys: + [ + { + name: "basic", + value: "binary-value" + } + ] + ) { + id + keys { + name + value + } + } + } + """>>, + Res = test_query(Node, Query, Opts), + ?event({test_response, Res}), + ?assertMatch( + #{ <<"data">> := + #{ + <<"message">> := + #{ + <<"id">> := _, + <<"keys">> := + [ + #{ + <<"name">> := <<"basic">>, + <<"value">> := <<"binary-value">> + }, + #{ + <<"name">> := <<"basic-2">>, + <<"value">> := <<"binary-value-2">> + } + ] + } + } + }, + Res + ). + +%%% Tests for the GraphQL interface of the dev_query module. +%%% This test checks if the GraphQL query can be executed with variables. +%%% NEED_TO_BE_FIXED: due to `application:ensure_all_started(graphql)` in `run/4`, +%%% only one test can be run at a time, as it will load the schema and context. +lookup_with_vars_test() -> + {ok, Opts, _} = dev_query:test_setup(), + Node = hb_http_server:start_node(Opts), + {ok, Res} = + hb_http:post( + Node, + #{ + <<"path">> => <<"~query@1.0/graphql">>, + <<"content-type">> => <<"application/json">>, + <<"codec-device">> => <<"json@1.0">>, + <<"body">> => + hb_json:encode(#{ + <<"query">> => + <<""" + query GetMessage($keys: [KeyInput]) { + message( + keys: $keys + ) { + id + keys { + name + value + } + } + } + """>>, + <<"operationName">> => <<"GetMessage">>, + <<"variables">> => #{ + <<"keys">> => + [ + #{ + <<"name">> => <<"basic">>, + <<"value">> => <<"binary-value">> + } + ] + } + }) + }, + Opts + ), + Object = hb_json:decode(hb_maps:get(<<"body">>, Res, <<>>, Opts)), + ?event({test_response, Object}), + ?assertMatch( + #{ <<"data">> := + #{ + <<"message">> := + #{ + <<"id">> := _, + <<"keys">> := + [ + #{ + <<"name">> := <<"basic">>, + <<"value">> := <<"binary-value">> + }, + #{ + <<"name">> := <<"basic-2">>, + <<"value">> := <<"binary-value-2">> + } + ] + } + } + }, + Object + ). + +lookup_without_opname_test() -> + {ok, Opts, _} = dev_query:test_setup(), + Node = hb_http_server:start_node(Opts), + {ok, Res} = + hb_http:post( + Node, + #{ + <<"path">> => <<"~query@1.0/graphql">>, + <<"content-type">> => <<"application/json">>, + <<"codec-device">> => <<"json@1.0">>, + <<"body">> => + hb_json:encode(#{ + <<"query">> => + <<""" + query($keys: [KeyInput]) { + message( + keys: $keys + ) { + id + keys { + name + value + } + } + } + """>>, + <<"variables">> => #{ + <<"keys">> => + [ + #{ + <<"name">> => <<"basic">>, + <<"value">> => <<"binary-value">> + } + ] + } + }) + }, + Opts + ), + Object = hb_json:decode(hb_maps:get(<<"body">>, Res, <<>>, Opts)), + ?event({test_response, Object}), + ?assertMatch( + #{ <<"data">> := + #{ + <<"message">> := + #{ + <<"id">> := _, + <<"keys">> := + [ + #{ + <<"name">> := <<"basic">>, + <<"value">> := <<"binary-value">> + }, + #{ + <<"name">> := <<"basic-2">>, + <<"value">> := <<"binary-value-2">> + } + ] + } + } + }, + Object + ). \ No newline at end of file diff --git a/src/dev_query_test_vectors.erl b/src/dev_query_test_vectors.erl new file mode 100644 index 000000000..98b4715ca --- /dev/null +++ b/src/dev_query_test_vectors.erl @@ -0,0 +1,770 @@ +%%% @doc A suite of test queries and responses for the `~query@1.0' device's +%%% GraphQL implementation. +-module(dev_query_test_vectors). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% Test helpers. + +write_test_message(Opts) -> + hb_cache:write( + Msg = hb_message:commit( + #{ + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"type">> => <<"Message">>, + <<"action">> => <<"Eval">>, + <<"data">> => <<"test data">> + }, + Opts, + #{ + <<"commitment-device">> => <<"ans104@1.0">> + } + ), + Opts + ), + {ok, Msg}. + +%% @doc Populate the cache with three test blocks. +get_test_blocks(Node) -> + InitialHeight = 1745749, + FinalHeight = 1745750, + lists:foreach( + fun(Height) -> + {ok, _} = + hb_http:request( + <<"GET">>, + Node, + <<"/~arweave@2.9-pre/block=", (hb_util:bin(Height))/binary>>, + #{} + ) + end, + lists:seq(InitialHeight, FinalHeight) + ). + +%% Helper function to write test message with Recipient +write_test_message_with_recipient(Recipient, Opts) -> + hb_cache:write( + Msg = hb_message:commit( + #{ + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"type">> => <<"Message">>, + <<"action">> => <<"Eval">>, + <<"content-type">> => <<"text/plain">>, + <<"data">> => <<"test data">>, + <<"target">> => Recipient + }, + Opts, + #{ + <<"commitment-device">> => <<"ans104@1.0">> + } + ), + Opts + ), + {ok, Msg}. + +%%% Tests + +simple_blocks_query_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + get_test_blocks(Node), + Query = + <<""" + query { + blocks( + ids: ["V7yZNKPQLIQfUu8r8-lcEaz4o7idl6LTHn5AHlGIFF8TKfxIe7s_yFxjqan6OW45"] + ) { + edges { + node { + id + previous + height + timestamp + } + } + } + } + """>>, + ?assertMatch( + #{ + <<"data">> := #{ + <<"blocks">> := #{ + <<"edges">> := [ + #{ + <<"node">> := #{ + <<"id">> := _, + <<"previous">> := _, + <<"height">> := 1745749, + <<"timestamp">> := 1756866695 + } + } + ] + } + } + }, + dev_query_graphql:test_query(Node, Query, #{}, Opts) + ). + +block_by_height_query_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + get_test_blocks(Node), + Query = + <<""" + query { + blocks( height: {min: 1745749, max: 1745750} ) { + edges { + node { + id + previous + height + timestamp + } + } + } + } + """>>, + ?assertMatch( + #{ + <<"data">> := #{ + <<"blocks">> := #{ + <<"edges">> := [ + #{ + <<"node">> := #{ + <<"id">> := _, + <<"previous">> := _, + <<"height">> := 1745749, + <<"timestamp">> := 1756866695 + } + }, + #{ + <<"node">> := #{ + <<"id">> := _, + <<"previous">> := _, + <<"height">> := 1745750, + <<"timestamp">> := _ + } + } + ] + } + } + }, + dev_query_graphql:test_query(Node, Query, #{}, Opts) + ). + +simple_ans104_query_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + {ok, WrittenMsg} = write_test_message(Opts), + ?assertMatch( + {ok, [_]}, + hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) + ), + Query = + <<""" + query($owners: [String!]) { + transactions( + tags: + [ + {name: "type" values: ["Message"]}, + {name: "variant" values: ["ao.N.1"]} + ], + owners: $owners + ) { + edges { + node { + id, + tags { + name, + value + } + } + } + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{ + <<"owners">> => [hb:address()] + }, + Opts + ), + ExpectedID = hb_message:id(WrittenMsg, all, Opts), + ?event({expected_id, ExpectedID}), + ?event({simple_ans104_query_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := + [#{ + <<"node">> := + #{ + <<"id">> := ExpectedID, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + }] + } + } + } when ?IS_ID(ExpectedID), + Res + ). + +%% @doc Test transactions query with tags filter +transactions_query_tags_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + {ok, WrittenMsg} = write_test_message(Opts), + ?assertMatch( + {ok, [_]}, + hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) + ), + Query = + <<""" + query { + transactions( + tags: [ + {name: "type", values: ["Message"]}, + {name: "variant", values: ["ao.N.1"]} + ] + ) { + edges { + node { + id + tags { + name + value + } + } + } + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{}, + Opts + ), + ExpectedID = hb_message:id(WrittenMsg, all, Opts), + ?event({expected_id, ExpectedID}), + ?event({transactions_query_tags_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := + [#{ + <<"node">> := + #{ + <<"id">> := ExpectedID, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + }] + } + } + } when ?IS_ID(ExpectedID), + Res + ). + +%% @doc Test transactions query with owners filter +transactions_query_owners_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + {ok, WrittenMsg} = write_test_message(Opts), + ?assertMatch( + {ok, [_]}, + hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) + ), + Query = + <<""" + query($owners: [String!]) { + transactions( + owners: $owners + ) { + edges { + node { + id + tags { + name + value + } + } + } + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{ + <<"owners">> => [hb:address()] + }, + Opts + ), + ExpectedID = hb_message:id(WrittenMsg, all, Opts), + ?event({expected_id, ExpectedID}), + ?event({transactions_query_owners_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := + [#{ + <<"node">> := + #{ + <<"id">> := ExpectedID, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + }] + } + } + } when ?IS_ID(ExpectedID), + Res + ). + +%% @doc Test transactions query with recipients filter +transactions_query_recipients_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + Alice = ar_wallet:new(), + ?event({alice, Alice, {explicit, hb_util:human_id(Alice)}}), + AliceAddress = hb_util:human_id(Alice), + {ok, WrittenMsg} = write_test_message_with_recipient(AliceAddress, Opts), + ?assertMatch( + {ok, [_]}, + hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) + ), + Query = + <<""" + query($recipients: [String!]) { + transactions( + recipients: $recipients + ) { + edges { + node { + id + tags { + name + value + } + } + } + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{ + <<"recipients">> => [AliceAddress] + }, + Opts + ), + ExpectedID = hb_message:id(WrittenMsg, all, Opts), + ?event({expected_id, ExpectedID}), + ?event({transactions_query_recipients_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := + [#{ + <<"node">> := + #{ + <<"id">> := ExpectedID, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + }] + } + } + } when ?IS_ID(ExpectedID), + Res + ). + +%% @doc Test transactions query with ids filter +transactions_query_ids_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + {ok, WrittenMsg} = write_test_message(Opts), + ExpectedID = hb_message:id(WrittenMsg, all, Opts), + ?assertMatch( + {ok, [_]}, + hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) + ), + Query = + <<""" + query($ids: [ID!]) { + transactions( + ids: $ids + ) { + edges { + node { + id + tags { + name + value + } + } + } + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{ + <<"ids">> => [ExpectedID] + }, + Opts + ), + ?event({expected_id, ExpectedID}), + ?event({transactions_query_ids_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := + [#{ + <<"node">> := + #{ + <<"id">> := ExpectedID, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + }] + } + } + } when ?IS_ID(ExpectedID), + Res + ). + +%% @doc Test transactions query with combined filters +transactions_query_combined_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + {ok, WrittenMsg} = write_test_message(Opts), + ExpectedID = hb_message:id(WrittenMsg, all, Opts), + ?assertMatch( + {ok, [_]}, + hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) + ), + Query = + <<""" + query($owners: [String!], $ids: [ID!]) { + transactions( + owners: $owners, + ids: $ids, + tags: [ + {name: "type", values: ["Message"]} + ] + ) { + edges { + node { + id + tags { + name + value + } + } + } + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{ + <<"owners">> => [hb:address()], + <<"ids">> => [ExpectedID] + }, + Opts + ), + ?event({expected_id, ExpectedID}), + ?event({transactions_query_combined_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := + [#{ + <<"node">> := + #{ + <<"id">> := ExpectedID, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + }] + } + } + } when ?IS_ID(ExpectedID), + Res + ). + + +%% @doc Test single transaction query by ID +transaction_query_by_id_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + {ok, WrittenMsg} = write_test_message(Opts), + ExpectedID = hb_message:id(WrittenMsg, all, Opts), + ?assertMatch( + {ok, [_]}, + hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) + ), + Query = + <<""" + query($id: ID!) { + transaction(id: $id) { + id + tags { + name + value + } + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{ + <<"id">> => ExpectedID + }, + Opts + ), + ?event({expected_id, ExpectedID}), + ?event({transaction_query_by_id_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transaction">> := #{ + <<"id">> := ExpectedID, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + } + } when ?IS_ID(ExpectedID), + Res + ). + +%% @doc Test single transaction query with more fields +transaction_query_full_test() -> + Opts = + #{ + priv_wallet => SenderKey = hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + Alice = ar_wallet:new(), + ?event({alice, Alice, {explicit, hb_util:human_id(Alice)}}), + AliceAddress = hb_util:human_id(Alice), + SenderAddress = hb_util:human_id(SenderKey), + SenderPubKey = hb_util:encode(ar_wallet:to_pubkey(SenderKey)), + {ok, WrittenMsg} = write_test_message_with_recipient(AliceAddress, Opts), + ExpectedID = hb_message:id(WrittenMsg, all, Opts), + ?assertMatch( + {ok, [_]}, + hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) + ), + Query = + <<""" + query($id: ID!) { + transaction(id: $id) { + id + anchor + signature + recipient + owner { + address + key + } + tags { + name + value + } + data { + size + type + } + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{ + <<"id">> => ExpectedID + }, + Opts + ), + ?event({expected_id, ExpectedID}), + ?event({transaction_query_full_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transaction">> := #{ + <<"id">> := ExpectedID, + <<"recipient">> := AliceAddress, + <<"anchor">> := <<"">>, + <<"owner">> := #{ + <<"address">> := SenderAddress, + <<"key">> := SenderPubKey + }, + <<"data">> := #{ + <<"size">> := <<"9">>, + <<"type">> := <<"text/plain">> + }, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + % Note: other fields may be "Not implemented." for now + } + } + } when ?IS_ID(ExpectedID), + Res + ). + +%% @doc Test single transaction query with non-existent ID +transaction_query_not_found_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Res = + dev_query_graphql:test_query( + hb_http_server:start_node(Opts), + <<""" + query($id: ID!) { + transaction(id: $id) { + id + tags { + name + value + } + } + } + """>>, + #{ + <<"id">> => hb_util:encode(crypto:strong_rand_bytes(32)) + }, + Opts + ), + % Should return null for non-existent transaction + ?assertMatch( + #{ + <<"data">> := #{ + <<"transaction">> := null + } + }, + Res + ). + +%% @doc Test parsing, storing, and querying a transaction with an anchor. +transaction_query_with_anchor_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + {ok, ID} = + hb_cache:write( + hb_message:convert( + ar_bundles:sign_item( + #tx { + anchor = AnchorID = crypto:strong_rand_bytes(32), + data = <<"test-data">> + }, + hb:wallet() + ), + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ), + Opts + ), + EncodedAnchor = hb_util:encode(AnchorID), + Query = + <<""" + query($id: ID!) { + transaction(id: $id) { + data { + size + type + } + anchor + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{ + <<"id">> => ID + }, + Opts + ), + ?event({transaction_query_with_anchor_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transaction">> := #{ + <<"anchor">> := EncodedAnchor + } + } + }, + Res + ). \ No newline at end of file diff --git a/src/dev_relay.erl b/src/dev_relay.erl new file mode 100644 index 000000000..fe4602090 --- /dev/null +++ b/src/dev_relay.erl @@ -0,0 +1,300 @@ +%%% @doc This module implements the relay device, which is responsible for +%%% relaying messages between nodes and other HTTP(S) endpoints. +%%% +%%% It can be called in either `call' or `cast' mode. In `call' mode, it +%%% returns a `{ok, Result}' tuple, where `Result' is the response from the +%%% remote peer to the message sent. In `cast' mode, the invocation returns +%%% immediately, and the message is relayed asynchronously. No response is given +%%% and the device returns `{ok, <<"OK">>}'. +%%% +%%% Example usage: +%%% +%%% 
+%%%     curl /~relay@.1.0/call?method=GET?0.path=https://www.arweave.net/
+%%%
+-module(dev_relay). +%%% Execute synchronous and asynchronous relay requests. +-export([call/3, cast/3]). +%%% Re-route requests that would be executed locally to other peers, according +%%% to the node's routing table. +-export([request/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Execute a `call' request using a node's routes. +%% +%% Supports the following options: +%% - `target': The target message to relay. Defaults to the original message. +%% - `relay-path': The path to relay the message to. Defaults to the original path. +%% - `method': The method to use for the request. Defaults to the original method. +%% - `commit-request': Whether the request should be committed before dispatching. +%% Defaults to `false'. +call(M1, RawM2, Opts) -> + ?event({relay_call, {m1, M1}, {raw_m2, RawM2}}), + {ok, BaseTarget} = hb_message:find_target(M1, RawM2, Opts), + ?event({relay_call, {message_to_relay, BaseTarget}}), + RelayPath = + hb_ao:get_first( + [ + {M1, <<"path">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"path">>}, + {RawM2, <<"relay-path">>}, + {M1, <<"relay-path">>} + ], + Opts + ), + RelayDevice = + hb_ao:get_first( + [ + {M1, <<"relay-device">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"relay-device">>}, + {RawM2, <<"relay-device">>} + ], + Opts + ), + RelayPeer = + hb_ao:get_first( + [ + {M1, <<"peer">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"peer">>}, + {RawM2, <<"peer">>} + ], + Opts + ), + RelayMethod = + hb_ao:get_first( + [ + {M1, <<"method">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"method">>}, + {RawM2, <<"relay-method">>}, + {M1, <<"relay-method">>}, + {RawM2, <<"method">>} + ], + Opts + ), + RelayBody = + hb_ao:get_first( + [ + {M1, <<"body">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"body">>}, + {RawM2, <<"relay-body">>}, + {M1, <<"relay-body">>}, + {RawM2, <<"body">>} + ], + Opts + ), + Commit = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, BaseTarget}, <<"commit-request">>}, + {RawM2, <<"relay-commit-request">>}, + {M1, <<"relay-commit-request">>}, + {RawM2, <<"commit-request">>}, + {M1, <<"commit-request">>} + ], + false, + Opts + ), + TargetMod1 = + if RelayBody == not_found -> BaseTarget; + true -> BaseTarget#{<<"body">> => RelayBody} + end, + TargetMod2 = + TargetMod1#{ + <<"method">> => RelayMethod, + <<"path">> => RelayPath + }, + TargetMod3 = + case RelayDevice of + not_found -> hb_maps:without([<<"device">>], TargetMod2); + _ -> TargetMod2#{<<"device">> => RelayDevice} + end, + TargetMod4 = + case Commit of + true -> + case hb_opts:get(relay_allow_commit_request, false, Opts) of + true -> + ?event(debug_relay, {recommitting, TargetMod3}, Opts), + Committed = hb_message:commit(TargetMod3, Opts), + ?event(debug_relay, {relay_call, {committed, Committed}}, Opts), + true = hb_message:verify(Committed, all), + Committed; + false -> + throw(relay_commit_request_not_allowed) + end; + false -> TargetMod3 + end, + ?event(debug_relay, {relay_call, {without_http_params, TargetMod3}}), + ?event(debug_relay, {relay_call, {with_http_params, TargetMod4}}), + true = hb_message:verify(TargetMod4), + ?event(debug_relay, {relay_call, {verified, true}}), + Client = + case hb_maps:get(<<"http-client">>, BaseTarget, not_found, Opts) of + not_found -> hb_opts:get(relay_http_client, Opts); + RequestedClient -> RequestedClient + end, + % Let `hb_http:request/2' handle finding the peer and dispatching the + % request, unless the peer is explicitly given. + HTTPOpts = Opts#{ http_client => Client, http_only_result => false }, + Res = case RelayPeer of + not_found -> + hb_http:request(TargetMod4, HTTPOpts); + _ -> + ?event(debug_relay, {relaying_to_peer, RelayPeer}), + hb_http:request( + RelayMethod, + RelayPeer, + RelayPath, + TargetMod4, + HTTPOpts + ) + end, + case Res of + {ok, R} -> + {ok, hb_maps:without([<<"set-cookie">>], R)}; + Err -> Err + end. + + +%% @doc Execute a request in the same way as `call/3', but asynchronously. Always +%% returns `<<"OK">>'. +cast(M1, M2, Opts) -> + spawn(fun() -> call(M1, M2, Opts) end), + {ok, <<"OK">>}. + +%% @doc Preprocess a request to check if it should be relayed to a different node. +request(_Msg1, Msg2, Opts) -> + {ok, + #{ + <<"body">> => + [ + #{ <<"device">> => <<"relay@1.0">> }, + #{ + <<"path">> => <<"call">>, + <<"target">> => <<"body">>, + <<"body">> => + hb_ao:get(<<"request">>, Msg2, Opts#{ hashpath => ignore }) + } + ] + } + }. + + +%%% Tests + +call_get_test() -> + application:ensure_all_started([hb]), + {ok, #{<<"body">> := Body}} = + hb_ao:resolve( + #{ + <<"device">> => <<"relay@1.0">>, + <<"method">> => <<"GET">>, + <<"path">> => <<"https://www.google.com/">> + }, + <<"call">>, + #{ protocol => http2 } + ), + ?assertEqual(true, byte_size(Body) > 10_000). + +relay_nearest_test() -> + Peer1 = hb_http_server:start_node(#{ priv_wallet => W1 = ar_wallet:new() }), + Peer2 = hb_http_server:start_node(#{ priv_wallet => W2 = ar_wallet:new() }), + Address1 = hb_util:human_id(ar_wallet:to_address(W1)), + Address2 = hb_util:human_id(ar_wallet:to_address(W2)), + Peers = [Address1, Address2], + Node = + hb_http_server:start_node(Opts = #{ + store => hb_opts:get(store), + priv_wallet => ar_wallet:new(), + routes => [ + #{ + <<"template">> => <<"/.*">>, + <<"strategy">> => <<"Nearest">>, + <<"nodes">> => [ + #{ + <<"prefix">> => Peer1, + <<"wallet">> => Address1 + }, + #{ + <<"prefix">> => Peer2, + <<"wallet">> => Address2 + } + ] + } + ] + }), + {ok, RelayRes} = + hb_http:get( + Node, + <<"/~relay@1.0/call?relay-path=/~meta@1.0/info">>, + Opts#{ http_only_result => false } + ), + ?event( + {relay_res, + {response, RelayRes}, + {signer, hb_message:signers(RelayRes, Opts)}, + {peers, Peers} + } + ), + HasValidSigner = + lists:any( + fun(Peer) -> + lists:member(Peer, hb_message:signers(RelayRes, Opts)) + end, + Peers + ), + ?assert(HasValidSigner). + +%% @doc Test that a `relay@1.0/call' correctly commits requests as specified. +%% We validate this by configuring two nodes: One that will execute a given +%% request from a user, but only if the request is committed. The other node +%% re-routes all requests to the first node, using `call`'s `commit-request' +%% key to sign the request during proxying. The initial request is not signed, +%% such that the first node would otherwise reject the request outright. +commit_request_test() -> + Port = 10000 + rand:uniform(10000), + Wallet = ar_wallet:new(), + Executor = + hb_http_server:start_node( + #{ + port => Port, + force_signed_requests => true + } + ), + Node = + hb_http_server:start_node(#{ + priv_wallet => Wallet, + relay_allow_commit_request => true, + routes => + [ + #{ + <<"template">> => <<"/test-key">>, + <<"strategy">> => <<"Nearest">>, + <<"nodes">> => [ + #{ + <<"wallet">> => hb_util:human_id(Wallet), + <<"prefix">> => Executor + } + ] + } + ], + on => #{ + <<"request">> => + #{ + <<"device">> => <<"router@1.0">>, + <<"path">> => <<"preprocess">>, + <<"commit-request">> => true + } + } + }), + {ok, Res} = + hb_http:get( + Node, + #{ + <<"path">> => <<"test-key">>, + <<"test-key">> => <<"value">> + }, + #{} + ), + ?event({res, Res}), + ?assertEqual(<<"value">>, Res). \ No newline at end of file diff --git a/src/dev_router.erl b/src/dev_router.erl index a813aec40..ed19e6705 100644 --- a/src/dev_router.erl +++ b/src/dev_router.erl @@ -3,135 +3,435 @@ %%% routed to a single process per node, which then load-balances them %%% between downstream workers that perform the actual requests. %%% -%%% The routes for the router are defined in the `routes` key of the `Opts`, +%%% The routes for the router are defined in the `routes' key of the `Opts', %%% as a precidence-ordered list of maps. The first map that matches the %%% message will be used to determine the route. %%% %%% Multiple nodes can be specified as viable for a single route, with the -%%% `Choose` key determining how many nodes to choose from the list (defaulting -%%% to 1). The `Strategy` key determines the load distribution strategy, -%%% which can be one of `Random`, `By-Base`, or `Nearest`. The route may also +%%% `Choose' key determining how many nodes to choose from the list (defaulting +%%% to 1). The `Strategy' key determines the load distribution strategy, +%%% which can be one of `Random', `By-Base', or `Nearest'. The route may also %%% define additional parallel execution parameters, which are used by the -%%% `hb_http` module to manage control of requests. +%%% `hb_http' module to manage control of requests. %%% %%% The structure of the routes should be as follows: -%%% ``` +%%% 
 %%%     Node?: The node to route the message to.
 %%%     Nodes?: A list of nodes to route the message to.
 %%%     Strategy?: The load distribution strategy to use.
 %%%     Choose?: The number of nodes to choose from the list.
 %%%     Template?: A message template to match the message against, either as a
 %%%                map or a path regex.
-%%% '''
+%%%
-module(dev_router). -%%% Device API: --export([routes/3]). -%%% Public utilities: --export([find_route/2, find_route/3, match_routes/3]). +-export([info/1, info/3, routes/3, route/2, route/3, preprocess/3]). +-export([match/3, register/3]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). +%% @doc Exported function for getting device info, controls which functions are +%% exposed via the device API. +info(_) -> + #{ exports => [info, routes, route, match, register, preprocess] }. + +%% @doc HTTP info response providing information about this device +info(_Msg1, _Msg2, _Opts) -> + InfoBody = #{ + <<"description">> => <<"Router device for handling outbound message routing">>, + <<"version">> => <<"1.0">>, + <<"api">> => #{ + <<"info">> => #{ + <<"description">> => <<"Get device info">> + }, + <<"routes">> => #{ + <<"description">> => <<"Get or add routes">>, + <<"method">> => <<"GET or POST">> + }, + <<"route">> => #{ + <<"description">> => <<"Find a route for a message">>, + <<"required_params">> => #{ + <<"route-path">> => <<"Path to route">> + } + }, + <<"match">> => #{ + <<"description">> => <<"Match a message against available routes">> + }, + <<"register">> => #{ + <<"description">> => <<"Register a route with a remote router node">>, + <<"node-message">> => #{ + <<"routes">> => + [ + #{ + <<"registration-peer">> => <<"Location of the router peer">>, + <<"prefix">> => <<"Prefix for the route">>, + <<"price">> => <<"Price for the route">>, + <<"template">> => <<"Template to match the route">> + } + ] + } + }, + <<"preprocess">> => #{ + <<"description">> => <<"Preprocess a request to check if it should be relayed">> + } + } + }, + {ok, InfoBody}. + +%% @doc Register function that allows telling the current node to register +%% a new route with a remote router node. This function should also be idempotent. +%% so that it can be called only once. +register(_M1, M2, Opts) -> + %% Extract all required parameters from options + %% These values will be used to construct the registration message + RouterOpts = hb_opts:get(router_opts, #{}, Opts), + RouterRegMsgs = + case hb_maps:get(<<"offered">>, RouterOpts, #{}, Opts) of + RegList when is_list(RegList) -> RegList; + RegMsg when is_map(RegMsg) -> [RegMsg] + end, + lists:foreach( + fun(RegMsg) -> + RouterNode = + hb_ao:get( + <<"registration-peer">>, + RegMsg, + not_found, + Opts + ), + {ok, SigOpts} = + case hb_ao:get(<<"as">>, M2, not_found, Opts) of + not_found -> {ok, Opts}; + AsID -> hb_opts:as(AsID, Opts) + end, + % Post registration request to the router node + % The message includes our route details and attestation + % for verification + {ok, Res} = + hb_http:post( + RouterNode, + <<"/~router@1.0/routes">>, + hb_message:commit( + #{ + <<"subject">> => <<"self">>, + <<"action">> => <<"register">>, + <<"route">> => RegMsg + }, + SigOpts + ), + Opts + ), + ?event({registered, {msg, M2}, {res, Res}}), + {ok, <<"Route registered.">>} + end, + RouterRegMsgs + ), + {ok, <<"Routes registered.">>}. + %% @doc Device function that returns all known routes. routes(M1, M2, Opts) -> - ?event(debug, {routes_msg, M1, M2}), - Routes = hb_opts:get(routes, [], Opts), - case hb_converge:get(method, M2, Opts) of + ?event({routes_msg, M1, M2}), + Routes = load_routes(Opts), + ?event({routes, Routes}), + case hb_ao:get(<<"method">>, M2, Opts) of <<"POST">> -> - Owner = hb_opts:get(owner, undefined, Opts), - RouteOwners = hb_opts:get(route_owners, [Owner], Opts), - Signers = hb_message:signers(M2), - IsTrusted = - lists:any( - fun(Signer) -> lists:member(Signer, Signers) end, - RouteOwners - ), - case IsTrusted of - true -> - Priority = hb_converge:get(<<"Priority">>, M2, Opts), - NewRoutes = - lists:sort(fun(X, Y) -> X > Y end, [Priority|Routes]), - hb_http_server:set_opts(Opts#{ routes => NewRoutes }), - {ok, <<"Route added.">>}; - false -> {error, not_authorized} + RouterOpts = hb_opts:get(router_opts, #{}, Opts), + ?event(debug_route_reg, {router_opts, RouterOpts}), + case hb_maps:get(<<"registrar">>, RouterOpts, not_found, Opts) of + not_found -> + % There is no registrar; register if and only if the message + % is signed by an authorized operator. + ?event(debug_route_reg, no_registrar), + Owner = hb_opts:get(operator, undefined, Opts), + RouteOwners = hb_opts:get(route_owners, [Owner], Opts), + Signers = hb_message:signers(M2, Opts), + IsTrusted = + lists:any( + fun(Signer) -> lists:member(Signer, Signers) end, + RouteOwners + ), + case IsTrusted of + true -> + % Minimize the work performed by AO-Core to make the sort + % more efficient. + SortOpts = Opts#{ hashpath => ignore }, + NewRoutes = + lists:sort( + fun(X, Y) -> + hb_ao:get(<<"priority">>, X, SortOpts) + < hb_ao:get(<<"priority">>, Y, SortOpts) + end, + [M2|Routes] + ), + ok = hb_http_server:set_opts(Opts#{ routes => NewRoutes }), + {ok, <<"Route added.">>}; + false -> {error, not_authorized} + end; + Registrar -> + % Parse the registrar message and execute the route + % registration against it. + RegistrarPath = + hb_maps:get( + <<"registrar-path">>, + RouterOpts, + not_found, + Opts + ), + ?event(debug_route_reg, + {registrar_found, {msg, Registrar}, {path, RegistrarPath}} + ), + RegReq = + case RegistrarPath of + not_found -> M2; + RegPath -> + M2#{ <<"path">> => RegPath } + end, + RegistrarMsgs = hb_singleton:from(Registrar, Opts) ++ [RegReq], + ?event(debug_route_reg, {registrar_msgs, RegistrarMsgs}), + case hb_ao:resolve_many(RegistrarMsgs, Opts) of + {ok, _} -> + {ok, <<"Route added.">>}; + {error, Error} -> + {error, Error} + end end; - _ -> {ok, Routes} + _ -> + {ok, Routes} end. -%% @doc If we have a route that has multiple resolving nodes, check +%% @doc Find the appropriate route for the given message. If we are able to +%% resolve to a single host+path, we return that directly. Otherwise, we return +%% the matching route (including a list of nodes under `nodes') from the list of +%% routes. +%% +%% If we have a route that has multiple resolving nodes, check %% the load distribution strategy and choose a node. Supported strategies: -%% ``` -%% Random: Distribute load evenly across all nodes, non-deterministically. +%% 
+%%           All: Return all nodes (default).
+%%        Random: Distribute load evenly across all nodes, non-deterministically.
 %%       By-Base: According to the base message's hashpath.
+%%     By-Weight: According to the node's `weight' key.
 %%       Nearest: According to the distance of the node's wallet address to the
 %%                base message's hashpath.
-%% '''
-%% `By-Base` will ensure that all traffic for the same hashpath is routed to the
-%% same node, minimizing work duplication, while `Random` ensures a more even
+%%
+%% `By-Base' will ensure that all traffic for the same hashpath is routed to the +%% same node, minimizing work duplication, while `Random' ensures a more even %% distribution of the requests. %% -%% Can operate as a `Router/1.0` device, which will ignore the base message, +%% Can operate as a `~router@1.0' device, which will ignore the base message, %% routing based on the Opts and request message provided, or as a standalone -%% function, taking only the request message and the `Opts` map. -find_route(Msg, Opts) -> find_route(undefined, Msg, Opts). -find_route(_, Msg, Opts) -> - Routes = hb_opts:get(routes, [], Opts), +%% function, taking only the request message and the `Opts' map. +route(Msg, Opts) -> route(undefined, Msg, Opts). +route(_, Msg, Opts) -> + Routes = load_routes(Opts), R = match_routes(Msg, Routes, Opts), - case (R =/= no_matches) andalso hb_converge:get(<<"Node">>, R, Opts) of - false -> no_matches; + ?event({find_route, {msg, Msg}, {routes, Routes}, {res, R}}), + case (R =/= no_matches) andalso hb_ao:get(<<"node">>, R, Opts) of + false -> {error, no_matches}; Node when is_binary(Node) -> {ok, Node}; + Node when is_map(Node) -> apply_route(Msg, Node, Opts); not_found -> - Nodes = hb_converge:get(<<"Peers">>, R, Opts), - case hb_converge:get(<<"Strategy">>, R, Opts) of - not_found -> {ok, Nodes}; + ModR = apply_routes(Msg, R, Opts), + case hb_ao:get(<<"strategy">>, R, Opts) of + not_found -> {ok, ModR}; + <<"All">> -> {ok, ModR}; Strategy -> - ChooseN = hb_converge:get(<<"Choose">>, R, 1, Opts), - Hashpath = hb_path:from_message(hashpath, R), - Chosen = choose(ChooseN, Strategy, Hashpath, Nodes, Opts), + ChooseN = hb_ao:get(<<"choose">>, R, 1, Opts), + % Get the first element of the path -- the `base' message + % of the request. + Base = extract_base(Msg, Opts), + Nodes = hb_ao:get(<<"nodes">>, ModR, Opts), + Chosen = choose(ChooseN, Strategy, Base, Nodes, Opts), + ?event({choose, + {strategy, Strategy}, + {choose_n, ChooseN}, + {base, Base}, + {nodes, Nodes}, + {chosen, Chosen} + }), case Chosen of - [X] when is_map(X) -> - {ok, hb_converge:get(<<"Host">>, X, Opts)}; - [X] -> {ok, X}; - _ -> - {ok, hb_converge:set(<<"Peers">>, Chosen, Opts)} + [Node] when is_map(Node) -> + apply_route(Msg, Node, Opts); + [NodeURI] -> {ok, NodeURI}; + _ChosenNodes -> + {ok, + hb_ao:set( + <<"nodes">>, + hb_maps:map( + fun(Node) -> + hb_util:ok(apply_route(Msg, Node, Opts)) + end, + Chosen, + Opts + ), + Opts + ) + } end end end. -%% @doc Find the first matching template in a list of known routes. +%% @doc Load the current routes for the node. Allows either explicit routes from +%% the node message's `routes' key, or dynamic routes generated by resolving the +%% `<<"provider">>' message. +load_routes(Opts) -> + RouterOpts = hb_opts:get(router_opts, #{}, Opts), + case hb_maps:get(<<"provider">>, RouterOpts, not_found, Opts) of + not_found -> hb_opts:get(routes, [], Opts); + RoutesProvider -> + ProviderMsgs = hb_singleton:from(RoutesProvider, Opts), + ?event({<<"provider">>, ProviderMsgs}), + case hb_ao:resolve_many(ProviderMsgs, Opts) of + {ok, Routes} -> hb_cache:ensure_all_loaded(Routes, Opts); + {error, Error} -> throw({routes, routes_provider_failed, Error}) + end + end. + +%% @doc Extract the base message ID from a request message. Produces a single +%% binary ID that can be used for routing decisions. +extract_base(#{ <<"path">> := Path }, Opts) -> + extract_base(Path, Opts); +extract_base(RawPath, Opts) when is_binary(RawPath) -> + BasePath = hb_path:hd(#{ <<"path">> => RawPath }, Opts), + case ?IS_ID(BasePath) of + true -> BasePath; + false -> + case binary:split(BasePath, [<<"\~">>, <<"?">>, <<"&">>], [global]) of + [BaseMsgID|_] when ?IS_ID(BaseMsgID) -> BaseMsgID; + _ -> hb_crypto:sha256(BasePath) + end + end. + +%% @doc Generate a `uri' key for each node in a route. +apply_routes(Msg, R, Opts) -> + Nodes = hb_ao:get(<<"nodes">>, R, Opts), + NodesWithRouteApplied = + lists:map( + fun(N) -> + ?event({apply_route, {msg, Msg}, {node, N}}), + case apply_route(Msg, N, Opts) of + {ok, URI} when is_binary(URI) -> N#{ <<"uri">> => URI }; + {ok, RMsg} -> hb_maps:merge(N, RMsg); + {error, _} -> N + end + end, + hb_util:message_to_ordered_list(Nodes, Opts) + ), + ?event({nodes_after_apply, NodesWithRouteApplied}), + R#{ <<"nodes">> => NodesWithRouteApplied }. + +%% @doc Apply a node map's rules for transforming the path of the message. +%% Supports the following keys: +%% - `opts': A map of options to pass to the request. +%% - `prefix': The prefix to add to the path. +%% - `suffix': The suffix to add to the path. +%% - `match' and `with': A regex to replace in the path. +apply_route(Msg, Route, Opts) -> + % LoadedRoute = hb_cache:ensure_all_loaded(Route, Opts), + RouteOpts = hb_maps:get(<<"opts">>, Route, #{}), + {ok, #{ + <<"opts">> => RouteOpts, + <<"uri">> => + hb_util:ok( + do_apply_route( + Msg, + hb_maps:without([<<"opts">>], Route, Opts), + Opts + ) + ) + }}. +do_apply_route(#{ <<"route-path">> := Path }, R, Opts) -> + do_apply_route(#{ <<"path">> => Path }, R, Opts); +do_apply_route(#{ <<"path">> := RawPath }, #{ <<"prefix">> := RawPrefix }, Opts) -> + Path = hb_cache:ensure_loaded(RawPath, Opts), + Prefix = hb_cache:ensure_loaded(RawPrefix, Opts), + {ok, <>}; +do_apply_route(#{ <<"path">> := RawPath }, #{ <<"suffix">> := RawSuffix }, Opts) -> + Path = hb_cache:ensure_loaded(RawPath, Opts), + Suffix = hb_cache:ensure_loaded(RawSuffix, Opts), + {ok, <>}; +do_apply_route( + #{ <<"path">> := RawPath }, + #{ <<"match">> := RawMatch, <<"with">> := RawWith }, + Opts) -> + Path = hb_cache:ensure_loaded(RawPath, Opts), + Match = hb_cache:ensure_loaded(RawMatch, Opts), + With = hb_cache:ensure_loaded(RawWith, Opts), + % Apply the regex to the path and replace the first occurrence. + case re:replace(Path, Match, With, [global, {return, binary}]) of + NewPath when is_binary(NewPath) -> + {ok, NewPath}; + _ -> + {error, invalid_replace_args} + end. + +%% @doc Find the first matching template in a list of known routes. Allows the +%% path to be specified by either the explicit `path' (for internal use by this +%% module), or `route-path' for use by external devices and users. +match(Base, Req, Opts) -> + ?event(debug_preprocess, + {matching_routes, + {base, Base}, + {req, Req} + } + ), + TargetPath = hb_util:find_target_path(Req, Opts), + Match = + match_routes( + Req#{ <<"path">> => TargetPath }, + hb_ao:get(<<"routes">>, {as, <<"message@1.0">>, Base}, [], Opts), + Opts + ), + case Match of + no_matches -> {error, no_matching_route}; + _ -> {ok, Match} + end. + match_routes(ToMatch, Routes, Opts) -> match_routes( - ToMatch, - Routes, - hb_converge:keys(Routes), + hb_cache:ensure_all_loaded(ToMatch, Opts), + hb_cache:ensure_all_loaded(Routes, Opts), + hb_ao:keys(hb_ao:normalize_keys(Routes, Opts)), Opts ). +match_routes(#{ <<"path">> := Explicit = <<"http://", _/binary>> }, _, _, _) -> + % If the route is an explicit HTTP URL, we can match it directly. + #{ <<"node">> => Explicit, <<"reference">> => <<"explicit">> }; +match_routes(#{ <<"path">> := Explicit = <<"https://", _/binary>> }, _, _, _) -> + #{ <<"node">> => Explicit, <<"reference">> => <<"explicit">> }; match_routes(_, _, [], _) -> no_matches; match_routes(ToMatch, Routes, [XKey|Keys], Opts) -> - XM = hb_converge:get(XKey, Routes, Opts), + XM = hb_ao:get(XKey, Routes, Opts), Template = - hb_converge:get( - <<"Template">>, + hb_ao:get( + <<"template">>, XM, #{}, Opts#{ hashpath => ignore } ), - case template_matches(ToMatch, Template) of - true -> XM; + case hb_util:template_matches(ToMatch, Template, Opts) of + true -> XM#{ <<"reference">> => hb_path:to_binary([<<"routes">>, XKey]) }; false -> match_routes(ToMatch, Routes, Keys, Opts) end. -%% @doc Check if a message matches a message template or path regex. -template_matches(ToMatch, Template) when is_map(Template) -> - hb_message:match(Template, ToMatch, primary); -template_matches(ToMatch, Regex) when is_binary(Regex) -> - MsgPath = (hb_path:from_message(request, ToMatch)), - hb_path:regex_matches(MsgPath, Regex). - %% @doc Implements the load distribution strategies if given a cluster. choose(0, _, _, _, _) -> []; choose(N, <<"Random">>, _, Nodes, _Opts) -> Node = lists:nth(rand:uniform(length(Nodes)), Nodes), [Node | choose(N - 1, <<"Random">>, nop, lists:delete(Node, Nodes), _Opts)]; +choose(N, <<"By-Weight">>, _, Nodes, Opts) -> + ?event({nodes, Nodes}), + NodesWithWeight = + [ + { Node, hb_util:float(hb_ao:get(<<"weight">>, Node, Opts)) } + || + Node <- Nodes + ], + Node = hb_util:weighted_random(NodesWithWeight), + [ + Node + | + choose(N - 1, <<"By-Weight">>, nop, lists:delete(Node, Nodes), Opts) + ]; choose(N, <<"By-Base">>, Hashpath, Nodes, Opts) when is_binary(Hashpath) -> choose(N, <<"By-Base">>, binary_to_bignum(Hashpath), Nodes, Opts); choose(N, <<"By-Base">>, HashInt, Nodes, Opts) -> @@ -152,15 +452,13 @@ choose(N, <<"Nearest">>, HashPath, Nodes, Opts) -> NodesWithDistances = lists:map( fun(Node) -> - Wallet = hb_converge:get(<<"Wallet">>, Node, Opts), + Wallet = hb_ao:get(<<"wallet">>, Node, Opts), DistanceScore = - hb_crypto:sha256( - << - Wallet/binary, - BareHashPath/binary - >> + field_distance( + hb_util:native_id(Wallet), + BareHashPath ), - {Node, binary_to_bignum(DistanceScore)} + {Node, DistanceScore} end, Nodes ), @@ -177,6 +475,17 @@ choose(N, <<"Nearest">>, HashPath, Nodes, Opts) -> ) ). +%% @doc Calculate the minimum distance between two numbers +%% (either progressing backwards or forwards), assuming a +%% 256-bit field. +field_distance(A, B) when is_binary(A) -> + field_distance(binary_to_bignum(A), B); +field_distance(A, B) when is_binary(B) -> + field_distance(A, binary_to_bignum(B)); +field_distance(A, B) -> + AbsDiff = abs(A - B), + min(AbsDiff, (1 bsl 256) - AbsDiff). + %% @doc Find the node with the lowest distance to the given hashpath. lowest_distance(Nodes) -> lowest_distance(Nodes, {undefined, infinity}). lowest_distance([], X) -> X; @@ -193,8 +502,780 @@ binary_to_bignum(Bin) when ?IS_ID(Bin) -> << Num:256/unsigned-integer >> = hb_util:native_id(Bin), Num. +%% @doc Preprocess a request to check if it should be relayed to a different node. +preprocess(Msg1, Msg2, Opts) -> + Req = hb_ao:get(<<"request">>, Msg2, Opts#{ hashpath => ignore }), + ?event(debug_preprocess, {called_preprocess,Req}), + TemplateRoutes = load_routes(Opts), + ?event(debug_preprocess, {template_routes, TemplateRoutes}), + Res = hb_http:message_to_request(Req, Opts), + ?event(debug_preprocess, {match, Res}), + case Res of + {error, _} -> + ?event(debug_preprocess, preprocessor_did_not_match), + case hb_opts:get(router_preprocess_default, <<"local">>, Opts) of + <<"local">> -> + ?event(debug_preprocess, executing_locally), + {ok, #{ + <<"body">> => + hb_ao:get(<<"body">>, Msg2, Opts#{ hashpath => ignore }) + }}; + <<"error">> -> + ?event(debug_preprocess, preprocessor_returning_error), + {ok, #{ + <<"body">> => + [#{ + <<"status">> => 404, + <<"message">> => + <<"No matching template found in the given routes.">> + }] + }} + end; + {ok, _Method, Node, _Path, _MsgWithoutMeta, _ReqOpts} -> + ?event(debug_preprocess, {matched_route, {explicit, Res}}), + CommitRequest = + hb_util:atom( + hb_ao:get_first( + [ + {Msg1, <<"commit-request">>} + ], + false, + Opts + ) + ), + MaybeCommit = + case CommitRequest of + true -> #{ <<"commit-request">> => true }; + false -> #{} + end, + % Construct a request to `relay@1.0/call' which will proxy a request + % to `apply@1.0/body' with the original request body as the argument. + % This allows us to potentially sign the request before sending it, + % letting the recipient node charge/verify us as necessary, without + % explicitly signing the user's request itself. + % + % We additionally ensure that the request itself has a commitment, + % such that headers added by the relaying node are not added to the + % user's request. + UserReqWithCommit = + case hb_message:signers(Req, Opts) of + [] -> + hb_message:commit( + Req, + Opts, + #{ + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"type">> => <<"unsigned">> + } + ); + _ -> + Req + end, + RelayReq = + #{ + <<"device">> => <<"apply@1.0">>, + <<"path">> => <<"user-path">>, + <<"source">> => <<"user-message">>, + <<"user-path">> => hb_maps:get(<<"path">>, Req, Opts), + <<"user-message">> => UserReqWithCommit + }, + ?event(debug_preprocess, {prepared_relay_req, RelayReq}), + { + ok, + #{ + <<"body">> => + [ + MaybeCommit#{ + <<"device">> => <<"relay@1.0">>, + <<"relay-device">> => <<"apply@1.0">>, + <<"method">> => <<"POST">>, + <<"peer">> => Node + }, + #{ + <<"path">> => <<"call">>, + <<"target">> => <<"proxy-message">>, + <<"proxy-message">> => RelayReq + } + ] + } + } + end. + %%% Tests +test_provider_test() -> + Node = + hb_http_server:start_node(Opts = + #{ + router_opts => #{ + <<"provider">> => #{ + <<"path">> => <<"/test-key/routes">>, + <<"test-key">> => #{ + <<"routes">> => [ + #{ + <<"template">> => <<"*">>, + <<"node">> => <<"testnode">> + } + ] + } + } + }, + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + } + } + ), + ?assertEqual( + {ok, <<"testnode">>}, + hb_http:get(Node, <<"/~router@1.0/routes/1/node">>, Opts) + ). + +dynamic_provider_test() -> + {ok, Script} = file:read_file("test/test.lua"), + Node = hb_http_server:start_node(#{ + router_opts => #{ + <<"provider">> => #{ + <<"device">> => <<"lua@5.3a">>, + <<"path">> => <<"provider">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Script + }, + <<"node">> => <<"test-dynamic-node">> + } + }, + priv_wallet => ar_wallet:new() + }), + ?assertEqual( + {ok, <<"test-dynamic-node">>}, + hb_http:get(Node, <<"/~router@1.0/routes/1/node">>, #{}) + ). + +local_process_provider_test_() -> + {timeout, 30, fun local_process_provider/0}. +local_process_provider() -> + {ok, Script} = file:read_file("test/test.lua"), + Node = hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new(), + router_opts => #{ + <<"provider">> => #{ + <<"path">> => <<"/router~node-process@1.0/now/known-routes">> + } + }, + node_processes => #{ + <<"router">> => #{ + <<"device">> => <<"process@1.0">>, + <<"execution-device">> => <<"lua@5.3a">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"body">> => Script + }, + <<"node">> => <<"router-node">>, + <<"function">> => <<"compute_routes">> + } + } + }), + ?assertEqual( + {ok, <<"test1">>}, + hb_http:get(Node, <<"/~router@1.0/routes/1/template">>, #{}) + ), + % Query the route 10 times with the same path. This should yield 2 different + % results, as the route provider should choose 1 node of a set of 2 at random. + Responses = + lists:map( + fun(_) -> + hb_util:ok( + hb_http:get( + Node, + <<"/~router@1.0/route&route-path=test2/uri">>, + #{} + ) + ) + end, + lists:seq(1, 10) + ), + ?event({responses, Responses}), + ?assertEqual(2, length(hb_util:unique(Responses))). + +%% @doc Example of a Lua module being used as the `<<"provider">>' for a +%% HyperBEAM node. The module utilized in this example dynamically adjusts the +%% likelihood of routing to a given node, depending upon price and performance. +local_dynamic_router_test_() -> + {timeout, 60, fun local_dynamic_router/0}. +local_dynamic_router() -> + BenchRoutes = 50, + TestNodes = 5, + {ok, Module} = file:read_file(<<"scripts/dynamic-router.lua">>), + Node = hb_http_server:start_node(Opts = #{ + store => hb_test_utils:test_store(), + priv_wallet => ar_wallet:new(), + router_opts => #{ + <<"registrar">> => #{ + <<"device">> => <<"router@1.0">>, + <<"path">> => <<"/router1~node-process@1.0/schedule">> + }, + <<"provider">> => #{ + <<"path">> => + RouteProvider = + <<"/router1~node-process@1.0/compute/routes~message@1.0">> + } + }, + node_processes => #{ + <<"router1">> => #{ + <<"device">> => <<"process@1.0">>, + <<"execution-device">> => <<"lua@5.3a">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"name">> => <<"dynamic-router">>, + <<"body">> => Module + }, + % Set module-specific factors for the test + <<"pricing-weight">> => 9, + <<"performance-weight">> => 1, + <<"score-preference">> => 4 + } + } + }), + Store = hb_opts:get(store, no_store, Opts), + ?event(debug_dynrouter, {store, Store}), + % Register workers with the dynamic router with varied prices. + lists:foreach( + fun(X) -> + hb_http:post( + Node, + #{ + <<"path">> => <<"/router1~node-process@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + #{ + <<"path">> => <<"register">>, + <<"route">> => + #{ + <<"prefix">> => + << + "https://test-node-", + (hb_util:bin(X))/binary, + ".com" + >>, + <<"template">> => <<"/.*~process@1.0/.*">>, + <<"price">> => X * 250 + } + }, + Opts + ) + }, + Opts + ) + end, + lists:seq(1, TestNodes) + ), + % Force computation of the current state. This should be done with a + % background worker (ex: a `~cron@1.0/every' task). + hb_http:get(Node, <<"/router~node-process@1.0/now">>, #{}), + {ok, Routes} = hb_http:get(Node, RouteProvider, Opts), + ?event(debug_dynrouter, {got_routes, Routes}), + % Query the route 10 times with the same path. This should yield 2 different + % results, as the route provider should choose 1 node of a set of 2 at random. + BeforeExec = os:system_time(millisecond), + Responses = + lists:map( + fun(_) -> + hb_util:ok( + hb_http:get( + Node, + <<"/~router@1.0/route/uri?route-path=/procID~process@1.0/now">>, + Opts + ) + ) + end, + lists:seq(1, BenchRoutes) + ), + AfterExec = os:system_time(millisecond), + hb_format:eunit_print( + "Calculated ~p routes in ~ps (~.2f routes/s)", + [ + BenchRoutes, + (AfterExec - BeforeExec) / 1000, + BenchRoutes / ((AfterExec - BeforeExec) / 1000) + ] + ), + % Calculate the distribution of the responses. + UniqueResponses = sets:to_list(sets:from_list(Responses)), + Dist = + [ + { + Resp, + hb_util:count(Resp, Responses) / length(Responses) + } + || + Resp <- UniqueResponses + ], + ?event(debug_distribution, {distribution_of_responses, Dist}), + ?assert(length(UniqueResponses) > 1). + +%% @doc Test that verifies dynamic router functionality and template-based pricing. +%% Sets up a two-node system: an execution node with p4@1.0 processing and a proxy +%% node with router@1.0 for dynamic routing. The test confirms that: +%% - dev_simple_pay correctly uses template matching via <<"router@1.0">> -> routes +%% to determine pricing for different routes (e.g., "/c" route with price 0) +%% - Dynamic routing works with Lua-based route providers that adjust routing +%% likelihood based on price and performance factors +%% - Request preprocessing and routing happens correctly between nodes +%% - Non-chargeable routes are properly handled via template patterns +dynamic_router_pricing_test_() -> + {timeout, 30, fun dynamic_router_pricing/0}. +dynamic_router_pricing() -> + {ok, Module} = file:read_file(<<"scripts/dynamic-router.lua">>), + {ok, ClientScript} = file:read_file("scripts/hyper-token-p4-client.lua"), + {ok, TokenScript} = file:read_file("scripts/hyper-token.lua"), + {ok, ProcessScript} = file:read_file("scripts/hyper-token-p4.lua"), + ExecWallet = hb:wallet(<<"test/admissible-report-wallet.json">>), + ProxyWallet = ar_wallet:new(), + ExecNodeAddr = hb_util:human_id(ar_wallet:to_address(ExecWallet)), + Processor = + #{ + <<"device">> => <<"p4@1.0">>, + <<"ledger-device">> => <<"lua@5.3a">>, + <<"pricing-device">> => <<"simple-pay@1.0">>, + <<"ledger-path">> => <<"/ledger2~node-process@1.0">>, + <<"module">> => #{ + <<"content-type">> => <<"text/x-lua">>, + <<"name">> => <<"scripts/hyper-token-p4-client.lua">>, + <<"body">> => ClientScript + } + }, + ExecNode = + hb_http_server:start_node( + ExecOpts = #{ + priv_wallet => ExecWallet, + port => 10009, + store => hb_test_utils:test_store(), + node_processes => #{ + <<"ledger2">> => #{ + <<"device">> => <<"process@1.0">>, + <<"execution-device">> => <<"lua@5.3a">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"authority-match">> => 1, + <<"admin">> => ExecNodeAddr, + <<"token">> => + <<"iVplXcMZwiu5mn0EZxY-PxAkz_A9KOU0cmRE0rwej3E">>, + <<"module">> => [ + #{ + <<"content-type">> => <<"text/x-lua">>, + <<"name">> => <<"scripts/hyper-token.lua">>, + <<"body">> => TokenScript + }, + #{ + <<"content-type">> => <<"text/x-lua">>, + <<"name">> => <<"scripts/hyper-token-p4.lua">>, + <<"body">> => ProcessScript + } + ], + <<"authority">> => ExecNodeAddr + } + }, + p4_recipient => ExecNodeAddr, + p4_non_chargable_routes => [ + #{ <<"template">> => <<"/*~node-process@1.0/*">> }, + #{ <<"template">> => <<"/*~router@1.0/*">> } + ], + on => #{ + <<"request">> => Processor, + <<"response">> => Processor + }, + node_process_spawn_codec => <<"ans104@1.0">>, + router_opts => #{ + <<"offered">> => [ + #{ + <<"registration-peer">> => <<"http://localhost:10010">>, + <<"template">> => <<"/c">>, + <<"prefix">> => <<"http://localhost:10009">>, + <<"price">> => 0 + }, + #{ + <<"registration-peer">> => <<"http://localhost:10010">>, + <<"template">> => <<"/b">>, + <<"prefix">> => <<"http://localhost:10009">>, + <<"price">> => 1 + } + ] + } + } + ), + RouterNode = hb_http_server:start_node(#{ + port => 10010, + store => hb_test_utils:test_store(), + priv_wallet => ProxyWallet, + on => + #{ + <<"request">> => #{ + <<"device">> => <<"router@1.0">>, + <<"path">> => <<"preprocess">>, + <<"commit-request">> => true + } + }, + router_opts => #{ + <<"provider">> => #{ + <<"path">> => + <<"/router2~node-process@1.0/compute/routes~message@1.0">> + }, + <<"registrar">> => #{ + <<"path">> => <<"/router2~node-process@1.0">> + }, + <<"registrar-path">> => <<"schedule">> + }, + relay_allow_commit_request => true, + node_processes => #{ + <<"router2">> => #{ + <<"type">> => <<"Process">>, + <<"device">> => <<"process@1.0">>, + <<"execution-device">> => <<"lua@5.3a">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"module">> => <<"dynamic-router">>, + <<"body">> => Module + }, + % Set module-specific factors for the test + <<"pricing-weight">> => 9, + <<"performance-weight">> => 1, + <<"score-preference">> => 4, + <<"is-admissible">> => #{ + <<"path">> => <<"default">>, + <<"default">> => <<"false">> + }, + <<"trusted-peer">> => ExecNodeAddr + } + } + }), + ?event( + debug_load_routes, + {node_message, hb_http:get(RouterNode, <<"/~meta@1.0/info">>, #{})} + ), + % Register workers with the dynamic router with varied prices. + {ok, <<"Routes registered.">>} = + hb_http:post( + ExecNode, + <<"/~router@1.0/register">>, + #{} + ), + % Force computation of the current state. + {Status, _NodeRoutes} = + hb_http:get( + RouterNode, + <<"/router2~node-process@1.0/now/at-slot">>, + #{} + ), + ?assertEqual(ok, Status), + % Check that path /c is free + {ok, CRes} = hb_http:get(RouterNode, <<"/c?c+list=1">>, #{}), + ?event(debug_dynrouter, {res_msg, CRes}), + ?assertEqual(1, hb_maps:get(<<"1">>, CRes, not_found)), + % Check that path /b is not free and returns Insufficient funds + {error, BRes} = hb_http:get(RouterNode, <<"/b?b+list=1">>, #{}), + ?event(debug_dynrouter, {res_msg, BRes}), + ?assertEqual(<<"Insufficient funds">>, hb_maps:get(<<"body">>, BRes, not_found)). + + +%% @doc Example of a Lua module being used as the `<<"provider">>' for a +%% HyperBEAM node. The module utilized in this example dynamically adjusts the +%% likelihood of routing to a given node, depending upon price and performance. +%% also include preprocessing support for routing +dynamic_router_test_() -> + {timeout, 30, fun dynamic_router/0}. +dynamic_router() -> + {ok, Module} = file:read_file(<<"scripts/dynamic-router.lua">>), + ExecWallet = hb:wallet(<<"test/admissible-report-wallet.json">>), + ProxyWallet = ar_wallet:new(), + ExecNode = + hb_http_server:start_node( + ExecOpts = #{ priv_wallet => ExecWallet, store => hb_test_utils:test_store() } + ), + Node = hb_http_server:start_node(ProxyOpts = #{ + snp_trusted => [ + #{ + <<"vcpus">> => 32, + <<"vcpu_type">> => 5, + <<"vmm_type">> => 1, + <<"guest_features">> => 1, + <<"firmware">> => + <<"b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510">>, + <<"kernel">> => + <<"69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576">>, + <<"initrd">> => + <<"544045560322dbcd2c454bdc50f35edf0147829ec440e6cb487b4a1503f923c1">>, + <<"append">> => + <<"95a34faced5e487991f9cc2253a41cbd26b708bf00328f98dddbbf6b3ea2892e">> + } + ], + store => hb_test_utils:test_store(), + priv_wallet => ProxyWallet, + on => + #{ + <<"request">> => #{ + <<"device">> => <<"router@1.0">>, + <<"path">> => <<"preprocess">> + } + }, + router_opts => #{ + <<"provider">> => #{ + <<"path">> => <<"/router~node-process@1.0/compute/routes~message@1.0">> + } + }, + node_processes => #{ + <<"router">> => #{ + <<"type">> => <<"Process">>, + <<"device">> => <<"process@1.0">>, + <<"execution-device">> => <<"lua@5.3a">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"module">> => <<"dynamic-router">>, + <<"body">> => Module + }, + % Set module-specific factors for the test + <<"pricing-weight">> => 9, + <<"performance-weight">> => 1, + <<"score-preference">> => 4, + <<"is-admissible">> => #{ + <<"device">> => <<"snp@1.0">>, + <<"path">> => <<"verify">> + } + } + } + }), % mergeRight this takes our defined Opts and merges them into the + % node opts configs. + Store = hb_opts:get(store, no_store, ProxyOpts), + ?event(debug_dynrouter, {store, Store}), + % Register workers with the dynamic router with varied prices. + {ok, [Req]} = file:consult(<<"test/admissible-report.eterm">>), + lists:foreach(fun(X) -> + {ok, Res} = + hb_http:post( + Node, + #{ + <<"path">> => <<"/router~node-process@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + #{ + <<"path">> => <<"register">>, + <<"route">> => + #{ + <<"prefix">> => ExecNode, + <<"template">> => <<"/c">>, + <<"price">> => X * 250 + }, + <<"body">> => hb_message:commit(Req, ExecOpts) + }, + ExecOpts + ) + }, + ExecOpts + ), + Res + end, lists:seq(1, 1)), + % Force computation of the current state. This should be done with a + % background worker (ex: a `~cron@1.0/every' task). + {Status, NodeRoutes} = hb_http:get(Node, <<"/router~node-process@1.0/now/at-slot">>, #{}), + ?event(debug_dynrouter, {got_node_routes, NodeRoutes}), + ?assertEqual(ok, Status), + ProxyWalletAddr = hb_util:human_id(ar_wallet:to_address(ProxyWallet)), + ExecNodeAddr = hb_util:human_id(ar_wallet:to_address(ExecWallet)), + % Ensure that the `~meta@1.0/info/address' response is produced by the + % proxy wallet. + ?event(debug_dynrouter, + {addresses, + {proxy_wallet_addr, ProxyWalletAddr}, + {exec_node_addr, ExecNodeAddr} + } + ), + ?assertEqual( + {ok, ProxyWalletAddr}, + hb_http:get(Node, <<"/~meta@1.0/info/address">>, ProxyOpts) + ), + % Ensure that computation is done by the exec node. + {ok, ResMsg} = hb_http:get(Node, <<"/c?c+list=1">>, ExecOpts), + ?assertEqual([ExecNodeAddr], hb_message:signers(ResMsg, ExecOpts)). + +%% @doc Demonstrates routing tables being dynamically created and adjusted +%% according to the real-time performance of nodes. This test utilizes the +%% `dynamic-router' script to manage routes and recalculate weights based on the +%% reported performance. +dynamic_routing_by_performance_test_() -> + {timeout, 60, fun dynamic_routing_by_performance/0}. +dynamic_routing_by_performance() -> + % Setup test parameters + TestNodes = 4, + BenchRoutes = 16, + TestPath = <<"/worker">>, + % Start the main node for the test, loading the `dynamic-router' script and + % the http_monitor to generate performance messages. + {ok, Script} = file:read_file(<<"scripts/dynamic-router.lua">>), + Node = hb_http_server:start_node(Opts = #{ + relay_http_client => gun, + store => hb_test_utils:test_store(), + priv_wallet => ar_wallet:new(), + router_opts => #{ + <<"provider">> => #{ + <<"path">> => + <<"/perf-router~node-process@1.0/compute/routes~message@1.0">> + } + }, + node_processes => #{ + <<"perf-router">> => #{ + <<"device">> => <<"process@1.0">>, + <<"execution-device">> => <<"lua@5.3a">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"module">> => #{ + <<"content-type">> => <<"application/lua">>, + <<"name">> => <<"dynamic-router">>, + <<"body">> => Script + }, + % Set module-specific factors for the test + <<"pricing-weight">> => 1, + <<"performance-weight">> => 99, + <<"score-preference">> => 4, + <<"performance-period">> => 2, % Adjust quickly + <<"initial-performance">> => 1000 + } + }, + % Define the request that should be called in order to record performance + % information into the process. The `body' of the `http_monitor' message + % is filled with the signed performance report. + http_monitor => #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"/perf-router~node-process@1.0/schedule">> + } + }), + % Start and add a series of nodes with decreasing performance, via lag + % introduced with a hook set to `~test@1.0/delay'. + _XNodes = + lists:map( + fun(X) -> + % Start the node, applying a delay that increases for each additional + % node. + XNode = + hb_http_server:start_node( + #{ + store => hb_test_utils:test_store(), + on => + #{ + <<"request">> => #{ + <<"device">> => <<"test-device@1.0">>, + <<"path">> => <<"delay">>, + <<"duration">> => (X - 1) * 100, + <<"return">> => #{ + <<"body">> => [ + #{ <<"worker">> => X }, + <<"worker">> + ] + } + } + } + } + ), + % Register the node with the router. + hb_http:post( + Node, + #{ + <<"path">> => <<"/perf-router~node-process@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + #{ + <<"path">> => <<"register">>, + <<"route">> => + #{ + <<"prefix">> => XNode, + <<"template">> => TestPath, + <<"price">> => 1000 + X + } + }, + Opts + ) + }, + Opts + ), + XNode + end, + lists:seq(1, TestNodes) + ), + % Force calculation of the process state. + {ok, ResBefore} = + hb_http:get( + Node, + PerfPath = + <<"/perf-router~node-process@1.0/now/routes~message@1.0/1/nodes">>, + Opts + ), + ?event(debug_dynrouter, {nodes_before, ResBefore}), + % Send `BenchRoutes' request messages to the nodes. + lists:foreach( + fun(_XNode) -> + % We send the requests to the main node's `relay@1.0' device, which + % will then apply the routes and the request to the test node set. + Res = hb_http:get( + Node, + << "/~relay@1.0/call?relay-path=/worker" >>, + Opts + ), + ?event(debug_dynrouter, {recvd, Res}) + end, + lists:seq(1, BenchRoutes) + ), + % Call `recalculate' on the router process and get the resulting weight + % table. + hb_http:post( + Node, + #{ + <<"path">> => <<"/perf-router~node-process@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit(#{ <<"path">> => <<"recalculate">> }, Opts) + }, + Opts + ), + % Get the new weights + {ok, After} = hb_http:get(Node, PerfPath, Opts), + WeightsByWorker = + maps:from_list( + lists:map( + fun(N) -> + { + N, + hb_ao:get( + <<(integer_to_binary(N))/binary, "/weight">>, + After, + Opts + ) + } + end, + lists:seq(1, TestNodes) + ) + ), + ?event(debug_dynrouter, {worker_weights, {explicit, WeightsByWorker}}), + ?assert(maps:get(1, WeightsByWorker) > 0.5), + ?assert(maps:get(TestNodes, WeightsByWorker) < 0.5), + ok. + +weighted_random_strategy_test() -> + Nodes = + [ + #{ <<"host">> => <<"1">>, <<"weight">> => 1 }, + #{ <<"host">> => <<"2">>, <<"weight">> => 99 } + ], + SimRes = simulate(1000, 1, Nodes, <<"By-Weight">>), + [HitsOnFirstHost, _] = simulation_distribution(SimRes, Nodes), + ProportionOfFirstHost = HitsOnFirstHost / 1000, + ?event(debug_weighted_random, {proportion_of_first_host, ProportionOfFirstHost}), + ?assert(ProportionOfFirstHost < 0.05), + ?assert(ProportionOfFirstHost >= 0.0001). + strategy_suite_test_() -> lists:map( fun(Strategy) -> @@ -218,7 +1299,7 @@ strategy_suite_test_() -> [<<"Random">>, <<"By-Base">>, <<"Nearest">>] ). -%% @doc Ensure that `By-Base` always chooses the same node for the same +%% @doc Ensure that `By-Base' always chooses the same node for the same %% hashpath. by_base_determinism_test() -> FirstN = 5, @@ -236,7 +1317,7 @@ unique_test(Strategy) -> unique_nodes(Simulation). choose_1_test(Strategy) -> - TestSize = 3750, + TestSize = 1500, Nodes = generate_nodes(20), Simulation = simulate(TestSize, 1, Nodes, Strategy), within_norms(Simulation, Nodes, TestSize). @@ -265,113 +1346,232 @@ unique_nodes(Simulation) -> route_template_message_matches_test() -> Routes = [ #{ - <<"Template">> => #{ <<"Other-Key">> => <<"Other-Value">> }, - <<"Node">> => <<"incorrect">> + <<"template">> => #{ <<"other-key">> => <<"other-value">> }, + <<"node">> => <<"incorrect">> }, #{ - <<"Template">> => #{ <<"Special-Key">> => <<"Special-Value">> }, - <<"Node">> => <<"correct">> + <<"template">> => #{ <<"special-key">> => <<"special-value">> }, + <<"node">> => <<"correct">> } ], ?assertEqual( {ok, <<"correct">>}, - find_route( - #{ path => <<"/">>, <<"Special-Key">> => <<"Special-Value">> }, + route( + #{ <<"path">> => <<"/">>, <<"special-key">> => <<"special-value">> }, #{ routes => Routes } ) ), ?assertEqual( - no_matches, - find_route( - #{ path => <<"/">>, <<"Special-Key">> => <<"Special-Value2">> }, + {error, no_matches}, + route( + #{ <<"path">> => <<"/">>, <<"special-key">> => <<"special-value2">> }, #{ routes => Routes } ) ), ?assertEqual( {ok, <<"fallback">>}, - find_route( - #{ path => <<"/">> }, - #{ routes => Routes ++ [#{ <<"Node">> => <<"fallback">> }] } + route( + #{ <<"path">> => <<"/">> }, + #{ routes => Routes ++ [#{ <<"node">> => <<"fallback">> }] } ) ). route_regex_matches_test() -> Routes = [ #{ - <<"Template">> => <<"/.*/Compute">>, - <<"Node">> => <<"incorrect">> + <<"template">> => <<"/.*/compute">>, + <<"node">> => <<"incorrect">> }, #{ - <<"Template">> => <<"/.*/Schedule">>, - <<"Node">> => <<"correct">> + <<"template">> => <<"/.*/schedule">>, + <<"node">> => <<"correct">> } ], ?assertEqual( {ok, <<"correct">>}, - find_route(#{ path => <<"/abc/Schedule">> }, #{ routes => Routes }) + route(#{ <<"path">> => <<"/abc/schedule">> }, #{ routes => Routes }) ), ?assertEqual( {ok, <<"correct">>}, - find_route(#{ path => <<"/a/b/c/Schedule">> }, #{ routes => Routes }) + route(#{ <<"path">> => <<"/a/b/c/schedule">> }, #{ routes => Routes }) + ), + ?assertEqual( + {error, no_matches}, + route(#{ <<"path">> => <<"/a/b/c/bad-key">> }, #{ routes => Routes }) + ). + +explicit_route_test() -> + Routes = [ + #{ + <<"template">> => <<"*">>, + <<"node">> => <<"unimportant">> + } + ], + ?assertEqual( + {ok, <<"https://google.com">>}, + route( + #{ <<"path">> => <<"https://google.com">> }, + #{ routes => Routes } + ) ), ?assertEqual( - no_matches, - find_route(#{ path => <<"/a/b/c/BadKey">> }, #{ routes => Routes }) + {ok, <<"http://google.com">>}, + route( + #{ <<"path">> => <<"http://google.com">> }, + #{ routes => Routes } + ) + ), + % Test that `route-path' can also be used to specify the path, via an AO + % call. + ?assertMatch( + {ok, #{ <<"node">> := <<"http://google.com">> }}, + hb_ao:resolve( + #{ <<"device">> => <<"router@1.0">>, routes => Routes }, + #{ + <<"path">> => <<"match">>, + <<"route-path">> => <<"http://google.com">> + }, + #{} + ) ). +device_call_from_singleton_test() -> + % Try with a real-world example, taken from a GET request to the router. + NodeOpts = #{ routes => Routes = [#{ + <<"template">> => <<"/some/path">>, + <<"node">> => <<"old">>, + <<"priority">> => 10 + }]}, + Msgs = hb_singleton:from(#{ <<"path">> => <<"~router@1.0/routes">> }, NodeOpts), + ?event({msgs, Msgs}), + ?assertEqual( + {ok, Routes}, + hb_ao:resolve_many(Msgs, NodeOpts) + ). + + get_routes_test() -> - Node = hb_http_server:start_test_node( + Node = hb_http_server:start_node( #{ force_signed => false, - routes => Routes = [ + routes => [ #{ - <<"Template">> => <<"*">>, - <<"Node">> => <<"our_node">>, - <<"Priority">> => 10 + <<"template">> => <<"*">>, + <<"node">> => <<"our_node">>, + <<"priority">> => 10 } ] } ), - Res = hb_client:routes(Node), - ?event(debug, {get_routes_test, Res}), - {ok, RecvdRoutes} = Res, - ?assert(hb_message:match(Routes, RecvdRoutes)). + Res = hb_http:get(Node, <<"/~router@1.0/routes/1/node">>, #{}), + ?event({get_routes_test, Res}), + {ok, Recvd} = Res, + ?assertMatch(<<"our_node">>, Recvd). add_route_test() -> - Node = hb_http_server:start_test_node( + Owner = ar_wallet:new(), + Node = hb_http_server:start_node( #{ force_signed => false, - routes => Routes = [ + routes => [ #{ - <<"Template">> => <<"/Some/Path">>, - <<"Node">> => <<"old">>, - <<"Priority">> => 10 + <<"template">> => <<"/some/path">>, + <<"node">> => <<"old">>, + <<"priority">> => 10 } - ] + ], + operator => hb_util:encode(ar_wallet:to_address(Owner)) } ), - Res = hb_client:add_route(Node, - NewRoute = #{ - <<"Template">> => <<"/Some/New/Path">>, - <<"Node">> => <<"new">>, - <<"Priority">> => 15 - } + Res = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/~router@1.0/routes">>, + <<"template">> => <<"/some/new/path">>, + <<"node">> => <<"new">>, + <<"priority">> => 15 + }, + Owner + ), + #{} + ), + ?event({post_res, Res}), + ?assertMatch({ok, <<"Route added.">>}, Res), + GetRes = hb_http:get(Node, <<"/~router@1.0/routes/2/node">>, #{}), + ?event({get_res, GetRes}), + {ok, Recvd} = GetRes, + ?assertMatch(<<"new">>, Recvd). + +%% @doc Test that the `preprocess/3' function re-routes a request to remote +%% peers via `~relay@1.0', according to the node's routing table. +request_hook_reroute_to_nearest_test() -> + Peer1 = hb_http_server:start_node(#{ priv_wallet => W1 = ar_wallet:new() }), + Peer2 = hb_http_server:start_node(#{ priv_wallet => W2 = ar_wallet:new() }), + Address1 = hb_util:human_id(ar_wallet:to_address(W1)), + Address2 = hb_util:human_id(ar_wallet:to_address(W2)), + Peers = [Address1, Address2], + Node = + hb_http_server:start_node(Opts = #{ + priv_wallet => ar_wallet:new(), + routes => + [ + #{ + <<"template">> => <<"/.*/.*/.*">>, + <<"strategy">> => <<"Nearest">>, + <<"nodes">> => + lists:map( + fun({Address, Node}) -> + #{ + <<"prefix">> => Node, + <<"wallet">> => Address + } + end, + [ + {Address1, Peer1}, + {Address2, Peer2} + ] + ) + } + ], + on => #{ <<"request">> => #{ <<"device">> => <<"relay@1.0">> } } + }), + Res = + lists:map( + fun(_) -> + hb_util:ok( + hb_http:get( + Node, + <<"/~meta@1.0/info/address">>, + Opts#{ http_only_result => true } + ) + ) + end, + lists:seq(1, 3) + ), + ?event(debug_test, + {res, { + {response, Res}, + {signers, hb_message:signers(Res, Opts)} + }} + ), + HasValidSigner = lists:any( + fun(Peer) -> + lists:member(Peer, Res) + end, + Peers ), - ?event(debug, {add_route_test, Res}), - ?assertEqual({ok, <<"Route added.">>}, Res), - Res2 = hb_client:routes(Node), - ?event(debug, {new_routes, Res2}), - {ok, RecvdRoutes} = Res2, - ?assert(hb_message:match(Routes ++ [NewRoute], RecvdRoutes)). + ?assert(HasValidSigner). %%% Statistical test utilities generate_nodes(N) -> [ #{ - <<"Host">> => + <<"host">> => <<"http://localhost:", (integer_to_binary(Port))/binary>>, - <<"Wallet">> => hb_util:encode(crypto:strong_rand_bytes(32)) + <<"wallet">> => hb_util:encode(crypto:strong_rand_bytes(32)) } || Port <- lists:seq(1, N) @@ -403,7 +1603,7 @@ simulation_occurences(SimRes, Nodes) -> fun(NearestNodes, Acc) -> lists:foldl( fun(Node, Acc2) -> - Acc2#{ Node => maps:get(Node, Acc2) + 1 } + Acc2#{ Node => hb_maps:get(Node, Acc2, 0, #{}) + 1 } end, Acc, NearestNodes @@ -414,11 +1614,11 @@ simulation_occurences(SimRes, Nodes) -> ). simulation_distribution(SimRes, Nodes) -> - maps:values(simulation_occurences(SimRes, Nodes)). + hb_maps:values(simulation_occurences(SimRes, Nodes), #{}). within_norms(SimRes, Nodes, TestSize) -> Distribution = simulation_distribution(SimRes, Nodes), - % Check that the mean is `TestSize/length(Nodes)` + % Check that the mean is `TestSize/length(Nodes)' Mean = hb_util:mean(Distribution), ?assert(Mean == (TestSize / length(Nodes))), % Check that the highest count is not more than 3 standard deviations diff --git a/src/dev_scheduler.erl b/src/dev_scheduler.erl index 76f70bc06..13d8c9659 100644 --- a/src/dev_scheduler.erl +++ b/src/dev_scheduler.erl @@ -1,32 +1,37 @@ %%% @doc A simple scheduler scheme for AO. %%% This device expects a message of the form: -%%% Process: #{ id, Scheduler: #{ Authority } } -%%% ``` +%%% Process: `#{ id, Scheduler: #{ Authority } }' +%%% 
 %%% It exposes the following keys for scheduling:
-%%%     #{ method: GET, path: <<"/info">> } ->
+%%%     `#{ method: GET, path: <<"/info">> }' ->
 %%%         Returns information about the scheduler.
-%%%     #{ method: GET, path: <<"/slot">> } -> slot(Msg1, Msg2, Opts)
+%%%     `#{ method: GET, path: <<"/slot">> }' -> `slot(Msg1, Msg2, Opts)'
 %%%         Returns the current slot for a process.
-%%%     #{ method: GET, path: <<"/schedule">> } -> get_schedule(Msg1, Msg2, Opts)
+%%%     `#{ method: GET, path: <<"/schedule">> }' -> `get_schedule(Msg1, Msg2, Opts)'
 %%%         Returns the schedule for a process in a cursor-traversable format.
-%%%     #{ method: POST, path: <<"/schedule">> } -> post_schedule(Msg1, Msg2, Opts)
-%%%         Schedules a new message for a process.'''
+%%%    ` #{ method: POST, path: <<"/schedule">> }' -> `post_schedule(Msg1, Msg2, Opts)'
+%%%         Schedules a new message for a process, or starts a new scheduler
+%%%         for the given message.
+%%%
-module(dev_scheduler). -%%% Converge API functions: +%%% AO-Core API functions: -export([info/0]). %%% Local scheduling functions: --export([schedule/3, append/3]). +-export([schedule/3, router/4, location/3]). %%% CU-flow functions: -export([slot/3, status/3, next/3]). --export([start/0, init/3, end_of_schedule/3, checkpoint/1]). +-export([start/0, checkpoint/1]). +%%% Utility functions: +-export([parse_schedulers/1]). %%% Test helper exports: -export([test_process/0]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). - - +%%% The maximum number of assignments that we will query/return at a time. -define(MAX_ASSIGNMENT_QUERY_LEN, 1000). +%%% The timeout for a lookahead worker. +-define(LOOKAHEAD_TIMEOUT, 1500). %% @doc Helper to ensure that the environment is started. start() -> @@ -43,165 +48,634 @@ info() -> #{ exports => [ + location, status, next, schedule, - append, slot, init, - end_of_schedule, checkpoint ], - excludes => [set, keys] + excludes => [set, keys], + default => fun router/4 }. +%% @doc General utility functions that are available to other modules. +parse_schedulers(SchedLoc) when is_list(SchedLoc) -> SchedLoc; +parse_schedulers(SchedLoc) when is_binary(SchedLoc) -> + binary:split( + binary:replace(SchedLoc, <<"\"">>, <<"">>, [global]), + <<",">>, + [global, trim_all] + ). + +%% @doc The default handler for the scheduler device. +router(_, Msg1, Msg2, Opts) -> + ?event({scheduler_router_called, {msg2, Msg2}, {opts, Opts}}), + schedule(Msg1, Msg2, Opts). + %% @doc Load the schedule for a process into the cache, then return the next -%% assignment. Assumes that Msg1 is a `dev_process` or similar message, having +%% assignment. Assumes that Msg1 is a `dev_process' or similar message, having %% a `Current-Slot' key. It stores a local cache of the schedule in the %% `priv/To-Process' key. next(Msg1, Msg2, Opts) -> - ?event({scheduler_next_called, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), - Schedule = - hb_private:get( - <<"priv/Scheduler/Assignments">>, + ?event(debug_next, {scheduler_next_called, {msg1, Msg1}, {msg2, Msg2}}), + ?event(next, started_next), + ?event(next_profiling, started_next), + Schedule = message_cached_assignments(Msg1, Opts), + LastProcessed = + hb_util:int( + hb_ao:get( + <<"at-slot">>, + Msg1, + Opts#{ hashpath => ignore } + ) + ), + ?event(next_profiling, got_last_processed), + ?event(debug_next, {in_message_cache, {schedule, Schedule}}), + ?event(next, {last_processed, LastProcessed, {message_cache, length(Schedule)}}), + % Get the assignments from the message cache, local cache, or fetch from + % the SU. Returns an ordered list of assignments. + NextAssignment = + find_next_assignment( Msg1, + Msg2, + Schedule, + LastProcessed, Opts ), - LastProcessed = hb_converge:get(<<"Current-Slot">>, Msg1, Opts), - ?event({local_schedule_cache, {schedule, Schedule}}), - Assignments = - case Schedule of - X when is_map(X) and map_size(X) > 0 -> X; - _ -> - {ok, RecvdAssignments} = - hb_converge:resolve( - Msg1, - #{ - <<"Method">> => <<"GET">>, - path => <<"Schedule/Assignments">>, - <<"From">> => LastProcessed - }, - Opts - ), - RecvdAssignments + ?event(next_profiling, got_assignments), + case NextAssignment of + {error, Reason} -> + ?event(next_profiling, got_no_assignments), + {error, Reason}; + {ok, [], _} -> + {error, #{ + <<"status">> => 404, + <<"reason">> => + <<"Requested slot not yet available in schedule.">> + } + }; + {ok, Assignments, Lookahead} -> + ?event(next_profiling, got_assignments), + validate_next_slot(Msg1, Assignments, Lookahead, LastProcessed, Opts) + end. + +%% @doc Validate the `next` slot generated by `find_next_assignment`. +validate_next_slot(Msg1, [NextAssignment|Assignments], Lookahead, Last, Opts) -> + % Paranoia: Get the slot of the next assignment, to ensure that it is the + % last processed slot + 1. + NextAssignmentSlot = + try hb_util:int( + hb_ao:get( + <<"slot">>, + NextAssignment, + Opts#{ hashpath => ignore } + ) + ) + catch + error:badarg -> slot_not_processable end, - ValidKeys = - lists:filter( - fun(Slot) -> - try - binary_to_integer(Slot) > LastProcessed - catch - _:_ -> false - end - end, - maps:keys(Assignments) - ), + ?event(next_profiling, found_next_assignment_slot), + ?event(debug_next, {norm_assignments, Assignments}), + ?event(next, {assignments_to_cache, length(Assignments)}), % Remove assignments that are below the last processed slot. - FilteredAssignments = maps:with(ValidKeys, Assignments), - ?event({filtered_assignments, FilteredAssignments}), - Slot = lists:min([ binary_to_integer(S) || S <- ValidKeys ]), - ?event({next_slot_to_process, Slot, {last_processed, LastProcessed}}), - case (LastProcessed + 1) == Slot of - true -> - NextMessage = - hb_converge:get( - integer_to_binary(Slot), - FilteredAssignments, - Opts - ), + ?event(debug_next, + {calculating_next_from_assignments, + {last_processed, Last}, + {next_slot_from_assignment, NextAssignmentSlot}, + {assignments_received, length(Assignments)} + }), + ExpectedSlot = Last + 1, + case NextAssignmentSlot of + ExpectedSlot -> + ?event(next_profiling, setting_cache), + ?event(next, {setting_cache, {assignments, length(Assignments)}}), NextState = - hb_private:set( - Msg1, - <<"Schedule/Assignments">>, - hb_converge:remove(FilteredAssignments, Slot), - Opts - ), - ?event( - {next_returning, {slot, Slot}, {message, NextMessage}}), - {ok, #{ <<"Message">> => NextMessage, <<"State">> => NextState }}; - false -> + case hb_util:atom(hb_opts:get(scheduler_in_memory_cache, true, Opts)) of + true -> + hb_private:set( + Msg1, + #{ <<"scheduler@1.0">> => #{ + <<"assignments">> => Assignments, + <<"lookahead-worker">> => Lookahead + }}, + Opts + ); + false -> + Msg1#{ + <<"scheduler@1.0">> => #{ + <<"lookahead-worker">> => Lookahead + } + } + end, + ?event(debug_next, + {next_returning, + {slot, NextAssignmentSlot}, + {message, NextAssignment} + } + ), + ?event(next, {next_returning, {slot, NextAssignmentSlot}}), + ?event(next_profiling, returning), + {ok, #{ <<"body">> => NextAssignment, <<"state">> => NextState }}; + slot_not_processable -> {error, #{ - <<"Status">> => <<"Service Unavailable">>, - <<"Body">> => <<"No assignment found for next slot.">> + <<"status">> => 500, + <<"reason">> => + <<"Unprocessable slot value received in assignment.">> } + }; + UnexpectedSlot -> + {error, + #{ + <<"status">> => 404, + <<"reason">> => + <<"Received assignment slot does not match expected slot.">>, + <<"unexpected-slot">> => UnexpectedSlot, + <<"expected-slot">> => ExpectedSlot + } + } + end. + +%% @doc Get the assignments for a process from the message cache, local cache, +%% or the inbox (thanks to a lookahead-worker). +find_next_assignment(_Msg1, _Msg2, Schedule = [_Next|_], _LastSlot, _Opts) -> + {ok, Schedule, undefined}; +find_next_assignment(Msg1, Msg2, _Schedule, LastSlot, Opts) -> + ProcID = dev_process:process_id(Msg1, Msg2, Opts), + LocalCacheRes = + case hb_util:atom(hb_opts:get(scheduler_ignore_local_cache, false, Opts)) of + true -> not_found; + false -> + check_lookahead_and_local_cache(Msg1, ProcID, LastSlot + 1, Opts) + end, + case LocalCacheRes of + {ok, Worker, Assignment} -> + ?event(next_debug, + {in_cache, + {slot, LastSlot + 1}, + {assignment, Assignment} + } + ), + ?event(next_profiling, read_assignment), + {ok, [Assignment], Worker}; + not_found -> + {ok, RecvdAssignments} = + hb_ao:resolve( + Msg1, + #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"schedule/assignments">>, + <<"from">> => LastSlot + }, + Opts#{ scheduler_follow_redirects => true } + ), + % Convert the assignments to an ordered list of messages, + % after removing all keys before the last processed slot. + { + ok, + hb_util:message_to_ordered_list( + maps:filter( + fun(<<"priv">>, _) -> false; + (<<"commitments">>, _) -> false; + (Slot, _) -> hb_util:int(Slot) > LastSlot + end, + RecvdAssignments + ) + ), + undefined } end. +%% @doc Non-device exported helper to get the cached assignments held in a +%% process. +message_cached_assignments(Msg, Opts) -> + hb_private:get( + <<"scheduler@1.0/assignments">>, + Msg, + [], + Opts + ). + +%% @doc Spawn a new Erlang process to fetch the next assignments from the local +%% cache, if we have them available. +spawn_lookahead_worker(ProcID, Slot, Opts) -> + Caller = self(), + spawn( + fun() -> + ?event(next_lookahead, + {looking_ahead, + {proc_id, ProcID}, + {slot, Slot}, + {caller, Caller} + } + ), + case dev_scheduler_cache:read(ProcID, Slot, Opts) of + {ok, Assignment} -> + LoadedAssignment = hb_cache:ensure_all_loaded(Assignment, Opts), + Caller ! {assignment, ProcID, Slot, LoadedAssignment}; + not_found -> + fail + end + end + ). + +%% @doc Check if we have a result from a lookahead worker or from our local +%% cache. If we have a result in the local cache, we may also start a new +%% lookahead worker to fetch the next assignments if we have them locally, +%% ahead of time. This can be enabled/disabled with the `scheduler_lookahead' +%% option. +check_lookahead_and_local_cache(Msg1, ProcID, TargetSlot, Opts) when is_map(Msg1) -> + case hb_private:get(<<"scheduler@1.0/lookahead-worker">>, Msg1, Opts) of + not_found -> + check_lookahead_and_local_cache(undefined, ProcID, TargetSlot, Opts); + LookaheadWorker -> + check_lookahead_and_local_cache(LookaheadWorker, ProcID, TargetSlot, Opts) + end; +check_lookahead_and_local_cache(Worker, ProcID, TargetSlot, Opts) when is_pid(Worker) -> + receive + {assignment, ProcID, OldSlot, _Assignment} when OldSlot < TargetSlot -> + % The lookahead worker has found an assignment for a slot that is + % before the target slot. We remove it from the cache and continue + % searching. + ?event(next_lookahead, + {received_expired_assignment, + {slot, OldSlot}, + {target_slot, TargetSlot} + } + ), + check_lookahead_and_local_cache(Worker, ProcID, TargetSlot, Opts); + {assignment, ProcID, TargetSlot, Assignment} -> + % The lookahead worker has found an assignment for the target slot. + % We return the assignment and stop searching. We will start a new + % lookahead worker to fetch the next slot in the background again. + NewWorker = spawn_lookahead_worker(ProcID, TargetSlot + 1, Opts), + ?event(next_lookahead, + {lookahead_worker_succeeded, {slot, TargetSlot}} + ), + {ok, NewWorker, Assignment} + after ?LOOKAHEAD_TIMEOUT -> + ?event(next_lookahead, {lookahead_read_timeout, {slot, TargetSlot}}), + erlang:exit(Worker, timeout), + check_lookahead_and_local_cache(undefined, ProcID, TargetSlot, Opts) + end; +check_lookahead_and_local_cache(undefined, ProcID, TargetSlot, Opts) -> + % The lookahead worker has not found an assignment for the target + % slot yet, so we check our local cache. + ?event(next_lookahead, {reading_local_cache, {slot, TargetSlot}}), + case dev_scheduler_cache:read(ProcID, TargetSlot, Opts) of + not_found -> not_found; + {ok, Assignment} -> + % We have an assignment in our local cache, so we return it. + % Depending on the `scheduler_lookahead' option, we may also + % start a new lookahead worker to fetch the next assignments + % if we have them locally, ahead of time. + Worker = + case hb_opts:get(scheduler_lookahead, true, Opts) of + false -> undefined; + true -> + % We found the assignment in our local cache, so + % optionally spawn a new Erlang process to fetch + % the next assignments if we have them locally, + % ahead of time. + spawn_lookahead_worker(ProcID, TargetSlot + 1, Opts) + end, + {ok, Worker, hb_cache:ensure_all_loaded(Assignment, Opts)} + end. + %% @doc Returns information about the entire scheduler. status(_M1, _M2, _Opts) -> ?event(getting_scheduler_status), Wallet = dev_scheduler_registry:get_wallet(), {ok, #{ - <<"Address">> => hb_util:id(ar_wallet:to_address(Wallet)), - <<"Processes">> => + <<"address">> => hb_util:id(ar_wallet:to_address(Wallet)), + <<"processes">> => lists:map( fun hb_util:id/1, dev_scheduler_registry:get_processes() ), - <<"Cache-Control">> => <<"no-store">> + <<"cache-control">> => <<"no-store">> } }. +%% @doc Router for `record' requests. Expects either a `POST' or `GET' request. +location(Msg1, Msg2, Opts) -> + case hb_ao:get(<<"method">>, Msg2, <<"GET">>, Opts) of + <<"POST">> -> post_location(Msg1, Msg2, Opts); + <<"GET">> -> get_location(Msg1, Msg2, Opts) + end. + +%% @doc Search for the location of the scheduler in the scheduler-location +%% cache. If an address is provided, we search for the location of that +%% specific scheduler. Otherwise, we return the location record for the current +%% node's scheduler, if it has been established. +get_location(_Msg1, Req, Opts) -> + % Get the address of the scheduler from the request. + Address = + hb_ao:get( + <<"address">>, + Req, + hb_util:human_id(ar_wallet:to_address( + hb_opts:get(priv_wallet, hb:wallet(), Opts) + )), + Opts + ), + % Search for the location of the scheduler in the scheduler-location cache. + case dev_scheduler_cache:read_location(Address, Opts) of + not_found -> + {ok, + #{ + <<"status">> => 404, + <<"body">> => + <<"No location found for address: ", Address/binary>> + } + }; + {ok, Location} -> {ok, #{ <<"body">> => Location }} + end. + +%% @doc Generate a new scheduler location record and register it. We both send +%% the new scheduler-location to the given registry, and return it to the caller. +post_location(Msg1, RawReq, RawOpts) -> + Opts = + case dev_whois:ensure_host(RawOpts) of + {ok, NewOpts} -> NewOpts; + _ -> RawOpts + end, + % Ensure that the request is signed by the operator. + Req = + case hb_ao:get_first( + [{Msg1, <<"target">>}, {RawReq, <<"target">>}], + not_found, + Opts + ) of + not_found -> RawReq; + <<"self">> -> Msg1; + <<"request">> -> RawReq; + Target -> hb_ao:get(Target, RawReq, not_found, Opts) + end, + {ok, OnlyCommitted} = hb_message:with_only_committed(Req, Opts), + ?event(scheduler_location, + {scheduler_location_registration_request, OnlyCommitted} + ), + % Gather metadata for request validation. + Signers = hb_message:signers(OnlyCommitted, Opts), + Self = + hb_util:human_id( + ar_wallet:to_address( + hb_opts:get(priv_wallet, hb:wallet(), Opts) + ) + ), + ExistingNonce = + case hb_gateway_client:scheduler_location(Self, Opts) of + {ok, SchedulerLocation} -> + hb_ao:get(<<"nonce">>, SchedulerLocation, 0, Opts); + {error, _} -> -1 + end, + NewNonce = hb_ao:get(<<"nonce">>, OnlyCommitted, ExistingNonce + 1, Opts), + case {NewNonce > ExistingNonce, lists:member(Self, Signers)} of + {false, _} -> + % Invalid request: Known nonce is already higher than requested nonce + % for the given operator. + {ok, + #{ + <<"status">> => 400, + <<"body">> => <<"Known nonce higher than requested nonce.">>, + <<"requested-nonce">> => NewNonce, + <<"existing-nonce">> => ExistingNonce, + <<"signers">> => Signers + } + }; + {true, false} -> + % Received request to store a new scheduler location from a peer + % that is not the operator. + case dev_scheduler_cache:write_location(OnlyCommitted, Opts) of + ok -> + ?event(scheduler_location, + {cached_foreign_peer_location, OnlyCommitted} + ), + {ok, OnlyCommitted}; + {error, Reason} -> + {error, + #{ + <<"status">> => 400, + <<"body">> => + <<"Failed to store new scheduler location.">>, + <<"reason">> => Reason + } + } + end; + {true, true} -> + % The operator has asked to replace the scheduler location. Get the + % details and register the new location. Registration occurs in the + % following steps: + % 1. Generate a new scheduler location message. + % 2. Sign the message. + % 3. Upload the message to Arweave. + % 4. Post the message to the peers specified in the + % `scheduler_location_notify_peers' option. + TimeToLive = + hb_ao:get_first( + [ + {Msg1, <<"time-to-live">>}, + {OnlyCommitted, <<"time-to-live">>} + ], + hb_opts:get(scheduler_location_ttl, 1000 * 60 * 60, Opts), + Opts + ), + URL = + case hb_ao:get(<<"url">>, OnlyCommitted, Opts) of + not_found -> + Port = hb_util:bin(hb_opts:get(port, 8734, Opts)), + Host = hb_opts:get(host, <<"localhost">>, Opts), + Protocol = hb_opts:get(protocol, http1, Opts), + ProtoStr = + case Protocol of + http1 -> <<"http">>; + _ -> <<"https">> + end, + <>; + GivenURL -> GivenURL + end, + % Construct the new scheduler location message. + Codec = + hb_ao:get_first( + [ + {Msg1, <<"require-codec">>}, + {OnlyCommitted, <<"require-codec">>} + ], + <<"httpsig@1.0">>, + Opts + ), + NewSchedulerLocation = + #{ + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"type">> => <<"scheduler-location">>, + <<"url">> => URL, + <<"nonce">> => NewNonce, + <<"time-to-live">> => TimeToLive, + <<"codec-device">> => Codec + }, + Signed = hb_message:commit(NewSchedulerLocation, Opts, Codec), + dev_scheduler_cache:write_location(Signed, Opts), + ?event(scheduler_location, + {uploading_signed_scheduler_location, Signed} + ), + % Asynchronously upload the location record to Arweave. + spawn( + fun() -> + hb_client:upload(Signed, Opts) + end + ), + % Post the new scheduler location to the peers specified in the + % `scheduler_location_notify_peers' option. + Results = + lists:map( + fun(Node) -> + PostRes = hb_http:post( + Node, + <<"/~scheduler@1.0/record">>, + Signed, + Opts + ), + ?event(scheduler_location, + {outbound_request, {res, PostRes}} + ) + end, + hb_opts:get(scheduler_location_notify_peers, [], Opts) + ), + ?event(scheduler_location, + {scheduler_location_registration_success, + {arweave_publication, async_upload_initiated}, + {foreign_peers_notified, length(Results)} + } + ), + {ok, Signed} + end. + %% @doc A router for choosing between getting the existing schedule, or %% scheduling a new message. schedule(Msg1, Msg2, Opts) -> ?event({resolving_schedule_request, {msg2, Msg2}, {state_msg, Msg1}}), - case hb_converge:get(<<"Method">>, Msg2, <<"GET">>, Opts) of - <<"POST">> -> post_schedule(Msg1, Msg2, Opts); - <<"GET">> -> get_schedule(Msg1, Msg2, Opts) + case hb_util:key_to_atom(hb_ao:get(<<"method">>, Msg2, <<"GET">>, Opts)) of + post -> post_schedule(Msg1, Msg2, Opts); + get -> get_schedule(Msg1, Msg2, Opts) end. -%% @doc Alternate access path for scheduling a message, for situations in which -%% the user cannot modify the `Method`. -append(Msg1, Msg2, Opts) -> - post_schedule(Msg1, Msg2, Opts). - -%% @doc Schedules a new message on the SU. +%% @doc Schedules a new message on the SU. Searches Msg1 for the appropriate ID, +%% then uses the wallet address of the scheduler to determine if the message is +%% for this scheduler. If so, it schedules the message and returns the assignment. post_schedule(Msg1, Msg2, Opts) -> ?event(scheduling_message), - ToSched = hb_converge:get(message, Msg2, Opts#{ hashpath => ignore }), - ToSchedID = hb_converge:get(id, ToSched), - Proc = hb_converge:get(process, Msg1, Opts#{ hashpath => ignore }), - ?no_prod("Once we have GQL, get the scheduler location record. " - "For now, we'll just use the address of the wallet."), - SchedulerLocation = - hb_converge:get(<<"Process/Scheduler-Location">>, - Msg1, Opts#{ hashpath => ignore }), - ProcID = hb_converge:get(id, Proc), - PID = dev_scheduler_registry:find(ProcID, true), - #{ wallet := Wallet } = dev_scheduler_server:info(PID), - WalletAddress = hb_util:id(ar_wallet:to_address(Wallet)), - ?event( - {post_schedule, - {process_id, ProcID}, - {process, Proc}, - {message_id, ToSchedID}, - {message, ToSched} - } - ), - ?no_prod("SU does not validate item before writing into stream."), - %case {ar_bundles:verify_item(ToSched), hb_converge:get(type, ToSched)} of - case {WalletAddress == SchedulerLocation, true, hb_converge:get(type, ToSched)} of - {false, _, _} -> - {ok, + % Find the target message to schedule: + ToSched = find_message_to_schedule(Msg1, Msg2, Opts), + ?event({to_sched, ToSched}), + % Find the ProcessID of the target message: + % - If it is a Process, use the ID of the message. + % - If not, use the target as the ProcessID. + ProcID = + case hb_ao:get(<<"type">>, ToSched, not_found, Opts) of + <<"Process">> -> hb_message:id(ToSched, all, Opts); + _ -> + case hb_ao:get(<<"target">>, ToSched, not_found, Opts) of + not_found -> find_target_id(Msg1, Msg2, Opts); + Target -> hb_util:human_id(Target) + end + end, + ?event({proc_id, ProcID}), + % Filter all unsigned keys from the source message. + case hb_message:with_only_committed(ToSched, Opts) of + {ok, OnlyCommitted} -> + ?event( + {post_schedule, + {schedule_id, ProcID}, + {message, ToSched} + } + ), + % Find the relevant scheduler server for the given process and + % message, start a new one if necessary, or return a redirect to the + % correct remote scheduler. + case find_server(ProcID, Msg1, ToSched, Opts) of + {local, PID} -> + ?event({scheduling_locally, {proc_id, ProcID}, {pid, PID}}), + do_post_schedule(ProcID, PID, OnlyCommitted, Opts); + {redirect, Redirect} -> + ?event({process_is_remote, {redirect, Redirect}}), + case hb_opts:get(scheduler_follow_redirects, true, Opts) of + true -> + ?event({proxying_to_remote_scheduler, + {redirect, Redirect}, + {msg, OnlyCommitted} + }), + post_remote_schedule( + ProcID, + Redirect, + OnlyCommitted, + Opts + ); + false -> {ok, Redirect} + end; + {error, Error} -> + ?event({error_finding_scheduler, {error, Error}}), + {error, Error} + end; + {error, Err} -> + {error, #{ - <<"Status">> => <<"Failed">>, - <<"Body">> => <<"Scheduler location does not match wallet address.">> + <<"status">> => 400, + <<"body">> => <<"Message invalid: ", + "Committed components cannot be validated.">>, + <<"reason">> => Err } - }; - {true, false, _} -> - {ok, + } + end. + +%% @doc Post schedule the message. `Msg2' by this point has been refined to only +%% committed keys, and to only include the `target' message that is to be +%% scheduled. +do_post_schedule(ProcID, PID, Msg2, Opts) -> + % Should we verify the message again before scheduling? + Verified = + case hb_opts:get(verify_assignments, true, Opts) of + true -> + ?event(debug_scheduler_verify, + {verifying_message_before_scheduling, Msg2} + ), + Res = length(hb_message:signers(Msg2, Opts)) > 0 + andalso hb_message:verify(Msg2, signers, Opts), + ?event(debug_scheduler_verify, {verified, Res}), + Res; + accept_unsigned -> + ?event( + debug_scheduler_verify, + {accepting_unsigned_message_before_scheduling, Msg2} + ), + hb_message:verify(Msg2, signers, Opts); + false -> true + end, + ?event({verified, Verified}), + % Handle scheduling of the message if the message is valid. + case {Verified, hb_ao:get(<<"type">>, Msg2, Opts)} of + {false, _} -> + {error, #{ - <<"Status">> => <<"Failed">>, - <<"Body">> => <<"Data item is not valid.">> + <<"status">> => 400, + <<"body">> => <<"Message is not valid.">>, + <<"reason">> => <<"Given message is invalid.">> } }; - {true, true, <<"Process">>} -> - ?no_prod("SU does not write to cache or upload to bundler."), - hb_cache:write(ToSched, Opts), - hb_client:upload(ToSched), + {true, <<"Process">>} -> + {ok, _} = hb_cache:write(Msg2, Opts), + spawn( + fun() -> + {ok, Results} = hb_client:upload(Msg2, Opts), + ?event( + {uploaded_process, {proc_id, ProcID}, {results, Results}} + ) + end + ), ?event( {registering_new_process, {proc_id, ProcID}, @@ -209,78 +683,880 @@ post_schedule(Msg1, Msg2, Opts) -> {is_alive, is_process_alive(PID)} } ), - {ok, - #{ - <<"Status">> => <<"OK">>, - <<"Assignment">> => <<"0">>, - <<"Process">> => ProcID + {ok, dev_scheduler_server:schedule(PID, Msg2)}; + {true, _} -> + ?event( + {scheduling_message, + {proc_id, ProcID}, + {pid, PID}, + {is_alive, is_process_alive(PID)} } - }; - {true, true, _} -> + ), % If Message2 is not a process, use the ID of Message1 as the PID - {ok, - dev_scheduler_server:schedule( - dev_scheduler_registry:find(ProcID, true), - ToSched + {ok, dev_scheduler_server:schedule(PID, Msg2)} + end. + +%% @doc Locate the correct scheduling server for a given process. +find_server(ProcID, Msg1, Opts) -> + find_server(ProcID, Msg1, undefined, Opts). +find_server(ProcID, Msg1, ToSched, Opts) -> + case get_hint(ProcID, Opts) of + {ok, Hint} -> + ?event({found_hint_in_proc_id, Hint}), + generate_redirect(ProcID, Hint, Opts); + not_found -> + ?event({no_hint_in_proc_id, ProcID}), + case dev_scheduler_registry:find(ProcID, false, Opts) of + PID when is_pid(PID) -> + ?event({found_pid_in_local_registry, PID}), + {local, PID}; + not_found -> + ?event({no_pid_in_local_registry, ProcID}), + Proc = find_process_message(ProcID, Msg1, ToSched, Opts), + ?event({found_process, {process, Proc}, {msg1, Msg1}}), + SchedLoc = + hb_ao:get_first( + [ + {Proc, <<"scheduler">>}, + {Proc, <<"scheduler-location">>} + ] ++ + case ToSched of + undefined -> []; + _ -> [{ToSched, <<"scheduler-location">>}] + end, + not_found, + Opts#{ hashpath => ignore } + ), + ?event({sched_loc, SchedLoc}), + case SchedLoc of + not_found -> + {error, <<"No scheduler information provided.">>}; + _ -> + ?event( + {confirming_if_scheduler_is_local, + {addr, SchedLoc} + } + ), + ParsedLoc = parse_schedulers(SchedLoc), + case is_local_scheduler(ProcID, Proc, ParsedLoc, Opts) of + {ok, PID} -> + % We are the scheduler. Start the server if + % it has not already been started, with the + % given options. + {local, PID}; + false -> + % We are not the scheduler. Find it and + % return a redirect. + find_remote_scheduler(ProcID, ParsedLoc, Opts) + end + end + end + end. + +%% @doc Find the process message for a given process ID and base message. +find_process_message(ProcID, Msg1, ToSched, Opts) -> + % Find the process from the message. + MaybeProcessMsg = + hb_ao:get( + <<"process">>, + Msg1, + not_found, + Opts#{ hashpath => ignore } + ), + case MaybeProcessMsg of + not_found -> + ToSchedIsProc = + (ToSched =/= undefined) + andalso (hb_message:id(ToSched, all) == ProcID), + case ToSchedIsProc of + true -> ToSched; + false -> + ?event( + {reading_cache, + {proc_id, ProcID}, + {store, hb_opts:get(store, Opts)} + } + ), + case hb_message:id(Msg1, all) of + ProcID -> Msg1; + _ -> + case hb_cache:read(ProcID, Opts) of + {ok, P} -> P; + not_found -> + throw({ + process_not_available, + ProcID + }) + end + end + end; + P -> P + end. + +%% @doc Determine if a scheduler is local. If so, return the PID and options. +%% We start the local server if we _can_ be the scheduler and it does not already +%% exist. +is_local_scheduler(_, _, [], _Opts) -> false; +is_local_scheduler(ProcID, ProcMsg, [Scheduler | Rest], Opts) -> + case is_local_scheduler(ProcID, ProcMsg, Scheduler, Opts) of + {ok, PID} -> {ok, PID}; + false -> is_local_scheduler(ProcID, ProcMsg, Rest, Opts) + end; +is_local_scheduler(ProcID, ProcMsg, Scheduler, Opts) -> + case hb_opts:as(Scheduler, Opts) of + {ok, _} -> + { + ok, + dev_scheduler_registry:find(ProcID, ProcMsg, Opts) + }; + {error, _} -> false + end. + +%% @doc If a hint is present in the string, return it. Else, return not_found. +get_hint(Str, Opts) when is_binary(Str) -> + case hb_opts:get(scheduler_follow_hints, true, Opts) of + true -> + case binary:split(Str, <<"?">>, [global]) of + [_, QS] -> + QueryMap = hb_maps:from_list(uri_string:dissect_query(QS)), + case hb_maps:get(<<"hint">>, QueryMap, not_found, Opts) of + not_found -> not_found; + Hint -> {ok, Hint} + end; + _ -> not_found + end; + false -> not_found + end; +get_hint(_Str, _Opts) -> not_found. + +%% @doc Generate a redirect message to a scheduler. +generate_redirect(ProcID, SchedulerLocation, Opts) -> + Variant = hb_ao:get(<<"variant">>, SchedulerLocation, <<"ao.N.1">>, Opts), + ?event({generating_redirect, {proc_id, ProcID}, {variant, Variant}}), + RedirectLocation = + case is_binary(SchedulerLocation) of + true -> SchedulerLocation; + false -> + hb_ao:get_first( + [ + {SchedulerLocation, <<"url">>}, + {SchedulerLocation, <<"location">>} + ], + <<"/">>, + Opts ) - } + end, + {redirect, + #{ + <<"status">> => 307, + <<"location">> => RedirectLocation, + <<"body">> => + <<"Redirecting to scheduler: ", RedirectLocation/binary>>, + <<"variant">> => Variant + } + }. + +%% @doc Take a process ID or target with a potential hint and return just the +%% process ID. +without_hint(Target) when ?IS_ID(Target) -> + hb_util:human_id(Target); +without_hint(Target) -> + case binary:split(Target, [<<"?">>, <<"&">>], [global]) of + [ProcID] when ?IS_ID(ProcID) -> hb_util:human_id(ProcID); + _ -> throw({invalid_operation_target, Target}) + end. + +%% @doc Use the SchedulerLocation to find the remote path and return a redirect. +%% If there are multiple locations, try each one in turn until we find the first +%% that matches. +find_remote_scheduler(_ProcID, [], _Opts) -> {error, not_found}; +find_remote_scheduler(ProcID, [Scheduler | Rest], Opts) -> + case find_remote_scheduler(ProcID, Rest, Opts) of + {error, not_found} -> + find_remote_scheduler(ProcID, Scheduler, Opts); + {ok, Redirect} -> + {ok, Redirect} + end; +find_remote_scheduler(ProcID, Scheduler, Opts) -> + % Parse the scheduler location to see if it has a hint. If there is a hint, + % we will use it to construct a redirect message. + case get_hint(Scheduler, Opts) of + {ok, Hint} -> + % We have a hint. Construct a redirect message. + generate_redirect(ProcID, Hint, Opts); + not_found -> + case dev_scheduler_cache:read_location(Scheduler, Opts) of + {ok, SchedMsg} -> + % We have a cached scheduler location. Use it to construct a + % redirect message. + generate_redirect(ProcID, SchedMsg, Opts); + not_found -> + % We have not yet cached the location for this address. + % Find it via the gateway. + case hb_gateway_client:scheduler_location(Scheduler, Opts) of + {ok, SchedMsg} -> + % We have found the location. Cache it and use it to + % construct a redirect message. + Res = + dev_scheduler_cache:write_location( + SchedMsg, + Opts + ), + ?event(scheduler_location, + {cached_scheduler_location, {res, Res}} + ), + generate_redirect(ProcID, SchedMsg, Opts); + {error, Res} -> + ?event( + scheduler_location, + {failed_to_find_scheduler_location_from_gateway, + {error, Res} + } + ), + {error, Res} + end + end end. %% @doc Returns information about the current slot for a process. -slot(M1, _M2, Opts) -> +slot(M1, M2, Opts) -> ?event({getting_current_slot, {msg, M1}}), - Proc = hb_converge:get( - process, - {as, dev_message, M1}, - Opts#{ hashpath => ignore } - ), - ProcID = hb_converge:get(id, Proc), - ?event({getting_current_slot, {proc_id, ProcID}, {process, Proc}}), - {Timestamp, Hash, Height} = ar_timestamp:get(), - #{ current := CurrentSlot, wallet := Wallet } = - dev_scheduler_server:info( - dev_scheduler_registry:find(ProcID) - ), - {ok, #{ - <<"Process">> => ProcID, - <<"Current-Slot">> => CurrentSlot, - <<"Timestamp">> => Timestamp, - <<"Block-Height">> => Height, - <<"Block-Hash">> => Hash, - <<"Cache-Control">> => <<"no-store">>, - <<"Wallet-Address">> => hb_util:human_id(ar_wallet:to_address(Wallet)) - }}. + ProcID = find_target_id(M1, M2, Opts), + case find_server(ProcID, M1, Opts) of + {local, PID} -> + ?event({getting_current_slot, {proc_id, ProcID}}), + {Timestamp, Height, Hash} = ar_timestamp:get(), + #{ current := CurrentSlot, wallets := Wallets } = + dev_scheduler_server:info(PID), + {ok, #{ + <<"process">> => ProcID, + <<"current">> => CurrentSlot, + <<"timestamp">> => Timestamp, + <<"block-height">> => Height, + <<"block-hash">> => Hash, + <<"cache-control">> => <<"no-store">>, + <<"addresses">> => lists:map(fun hb_util:human_id/1, Wallets) + }}; + {redirect, Redirect} -> + case hb_opts:get(scheduler_follow_redirects, true, Opts) of + false -> {ok, Redirect}; + true -> remote_slot(ProcID, Redirect, Opts) + end + end. + +%% @doc Get the current slot from a remote scheduler. +remote_slot(ProcID, Redirect, Opts) -> + ?event({getting_remote_slot, {proc_id, ProcID}, {redirect, {explicit, Redirect}}}), + Node = node_from_redirect(Redirect, Opts), + ?event({getting_slot_from_node, {string, Node}}), + remote_slot( + hb_ao:get(<<"variant">>, Redirect, <<"ao.N.1">>, Opts), + ProcID, + Node, + Opts + ). + +%% @doc Get the current slot from a remote scheduler, based on the variant of +%% the process's scheduler. +remote_slot(<<"ao.N.1">>, ProcID, Node, Opts) -> + % The process is running on a mainnet AO-Core scheduler, so we can just + % use the `/slot' endpoint to get the current slot. + ?event({getting_slot_from_ao_core_remote, + {path, {string, <<"/", ProcID/binary, "/slot">>}}}), + hb_http:get(Node, <>, Opts); +remote_slot(<<"ao.TN.1">>, ProcID, Node, Opts) -> + % The process is running on a testnet AO-Core scheduler, so we need to use + % `/processes/procID/latest' to get the current slot. + Path = << ProcID/binary, "/latest?proc-id=", ProcID/binary>>, + ?event({getting_slot_from_ao_core_remote, {path, {string, Path}}}), + case hb_http:get(Node, Path, Opts#{ http_client => httpc }) of + {ok, Res} -> + ?event({remote_slot_result, {res, Res}}), + case hb_util:int(hb_ao:get(<<"status">>, Res, 200, Opts)) of + 200 -> + Body = hb_ao:get(<<"body">>, Res, Opts), + JSON = hb_json:decode(Body), + ?event({got_slot_response, {json, JSON}}), + % Convert the JSON object for the latest assignment into the + % standardized `~scheduler@1.0' format. + A = + dev_scheduler_formats:aos2_to_assignment( + JSON, + Opts + ), + ?event({got_slot_response, {assignment, A}}), + {ok, #{ + <<"process">> => ProcID, + <<"current">> => hb_maps:get(<<"slot">>, A, undefined, Opts), + <<"timestamp">> => hb_maps:get(<<"timestamp">>, A, undefined, Opts), + <<"block-height">> => hb_maps:get(<<"block-height">>, A, undefined, Opts), + <<"block-hash">> => hb_util:encode(<<0:256>>), + <<"cache-control">> => <<"no-store">> + }}; + 307 -> + ?event({generating_new_redirect, {redirect, Res}}), + % Maintain the same variant, but generate the redirect using + % the new location. + NewRedirect = + generate_redirect( + ProcID, + Res#{ <<"variant">> => <<"ao.TN.1">> }, + Opts + ), + ?event({recursing_on_new_redirect, {redirect, NewRedirect}}), + remote_slot(ProcID, NewRedirect, Opts); + _ -> + {error, Res} + end; + {error, Res} -> + ?event({remote_slot_error, {error, Res}}), + {error, Res} + end. +%% @doc Generate and return a schedule for a process, optionally between +%% two slots -- labelled as `from' and `to'. If the schedule is not local, +%% we redirect to the remote scheduler or proxy based on the node opts. get_schedule(Msg1, Msg2, Opts) -> - Proc = hb_converge:get( - process, - {as, dev_message, Msg1}, - Opts#{ hashpath => ignore } - ), - ProcID = hb_converge:get(id, Proc), + ProcID = hb_util:human_id(find_target_id(Msg1, Msg2, Opts)), From = - case hb_converge:get(from, Msg2, not_found, Opts) of + case hb_ao:get(<<"from">>, Msg2, not_found, Opts) of not_found -> 0; X when X < 0 -> 0; - FromRes -> FromRes + FromRes -> hb_util:int(FromRes) end, To = - case hb_converge:get(to, Msg2, not_found, Opts) of - not_found -> - ?event({getting_current_slot, {proc_id, ProcID}}), - maps:get(current, - dev_scheduler_server:info( - dev_scheduler_registry:find(ProcID) + case hb_ao:get(<<"to">>, Msg2, not_found, Opts) of + not_found -> undefined; + ToRes -> hb_util:int(ToRes) + end, + Format = hb_ao:get(<<"accept">>, Msg2, <<"application/http">>, Opts), + ?event( + {parsed_get_schedule, + {process, ProcID}, + {from, From}, + {to, To}, + {format, Format} + } + ), + case find_server(ProcID, Msg1, Opts) of + {local, _PID} -> + generate_local_schedule(Format, ProcID, From, To, Opts); + {redirect, Redirect} -> + ?event({redirect_received, {redirect, Redirect}}), + case hb_opts:get(scheduler_follow_redirects, true, Opts) of + true -> + case get_remote_schedule(ProcID, From, To, Redirect, Opts) of + {ok, Res} -> + case uri_string:percent_decode(Format) of + <<"application/aos-2">> -> + dev_scheduler_formats:assignments_to_aos2( + ProcID, + hb_ao:get( + <<"assignments">>, Res, [], Opts), + hb_util:atom(hb_ao:get( + <<"continues">>, Res, false, Opts)), + Opts + ); + _ -> + {ok, Res} + end; + {error, Res} -> + {error, Res} + end; + false -> + {ok, Redirect} + end + end. + +%% @doc Get a schedule from a remote scheduler, but first read all of the +%% assignments from the local cache that we already know about. +get_remote_schedule(RawProcID, From, To, Redirect, Opts) -> + % If we are responding to a legacy scheduler request we must add one to the + % `from' slot to account for the fact that the legacy scheduler gives us + % the slots _after_ the stated nonce. + ProcID = without_hint(RawProcID), + {FromLocalCache, _} = get_local_assignments(ProcID, From, To, Opts), + ?event(debug_sched, + {from_local_cache, + {from, From}, + {to, To}, + {read, length(FromLocalCache)} + } + ), + do_get_remote_schedule( + ProcID, + FromLocalCache, + From + length(FromLocalCache), + To, + Redirect, + Opts + ). + +%% @doc Get a schedule from a remote scheduler, unless we already have already +%% read all of the assignments from the local cache. +do_get_remote_schedule(ProcID, LocalAssignments, From, To, _, Opts) + when (To =/= undefined) andalso (From >= To) -> + % We already have all of the assignments from the local cache. Return them + % as a bundle. We set the 'more' to `undefined' to indicate that there may + % be more assignments to fetch, but we don't know for sure. + Res = + dev_scheduler_formats:assignments_to_bundle( + ProcID, + LocalAssignments, + undefined, + Opts + ), + ?event(debug_sched, + {returning_remote_schedule_from_only_cache, + {length, length(LocalAssignments)}, + {from_after_local_cache, From}, + {original_to, To} + }), + Res; +do_get_remote_schedule(ProcID, LocalAssignments, From, To, Redirect, Opts) -> + % We don't have all of the assignments from the local cache, so we need to + % fetch the rest from the remote scheduler. + Node = node_from_redirect(Redirect, Opts), + Variant = hb_ao:get(<<"variant">>, Redirect, <<"ao.N.1">>, Opts), + ?event( + {getting_remote_schedule, + {node, {string, Node}}, + {proc_id, {string, ProcID}}, + {from, From}, + {to, To} + } + ), + MaybeNonce = + if Variant == <<"ao.TN.1">> -> <<"-nonce">>; + true -> <<>> + end, + FromBin = + case From of + undefined -> <<>>; + From -> + % The legacy scheduler gives us the slots _after_ the stated + % nonce. So we need to subtract one from the nonce to get the + % correct slot. + ModFrom = + if Variant == <<"ao.TN.1">> -> From - 1; + true -> From + end, + << + "&from", + MaybeNonce/binary, + "=", + (integer_to_binary(ModFrom))/binary + >> + end, + ToParam = + case To of + undefined -> <<>>; + To -> + << + "&to", MaybeNonce/binary, "=", (integer_to_binary(To))/binary + >> + end, + Path = + case Variant of + <<"ao.N.1">> -> + << + ProcID/binary, + "/schedule?from=", FromBin/binary, ToParam/binary + >>; + <<"ao.TN.1">> -> + << + ProcID/binary, "?proc-id=", ProcID/binary, + FromBin/binary, ToParam/binary, + "&limit=", (hb_util:bin(?MAX_ASSIGNMENT_QUERY_LEN))/binary + >> + end, + ?event({getting_remote_schedule, {node, {string, Node}}, {path, {string, Path}}}), + case hb_http:get(Node, Path, Opts#{ http_client => httpc, protocol => http2 }) of + {ok, Res} -> + case hb_util:int(hb_ao:get(<<"status">>, Res, 200, Opts)) of + 200 -> + {ok, NormSched} = + case Variant of + <<"ao.N.1">> -> + cache_remote_schedule(Variant, ProcID, Res, Opts), + {ok, Res}; + <<"ao.TN.1">> -> + JSONRes = + hb_json:decode( + hb_ao:get( + <<"body">>, + Res, + <<"">>, + Opts#{ hashpath => ignore } + ) + ), + cache_remote_schedule(Variant, ProcID, JSONRes, Opts), + ?event(debug_aos2, {json_res, {json, JSONRes}}), + Filtered = filter_json_assignments(JSONRes, To, From, Opts), + dev_scheduler_formats:aos2_to_assignments( + ProcID, + Filtered, + Opts + ) + end, + % Add existing local assignments we read to the remote schedule. + % In order to do this, we need to first convert the remote + % assignments to a list, maintaining the order of the keys. + RemoteAssignments = + hb_util:message_to_ordered_list( + hb_ao:normalize_keys( + hb_ao:get( + <<"assignments">>, + NormSched, + Opts + ), + Opts + ) + ), + % Merge the local assignments with the remote assignments, + % and normalize the keys. + Merged = + dev_scheduler_formats:assignments_to_bundle( + ProcID, + MergedAssignments = LocalAssignments ++ RemoteAssignments, + hb_ao:get(<<"continues">>, NormSched, false, Opts), + Opts + ), + ?event(debug_sched, + {returning_remote_schedule, + {length, length(MergedAssignments)}, + {from_local_cache, length(LocalAssignments)}, + {from_remote_cache, length(RemoteAssignments)} + } + ), + Merged; + 307 -> + % NOTE: Shouldn't this be using the `Res' location key to + % regenerate the redirect and recurse on that, instead of + % just using the same redirect? + ?event({recursing_on_same_redirect, {redirect, Redirect}}), + do_get_remote_schedule( + ProcID, + LocalAssignments, + From, + To, + Redirect, + Opts + ) + end; + {error, Res} -> + ?event(push, {remote_schedule_result, {res, Res}}, Opts), + {error, Res} + end. + +%% @doc Cache a schedule received from a remote scheduler. +cache_remote_schedule(<<"ao.TN.1">>, ProcID, Schedule, Opts) -> + % If the schedule has a variant of ao.TN.1, we add this to the raw assignment + % before caching it. + ModSchedule = + lists:map( + fun(Assignment) -> + Assignment#{ + <<"variant">> => <<"ao.TN.1">>, + <<"slot">> => + hb_maps:get(<<"cursor">>, Assignment, undefined, Opts), + <<"process">> => ProcID + } + end, + hb_util:ok(hb_maps:find(<<"edges">>, Schedule, Opts)) + ), + cache_remote_schedule(common, ProcID, ModSchedule, Opts); +cache_remote_schedule(<<"ao.N.1">>, ProcID, Schedule, Opts) -> + Assignments = + hb_ao:get( + <<"assignments">>, + Schedule, + Opts#{ hashpath => ignore } + ), + cache_remote_schedule(common, ProcID, Assignments, Opts); +cache_remote_schedule(_, _ProcID, Schedule, Opts) -> + Cacher = + fun() -> + ?event(debug_sched, {caching_remote_schedule, {schedule, Schedule}}), + lists:foreach( + fun(Assignment) -> + % We do not care about the result of the write because it is only + % an additional cache. + ?event(debug_sched, + {writing_assignment, + {assignment, hb_maps:get(<<"slot">>, Assignment, undefined, Opts)} + } + ), + dev_scheduler_cache:write(Assignment, Opts) + end, + AssignmentList = + hb_util:message_to_ordered_list( + hb_maps:without( + [<<"priv">>], + hb_ao:normalize_keys(Schedule, Opts), + Opts + ) ) - ); - ToRes -> ToRes + ), + ?event(debug_sched, + {caching_remote_schedule, {assignments, length(AssignmentList)}} + ) + end, + case hb_opts:get(scheduler_async_remote_cache, true, Opts) of + true -> spawn(Cacher); + false -> Cacher() + end. + +%% @doc Get the node URL from a redirect. +node_from_redirect(Redirect, Opts) -> + uri_string:recompose( + ( + hb_maps:remove( + query, + uri_string:parse( + hb_ao:get(<<"location">>, Redirect, Opts) + ), + Opts + ) + )#{path => <<"/">>} + ). + +%% @doc Filter JSON assignment results from a remote legacy scheduler. +filter_json_assignments(JSONRes, To, From, Opts) -> + Edges = hb_maps:get(<<"edges">>, JSONRes, [], Opts), + Filtered = + lists:filter( + fun(Edge) -> + Node = hb_maps:get(<<"node">>, Edge, undefined, Opts), + Assignment = hb_maps:get(<<"assignment">>, Node, undefined, Opts), + Tags = hb_maps:get(<<"tags">>, Assignment, undefined, Opts), + Nonces = + lists:filtermap( + fun(#{ <<"name">> := <<"Nonce">>, <<"value">> := Nonce }) -> + {true, hb_util:int(Nonce)}; + (_) -> false + end, + Tags + ), + Nonce = hd(Nonces), + ?event({filter, {nonce, Nonce}, {from, From}, {to, To}}), + Nonce >= From andalso Nonce =< To + end, + Edges + ), + ?event({filtered, {length, length(Filtered)}, {edges, Filtered}}), + JSONRes#{ <<"edges">> => Filtered }. + +post_remote_schedule(RawProcID, Redirect, OnlyCommitted, Opts) -> + RemoteOpts = Opts#{ http_client => httpc }, + ProcID = without_hint(RawProcID), + Location = hb_ao:get(<<"location">>, Redirect, Opts), + Parsed = uri_string:parse(Location), + Node = uri_string:recompose((hb_maps:remove(query, Parsed, Opts))#{path => <<"/">>}), + Variant = hb_ao:get(<<"variant">>, Redirect, <<"ao.N.1">>, Opts), + case Variant of + <<"ao.N.1">> -> + PostMsg = #{ + <<"path">> => << ProcID/binary, "/schedule">>, + <<"body">> => OnlyCommitted, + <<"method">> => <<"POST">> + }, + hb_http:post(Node, PostMsg, RemoteOpts); + <<"ao.TN.1">> -> + % Ensure that the message is signed with ANS-104. + WithANS104Comms = + hb_message:with_commitments( + #{ <<"commitment-device">> => <<"ans104@1.0">> }, + OnlyCommitted, + Opts + ), + ?event(debug_downgrade, + {with_ans104_comms, + {only_committed, OnlyCommitted}, + {with_only_ans104_comms, WithANS104Comms} + } + ), + case hb_message:signers(WithANS104Comms, Opts) of + [] -> + {error, #{ + <<"status">> => 422, + <<"body">> => + << + "Process resides on legacy scheduler. ", + "Message must be signed with ANS-104." + >> + }}; + _ -> + % The message is signed with ANS-104, so we can post it to + % the legacy scheduler. + post_legacy_schedule(ProcID, WithANS104Comms, Node, RemoteOpts) + end + end. + +post_legacy_schedule(ProcID, OnlyCommitted, Node, Opts) -> + ?event({encoding_for_legacy_scheduler, {node, {string, Node}}}), + Encoded = + try + Item = + hb_message:convert( + OnlyCommitted, + <<"ans104@1.0">>, + Opts + ), + ?event( + {encoded_for_legacy_scheduler, + {item, Item}, + {exact, {explicit, Item}} + } + ), + {ok, ar_bundles:serialize(Item)} + catch + Class:Reason -> + {error, + #{ + <<"status">> => 422, + <<"body">> => + << + "Failed to encode message for legacy scheduler on ", + Node/binary, + ". Try different encoding?" + >>, + <<"class">> => Class, + <<"reason">> => + iolist_to_binary(io_lib:format("~p", [Reason])) + } + } end, - gen_schedule(ProcID, From, To, Opts). + case Encoded of + {error, EncodingErr} -> + ?event({could_not_encode_for_legacy_scheduler, {error, EncodingErr}}), + {error, #{ + <<"status">> => 422, + <<"body">> => + <<"Incorrect encoding. Scheduler has variant: ao.TN.1">> + } + }; + {ok, Body} -> + ?event({encoded_for_legacy_scheduler, {encoded, Body}}), + PostMsg = #{ + <<"path">> => P = <<"/?proc-id=", ProcID/binary>>, + <<"body">> => Body, + <<"method">> => <<"POST">> + }, + ?event({posting_to_remote_legacy_scheduler, + {node, {string, Node}}, + {path, {string, P}}, + {process_id, {string, ProcID}} + }), + LegacyOpts = Opts#{ protocol => http2 }, + case hb_http:post(Node, PostMsg, LegacyOpts) of + {ok, PostRes} -> + ?event({remote_schedule_result, PostRes}), + JSONRes = + hb_json:decode( + hb_ao:get(<<"body">>, PostRes, Opts) + ), + % Legacy SUs return only the ID of the assignment, so we need + % to read and return it. + ID = hb_maps:get(<<"id">>, JSONRes, undefined, Opts), + ?event({remote_schedule_result_id, ID, {json, JSONRes}}), + LegacyPath = << ID/binary, "?process-id=", ProcID/binary>>, + case hb_http:get(Node, LegacyPath, LegacyOpts) of + {ok, AssignmentRes} -> + ?event({received_full_assignment, AssignmentRes}), + AssignmentJSON = + hb_json:decode( + hb_ao:get(<<"body">>, AssignmentRes, Opts) + ), + Assignment = + dev_scheduler_formats:aos2_to_assignment( + AssignmentJSON, + Opts + ), + {ok, Assignment}; + {error, PostErr} -> {error, PostErr} + end; + {error, Resp = #{ <<"status">> := 404 }} -> + ?event( + {legacy_scheduler_not_found, + {url, {string, P}}, + {resp, Resp} + } + ), + {error, Resp}; + {error, PostRes} -> + ?event({remote_schedule_proxy_error, {error, PostRes}}), + {error, PostRes} + end + end. -%% Private methods +%%% Private methods + +%% @doc Find the schedule ID from a given request. The precidence order for +%% search is as follows: +%% [1. `ToSched/id' -- in the case of `POST schedule', handled locally] +%% 2. `Msg2/target' +%% 3. `Msg2/id' when `Msg2' has `type: Process' +%% 4. `Msg1/process/id' +%% 5. `Msg1/id' when `Msg1' has `type: Process' +%% 6. `Msg2/id' +find_target_id(Msg1, Msg2, Opts) -> + TempOpts = Opts#{ hashpath => ignore }, + Res = case hb_ao:resolve(Msg2, <<"target">>, TempOpts) of + {ok, Target} -> + % ID found at Msg2/target + Target; + _ -> + case hb_ao:resolve(Msg2, <<"type">>, TempOpts) of + {ok, <<"Process">>} -> + % Msg2 is a Process, so the ID is at Msg2/id + hb_message:id(Msg2, all, Opts); + _ -> + case hb_ao:resolve(Msg1, <<"process">>, TempOpts) of + {ok, Process} -> + % ID found at Msg1/process/id + hb_message:id(Process, all, Opts); + _ -> + % Does the message have a type of Process? + case hb_ao:get(<<"type">>, Msg1, TempOpts) of + <<"Process">> -> + % Yes, so try Msg1/id + hb_message:id(Msg1, all, Opts); + _ -> + % No, so the ID is at Msg2/id + hb_message:id(Msg2, all, Opts) + end + end + end + end, + ?event({found_id, {id, Res}, {msg1, Msg1}, {msg2, Msg2}}), + Res. + +%% @doc Search the given base and request message pair to find the message to +%% schedule. The precidence order for search is as follows: +%% 1. A key in `Msg2' with the value `self', indicating that the entire message +%% is the subject. +%% 2. A key in `Msg2' with another value, present in that message. +%% 3. The body of the message. +%% 4. The message itself. +find_message_to_schedule(_Msg1, Msg2, Opts) -> + Subject = + hb_ao:get( + <<"subject">>, + Msg2, + not_found, + Opts#{ hashpath => ignore } + ), + case Subject of + <<"self">> -> Msg2; + not_found -> + hb_ao:get(<<"body">>, Msg2, Msg2, Opts#{ hashpath => ignore }); + Subject -> + hb_ao:get(Subject, Msg2, Opts#{ hashpath => ignore }) + end. -gen_schedule(ProcID, From, To, Opts) -> - {Timestamp, Height, Hash} = ar_timestamp:get(), +%% @doc Generate a `GET /schedule' response for a process. +generate_local_schedule(Format, ProcID, From, To, Opts) -> ?event( {servicing_request_for_assignments, {proc_id, ProcID}, @@ -288,287 +1564,712 @@ gen_schedule(ProcID, From, To, Opts) -> {to, To} } ), - {Assignments, More} = get_assignments( - ProcID, - From, - To, - Opts - ), + ?event(generating_schedule_from_local_server), + {Assignments, More} = get_local_assignments(ProcID, From, To, Opts), ?event({got_assignments, length(Assignments), {more, More}}), - Bundle = #{ - <<"Type">> => <<"Schedule">>, - <<"Process">> => ProcID, - <<"Continues">> => atom_to_binary(More, utf8), - <<"Timestamp">> => list_to_binary(integer_to_list(Timestamp)), - <<"Block-Height">> => list_to_binary(integer_to_list(Height)), - <<"Block-Hash">> => Hash, - <<"Assignments">> => assignment_bundle(Assignments, Opts) - }, - ?event(assignments_bundle_outbound), - Signed = hb_message:sign(Bundle, hb:wallet()), - {ok, Signed}. + % Determine and apply the formatting function to use for generation + % of the response, based on the `Accept' header. + FormatterFun = + case uri_string:percent_decode(Format) of + <<"application/aos-2">> -> + fun dev_scheduler_formats:assignments_to_aos2/4; + _ -> + fun dev_scheduler_formats:assignments_to_bundle/4 + end, + Res = FormatterFun(ProcID, Assignments, More, Opts), + ?event({assignments_bundle_outbound, {format, Format}, {res, Res}}), + Res. %% @doc Get the assignments for a process, and whether the request was truncated. -get_assignments(ProcID, From, RequestedTo, Opts) -> - ?event({handling_req_to_get_assignments, ProcID, From, RequestedTo}), +get_local_assignments(ProcID, From, undefined, Opts) -> + case dev_scheduler_cache:latest(ProcID, Opts) of + not_found -> + % No assignments in cache. + {[], false}; + {Slot, _} -> + get_local_assignments(ProcID, From, Slot, Opts) + end; +get_local_assignments(ProcID, From, RequestedTo, Opts) -> + ?event({handling_req_to_get_assignments, ProcID, {from, From}, {to, RequestedTo}}), ComputedTo = case (RequestedTo - From) > ?MAX_ASSIGNMENT_QUERY_LEN of - true -> RequestedTo + ?MAX_ASSIGNMENT_QUERY_LEN; + true -> From + ?MAX_ASSIGNMENT_QUERY_LEN; false -> RequestedTo end, - {do_get_assignments(ProcID, From, ComputedTo, Opts), ComputedTo =/= RequestedTo }. + { + read_local_assignments(ProcID, From, ComputedTo, Opts), + ComputedTo < RequestedTo + }. -do_get_assignments(_ProcID, From, To, _Opts) when From > To -> +%% @doc Get the assignments for a process. +read_local_assignments(_ProcID, From, To, _Opts) when From > To -> []; -do_get_assignments(ProcID, From, To, Opts) -> - case dev_scheduler_cache:read(ProcID, From, Opts) of +read_local_assignments(ProcID, CurrentSlot, To, Opts) -> + case dev_scheduler_cache:read(ProcID, CurrentSlot, Opts) of not_found -> + % No assignment found in cache. []; {ok, Assignment} -> [ Assignment - | do_get_assignments( + | read_local_assignments( ProcID, - From + 1, + CurrentSlot + 1, To, Opts ) ] end. -assignment_bundle(Assignments, Opts) -> - assignment_bundle(Assignments, #{}, Opts). -assignment_bundle([], Bundle, _Opts) -> - Bundle; -assignment_bundle([Assignment | Assignments], Bundle, Opts) -> - Slot = hb_converge:get(<<"Slot">>, Assignment, Opts#{ hashpath => ignore }), - MessageID = - hb_converge:get(<<"Message">>, Assignment, Opts#{ hashpath => ignore }), - {ok, Message} = hb_cache:read(MessageID, Opts), - ?event( - {adding_assignment_to_bundle, - Slot, - {requested, MessageID}, - hb_util:id(Assignment, signed), - hb_util:id(Assignment, unsigned) - } - ), - assignment_bundle( - Assignments, - Bundle#{ - Slot => - hb_message:sign( - #{ - path => <<"Compute">>, - <<"Assignment">> => Assignment, - <<"Message">> => Message - }, - hb:wallet() - ) - }, - Opts - ). - -%%% Compute-stack flow functions. -%%% These keys are used during the compute phase for a process to interact with -%%% the scheduler. - -%% @doc Initializes the scheduler state. -init(M1, M2, Opts) -> - update_schedule(M1, M2, Opts). - -%% @doc Updates the schedule for a process. -end_of_schedule(M1, M2, Opts) -> update_schedule(M1, M2, Opts). - -%% @doc Abstracted function for updating the schedule for a process the current -%% schedule is in the `/priv/Scheduler/*' private map. -update_schedule(M1, M2, Opts) -> - Proc = hb_converge:get(process, M1, Opts), - ProcID = hb_util:id(Proc), - CurrentSlot = - hb_converge:get(<<"Current-Slot">>, M1, Opts, 0), - ToSlot = - hb_converge:get(<<"Slot">>, M2, Opts, undefined), - ?event({updating_schedule_current, CurrentSlot, to, ToSlot}), - Assignments = - hb_client:get_assignments( - ProcID, - CurrentSlot, - ToSlot - ), - ?event({got_assignments_from_su, - [ - { - element(2, lists:keyfind(<<"Assignment">>, 1, A#tx.tags)), - hb_util:id(A, signed), - hb_util:id(A, unsigned) - } - || - A <- Assignments - ]}), - lists:foreach( - fun(Assignment) -> - ?event( - {writing_assignment_to_cache, - hb_util:id(Assignment, unsigned) - } - ), - hb_cache:write(Assignment, Opts) - end, - Assignments - ), - {ok, hb_private:set(M1, <<"priv/Schedule">>, Assignments, Opts)}. - %% @doc Returns the current state of the scheduler. checkpoint(State) -> {ok, State}. %%% Tests -%%% These tests assume that the process message has been transformed by the -%%% dev_process, such that the full process is found in `/process', but the -%%% scheduler is the device of the primary message. - %% @doc Generate a _transformed_ process message, not as they are generated %% by users. See `dev_process' for examples of AO process messages. -test_process() -> - Wallet = hb:wallet(), - Address = hb_util:human_id(ar_wallet:to_address(Wallet)), +test_process() -> test_process(#{ priv_wallet => hb:wallet()}). +test_process(#{ priv_wallet := Wallet}) -> + test_process(hb_util:human_id(ar_wallet:to_address(Wallet))); +test_process(Address) -> #{ - device => ?MODULE, - process => #{ - <<"Device-Stack">> => [dev_cron, dev_wasm, dev_poda], - <<"Image">> => <<"wasm-image-id">>, - <<"Type">> => <<"Process">>, - <<"Scheduler-Location">> => Address, - <<"Test-Random-Seed">> => rand:uniform(1337) - } + <<"device">> => <<"scheduler@1.0">>, + <<"device-stack">> => [<<"cron@1.0">>, <<"wasm-64@1.0">>, <<"poda@1.0">>], + <<"image">> => <<"wasm-image-id">>, + <<"type">> => <<"Process">>, + <<"scheduler-location">> => Address, + <<"test-random-seed">> => rand:uniform(1337) }. status_test() -> start(), ?assertMatch( - #{<<"Processes">> := Processes, - <<"Address">> := Address} + #{<<"processes">> := Processes, + <<"address">> := Address} when is_list(Processes) and is_binary(Address), - hb_converge:get(status, test_process()) + hb_ao:get(status, test_process()) ). register_new_process_test() -> start(), - Msg1 = test_process(), - Proc = hb_converge:get(process, Msg1, #{ hashpath => ignore }), - ProcID = hb_util:id(Proc), - ?event({test_registering_new_process, {id, ProcID}, {msg, Msg1}}), + Opts = #{ priv_wallet => hb:wallet() }, + Msg1 = hb_message:commit(test_process(Opts), Opts), + ?event({test_registering_new_process, {msg, Msg1}}), ?assertMatch({ok, _}, - hb_converge:resolve( + hb_ao:resolve( Msg1, #{ - <<"Method">> => <<"POST">>, - path => <<"Schedule">>, - <<"Message">> => Proc + <<"method">> => <<"POST">>, + <<"path">> => <<"schedule">>, + <<"body">> => Msg1 }, #{} ) ), + ?event({status_response, Msg1}), + Procs = hb_ao:get(<<"processes">>, hb_ao:get(status, Msg1)), + ?event({procs, Procs}), ?assert( lists:member( - ProcID, - hb_converge:get(processes, hb_converge:get(status, Msg1)) + hb_util:id(Msg1, all), + hb_ao:get(<<"processes">>, hb_ao:get(status, Msg1)) ) ). +%% @doc Test that a scheduler location is registered on boot. +register_location_on_boot_test() -> + NotifiedPeerWallet = ar_wallet:new(), + RegisteringNodeWallet = ar_wallet:new(), + start(), + NotifiedPeer = + hb_http_server:start_node(#{ + priv_wallet => NotifiedPeerWallet, + store => [ + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/scheduler-location-notified">> + } + ] + }), + RegisteringNode = hb_http_server:start_node( + #{ + priv_wallet => RegisteringNodeWallet, + on => + #{ + <<"start">> => #{ + <<"device">> => <<"scheduler@1.0">>, + <<"path">> => <<"location">>, + <<"method">> => <<"POST">>, + <<"target">> => <<"self">>, + <<"require-codec">> => <<"ans104@1.0">>, + <<"url">> => <<"https://hyperbeam-test-ignore.com">>, + <<"hook">> => #{ + <<"result">> => <<"ignore">>, + <<"commit-request">> => true + } + } + }, + scheduler_location_notify_peers => [NotifiedPeer] + } + ), + {ok, CurrentLocation} = + hb_http:get( + RegisteringNode, + #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"/~scheduler@1.0/location">>, + <<"address">> => + hb_util:human_id(ar_wallet:to_address(RegisteringNodeWallet)) + }, + #{} + ), + ?event({current_location, CurrentLocation}), + ?assertMatch( + #{ + <<"url">> := <<"https://hyperbeam-test-ignore.com">>, + <<"nonce">> := 0 + }, + hb_ao:get(<<"body">>, CurrentLocation, #{}) + ). + schedule_message_and_get_slot_test() -> start(), Msg1 = test_process(), - Proc = hb_converge:get(process, Msg1, #{ hashpath => ignore }), - ProcID = hb_util:id(Proc), Msg2 = #{ - path => <<"Schedule">>, - <<"Method">> => <<"POST">>, - <<"Message">> => - #{ - <<"Type">> => <<"Message">>, - <<"Test-Key">> => <<"true">> - } + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit(#{ + <<"type">> => <<"Message">>, + <<"test-key">> => <<"true">> + }, hb:wallet()) }, - ?assertMatch({ok, _}, hb_converge:resolve(Msg1, Msg2, #{})), - ?assertMatch({ok, _}, hb_converge:resolve(Msg1, Msg2, #{})), + ?assertMatch({ok, _}, hb_ao:resolve(Msg1, Msg2, #{})), + ?assertMatch({ok, _}, hb_ao:resolve(Msg1, Msg2, #{})), Msg3 = #{ - path => <<"Slot">>, - <<"Method">> => <<"GET">>, - <<"Process">> => ProcID + <<"path">> => <<"slot">>, + <<"method">> => <<"GET">>, + <<"process">> => hb_util:id(Msg1) }, ?event({pg, dev_scheduler_registry:get_processes()}), ?event({getting_schedule, {msg, Msg3}}), - ?assertMatch({ok, #{ <<"Current-Slot">> := CurrentSlot }} + ?assertMatch({ok, #{ <<"current">> := CurrentSlot }} when CurrentSlot > 0, - hb_converge:resolve(Msg1, Msg3, #{})). + hb_ao:resolve(Msg1, Msg3, #{})). + +redirect_to_hint_test() -> + start(), + RandAddr = hb_util:human_id(crypto:strong_rand_bytes(32)), + TestLoc = <<"http://test.computer">>, + Msg1 = test_process(<< RandAddr/binary, "?hint=", TestLoc/binary>>), + Msg2 = #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => Msg1 + }, + ?assertMatch( + {ok, #{ <<"location">> := Location }} when is_binary(Location), + hb_ao:resolve( + Msg1, + Msg2, + #{ + scheduler_follow_hints => true, + scheduler_follow_redirects => false + } + ) + ). + +redirect_from_graphql_test_() -> + {timeout, 60, fun redirect_from_graphql/0}. +redirect_from_graphql() -> + start(), + Opts = + #{ store => + [ + #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-mainnet">> }, + #{ <<"store-module">> => hb_store_gateway, <<"store">> => false } + ] + }, + {ok, Msg} = hb_cache:read(<<"0syT13r0s0tgPmIed95bJnuSqaD29HQNN8D3ElLSrsc">>, Opts), + ?assertMatch( + {ok, #{ <<"location">> := Location }} when is_binary(Location), + hb_ao:resolve( + Msg, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit(#{ + <<"type">> => <<"Message">>, + <<"target">> => + <<"0syT13r0s0tgPmIed95bJnuSqaD29HQNN8D3ElLSrsc">>, + <<"test-key">> => <<"Test-Val">> + }, + hb:wallet() + ) + }, + #{ + scheduler_follow_redirects => false + } + ) + ). -benchmark_test() -> +get_local_schedule_test() -> start(), - BenchTime = 4, Msg1 = test_process(), - Proc = hb_converge:get(process, Msg1, #{ hashpath => ignore }), - ProcID = hb_util:id(Proc), + Msg2 = #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit(#{ + <<"type">> => <<"Message">>, + <<"test-key">> => <<"Test-Val">> + }, hb:wallet()) + }, + Msg3 = #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit(#{ + <<"type">> => <<"Message">>, + <<"test-key">> => <<"Test-Val-2">> + }, hb:wallet()) + }, + ?assertMatch({ok, _}, hb_ao:resolve(Msg1, Msg2, #{})), + ?assertMatch({ok, _}, hb_ao:resolve(Msg1, Msg3, #{})), + ?assertMatch( + {ok, _}, + hb_ao:resolve(Msg1, #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"schedule">>, + <<"target">> => hb_util:id(Msg1) + }, + #{}) + ). + +%%% HTTP tests + +http_init() -> http_init(#{}). +http_init(Opts) -> + start(), + Wallet = ar_wallet:new(), + ExtendedOpts = Opts#{ + priv_wallet => Wallet, + store => [ + #{ + <<"store-module">> => hb_store_lmdb, + <<"name">> => <<"cache-mainnet/lmdb">> + }, + #{ <<"store-module">> => hb_store_gateway, <<"store">> => false } + ] + }, + Node = hb_http_server:start_node(ExtendedOpts), + {Node, ExtendedOpts}. + +register_scheduler_test() -> + start(), + {Node, Wallet} = http_init(), + Msg1 = hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/location">>, + <<"url">> => <<"https://hyperbeam-test-ignore.com">>, + <<"method">> => <<"POST">>, + <<"nonce">> => 1, + <<"require-codec">> => <<"ans104@1.0">> + }, Wallet), + {ok, Res} = hb_http:post(Node, Msg1, #{}), + ?assertMatch(#{ <<"url">> := Location } when is_binary(Location), Res). + +http_post_schedule_sign(Node, Msg, ProcessMsg, Wallet) -> + Msg1 = hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + Msg#{ + <<"target">> => + hb_util:human_id(hb_message:id(ProcessMsg, all)), + <<"type">> => <<"Message">> + }, + Wallet + ) + }, Wallet), + hb_http:post(Node, Msg1, #{}). + +http_get_slot(N, PMsg) -> + ID = hb_message:id(PMsg, all), + Wallet = hb:wallet(), + {ok, _} = hb_http:get(N, hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/slot">>, + <<"method">> => <<"GET">>, + <<"target">> => ID + }, Wallet), #{}). + +http_get_schedule(N, PMsg, From, To) -> + http_get_schedule(N, PMsg, From, To, <<"application/http">>). + +http_get_schedule(N, PMsg, From, To, Format) -> + ID = hb_message:id(PMsg, all), + Wallet = hb:wallet(), + {ok, _} = hb_http:get(N, hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"method">> => <<"GET">>, + <<"target">> => hb_util:human_id(ID), + <<"from">> => From, + <<"to">> => To, + <<"accept">> => Format + }, Wallet), #{}). + +http_get_schedule_redirect_test_() -> + {timeout, 60, fun http_get_schedule_redirect/0}. +http_get_schedule_redirect() -> + Opts = + #{ + store => + [ + #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-mainnet">> }, + #{ <<"store-module">> => hb_store_gateway, <<"opts">> => #{} } + ], + scheduler_follow_redirects => false + }, + {N, _Wallet} = http_init(Opts), + start(), + ProcID = <<"0syT13r0s0tgPmIed95bJnuSqaD29HQNN8D3ElLSrsc">>, + Res = hb_http:get(N, <<"/", ProcID/binary, "/schedule">>, Opts), + ?assertMatch({ok, #{ <<"location">> := Location }} when is_binary(Location), Res). + +http_post_schedule_test_() -> + {timeout, 60, fun http_post_schedule/0}. +http_post_schedule() -> + {N, Opts} = http_init(), + PMsg = hb_message:commit(test_process(Opts), Opts), + Msg1 = hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => PMsg + }, Opts), + {ok, _Res} = hb_http:post(N, Msg1, Opts), + {ok, Res2} = + http_post_schedule_sign( + N, + #{ <<"inner">> => <<"test-message">> }, + PMsg, + Opts + ), + ?assertEqual(<<"test-message">>, hb_ao:get(<<"body/inner">>, Res2, Opts)), + ?assertMatch({ok, #{ <<"current">> := 1 }}, http_get_slot(N, PMsg)). + +http_get_schedule_test_() -> + {timeout, 20, fun() -> + {Node, Opts} = http_init(), + PMsg = hb_message:commit(test_process(Opts), Opts), + Msg1 = hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => PMsg + }, Opts), + Msg2 = hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + #{ + <<"target">> => + hb_util:human_id( + hb_message:id(PMsg, all, Opts) + ), + <<"body">> => <<"test-message">>, + <<"type">> => <<"Message">> + }, + Opts + ) + }, Opts), + {ok, _} = hb_http:post(Node, Msg1, Opts), + lists:foreach( + fun(_) -> + {ok, Res} = hb_http:post(Node, Msg2, Opts), + ?event(debug_scheduler_test, {res, Res}) + end, + lists:seq(1, 10) + ), + ?assertMatch({ok, #{ <<"current">> := 10 }}, http_get_slot(Node, PMsg)), + ?debug_wait(5000), + {ok, Schedule} = http_get_schedule(Node, PMsg, 0, 10), + Assignments = hb_ao:get(<<"assignments">>, Schedule, Opts), + ?assertEqual( + 12, % +1 for the hashpath + hb_maps:size(Assignments, Opts) + ) + end}. + + +http_get_legacy_schedule_test_() -> + {timeout, 60, fun() -> + Target = <<"CtOVB2dBtyN_vw3BdzCOrvcQvd9Y1oUGT-zLit8E3qM">>, + {Node, Opts} = http_init(), + {ok, Res} = hb_http:get(Node, <<"/~scheduler@1.0/schedule&target=", Target/binary>>, Opts), + LoadedRes = hb_cache:ensure_all_loaded(Res, Opts), + ?assertMatch(#{ <<"assignments">> := As } when map_size(As) > 0, LoadedRes) + end}. + +http_get_legacy_slot_test_() -> + {timeout, 60, fun() -> + Target = <<"CtOVB2dBtyN_vw3BdzCOrvcQvd9Y1oUGT-zLit8E3qM">>, + {Node, Opts} = http_init(), + Res = hb_http:get(Node, <<"/~scheduler@1.0/slot&target=", Target/binary>>, Opts), + ?assertMatch({ok, #{ <<"current">> := Slot }} when Slot > 0, Res) + end}. + +http_get_legacy_schedule_slot_range_test_() -> + {timeout, 60, fun() -> + Target = <<"zrhm4OpfW85UXfLznhdD-kQ7XijXM-s2fAboha0V5GY">>, + {Node, Opts} = http_init(), + {ok, Res} = hb_http:get(Node, <<"/~scheduler@1.0/schedule&target=", Target/binary, + "&from=0&to=10">>, Opts), + LoadedRes = hb_cache:ensure_all_loaded(Res, Opts), + ?event({res, LoadedRes}), + ?assertMatch(#{ <<"assignments">> := As } when map_size(As) == 11, LoadedRes) + end}. + +http_get_legacy_schedule_as_aos2_test_() -> + {timeout, 60, fun() -> + Target = <<"CtOVB2dBtyN_vw3BdzCOrvcQvd9Y1oUGT-zLit8E3qM">>, + {Node, Opts} = http_init(), + {ok, Res} = + hb_http:get( + Node, + #{ + <<"path">> => <<"/~scheduler@1.0/schedule?target=", Target/binary>>, + <<"accept">> => <<"application/aos-2">>, + <<"method">> => <<"GET">> + }, + #{} + ), + Decoded = hb_json:decode(hb_ao:get(<<"body">>, Res, Opts)), + ?assertMatch(#{ <<"edges">> := As } when length(As) > 0, Decoded) + end}. + +http_post_legacy_schedule_test_() -> + {timeout, 60, fun() -> + {Node, Opts} = http_init(), + Target = <<"zrhm4OpfW85UXfLznhdD-kQ7XijXM-s2fAboha0V5GY">>, + Signed = + hb_message:commit( + #{ + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.TN.1">>, + <<"type">> => <<"Message">>, + <<"action">> => <<"ping">>, + <<"target">> => Target, + <<"test-from">> => hb_util:human_id(hb:address()) + }, + Opts, + <<"ans104@1.0">> + ), + WithMethodAndPath = + Signed#{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"method">> => <<"POST">> + }, + ?event(debug_downgrade, {signed, Signed}), + {Status, Res} = hb_http:post(Node, WithMethodAndPath, Opts), + ?event(debug_downgrade, {status, Status}), + ?event({res, Res}), + ?assertMatch( + {ok, #{ <<"slot">> := Slot }} when Slot > 0, + {Status, Res} + ) + end}. + +http_get_json_schedule_test_() -> + {timeout, 60, fun() -> + {Node, Opts} = http_init(), + PMsg = hb_message:commit(test_process(Opts), Opts), + Msg1 = hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => PMsg + }, Opts), + {ok, _} = hb_http:post(Node, Msg1, Opts), + Msg2 = hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + hb_message:commit( + #{ + <<"inner">> => <<"test">>, + <<"target">> => hb_util:human_id(hb_message:id(PMsg, all)) + }, + Opts + ) + }, + Opts + ), + lists:foreach( + fun(_) -> {ok, _} = hb_http:post(Node, Msg2, Opts) end, + lists:seq(1, 10) + ), + ?assertMatch({ok, #{ <<"current">> := 10 }}, http_get_slot(Node, PMsg)), + {ok, Schedule} = http_get_schedule(Node, PMsg, 0, 10, <<"application/aos-2">>), + ?event({schedule, Schedule}), + JSON = hb_ao:get(<<"body">>, Schedule, Opts), + Assignments = hb_json:decode(JSON), + ?assertEqual( + 11, % +1 for the hashpath + length(hb_maps:get(<<"edges">>, Assignments)) + ) + end}. + +%%% Benchmarks + +single_resolution(Opts) -> + start(), + BenchTime = 1, + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + Msg1 = test_process(Opts#{ priv_wallet => Wallet }), ?event({benchmark_start, ?MODULE}), - Iterations = hb:benchmark( - fun(X) -> + MsgToSchedule = hb_message:commit(#{ + <<"type">> => <<"Message">>, + <<"test-key">> => <<"test-val">> + }, Opts), + Iterations = hb_test_utils:benchmark( + fun(_) -> MsgX = #{ - path => <<"Schedule">>, - <<"Method">> => <<"POST">>, - <<"Message">> => - #{ - <<"Type">> => <<"Message">>, - <<"Test-Val">> => X - } + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => MsgToSchedule }, - ?assertMatch({ok, _}, hb_converge:resolve(Msg1, MsgX, #{})) + ?assertMatch({ok, _}, hb_ao:resolve(Msg1, MsgX, Opts)) end, BenchTime ), ?event(benchmark, {scheduled, Iterations}), Msg3 = #{ - path => <<"Slot">>, - <<"Method">> => <<"GET">>, - <<"Process">> => ProcID + <<"path">> => <<"slot">>, + <<"method">> => <<"GET">>, + <<"process">> => hb_util:human_id(hb_message:id(Msg1, all, Opts)) }, - ?assertMatch({ok, #{ <<"Current-Slot">> := CurrentSlot }} + ?assertMatch({ok, #{ <<"current">> := CurrentSlot }} when CurrentSlot == Iterations - 1, - hb_converge:resolve(Msg1, Msg3, #{})), - hb_util:eunit_print( - "Scheduled ~p messages through Converge in ~p seconds (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] + hb_ao:resolve(Msg1, Msg3, Opts)), + ?event(bench, {res, Iterations - 1}), + hb_test_utils:benchmark_print( + <<"Scheduled through AO-Core:">>, + <<"messages">>, + Iterations, + BenchTime ), - ?assert(Iterations > 100). + ?assert(Iterations > 3). -get_schedule_test() -> - start(), - Msg1 = test_process(), - Msg2 = #{ - path => <<"Schedule">>, - <<"Method">> => <<"POST">>, - <<"Message">> => - #{ - <<"Type">> => <<"Message">>, - <<"Test-Key">> => <<"Test-Val">> - } - }, - Msg3 = #{ - path => <<"Schedule">>, - <<"Method">> => <<"POST">>, - <<"Message">> => - #{ - <<"Type">> => <<"Message">>, - <<"Test-Key">> => <<"Test-Val-2">> - } - }, - ?assertMatch({ok, _}, hb_converge:resolve(Msg1, Msg2, #{})), - ?assertMatch({ok, _}, hb_converge:resolve(Msg1, Msg3, #{})), - ?assertMatch( - {ok, _}, - hb_converge:resolve(Msg1, #{ - <<"Method">> => <<"GET">>, - path => <<"Schedule">> +many_clients(Opts) -> + BenchTime = 1, + Processes = hb_opts:get(workers, 25, Opts), + {Node, Opts} = http_init(Opts), + PMsg = hb_message:commit(test_process(Opts), Opts), + Msg1 = hb_message:commit(#{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"method">> => <<"POST">>, + <<"process">> => PMsg, + <<"body">> => hb_message:commit(#{ <<"inner">> => <<"test">> }, Opts) + }, Opts), + {ok, _} = hb_http:post(Node, Msg1, Opts), + Iterations = hb_test_utils:benchmark( + fun(X) -> + {ok, _} = hb_http:post(Node, Msg1, Opts), + ?event(bench, {iteration, X, self()}) + end, + BenchTime, + Processes + ), + ?event({iterations, Iterations}), + hb_format:eunit_print( + "Scheduled ~p messages with ~p workers through HTTP in ~ps (~.2f msg/s)", + [Iterations, Processes, BenchTime, Iterations / BenchTime] + ), + {ok, Res} = http_get_slot(Node, PMsg), + ?event(bench, {res, Res}), + ?assert(Iterations > 10). + +benchmark_suite_test_() -> + {timeout, 10, fun() -> + rand:seed(exsplus, erlang:timestamp()), + Port = 30000 + rand:uniform(10000), + Bench = [ + {benchmark, "benchmark", fun single_resolution/1}, + {multihttp_benchmark, "multihttp_benchmark", fun many_clients/1} + ], + filelib:ensure_dir( + binary_to_list(Base = <<"cache-TEST/run-">>) + ), + hb_test_utils:suite_with_opts(Bench, benchmark_suite(Port, Base)) + end}. + +benchmark_suite(Port, Base) -> + PortBin = integer_to_binary(Port), + [ + #{ + name => fs, + requires => [hb_store_fs], + opts => #{ + store => #{ <<"store-module">> => hb_store_fs, + <<"name">> => <> + }, + scheduling_mode => local_confirmation, + port => Port + }, + desc => <<"FS store, local conf.">> }, - #{}) - ). \ No newline at end of file + #{ + name => fs_aggressive, + requires => [hb_store_fs], + opts => #{ + store => #{ <<"store-module">> => hb_store_fs, + <<"name">> => <> + }, + scheduling_mode => aggressive, + port => Port + 1 + }, + desc => <<"FS store, aggressive conf.">> + }, + #{ + name => rocksdb, + requires => [hb_store_rocksdb], + opts => #{ + store => #{ <<"store-module">> => hb_store_rocksdb, + <<"name">> => <> + }, + scheduling_mode => local_confirmation, + port => Port + 2 + }, + desc => <<"RocksDB store, local conf.">> + }, + #{ + name => rocksdb_aggressive, + requires => [hb_store_rocksdb], + opts => #{ + store => #{ <<"store-module">> => hb_store_rocksdb, + <<"name">> => <> + }, + scheduling_mode => aggressive, + port => Port + 3 + }, + desc => <<"RocksDB store, aggressive conf.">> + }, + #{ + name => rocksdb_extreme_aggressive_h3, + requires => [http3], + opts => #{ + store => #{ <<"store-module">> => hb_store_rocksdb, + <<"name">> => + << + Base/binary, + "run-", + (integer_to_binary(Port+4))/binary + >> + }, + scheduling_mode => aggressive, + protocol => http3, + workers => 100 + }, + desc => <<"100xRocksDB store, aggressive conf, http/3.">> + } + ]. diff --git a/src/dev_scheduler_cache.erl b/src/dev_scheduler_cache.erl index 1ea796a31..a7cf4f5b2 100644 --- a/src/dev_scheduler_cache.erl +++ b/src/dev_scheduler_cache.erl @@ -1,15 +1,34 @@ +%%% @doc A module that provides a cache for scheduler assignments and locations. -module(dev_scheduler_cache). --export([write/2, read/3, list/2]). +-export([write/2, write_spawn/2, write_location/2]). +-export([read/3, read_location/2]). +-export([list/2, latest/2]). -include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). -%%% Assignment cache functions +%%% The pseudo-path prefix which the scheduler cache should use. +-define(SCHEDULER_CACHE_PREFIX, <<"~scheduler@1.0">>). + +%% @doc Merge the scheduler store with the main store. Used before writing +%% to the cache. +opts(Opts) -> + Opts#{ + store => + hb_opts:get( + scheduler_store, + hb_opts:get(store, no_viable_store, Opts), + Opts + ) + }. %% @doc Write an assignment message into the cache. -write(Assignment, Opts) -> +write(RawAssignment, RawOpts) -> + Assignment = hb_cache:ensure_all_loaded(RawAssignment, RawOpts), + Opts = opts(RawOpts), Store = hb_opts:get(store, no_viable_store, Opts), % Write the message into the main cache - ProcID = hb_converge:get(<<"Process">>, Assignment), - Slot = hb_converge:get(<<"Slot">>, Assignment), + ProcID = hb_ao:get(<<"process">>, Assignment, Opts), + Slot = hb_ao:get(<<"slot">>, Assignment, Opts), ?event( {writing_assignment, {proc_id, ProcID}, @@ -17,46 +36,546 @@ write(Assignment, Opts) -> {assignment, Assignment} } ), - {ok, RootPath} = hb_cache:write(Assignment, Opts), - % Create symlinks from the message on the process and the - % slot on the process to the underlying data. - hb_store:make_link( - Store, - RootPath, - hb_store:path( - Store, - [ - <<"assignments">>, - hb_util:human_id(ProcID), - hb_converge:key_to_binary(Slot) - ] - ) - ), - ok. + case hb_cache:write(Assignment, Opts) of + {ok, RootPath} -> + % Create symlinks from the message on the process and the + % slot on the process to the underlying data. + hb_store:make_link( + Store, + RootPath, + hb_store:path( + Store, + [ + ?SCHEDULER_CACHE_PREFIX, + <<"assignments">>, + hb_util:human_id(ProcID), + hb_ao:normalize_key(Slot) + ] + ) + ), + ok; + {error, Reason} -> + ?event(error, {failed_to_write_assignment, {reason, Reason}}), + {error, Reason} + end. + +%% @doc Write the initial assignment message to the cache. +write_spawn(RawInitMessage, Opts) -> + InitMessage = hb_cache:ensure_all_loaded(RawInitMessage, Opts), + hb_cache:write(InitMessage, opts(Opts)). %% @doc Get an assignment message from the cache. read(ProcID, Slot, Opts) when is_integer(Slot) -> - read(ProcID, integer_to_list(Slot), Opts); -read(ProcID, Slot, Opts) -> + read(ProcID, hb_util:bin(Slot), Opts); +read(ProcID, Slot, RawOpts) -> + Opts = opts(RawOpts), Store = hb_opts:get(store, no_viable_store, Opts), ResolvedPath = P2 = hb_store:resolve( Store, P1 = hb_store:path(Store, [ + ?SCHEDULER_CACHE_PREFIX, "assignments", hb_util:human_id(ProcID), Slot ]) ), + ?event( + {read_assignment, + {proc_id, ProcID}, + {slot, Slot}, + {store, Store} + } + ), ?event({resolved_path, {p1, P1}, {p2, P2}, {resolved, ResolvedPath}}), - hb_cache:read(ResolvedPath, Opts). + case hb_cache:read(ResolvedPath, Opts) of + {ok, Assignment} -> + % If the slot key is not present, the format of the assignment is + % AOS2, so we need to convert it to the canonical format. + case hb_ao:get(<<"variant">>, Assignment, Opts) of + <<"ao.TN.1">> -> + Loaded = hb_cache:ensure_all_loaded(Assignment, Opts), + Norm = dev_scheduler_formats:aos2_to_assignment(Loaded, Opts), + ?event({normalized_aos2_assignment, Norm}), + {ok, Norm}; + <<"ao.N.1">> -> + {ok, hb_cache:ensure_all_loaded(Assignment, Opts)} + end; + not_found -> + ?event(debug_sched, {read_assignment, {res, not_found}}), + not_found + end. %% @doc Get the assignments for a process. -list(ProcID, Opts) -> +list(ProcID, RawOpts) -> + Opts = opts(RawOpts), hb_cache:list_numbered( hb_store:path(hb_opts:get(store, no_viable_store, Opts), [ + ?SCHEDULER_CACHE_PREFIX, "assignments", hb_util:human_id(ProcID) ]), Opts - ). \ No newline at end of file + ). + +%% @doc Get the latest assignment from the cache. +latest(ProcID, RawOpts) -> + Opts = opts(RawOpts), + ?event({getting_assignments_from_cache, {proc_id, ProcID}, {opts, Opts}}), + case dev_scheduler_cache:list(ProcID, Opts) of + [] -> + ?event({no_assignments_in_cache, {proc_id, ProcID}}), + not_found; + Assignments -> + AssignmentNum = lists:max(Assignments), + ?event( + {found_assignment_from_cache, + {proc_id, ProcID}, + {assignment_num, AssignmentNum} + } + ), + {ok, Assignment} = dev_scheduler_cache:read( + ProcID, + AssignmentNum, + Opts + ), + { + AssignmentNum, + hb_ao:get( + <<"hash-chain">>, Assignment, #{ hashpath => ignore }) + } + end. + +%% @doc Read the latest known scheduler location for an address. +read_location(Address, RawOpts) -> + Opts = opts(RawOpts), + Res = + hb_cache:read( + hb_store:path(hb_opts:get(store, no_viable_store, Opts), [ + ?SCHEDULER_CACHE_PREFIX, + "locations", + hb_util:human_id(Address) + ]), + Opts + ), + Event = + case Res of + {ok, _} -> found_in_store; + not_found -> not_found_in_store; + _ -> local_lookup_unexpected_result + end, + ?event(scheduler_location, {Event, {address, Address}, {res, Res}}), + Res. + +%% @doc Write the latest known scheduler location for an address. +write_location(LocationMsg, RawOpts) -> + Opts = opts(RawOpts), + Signers = hb_message:signers(LocationMsg, Opts), + ?event( + scheduler_location, + {caching_locally, + {signers, Signers}, + {location_msg, LocationMsg} + } + ), + case hb_cache:write(LocationMsg, Opts) of + {ok, RootPath} -> + lists:foreach( + fun(Signer) -> + hb_store:make_link( + hb_opts:get(store, no_viable_store, Opts), + RootPath, + hb_store:path( + hb_opts:get(store, no_viable_store, Opts), + [ + ?SCHEDULER_CACHE_PREFIX, + "locations", + hb_util:human_id(Signer) + ] + ) + ) + end, + Signers + ), + ok; + false -> + % The message is not valid, so we don't cache it. + {error, <<"Invalid scheduler location message. Not caching.">>}; + {error, Reason} -> + ?event(warning, {failed_to_cache_location_msg, {reason, Reason}}), + {error, Reason} + end. + +%%% Tests + +%% @doc Test that a volatile schedule is lost on restart. +volatile_schedule_test() -> + VolStore = hb_test_utils:test_store(hb_store_fs, <<"volatile-sched">>), + NonVolStore = hb_test_utils:test_store(hb_store_fs, <<"non-volatile-sched">>), + Opts = #{ + store => [NonVolStore], + scheduler_store => [VolStore] + }, + hb_store:start(VolStore), + hb_store:start(NonVolStore), + Assignment = #{ + <<"variant">> => <<"ao.N.1">>, + <<"process">> => ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + <<"slot">> => 1, + <<"hash-chain">> => <<"test-hash-chain">> + }, + ?assertEqual(ok, write(Assignment, Opts)), + ?assertMatch({1, _}, latest(ProcID, Opts)), + ?assertEqual({ok, Assignment}, read(ProcID, 1, Opts)), + hb_store:stop(VolStore), + hb_store:reset(VolStore), + hb_store:start(VolStore), + ?assertMatch(not_found, latest(ProcID, Opts)), + ?assertMatch(not_found, read(ProcID, 1, Opts)). + +%% @doc Test concurrent writes to scheduler store from multiple processes. +concurrent_scheduler_write_test() -> + VolStore = hb_test_utils:test_store(hb_store_fs, <<"concurrent-vol">>), + NonVolStore = hb_test_utils:test_store(hb_store_fs, <<"concurrent-nonvol">>), + Opts = #{ + store => [NonVolStore], + scheduler_store => [VolStore] + }, + hb_store:start(VolStore), + hb_store:start(NonVolStore), + Workers = 50, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Parent = self(), + lists:foreach(fun(Slot) -> + spawn_link(fun() -> + Assignment = #{ + <<"process">> => ProcID, + <<"slot">> => Slot, + <<"hash-chain">> => + <<"concurrent-test-", (integer_to_binary(Slot))/binary>> + }, + Result = write(Assignment, Opts), + Parent ! {write_result, Slot, Result} + end) + end, lists:seq(1, Workers)), + Results = + lists:map( + fun(Slot) -> + receive + {write_result, Slot, Result} -> + ?event(testing, {write_result, Slot, Result}), + Result + after 5000 -> + timeout + end + end, + lists:seq(1, Workers) + ), + ?event(testing, {concurrent_write_results, Results,Workers}), + ?assertEqual(lists:duplicate(Workers, ok), Results), + AllSlots = list(ProcID, Opts), + ?event(testing, {all_slots, AllSlots}), + ?assertEqual(Workers, length(AllSlots)), + ?assertEqual(lists:seq(1, Workers), lists:sort(AllSlots)). + +%% @doc Test concurrent reads during writes to detect race conditions. +concurrent_read_write_test() -> + VolStore = hb_test_utils:test_store(hb_store_fs, <<"race-vol">>), + NonVolStore = hb_test_utils:test_store(hb_store_fs, <<"race-nonvol">>), + Opts = #{ + store => [NonVolStore], + scheduler_store => [VolStore] + }, + hb_store:start(VolStore), + hb_store:start(NonVolStore), + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Parent = self(), + ?event(testing, {concurrent_test_proc_id, ProcID}), + spawn_link(fun() -> + lists:foreach(fun(Slot) -> + Assignment = #{ + <<"variant">> => <<"ao.N.1">>, + <<"process">> => ProcID, + <<"slot">> => Slot, + <<"hash-chain">> => <<"race-test-", (integer_to_binary(Slot))/binary>> + }, + write(Assignment, Opts), + timer:sleep(1) + end, lists:seq(1, 100)), + ?event(testing, {writer_completed}), + Parent ! writer_done + end), + lists:foreach( + fun(ReaderNum) -> + spawn_link(fun() -> + ReadResults = lists:map(fun(Slot) -> + timer:sleep(rand:uniform(5)), + case read(ProcID, Slot, Opts) of + {ok, _} -> success; + not_found -> not_found + end + end, lists:seq(1, 100)), + SuccessCount = length([R || R <- ReadResults, R == success]), + ?event(testing, {reader_done, ReaderNum, SuccessCount}), + Parent ! {reader_done, ReaderNum, ReadResults} + end) + end, + lists:seq(1, 10) + ), + receive + writer_done -> ok + after 15000 -> + ?assert(false) + end, + AllReaderResults = lists:map(fun(ReaderNum) -> + receive + {reader_done, ReaderNum, Results} -> Results + after 5000 -> + ?assert(false), + [] + end + end, lists:seq(1, 10)), + FinalSlots = list(ProcID, Opts), + ?event(testing, {final_verification, {slots_found, length(FinalSlots)}}), + ?assertEqual(100, length(FinalSlots)), + ?assertEqual(lists:seq(1, 100), lists:sort(FinalSlots)), + TotalSuccessfulReads = lists:sum([ + length([R || R <- Results, R == success]) || Results <- AllReaderResults + ]), + ?event(testing, { + concurrent_read_stats, + {total_successful_reads, TotalSuccessfulReads} + }), + ?assert(TotalSuccessfulReads > 0). + +%% @doc Test writing a large volume of assignments to stress memory. Helps +%% identify memory leaks and also, checks performance issues. +large_assignment_volume_test() -> + VolStore = hb_test_utils:test_store(hb_store_fs, <<"volume-vol">>), + NonVolStore = hb_test_utils:test_store(hb_store_fs, <<"volume-nonvol">>), + Opts = #{ + store => [NonVolStore], + scheduler_store => [VolStore] + }, + hb_store:start(VolStore), + hb_store:start(NonVolStore), + VolumeSize = 1000, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + StartTime = erlang:monotonic_time(millisecond), + lists:foreach( + fun(Slot) -> + Assignment = #{ + <<"variant">> => <<"ao.N.1">>, + <<"process">> => ProcID, + <<"slot">> => Slot, + <<"hash-chain">> => crypto:strong_rand_bytes(64) + }, + ?assertEqual(ok, write(Assignment, Opts)) + end, + lists:seq(1, VolumeSize) + ), + EndTime = erlang:monotonic_time(millisecond), + ?event(testing, {large_volume_write_time, EndTime - StartTime}), + AllSlots = list(ProcID, Opts), + ?assertEqual(VolumeSize, length(AllSlots)), + ?assertEqual(lists:seq(1, VolumeSize), lists:sort(AllSlots)), + ReadStartTime = erlang:monotonic_time(millisecond), + lists:foreach(fun(Slot) -> + ?assertMatch({ok, _}, read(ProcID, Slot, Opts)) + end, lists:seq(1, VolumeSize)), + ReadEndTime = erlang:monotonic_time(millisecond), + ?event(testing, {large_volume_read_time, ReadEndTime - ReadStartTime}). + +%% @doc Test rapid store restarts under load. +rapid_restart_test() -> + VolStore = hb_test_utils:test_store(hb_store_fs, <<"restart-vol">>), + NonVolStore = hb_test_utils:test_store(hb_store_fs, <<"restart-nonvol">>), + Opts = #{ + store => [NonVolStore], + scheduler_store => [VolStore] + }, + hb_store:start(VolStore), + hb_store:start(NonVolStore), + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + lists:foreach( + fun(Cycle) -> + lists:foreach( + fun(Slot) -> + Assignment = #{ + <<"variant">> => <<"ao.N.1">>, + <<"process">> => ProcID, + <<"slot">> => Slot + (Cycle * 10), + <<"hash-chain">> => + <<"restart-cycle-", (integer_to_binary(Cycle))/binary>> + }, + ?assertEqual(ok, write(Assignment, Opts)) + end, + lists:seq(1, 10) + ), + SlotsBeforeRestart = list(ProcID, Opts), + ?assertMatch([_|_], SlotsBeforeRestart), + ?event(testing, { + restart_cycle, Cycle, {slots_before, length(SlotsBeforeRestart)} + }), + hb_store:stop(VolStore), + timer:sleep(10), + hb_store:reset(VolStore), + hb_store:start(VolStore), + SlotsAfterRestart = list(ProcID, Opts), + ?assertEqual([], SlotsAfterRestart), + ?event({restart_verified, Cycle, {slots_after, length(SlotsAfterRestart)}}) + end, + lists:seq(1, 5) + ). + +%% @doc Test scheduler store behavior during reset store operations. +mixed_store_reset_operations_test() -> + VolStore = hb_test_utils:test_store(hb_store_fs, <<"mixed-vol">>), + NonVolStore = hb_test_utils:test_store(hb_store_fs, <<"mixed-nonvol">>), + Opts = #{ + store => [NonVolStore], + scheduler_store => [VolStore] + }, + hb_store:start(VolStore), + hb_store:start(NonVolStore), + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Assignment1 = #{ + <<"variant">> => <<"ao.N.1">>, + <<"process">> => ProcID, + <<"slot">> => 1, + <<"hash-chain">> => <<"mixed-test-1">> + }, + ?assertEqual(ok, write(Assignment1, Opts)), + ?event(testing, {assignment_written, ProcID}), + hb_store:reset(NonVolStore), + ReadAfterNonVolReset = read(ProcID, 1, Opts), + ?assertMatch({ok, _}, ReadAfterNonVolReset), + ?event(testing, {after_nonvol_reset, ReadAfterNonVolReset}), + hb_store:reset(VolStore), + ReadAfterVolReset = read(ProcID, 1, Opts), + ?assertEqual(not_found, ReadAfterVolReset), + ?event(testing, {after_vol_reset, ReadAfterVolReset}). + +%% @doc Test handling of invalid assignment data. +invalid_assignment_stress_test() -> + VolStore = hb_test_utils:test_store(hb_store_fs, <<"invalid-vol">>), + NonVolStore = hb_test_utils:test_store(hb_store_fs, <<"invalid-nonvol">>), + Opts = #{ + store => [NonVolStore], + scheduler_store => [VolStore] + }, + hb_store:start(VolStore), + hb_store:start(NonVolStore), + InvalidAssignments = [ + #{}, + #{<<"process">> => <<"invalid">>}, + #{<<"slot">> => 1}, + #{<<"process">> => <<>>, <<"slot">> => 1}, + #{<<"process">> => <<"valid">>, <<"slot">> => -1}, + #{<<"process">> => <<"valid">>, <<"slot">> => <<"not-integer">>} + ], + ?event(testing, {testing_invalid_assignments, length(InvalidAssignments)}), + Results = lists:map(fun(Assignment) -> + Result = try + write(Assignment, Opts) + catch + _:_ -> error + end, + ?assertNotEqual(ok, Result), + Result + end, InvalidAssignments), + + ErrorCount = length([R || R <- Results, R == error]), + ?event( + {invalid_assignment_results, + {errors, ErrorCount}, + {total, length(InvalidAssignments)} + } + ), + ?assertEqual(6, ErrorCount). + +%% @doc Test scheduler location operations under stress. +scheduler_location_stress_test() -> + VolStore = hb_test_utils:test_store(hb_store_fs, <<"location-vol">>), + NonVolStore = hb_test_utils:test_store(hb_store_fs, <<"location-nonvol">>), + Wallet = ar_wallet:new(), + Opts = #{ + store => [NonVolStore], + scheduler_store => [VolStore], + priv_wallet => Wallet + }, + hb_store:start(VolStore), + hb_store:start(NonVolStore), + LocationCount = 10, + ?event(testing, {location_stress_test_starting, LocationCount}), + Results = + lists:map( + fun(N) -> + LocationMsg = #{ + <<"scheduler">> => + hb_util:human_id(ar_wallet:to_address(Wallet)), + <<"location">> => + << + "http://scheduler", + (integer_to_binary(N))/binary, + ".com" + >>, + <<"timestamp">> => erlang:system_time(millisecond), + <<"ttl">> => 3600000 + }, + Result = + try + write_location(LocationMsg, Opts) + catch + Res -> + ?event(testing, {location_write_error, {error, Res}}), + ok + end, + ?assert(Result == ok orelse element(1, Result) == error), + Result + end, + lists:seq(1, LocationCount) + ), + SuccessCount = length([R || R <- Results, R == ok]), + ?event( + {location_stress_results, + {successes, SuccessCount}, + {total, LocationCount} + } + ). + +%% @doc Test system behavior with corrupted data in volatile store. +volatile_store_corruption_test() -> + VolStore = hb_test_utils:test_store(hb_store_fs, <<"corruption-vol">>), + NonVolStore = hb_test_utils:test_store(hb_store_fs, <<"corruption-nonvol">>), + Opts = #{ + store => [NonVolStore], + scheduler_store => [VolStore] + }, + hb_store:start(VolStore), + hb_store:start(NonVolStore), + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Assignment = #{ + <<"variant">> => <<"ao.N.1">>, + <<"process">> => ProcID, + <<"slot">> => 1, + <<"hash-chain">> => <<"corruption-test">> + }, + ?assertEqual(ok, write(Assignment, Opts)), + ReadBeforeCorruption = read(ProcID, 1, Opts), + ?assertMatch({ok, _}, ReadBeforeCorruption), + ?event(testing, {before_corruption, ReadBeforeCorruption}), + hb_store:reset(VolStore), + ?event(testing, {volatile_store_reset}), + ReadAfterCorruption = read(ProcID, 1, Opts), + SlotsAfterCorruption = list(ProcID, Opts), + LatestAfterCorruption = latest(ProcID, Opts), + ?assertEqual(not_found, ReadAfterCorruption), + ?assertEqual([], SlotsAfterCorruption), + ?assertEqual(not_found, LatestAfterCorruption), + ?event(testing, + { corruption_recovery_verified, + { read, ReadAfterCorruption }, + { list, length(SlotsAfterCorruption) }, + { latest, LatestAfterCorruption } + }). \ No newline at end of file diff --git a/src/dev_scheduler_formats.erl b/src/dev_scheduler_formats.erl new file mode 100644 index 000000000..864c601dc --- /dev/null +++ b/src/dev_scheduler_formats.erl @@ -0,0 +1,214 @@ +%%% @doc This module is used by dev_scheduler in order to produce outputs that +%%% are compatible with various forms of AO clients. It features two main formats: +%%% +%%% - `application/json' +%%% - `application/http' +%%% +%%% The `application/json' format is a legacy format that is not recommended for +%%% new integrations of the AO protocol. +-module(dev_scheduler_formats). +-export([assignments_to_bundle/4, assignments_to_aos2/4]). +-export([aos2_to_assignments/3, aos2_to_assignment/2]). +-export([aos2_normalize_types/1]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% @doc Generate a `GET /schedule' response for a process as HTTP-sig bundles. +assignments_to_bundle(ProcID, Assignments, More, Opts) -> + TimeInfo = ar_timestamp:get(), + assignments_to_bundle(ProcID, Assignments, More, TimeInfo, Opts). +assignments_to_bundle(ProcID, Assignments, More, TimeInfo, RawOpts) -> + Opts = format_opts(RawOpts), + {Timestamp, Height, Hash} = TimeInfo, + {ok, #{ + <<"type">> => <<"schedule">>, + <<"process">> => hb_util:human_id(ProcID), + <<"continues">> => hb_util:atom(More), + <<"timestamp">> => hb_util:int(Timestamp), + <<"block-height">> => hb_util:int(Height), + <<"block-hash">> => hb_util:human_id(Hash), + <<"assignments">> => + hb_maps:from_list( + lists:map( + fun(Assignment) -> + { + hb_ao:get( + <<"slot">>, + Assignment, + Opts#{ hashpath => ignore } + ), + Assignment + } + end, + Assignments + ) + ) + }}. + +%%% Return legacy net-SU compatible results. +assignments_to_aos2(ProcID, Assignments, More, RawOpts) when is_map(Assignments) -> + assignments_to_aos2( + ProcID, + hb_util:message_to_ordered_list(Assignments), + More, + format_opts(RawOpts) + ); +assignments_to_aos2(ProcID, Assignments, More, RawOpts) -> + Opts = format_opts(RawOpts), + {Timestamp, Height, Hash} = ar_timestamp:get(), + BodyStruct = + #{ + <<"page_info">> => + #{ + <<"process">> => hb_util:human_id(ProcID), + <<"has_next_page">> => More, + <<"timestamp">> => list_to_binary(integer_to_list(Timestamp)), + <<"block-height">> => list_to_binary(integer_to_list(Height)), + <<"block-hash">> => hb_util:human_id(Hash) + }, + <<"edges">> => + lists:map( + fun(Assignment) -> + #{ + <<"cursor">> => cursor(Assignment, Opts), + <<"node">> => assignment_to_aos2(Assignment, Opts) + } + end, + Assignments + ) + }, + Encoded = hb_json:encode(BodyStruct), + ?event({body_struct, BodyStruct}), + ?event({encoded, {explicit, Encoded}}), + {ok, + #{ + <<"content-type">> => <<"application/json">>, + <<"body">> => Encoded + } + }. + +%% @doc Generate a cursor for an assignment. This should be the slot number, at +%% least in the case of mainnet `ao.N.1' assignments. In the case of legacynet +%% (`ao.TN.1') assignments, we may want to use the assignment ID. +cursor(Assignment, RawOpts) -> + Opts = format_opts(RawOpts), + hb_ao:get(<<"slot">>, Assignment, Opts). +%% @doc Convert an assignment to an AOS2-compatible JSON structure. +assignment_to_aos2(Assignment, RawOpts) -> + Opts = format_opts(RawOpts), + Message = hb_ao:get(<<"body">>, Assignment, Opts), + AssignmentWithoutBody = hb_maps:without([<<"body">>], Assignment, Opts), + #{ + <<"message">> => + dev_json_iface:message_to_json_struct(Message, Opts), + <<"assignment">> => + dev_json_iface:message_to_json_struct(AssignmentWithoutBody, Opts) + }. + +%% @doc Convert an AOS2-style JSON structure to a normalized HyperBEAM +%% assignments response. +aos2_to_assignments(ProcID, Body, RawOpts) -> + Opts = format_opts(RawOpts), + Assignments = hb_maps:get(<<"edges">>, Body, Opts, Opts), + ?event({raw_assignments, Assignments}), + ParsedAssignments = + lists:map( + fun(A) -> aos2_to_assignment(A, Opts) end, + Assignments + ), + ?event({parsed_assignments, ParsedAssignments}), + TimeInfo = + case ParsedAssignments of + [] -> {0, 0, hb_util:encode(<<0:256>>)}; + _ -> + Last = lists:last(ParsedAssignments), + { + hb_ao:get(<<"timestamp">>, Last, Opts), + hb_ao:get(<<"block-height">>, Last, Opts), + hb_ao:get(<<"block-hash">>, Last, Opts) + } + end, + assignments_to_bundle(ProcID, ParsedAssignments, false, TimeInfo, Opts). + +%% @doc Create and normalize an assignment from an AOS2-style JSON structure. +%% NOTE: This method is destructive to the verifiability of the assignment. +aos2_to_assignment(A, RawOpts) -> + Opts = format_opts(RawOpts), + % Unwrap the node if it is provided + Node = hb_maps:get(<<"node">>, A, A, Opts), + ?event({node, Node}), + {ok, Assignment} = + hb_gateway_client:result_to_message( + aos2_normalize_data(hb_maps:get(<<"assignment">>, Node, undefined, Opts)), + Opts + ), + NormalizedAssignment = aos2_normalize_types(Assignment), + {ok, Message} = + case hb_maps:get(<<"message">>, Node, undefined, Opts) of + null -> + MessageID = hb_maps:get(<<"message">>, Assignment, undefined, Opts), + ?event(error, {scheduler_did_not_provide_message, MessageID}), + case hb_cache:read(MessageID, Opts) of + {ok, Msg} -> {ok, Msg}; + {error, _} -> + throw({error, + {message_not_given_by_scheduler_or_cache, + MessageID} + } + ) + end; + Body -> + hb_gateway_client:result_to_message( + aos2_normalize_data(Body), + Opts + ) + end, + NormalizedMessage = aos2_normalize_types(Message), + ?event({message, Message}), + NormalizedAssignment#{ <<"body">> => NormalizedMessage }. + +%% @doc The `hb_gateway_client' module expects all JSON structures to at least +%% have a `data' field. This function ensures that. +aos2_normalize_data(JSONStruct) -> + case JSONStruct of + #{<<"data">> := _} -> JSONStruct; + _ -> JSONStruct#{ <<"data">> => <<>> } + end. + +%% @doc Normalize an AOS2 formatted message to ensure that all field NAMES and +%% types are correct. This involves converting field names to integers and +%% specific field names to their canonical form. +%% NOTE: This will result in a message that is not verifiable! It is, however, +%% necessary for gaining compatibility with the AOS2-style scheduling API. +aos2_normalize_types(Msg = #{ <<"timestamp">> := TS }) when is_binary(TS) -> + aos2_normalize_types(Msg#{ <<"timestamp">> => hb_util:int(TS) }); +aos2_normalize_types(Msg = #{ <<"nonce">> := Nonce }) + when is_binary(Nonce) and not is_map_key(<<"slot">>, Msg) -> + aos2_normalize_types( + Msg#{ <<"slot">> => hb_util:int(Nonce) } + ); +aos2_normalize_types(Msg = #{ <<"epoch">> := DS }) when is_binary(DS) -> + aos2_normalize_types(Msg#{ <<"epoch">> => hb_util:int(DS) }); +aos2_normalize_types(Msg = #{ <<"slot">> := Slot }) when is_binary(Slot) -> + aos2_normalize_types(Msg#{ <<"slot">> => hb_util:int(Slot) }); +aos2_normalize_types(Msg) when not is_map_key(<<"block-hash">>, Msg) -> + ?event({missing_block_hash, Msg}), + aos2_normalize_types(Msg#{ <<"block-hash">> => hb_util:encode(<<0:256>>) }); +aos2_normalize_types(Msg) -> + ?event( + { + aos2_normalized_types, + {msg, Msg}, + {anchor, hb_ao:get(<<"anchor">>, Msg, <<>>, #{})} + } + ), + Msg. + +%% @doc For all scheduler format operations, we do not calculate hashpaths, +%% perform cache lookups, or await inprogress results. +format_opts(Opts) -> + Opts#{ + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>], + await_inprogress => false + }. \ No newline at end of file diff --git a/src/dev_scheduler_interface.erl b/src/dev_scheduler_interface.erl deleted file mode 100644 index 5637cc03d..000000000 --- a/src/dev_scheduler_interface.erl +++ /dev/null @@ -1,182 +0,0 @@ --module(dev_scheduler_interface). --export([handle/1]). --include("include/hb.hrl"). --include_lib("eunit/include/eunit.hrl"). - -%%% The SU device's API functions. Enables clients to read/write messages into -%%% the schedule for a process. - -handle(M) -> - (choose_handler(M))(M). - -choose_handler(M) -> - Method = hb_converge:get(<<"Method">>, M), - Action = hb_converge:get(<<"Action">>, M), - case {Method, Action} of - {{_, <<"GET">>}, {_, <<"Info">>}} -> fun info/1; - {{_, <<"GET">>}, {_, <<"Slot">>}} -> fun current_slot/1; - {{_, <<"GET">>}, {_, <<"Schedule">>}} -> fun current_schedule/1; - {{_, <<"POST">>}, _} -> fun schedule/1 - end. - -info(M) -> - Wallet = dev_scheduler_registry:get_wallet(), - {ok, - #{ - <<"Unit">> => <<"Scheduler">>, - <<"Address">> => hb_util:id(ar_wallet:to_address(Wallet)), - <<"Data">> => - jiffy:encode( - lists:map( - fun hb_util:id/1, - dev_scheduler_registry:get_processes() - ) - ) - } - }. - -current_slot(M) -> - {_, ProcID} = lists:keyfind(<<"Process">>, 1, M#tx.tags), - {Timestamp, Hash, Height} = ar_timestamp:get(), - {ok, #tx{ - tags = [ - {<<"Process">>, ProcID}, - {<<"Current-Slot">>, - dev_scheduler_server:get_current_slot( - dev_scheduler_registry:find(ProcID) - ) - }, - {<<"Timestamp">>, Timestamp}, - {<<"Block-Height">>, Height}, - {<<"Block-Hash">>, Hash} - ] - }}. - -current_schedule(M) -> - {_, ProcID} = lists:keyfind(<<"Process">>, 1, M#tx.tags), - send_schedule( - hb_opts:get(store), - ProcID, - lists:keyfind(<<"From">>, 1, M#tx.tags), - lists:keyfind(<<"To">>, 1, M#tx.tags) - ). - -schedule(CarrierM) -> - ?event(scheduling_message), - #{ <<"1">> := M } = CarrierM#tx.data, - %ar_bundles:print(M), - Store = hb_opts:get(store), - ?no_prod("SU does not validate item before writing into stream."), - case {ar_bundles:verify_item(M), lists:keyfind(<<"Type">>, 1, M#tx.tags)} of - % {false, _} -> - % {ok, - % #tx{ - % tags = [{<<"Status">>, <<"Failed">>}], - % data = <<"Data item is not valid.">> - % } - % }; - {_, {<<"Type">>, <<"Process">>}} -> - hb_cache:write(Store, M), - hb_client:upload(M), - {ok, - #tx{ - tags = - [ - {<<"Status">>, <<"OK">>}, - {<<"Initial-Assignment">>, <<"0">>}, - {<<"Process">>, hb_util:id(M, signed)} - ], - data = [] - } - }; - {_, _} -> - % If the process-id is not specified, use the target of the message as the process-id - AOProcID = - case lists:keyfind(<<"Process">>, 1, M#tx.tags) of - false -> binary_to_list(hb_util:id(M#tx.target)); - {_, ProcessID} -> ProcessID - end, - {ok, dev_scheduler_server:schedule(dev_scheduler_registry:find(AOProcID, true), M)} - end. - -%% Private methods - -send_schedule(Store, ProcID, false, To) -> - send_schedule(Store, ProcID, 0, To); -send_schedule(Store, ProcID, From, false) -> - send_schedule(Store, ProcID, From, dev_scheduler_server:get_current_slot(dev_scheduler_registry:find(ProcID))); -send_schedule(Store, ProcID, {<<"From">>, From}, To) -> - send_schedule(Store, ProcID, binary_to_integer(From), To); -send_schedule(Store, ProcID, From, {<<"To">>, To}) when byte_size(To) == 43 -> - send_schedule(Store, ProcID, From, To); -send_schedule(Store, ProcID, From, {<<"To">>, To}) -> - send_schedule(Store, ProcID, From, binary_to_integer(To)); -send_schedule(Store, ProcID, From, To) -> - {Timestamp, Height, Hash} = ar_timestamp:get(), - ?event({servicing_request_for_assignments, {proc_id, ProcID}, {from, From}, {to, To}}), - {Assignments, More} = dev_scheduler_server:get_assignments( - ProcID, - From, - To - ), - Bundle = #tx{ - tags = - [ - {<<"Type">>, <<"Schedule">>}, - {<<"Process">>, ProcID}, - {<<"Continues">>, atom_to_binary(More, utf8)}, - {<<"Timestamp">>, list_to_binary(integer_to_list(Timestamp))}, - {<<"Block-Height">>, list_to_binary(integer_to_list(Height))}, - {<<"Block-Hash">>, Hash} - ] ++ - case Assignments of - [] -> - []; - _ -> - {_, FromSlot} = lists:keyfind( - <<"Slot">>, 1, (hd(Assignments))#tx.tags - ), - {_, ToSlot} = lists:keyfind( - <<"Slot">>, 1, (lists:last(Assignments))#tx.tags - ), - [ - {<<"From">>, FromSlot}, - {<<"To">>, ToSlot} - ] - end, - data = assignments_to_bundle(Store, Assignments) - }, - ?event(assignments_bundle_outbound), - %ar_bundles:print(Bundle), - SignedBundle = ar_bundles:sign_item(Bundle, hb:wallet()), - {ok, SignedBundle}. - -assignments_to_bundle(Store, Assignments) -> - assignments_to_bundle(Store, Assignments, #{}). -assignments_to_bundle(_, [], Bundle) -> - Bundle; -assignments_to_bundle(Store, [Assignment | Assignments], Bundle) -> - {_, Slot} = lists:keyfind(<<"Slot">>, 1, Assignment#tx.tags), - {_, MessageID} = lists:keyfind(<<"Message">>, 1, Assignment#tx.tags), - {ok, Message} = hb_cache:read_message(Store, MessageID), - ?event({adding_assignment_to_bundle, Slot, {requested, MessageID}, hb_util:id(Assignment, signed), hb_util:id(Assignment, unsigned)}), - assignments_to_bundle( - Store, - Assignments, - Bundle#{ - Slot => - ar_bundles:sign_item( - #tx{ - tags = [ - {<<"Assignment">>, Slot}, - {<<"Message">>, MessageID} - ], - data = #{ - <<"Assignment">> => Assignment, - <<"Message">> => Message - } - }, - hb:wallet() - ) - } - ). \ No newline at end of file diff --git a/src/dev_scheduler_registry.erl b/src/dev_scheduler_registry.erl index d464616ee..3277a6d27 100644 --- a/src/dev_scheduler_registry.erl +++ b/src/dev_scheduler_registry.erl @@ -1,5 +1,5 @@ -module(dev_scheduler_registry). --export([start/0, find/1, find/2, get_wallet/0, get_processes/0]). +-export([start/0, find/1, find/2, find/3, get_wallet/0, get_processes/0]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -7,72 +7,105 @@ %%% only SU processes are supported. start() -> - pg:start(pg), + hb_name:start(), ok. get_wallet() -> % TODO: We might want to use a different wallet per SU later. hb:wallet(). +%% @doc Find a process associated with the processor ID in the local registry +%% If the process is not found, it will not create a new one find(ProcID) -> find(ProcID, false). -find(ProcID, GenIfNotHosted) -> - case pg:get_local_members({dev_scheduler, ProcID}) of - [] -> - maybe_new_proc(ProcID, GenIfNotHosted); - [Pid] -> - case is_process_alive(Pid) of - true -> Pid; - false -> - maybe_new_proc(ProcID, GenIfNotHosted) - end + +%% @doc Find a process associated with the processor ID in the local registry +%% If the process is not found and `GenIfNotHosted' is true, it attemps to +%% create a new one +find(ProcID, ProcMsgOrFalse) -> + find(ProcID, ProcMsgOrFalse, #{ priv_wallet => hb:wallet() }). + +%% @doc Same as `find/2' but with additional options passed when spawning a +%% new process (if needed) +find(ProcID, ProcMsgOrFalse, Opts) -> + case hb_name:lookup({<<"scheduler@1.0">>, ProcID}) of + undefined -> maybe_new_proc(ProcID, ProcMsgOrFalse, Opts); + Pid -> Pid end. +%% @doc Return a list of all currently registered ProcID. get_processes() -> - [ ProcID || {dev_scheduler, ProcID} <- pg:which_groups() ]. + ?event({getting_processes, hb_name:all()}), + [ ProcID || {{<<"scheduler@1.0">>, ProcID}, _} <- hb_name:all() ]. -maybe_new_proc(_ProcID, false) -> not_found; -maybe_new_proc(ProcID, _) -> - ?event({starting_scheduler_for, ProcID}), - Pid = dev_scheduler_server:start(ProcID, #{}), - try - pg:join({dev_scheduler, ProcID}, Pid), - Pid - catch - error:badarg -> - {error, registration_failed} - end. +maybe_new_proc(_ProcID, false, _Opts) -> not_found; +maybe_new_proc(ProcID, ProcMsg, Opts) -> + dev_scheduler_server:start(ProcID, ProcMsg, Opts). %%% Tests --define(TEST_PROC_ID1, <<0:256>>). --define(TEST_PROC_ID2, <<1:256>>). +test_opts() -> + #{ + store => hb_test_utils:test_store(), + priv_wallet => hb:wallet() + }. + +generate_test_procs(Opts) -> + [ + hb_message:commit( + #{ + <<"type">> => <<"Process">>, + <<"image">> => <<0:(1024*32)>> + }, + Opts + ), + hb_message:commit( + #{ + <<"type">> => <<"Process">>, + <<"image">> => <<0:(1024*32)>> + }, + Opts + ) + ]. find_non_existent_process_test() -> + Opts = test_opts(), + [Proc1, _Proc2] = generate_test_procs(Opts), start(), - ?assertEqual(not_found, ?MODULE:find(?TEST_PROC_ID1)). + ?assertEqual(not_found, ?MODULE:find(hb_message:id(Proc1, all))). create_and_find_process_test() -> + Opts = test_opts(), + [Proc1, _Proc2] = generate_test_procs(Opts), + ID = hb_message:id(Proc1, all, Opts), start(), - Pid1 = ?MODULE:find(?TEST_PROC_ID1, true), + Pid1 = ?MODULE:find(ID, Proc1), ?assert(is_pid(Pid1)), - ?assertEqual(Pid1, ?MODULE:find(?TEST_PROC_ID1)). + ?assertEqual(Pid1, ?MODULE:find(ID, Proc1)). create_multiple_processes_test() -> + Opts = test_opts(), + [Proc1, Proc2] = generate_test_procs(Opts), start(), - Pid1 = ?MODULE:find(?TEST_PROC_ID1, true), - Pid2 = ?MODULE:find(?TEST_PROC_ID2, true), + ID1 = hb_message:id(Proc1, all, Opts), + ID2 = hb_message:id(Proc2, all, Opts), + Pid1 = ?MODULE:find(ID1, Proc1), + Pid2 = ?MODULE:find(ID2, Proc2), ?assert(is_pid(Pid1)), ?assert(is_pid(Pid2)), ?assertNotEqual(Pid1, Pid2), - ?assertEqual(Pid1, ?MODULE:find(?TEST_PROC_ID1)), - ?assertEqual(Pid2, ?MODULE:find(?TEST_PROC_ID2)). + ?assertEqual(Pid1, ?MODULE:find(ID1, Proc1)), + ?assertEqual(Pid2, ?MODULE:find(ID2, Proc2)). get_all_processes_test() -> + Opts = test_opts(), + [Proc1, Proc2] = generate_test_procs(Opts), start(), - ?MODULE:find(?TEST_PROC_ID1, true), - ?MODULE:find(?TEST_PROC_ID2, true), + ID1 = hb_message:id(Proc1, all, Opts), + ID2 = hb_message:id(Proc2, all, Opts), + ?MODULE:find(ID1, Proc1), + ?MODULE:find(ID2, Proc2), Processes = ?MODULE:get_processes(), ?assert(length(Processes) >= 2), ?event({processes, Processes}), - ?assert(lists:member(?TEST_PROC_ID1, Processes)), - ?assert(lists:member(?TEST_PROC_ID2, Processes)). \ No newline at end of file + ?assert(lists:member(ID1, Processes)), + ?assert(lists:member(ID2, Processes)). \ No newline at end of file diff --git a/src/dev_scheduler_server.erl b/src/dev_scheduler_server.erl index 4c49db885..1595c83fe 100644 --- a/src/dev_scheduler_server.erl +++ b/src/dev_scheduler_server.erl @@ -2,19 +2,58 @@ %%% It acts as a deliberate 'bottleneck' to prevent the server accidentally %%% assigning multiple messages to the same slot. -module(dev_scheduler_server). --export([start/2, schedule/2]). +-export([start/3, schedule/2, stop/1]). -export([info/1]). -include_lib("eunit/include/eunit.hrl"). - -include("include/hb.hrl"). +%%% By default, we wait 10 seconds for a response from the scheduler before +%%% throwing an error on the client. If the scheduler is not able to sequence +%%% the message within this time, it will be discarded upon recipient by the +%%% server. This avoids situations in which the client did not receive +%%% confirmation of the assignment, but the scheduler still processes it. +-define(DEFAULT_TIMEOUT, 10000). + %% @doc Start a scheduling server for a given computation. -start(ProcID, Opts) -> - {CurrentSlot, HashChain} = slot_from_cache(ProcID, Opts), - spawn( +start(ProcID, Proc, Opts) -> + ?event(scheduling, {starting_scheduling_server, {proc_id, ProcID}}), + spawn_link( fun() -> + % Before we start, register the scheduler name. + case hb_name:register({<<"scheduler@1.0">>, ProcID}) of + ok -> ok; + error -> + throw( + {another_scheduler_is_already_registered, + {proc_id, ProcID} + } + ) + end, + % Write the process to the cache. We are the provider-of-last-resort + % for this data. + dev_scheduler_cache:write_spawn(Proc, Opts), + case hb_opts:get(scheduling_mode, disabled, Opts) of + disabled -> + throw({scheduling_disabled_on_node, {requested_for, ProcID}}); + _ -> ok + end, + {CurrentSlot, HashChain} = + case dev_scheduler_cache:latest(ProcID, Opts) of + not_found -> + ?event({starting_new_schedule, {proc_id, ProcID}}), + {-1, <<>>}; + {Slot, Chain} -> + ?event( + {continuing_schedule, + {proc_id, ProcID}, + {current_slot, Slot}, + {hash_chain, Chain} + } + ), + {Slot, Chain} + end, ?event( - {starting_scheduling_server, + {scheduler_got_process_info, {proc_id, ProcID}, {current, CurrentSlot}, {hash_chain, HashChain} @@ -24,68 +63,97 @@ start(ProcID, Opts) -> #{ id => ProcID, current => CurrentSlot, - wallet => hb_opts:get(priv_wallet, hb:wallet(), Opts), hash_chain => HashChain, + wallets => commitment_wallets(Proc, Opts), + mode => + hb_opts:get( + scheduling_mode, + remote_confirmation, + Opts + ), opts => Opts } ) end ). -%% @doc Get the current slot from the cache. -slot_from_cache(ProcID, Opts) -> - case dev_scheduler_cache:list(ProcID, Opts) of - [] -> - ?event({no_assignments_in_cache, {proc_id, ProcID}}), - {-1, <<>>}; - Assignments -> - AssignmentNum = lists:max(Assignments), - ?event( - {found_assignment_from_cache, - {proc_id, ProcID}, - {assignment_num, AssignmentNum} - } - ), - {ok, Assignment} = dev_scheduler_cache:read( - ProcID, - AssignmentNum, - Opts - ), - { - AssignmentNum, - hb_converge:get( - <<"Hash-Chain">>, Assignment, #{ hashpath => ignore }) - } - end. +%% @doc Determine the appropriate list of keys to use to commit assignments for +%% a process. +commitment_wallets(ProcMsg, Opts) -> + SchedulerVal = + hb_ao:get_first( + [ + {ProcMsg, <<"scheduler">>}, + {ProcMsg, <<"scheduler-location">>} + ], + [], + Opts + ), + lists:filtermap( + fun(Scheduler) -> + case hb_opts:as(Scheduler, Opts) of + {ok, #{ priv_wallet := Wallet }} -> {true, Wallet}; + _ -> false + end + end, + dev_scheduler:parse_schedulers(SchedulerVal) + ). %% @doc Call the appropriate scheduling server to assign a message. schedule(AOProcID, Message) when is_binary(AOProcID) -> schedule(dev_scheduler_registry:find(AOProcID), Message); schedule(ErlangProcID, Message) -> - ErlangProcID ! {schedule, Message, self()}, + ?event( + {scheduling_message, + {proc_id, ErlangProcID}, + {message, Message}, + {is_alive, is_process_alive(ErlangProcID)} + } + ), + AbortTime = scheduler_time() + ?DEFAULT_TIMEOUT, + ErlangProcID ! {schedule, Message, self(), AbortTime}, receive {scheduled, Message, Assignment} -> Assignment + after ?DEFAULT_TIMEOUT -> + throw({scheduler_timeout, {proc_id, ErlangProcID}, {message, Message}}) end. %% @doc Get the current slot from the scheduling server. info(ProcID) -> ?event({getting_info, {proc_id, ProcID}}), ProcID ! {info, self()}, - receive - {info, Info} -> - Info - end. + receive {info, Info} -> Info end. + +stop(ProcID) -> + ?event({stopping_scheduling_server, {proc_id, ProcID}}), + ProcID ! stop. %% @doc The main loop of the server. Simply waits for messages to assign and %% returns the current slot. server(State) -> receive - {schedule, Message, Reply} -> - server(assign(State, Message, Reply)); + {schedule, Message, Reply, AbortTime} -> + case SchedTime = scheduler_time() > AbortTime of + true -> + % Ignore scheduling requests if they are too old. The + % `abort-time' signals to us that the client has already + % given up on the request, so in order to maintain + % predictability we ignore it. + ?event(error, + {received_old_schedule_request, + {abort_time, AbortTime}, + {sched_time, SchedTime} + } + ), + server(State); + false -> + server(assign(State, Message, Reply)) + end; {info, Reply} -> Reply ! {info, State}, - server(State) + server(State); + stop -> ok end. %% @doc Assign a message to the next slot. @@ -95,140 +163,199 @@ assign(State, Message, ReplyPID) -> catch _Class:Reason:Stack -> ?event({error_scheduling, Reason, Stack}), - {error, State} + State end. %% @doc Generate and store the actual assignment message. do_assign(State, Message, ReplyPID) -> - ?event( - {assigning_message, - {id, hb_converge:get(id, Message)}, - {message, Message} - } - ), - HashChain = next_hashchain(maps:get(hash_chain, State), Message), + % Ensure that only committed keys from the message are included in the + % assignment. + {ok, OnlyAttested} = + hb_message:with_only_committed( + Message, + Opts = maps:get(opts, State) + ), + HashChain = + next_hashchain( + maps:get(hash_chain, State), + OnlyAttested, + Opts + ), NextSlot = maps:get(current, State) + 1, - % Run the signing of the assignment and writes to the disk in a separate process - spawn( + % Run the signing of the assignment and writes to the disk in a separate + % process. + AssignFun = fun() -> {Timestamp, Height, Hash} = ar_timestamp:get(), - Assignment = hb_message:sign(#{ - <<"Data-Protocol">> => <<"ao">>, - <<"Variant">> => <<"ao.TN.2">>, - <<"Process">> => hb_util:id(maps:get(id, State)), - <<"Epoch">> => <<"0">>, - <<"Slot">> => NextSlot, - <<"Message">> => hb_converge:get(id, Message), - <<"Block-Height">> => Height, - <<"Block-Hash">> => Hash, - <<"Block-Timestamp">> => Timestamp, - % Note: Local time on the SU, not Arweave - <<"Timestamp">> => erlang:system_time(millisecond), - <<"Hash-Chain">> => hb_util:id(HashChain) - }, maps:get(wallet, State)), - maybe_inform_recipient(aggressive, ReplyPID, Message, Assignment), + Assignment = + commit_assignment( + #{ + <<"path">> => + case hb_path:from_message(request, Message, Opts) of + undefined -> <<"compute">>; + Path -> Path + end, + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"process">> => hb_util:id(maps:get(id, State)), + <<"epoch">> => <<"0">>, + <<"slot">> => NextSlot, + <<"block-height">> => Height, + <<"block-hash">> => hb_util:human_id(Hash), + <<"block-timestamp">> => Timestamp, + % Note: Local time on the SU, not Arweave + <<"timestamp">> => scheduler_time(), + <<"hash-chain">> => hb_util:id(HashChain), + <<"body">> => OnlyAttested, + <<"type">> => <<"assignment">> + }, + State + ), + AssignmentID = hb_message:id(Assignment, all), + ?event(scheduling, + {assigned, + {proc_id, maps:get(id, State)}, + {slot, NextSlot}, + {assignment, AssignmentID} + } + ), + maybe_inform_recipient( + aggressive, + ReplyPID, + Message, + Assignment, + State + ), ?event(starting_message_write), - dev_scheduler_cache:write(Assignment, maps:get(opts, State)), - hb_cache:write(Message, maps:get(opts, State)), + ok = dev_scheduler_cache:write(Assignment, Opts), maybe_inform_recipient( local_confirmation, ReplyPID, Message, - Assignment + Assignment, + State ), ?event(writes_complete), ?event(uploading_assignment), - hb_client:upload(Assignment), - ?event(uploading_message), - hb_client:upload(Message), + hb_client:upload(Assignment, Opts), ?event(uploads_complete), maybe_inform_recipient( remote_confirmation, ReplyPID, Message, - Assignment + Assignment, + State ) - end - ), + end, + case hb_opts:get(scheduling_mode, sync, Opts) of + aggressive -> + spawn(AssignFun); + Other -> + ?event({scheduling_mode, Other}), + AssignFun() + end, State#{ current := NextSlot, hash_chain := HashChain }. -maybe_inform_recipient(Mode, ReplyPID, Message, Assignment) -> - case hb_opts:get(scheduling_mode, remote_confirmation) of +%% @doc Commit to the assignment using all of our appropriate wallets. +commit_assignment(BaseAssignment, State) -> + Wallets = maps:get(wallets, State), + Opts = maps:get(opts, State), + lists:foldr( + fun(Wallet, Assignment) -> + hb_message:commit(Assignment, Opts#{ priv_wallet => Wallet }) + end, + BaseAssignment, + Wallets + ). + +%% @doc Potentially inform the caller that the assignment has been scheduled. +%% The main assignment loop calls this function repeatedly at different stages +%% of the assignment process. The scheduling mode determines which stages +%% trigger an update. +maybe_inform_recipient(Mode, ReplyPID, Message, Assignment, State) -> + case maps:get(mode, State) of Mode -> ReplyPID ! {scheduled, Message, Assignment}; _ -> ok end. %% @doc Create the next element in a chain of hashes that links this and prior %% assignments. -next_hashchain(HashChain, Message) -> +next_hashchain(HashChain, Message, Opts) -> + ?event({creating_next_hashchain, {hash_chain, HashChain}, {message, Message}}), + ID = hb_message:id(Message, all, Opts), crypto:hash( sha256, - << HashChain/binary, (hb_util:id(Message, signed))/binary >> + << HashChain/binary, ID/binary >> ). +%% @doc Return the current time in milliseconds. +scheduler_time() -> + erlang:system_time(millisecond). + %% TESTS %% @doc Test the basic functionality of the server. new_proc_test() -> Wallet = ar_wallet:new(), - SignedItem = hb_message:sign( - #{ <<"Data">> => <<"test">>, <<"Random-Key">> => rand:uniform(10000) }, - Wallet + SignedItem = hb_message:commit( + #{ <<"data">> => <<"test">>, <<"random-key">> => rand:uniform(10000) }, + #{ priv_wallet => Wallet } ), - SignedItem2 = hb_message:sign( - #{ <<"Data">> => <<"test2">> }, - Wallet + SignedItem2 = hb_message:commit( + #{ <<"data">> => <<"test2">> }, + #{ priv_wallet => Wallet } ), - SignedItem3 = hb_message:sign( + SignedItem3 = hb_message:commit( #{ - <<"Data">> => <<"test2">>, - <<"Deep-Key">> => - #{ <<"Data">> => <<"test3">> } + <<"data">> => <<"test2">>, + <<"deep-key">> => + #{ <<"data">> => <<"test3">> } }, - Wallet + #{ priv_wallet => Wallet } ), - dev_scheduler_registry:find(hb_converge:get(id, SignedItem), true), - schedule(ID = hb_converge:get(id, SignedItem), SignedItem), + dev_scheduler_registry:find(hb_message:id(SignedItem, all), SignedItem), + schedule(ID = hb_message:id(SignedItem, all), SignedItem), schedule(ID, SignedItem2), schedule(ID, SignedItem3), ?assertMatch( #{ current := 2 }, dev_scheduler_server:info(dev_scheduler_registry:find(ID)) ). + -benchmark_test() -> - BenchTime = 1, - Wallet = ar_wallet:new(), - SignedItem = hb_message:sign( - #{ <<"Data">> => <<"test">>, <<"Random-Key">> => rand:uniform(10000) }, - Wallet - ), - dev_scheduler_registry:find(ID = hb_converge:get(id, SignedItem), true), - ?event({benchmark_start, ?MODULE}), - Iterations = hb:benchmark( - fun(X) -> - MsgX = #{ - path => <<"Schedule">>, - <<"Method">> => <<"POST">>, - <<"Message">> => - #{ - <<"Type">> => <<"Message">>, - <<"Test-Val">> => X - } - }, - schedule(ID, MsgX) - end, - BenchTime - ), - hb_util:eunit_print( - "Scheduled ~p messages in ~p seconds (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] - ), - ?assertMatch( - #{ current := X } when X == Iterations - 1, - dev_scheduler_server:info(dev_scheduler_registry:find(ID)) - ), - ?assert(Iterations > 30). +% benchmark_test() -> +% BenchTime = 1, +% Wallet = ar_wallet:new(), +% SignedItem = hb_message:commit( +% #{ <<"data">> => <<"test">>, <<"random-key">> => rand:uniform(10000) }, +% Wallet +% ), +% dev_scheduler_registry:find(ID = hb_ao:get(id, SignedItem), true), +% ?event({benchmark_start, ?MODULE}), +% Iterations = hb_test_utils:benchmark( +% fun(X) -> +% MsgX = #{ +% path => <<"Schedule">>, +% <<"method">> => <<"POST">>, +% <<"body">> => +% #{ +% <<"type">> => <<"Message">>, +% <<"test-val">> => X +% } +% }, +% schedule(ID, MsgX) +% end, +% BenchTime +% ), +% hb_formatter:eunit_print( +% "Scheduled ~p messages in ~p seconds (~.2f msg/s)", +% [Iterations, BenchTime, Iterations / BenchTime] +% ), +% ?assertMatch( +% #{ current := X } when X == Iterations - 1, +% dev_scheduler_server:info(dev_scheduler_registry:find(ID)) +% ), +% ?assert(Iterations > 30). diff --git a/src/dev_secret.erl b/src/dev_secret.erl new file mode 100644 index 000000000..ce01df21a --- /dev/null +++ b/src/dev_secret.erl @@ -0,0 +1,1147 @@ +%%% @doc A device that allows a node to create, export, and commit messages with +%%% secrets that are stored on the node itself. Users of this device must specify +%%% an `access-control' message which requests are validated against before +%%% access to secrets is granted. +%%% +%%% This device is intended for use in situations in which the node is trusted +%%% by the user, for example if it is running on their own machine or in a +%%% TEE-protected environment that they deem to be secure. +%%% +%%% # Authentication Flow +%%% +%%% Each secret is associated with an `access-control' message and a list of +%%% `controllers' that may access it. The `access-control' system is pluggable +%%% -- users may configure their messages to call any AO-Core device that is +%%% executable on the host node. The default `access-control' message uses the +%%% `~cookie@1.0' device's `generate' and `verify' keys to authenticate users. +%%% +%%% During secret generation: +%%% 1. This device creates the secret and determines its `committer' address. +%%% 2. The device invokes the caller's `access-control' message with the `commit' +%%% path and the `keyid' in the request. +%%% 3. The `access-control' message sets up authentication (e.g., creates cookies, +%%% secrets) and returns a response, containing a commitment with a `keyid' +%%% field. This `keyid' is used to identify the user's 'access secret' which +%%% grants them the ability to use the device's 'hidden' secret in the future. +%%% 4. This device stores both the secret and the initialized `access-control' +%%% message, as well as its other metadata. +%%% 5. This device returns the initialized `access-control' message with the +%%% secret's `keyid' added to the `body' field. +%%% +%%% During secret operations (commit, export, etc.): +%%% 1. This device retrieves the stored `access-control' message for the +%%% secret either from persistent storage or from the node message's private +%%% element. The keyid of the `access secret' is either provided by the +%%% user in the request, or is determined from a provided `secret' parameter +%%% in the request. +%%% 2. This device calls the `access-control' message with path `verify' and +%%% the user's request. +%%% 3. The `access-control' message verifies the request (e.g., checks cookies, +%%% provided authentication credentials, etc.). +%%% 4. If verification passes, the device performs the requested operation. +%%% 5. If verification fails, a 400 error is returned. +%%% +%%% # Access Control Message Requirements +%%% +%%% Access control messages are fully customizable by callers, but must support +%%% two paths: +%%% +%%% `/commit': Called during secret generation to bind the `access-control' +%%% template message to the given `keyid' (secret reference). +%%% - Input: Request message containing `keyid' field with the secret's `keyid' +%%% in the `body' field. +%%% - Output: Response message with authentication setup (cookies, tokens, etc.). +%%% This message will be used as the `Base' message for the `verify' +%%% path. +%%% +%%% `/verify': Called before allowing an operation that requires access to a +%%% secret to proceed. +%%% - Base: The initialized `access-control' message from the `commit' path. +%%% - Request: Caller's request message with authentication credentials. +%%% - Output: `false' if an error has occurred. If the request is valid, the +%%% `access-control' message should return either `true' or a modification +%%% of the request message which will be used for any subsequent +%%% operations. +%%% +%%% The default `access-control' message is `~cookie@1.0', which uses HTTP +%%% cookies with secrets to authenticate users. +%%% +%%% # Secret Generation Parameters +%%% +%%% The following parameters are supported by the `generate' key: +%%% +%%% ``` +%%% /generate +%%% - `access-control' (optional): The `access-control' message to use. +%%% Defaults to `#{<<"device">> => <<"cookie@1.0">>}'. +%%% - `keyid' (optional): The `keyid' of the secret to generate. If not +%%% provided, the secret's address will be used as the name. +%%% - `persist' (optional): How the node should persist the secret. Options: +%%% - `client': The secret is generated on the server, but not persisted. +%%% The full secret key is returned for the user to store. +%%% - `in-memory': The wallet is generated on the server and persisted only +%%% in local memory, never written to disk. +%%% - `non-volatile': The wallet is persisted to non-volatile storage on +%%% the node. The store used by this option is segmented from +%%% the node's main storage, configurable via the `priv_store' +%%% node message option. +%%% - `controllers' (optional): A list of controllers that may access the +%%% secret. Defaults to the node's `wallet_admin' option if set, +%%% or its operator address if not. +%%% - `required-controllers' (optional): The number of controllers that must +%%% sign the secret for it to be valid. Defaults to `1'. +%%% +%%% The response will contain authentication setup (such as cookies) from the +%%% `access-control' message, plus the secret's `keyid' in the `body' field. +%%% The secret's key is not returned to the user unless the `persist' option +%%% is set to `client'. If it is, the `~cookie@1.0' device will be employed +%%% to set the user's cookie with the secret. +%%% +%%% /import +%%% Parameters: +%%% - `key' (optional): The JSON-encoded secret to import. +%%% - `cookie' (optional): A structured-fields cookie containing a map with +%%% a `key' field which is a JSON-encoded secret. +%%% - `access-control' (optional): The `access-control' message to use. +%%% - `persist' (optional): How the node should persist the secret. The +%%% supported options are as with the `generate' key. +%%% +%%% Imports a secret for hosting from the user. Executes as `generate' does, +%%% except that it expects the key to store to be provided either directly +%%% via the `key' parameter as a `keyid' field in the cookie Structured-Fields +%%% map. Support for loading the key from the cookie is provided such that +%%% a previously-generated secret by the user can have its persistence mode +%%% changed. +%%% +%%% /list +%%% Parameters: +%%% - `keyids' (optional): A list of `keyid's to list. If not provided, +%%% all secrets will be listed via the `keyid' that is must be provided +%%% in order to access them. +%%% +%%% Lists all hosted secrets on the node by the `keyid' that is used to +%%% access them. If `keyids' is provided, only the secrets with those +%%% `keyid's will be listed. +%%% +%%% /commit +%%% Parameters: +%%% - `keyid' (optional): The `keyid' of the secret to commit with. +%%% - Authentication credentials as required by the `access-control' message. +%%% +%%% Commits the given message using the specified secret after authentication. +%%% If no `keyid' parameter is provided, the request's authentication data +%%% (such as cookies) must contain secret identification. +%%% +%%% /export +%%% Parameters: +%%% - `keyids' (optional): A list of `keyid's to export, or `all' to +%%% export all secrets for which the request passes authentication. +%%% +%%% Exports a given secret or set of secrets. If multiple secrets are +%%% requested, the result is a message with form `keyid => #{ `key` => +%%% JSON-encoded secret, `access-control' => `access-control' message, +%%% `controllers' => [address, ...], `required-controllers' => integer, +%%% `persist' => `client' | `in-memory' | `non-volatile' }'. +%%% +%%% A secret will be exported if: +%%% - The given request passes each requested secret's `access-control' +%%% message; or +%%% - The request passes each requested secret's `controllers' parameter +%%% checks. +%%% +%%% /sync +%%% Parameters: +%%% - `node': The peer node to pull secrets from. +%%% - `as' (optional): The identity it should use when signing its request +%%% to the remote peer. +%%% - `keyids' (optional): A list of `keyid's to export, or `all' to load +%%% every available secret. Defaults to `all'. +%%% +%%% Attempts to download all (or a given subset of) secrets from the given +%%% node and import them. If the `keyids' parameter is provided, only the +%%% secrets with those `keyid's will be imported. The `as' parameter is +%%% used to inform the node which key it should use to sign its request to +%%% the remote peer, such that its request validates against the secret's +%%% `access-control' messages on the remote peer. +%%% ''' +-module(dev_secret). +-export([generate/3, import/3, list/3, commit/3, export/3, sync/3]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +-define(DEFAULT_AUTH_DEVICE, <<"cookie@1.0">>). + +%% @doc Generate a new wallet for a user and register it on the node. If the +%% `committer' field is provided, we first check whether there is a wallet +%% already registered for it. If there is, we return the wallet details. +generate(Base, Request, Opts) -> + case request_to_wallets(Base, Request, Opts) of + [] -> + % No wallets found, create a new one. + Wallet = ar_wallet:new(), + register_wallet(Wallet, Base, Request, Opts); + [WalletDetails] -> + ?event({details, WalletDetails}), + % Wallets found, return them. + { + ok, + WalletDetails#{ + <<"body">> => + hb_maps:get( + <<"keyid">>, + Base, + Opts + ) + } + } + end. + +%% @doc Import a wallet for hosting on the node. Expects the keys to be either +%% provided as a list of keys, or a single key in the `key' field. If neither +%% are provided, the keys are extracted from the cookie. +import(Base, Request, Opts) -> + Wallets = + case hb_maps:find(<<"key">>, Request, Opts) of + {ok, Keys} when is_list(Keys) -> + [ wallet_from_key(Key) || Key <- Keys ]; + {ok, Key} -> + [ wallet_from_key(hb_escape:decode_quotes(Key)) ]; + error -> + request_to_wallets(Base, Request, Opts) + end, + case Wallets of + [] -> + {error, <<"No viable wallets found to import.">>}; + Wallets -> + import_wallets(Wallets, Base, Request, Opts) + end. + +%% @doc Register a series of wallets, returning a summary message with the +%% list of imported wallets, as well as merged cookies. +import_wallets(Wallets, Base, Request, Opts) -> + Res = + lists:foldl( + fun(Wallet, Acc) -> + case register_wallet(Wallet, Base, Request, Opts) of + {ok, RegRes} -> + % Merge the private element of the registration response + % into the accumulator. + WalletAddress = hb_maps:get(<<"wallet-address">>, RegRes, Opts), + OldImported = hb_maps:get(<<"imported">>, Acc, [], Opts), + Merged = + hb_private:merge( + Acc, + RegRes, + Opts + ), + Merged#{ + <<"imported">> => [ WalletAddress | OldImported ] + }; + {error, _} -> Acc + end + end, + #{}, + Wallets + ), + {ok, + Res#{ + <<"body">> => + addresses_to_binary(hb_maps:get(<<"imported">>, Res, [], Opts)) + } + }. + +%% @doc Transform a wallet key serialized form into a wallet. +wallet_from_key(Key) when is_binary(Key) -> + ar_wallet:from_json(Key); +wallet_from_key(Key) -> + Key. + +%% @doc Register a wallet on the node. +register_wallet(Wallet, Base, Request, Opts) -> + % Find the wallet's address. + {PrivKey, _} = Wallet, + Address = hb_util:human_id(ar_wallet:to_address(Wallet)), + % Determine how to persist the wallet. + PersistMode = hb_ao:get(<<"persist">>, Request, <<"in-memory">>, Opts), + % Get the authentication message from the request. If the message is a path + % or a message with a `path' field, we resolve it to get the base. + {ok, BaseAccessControl} = + case hb_ao:get(<<"access-control">>, Base, undefined, Opts) of + undefined -> + ?event( + debug_auth, + {defaulting_access_control, {base, Base}, {request, Request}} + ), + {ok, #{ <<"device">> => ?DEFAULT_AUTH_DEVICE }}; + AuthPath when is_binary(AuthPath) -> + hb_ao:resolve(AuthPath, Opts); + Msg -> + case hb_maps:is_key(<<"path">>, Msg, Opts) of + true -> hb_ao:resolve(Msg, Opts); + false -> {ok, Msg} + end + end, + AccessControl = + BaseAccessControl#{ + <<"wallet-address">> => hb_util:human_id(Address) + }, + Controllers = + hb_ao:get(<<"controllers">>, Request, default, Opts), + RequiredControllers = + hb_util:int(hb_ao:get(<<"required-controllers">>, Request, 1, Opts)), + % Call authentication device to set up auth. Pass the wallet address as the + % nonce. Some auth devices may use the nonce to track the messages that + % they have committed. + AuthRequest = + case hb_ao:get(<<"secret">>, Base, undefined, Opts) of + undefined -> + Request#{ + <<"path">> => <<"commit">> + }; + Secret -> + Request#{ + <<"path">> => <<"commit">>, + <<"secret">> => Secret + } + end, + ?event({register_wallet, {access_control, AccessControl}, {request, AuthRequest}}), + case hb_ao:resolve(AccessControl, AuthRequest, Opts) of + {ok, InitializedAuthMsg} -> + ?event({register_wallet_success, {initialized_auth_msg, InitializedAuthMsg}}), + % Find the new signer address. + PriorSigners = hb_message:signers(AccessControl, Opts), + NewSigners = hb_message:signers(InitializedAuthMsg, Opts), + [Committer] = NewSigners -- PriorSigners, + % Store wallet details. + WalletDetails = + #{ + <<"wallet">> => ar_wallet:to_json(PrivKey), + <<"address">> => hb_util:human_id(Address), + <<"persist">> => PersistMode, + <<"access-control">> => hb_private:reset(InitializedAuthMsg), + <<"committer">> => Committer, + <<"controllers">> => parse_controllers(Controllers, Opts), + <<"required-controllers">> => RequiredControllers + }, + persist_registered_wallet(WalletDetails, InitializedAuthMsg, Opts); + {error, Reason} -> + ?event({register_wallet_error, {reason, Reason}}), + {error, Reason} + end. + +%% @doc Persist a wallet and return the auth response. Optionally takes a +%% response base that is used as the message to build upon for the eventual +%% user-response. +persist_registered_wallet(WalletDetails, Opts) -> + persist_registered_wallet(WalletDetails, #{}, Opts). +persist_registered_wallet(WalletDetails, RespBase, Opts) -> + % Add the wallet address as the body of the response. + Address = hb_maps:get(<<"address">>, WalletDetails, undefined, Opts), + ?event({resp_base, RespBase, WalletDetails}), + AccessControl = hb_maps:get(<<"access-control">>, WalletDetails, #{}, Opts), + {ok, _, Commitment} = hb_message:commitment(#{}, AccessControl, Opts), + KeyID = hb_maps:get(<<"keyid">>, Commitment, Opts), + Base = RespBase#{ <<"body">> => KeyID }, + % Determine how to persist the wallet. + case hb_maps:get(<<"persist">>, WalletDetails, <<"in-memory">>, Opts) of + <<"client">> -> + ?event({wallet_details, WalletDetails}), + % Find the necessary wallet details to set the cookie on the client. + JSONKey = hb_maps:get(<<"wallet">>, WalletDetails, undefined, Opts), + % Don't store, set the cookie in the response. + hb_ao:resolve( + Base#{ <<"device">> => <<"cookie@1.0">> }, + #{ + <<"path">> => <<"store">>, + <<"wallet-", Address/binary>> => hb_escape:encode_quotes(JSONKey) + }, + Opts + ); + PersistMode -> + % Store wallet and return auth response with wallet info. + store_wallet( + hb_util:key_to_atom(PersistMode, existing), + KeyID, + WalletDetails, + Opts + ), + ?event( + {stored_and_returning, + {auth_response, Base}, + {wallet_details, WalletDetails} + } + ), + % Return auth response with wallet info added. + {ok, Base} + end. + +%% @doc List all hosted wallets +list(_Base, _Request, Opts) -> + {ok, list_wallets(Opts)}. + +%% @doc Sign a message with a wallet. +commit(Base, Request, Opts) -> + ?event({commit_invoked, {base, Base}, {request, Request}}), + case request_to_wallets(Base, Request, Opts) of + [] -> {error, <<"No wallets found to sign with.">>}; + WalletDetailsList -> + ?event( + {commit_signing, + {request, Request}, + {wallet_list, WalletDetailsList} + } + ), + { + ok, + lists:foldl( + fun(WalletDetails, Acc) -> + ?event( + {invoking_commit_message, + {message, Acc}, + {wallet, WalletDetails} + } + ), + commit_message(Acc, WalletDetails, Opts) + end, + Base, + WalletDetailsList + ) + } + end. + +%% @doc Take a request and return the wallets it references. Performs validation +%% of access rights for the wallets before returning them. +request_to_wallets(Base, Request, Opts) -> + % Get the wallet references or keys from the request or cookie. + ?event({request_to_wallets, {base, Base}, {request, Request}}), + Keys = + hb_ao:get_first( + [ + {Request, <<"secret">>}, + {Base, <<"secret">>} + ], + <<"all">>, + Opts + ), + ?event({request_to_wallets, {keys, Keys}}), + WalletKeyIDs = + case hb_maps:get(<<"keyids">>, Request, not_found, Opts) of + not_found -> + case Keys of + <<"all">> -> + % Get the wallet name from the cookie. + wallets_from_cookie(Request, Opts); + _ -> secrets_to_keyids(Keys) + end; + KeyIDs -> lists:map(fun(KeyID) -> + Wallet = find_wallet(KeyID, Opts), + {secret, KeyID, hb_maps:get(<<"wallet">>, Wallet, Opts) } + end, KeyIDs) + end, + ?event({attempting_to_load_wallets, {keyids, WalletKeyIDs}, {request, Request}}), + lists:filtermap( + fun(WalletKeyID) -> + case load_and_verify(WalletKeyID, Base, Request, Opts) of + {ok, WalletDetails} -> + ?event({request_to_wallets, {loaded_wallet, WalletDetails}}), + {true, WalletDetails}; + {error, Reason} -> + ?event( + {failed_to_load_wallet, + {keyid, WalletKeyID}, + {reason, Reason} + } + ), + false + end + end, + WalletKeyIDs + ). + +%% @doc Load a wallet from a keyid and verify we have the authority to access it. +load_and_verify({wallet, WalletKey}, _Base, _Request, _Opts) -> + % Return the wallet key. + Wallet = ar_wallet:from_json(WalletKey), + PubKey = ar_wallet:to_pubkey(Wallet), + Address = ar_wallet:to_address(PubKey), + {ok, #{ + <<"wallet">> => WalletKey, + <<"address">> => hb_util:human_id(Address), + <<"persist">> => <<"client">>, + <<"committer">> => << "publickey:", (hb_util:encode(PubKey))/binary >> + }}; +load_and_verify({secret, KeyID, _}, _Base, Request, Opts) -> + % Get the wallet from the node's options. + case find_wallet(KeyID, Opts) of + not_found -> {error, <<"Wallet not hosted on node.">>}; + WalletDetails -> + case verify_controllers(WalletDetails, Request, Opts) of + true -> + % If the request is already signed by an exporter + % return the request as-is with the wallet. + {ok, WalletDetails}; + false -> + case verify_auth(WalletDetails, Request, Opts) of + {ok, true} -> + {ok, WalletDetails}; + {ok, false} -> + {error, <<"Verification failed.">>}; + {error, Reason} -> + {error, Reason} + end + end + end. + +%% @doc Validate if a calling message has the required `controllers' for the +%% given wallet. +verify_controllers(WalletDetails, Request, Opts) -> + RequiredControllers = + hb_util:int(hb_maps:get(<<"required-controllers">>, WalletDetails, 1, Opts)), + Controllers = + parse_controllers( + hb_maps:get(<<"controllers">>, WalletDetails, [], Opts), + Opts + ), + PresentControllers = + lists:filter( + fun(Signer) -> + lists:member(Signer, Controllers) + end, + hb_message:signers(Request, Opts) + ), + length(PresentControllers) >= RequiredControllers. + +%% @doc Verify a wallet for a given request. +verify_auth(WalletDetails, Req, Opts) -> + AuthBase = hb_maps:get(<<"access-control">>, WalletDetails, #{}, Opts), + AuthRequest = + Req#{ + <<"path">> => <<"verify">>, + <<"committer">> => + hb_maps:get(<<"committer">>, WalletDetails, undefined, Opts) + }, + ?event({verify_wallet, {auth_base, AuthBase}, {request, AuthRequest}}), + hb_ao:resolve(AuthBase, AuthRequest, Opts). + +%% @doc Parse cookie from a message to extract wallets. +wallets_from_cookie(Msg, Opts) -> + % Parse the cookie as a Structured-Fields map. + ParsedCookie = + try dev_codec_cookie:extract(Msg, #{ <<"format">> => <<"cookie">> }, Opts) of + {ok, CookieMsg} -> CookieMsg + catch _:_ -> {error, <<"Invalid cookie format.">>} + end, + ?event({parsed_cookie, ParsedCookie}), + % Get the wallets that we should be able to access from the parsed cookie. + % We determine their type from the `type-' prefix of the key. + lists:flatten(lists:filtermap( + fun({<<"secret-", Address/binary >>, Key}) -> + DecodedKey = hb_escape:decode_quotes(Key), + ?event({wallet_from_cookie, {key, DecodedKey},{ address, Address}}), + {true, secrets_to_keyids(DecodedKey)}; + ({<<"wallet-", Address/binary >>, Key}) -> + DecodedKey = hb_escape:decode_quotes(Key), + ?event({wallet_from_cookie, {key, DecodedKey}, {address, Address}}), + {true, [{wallet, DecodedKey}]}; + ({_Irrelevant, _}) -> false + end, + hb_maps:to_list(ParsedCookie, Opts) + )). + +%% @doc Sign a message using hb_message:commit, taking either a wallet as a +%% JSON-encoded string or a wallet details message with a `key' field. +commit_message(Message, NonMap, Opts) when not is_map(NonMap) -> + commit_message(Message, #{ <<"wallet">> => NonMap }, Opts); +commit_message(Message, #{ <<"wallet">> := Key }, Opts) when is_binary(Key) -> + commit_message(Message, ar_wallet:from_json(Key), Opts); +commit_message(Message, #{ <<"wallet">> := Key }, Opts) -> + ?event({committing_with_proxy, {message, Message}, {wallet, Key}}), + hb_message:commit(Message, Opts#{ priv_wallet => Key }). + +%% @doc Export wallets from a request. The request should contain a source of +%% wallets (cookies, keys, or wallet names), or a specific list/name of a +%% wallet to authenticate and export. +export(Base, Request, Opts) -> + PrivOpts = priv_store_opts(Opts), + ModReq = + case hb_ao:get(<<"keyids">>, Request, not_found, Opts) of + <<"all">> -> + AllLocalWallets = list_wallets(Opts), + Request#{ <<"keyids">> => AllLocalWallets }; + _ -> Request + end, + ?event({export, {base, Base}, {request, ModReq}}), + case request_to_wallets(Base, ModReq, Opts) of + [] -> {error, <<"No wallets found to export.">>}; + Wallets -> + { + ok, + lists:map( + fun(Wallet) -> + Loaded = hb_cache:ensure_all_loaded(Wallet, PrivOpts), + ?event({exported, {wallet, Loaded}}), + Loaded + end, + Wallets + ) + } + end. + +%% @doc Sync wallets from a remote node +sync(_Base, Request, Opts) -> + case hb_ao:get(<<"node">>, Request, undefined, Opts) of + undefined -> + {error, <<"Node not specified.">>}; + Node -> + Wallets = hb_maps:get(<<"keyids">>, Request, <<"all">>, Opts), + SignAsOpts = + case hb_ao:get(<<"as">>, Request, undefined, Opts) of + undefined -> Opts; + SignAs -> hb_opts:as(SignAs, Opts) + end, + ExportRequest = + (hb_message:commit( + #{ <<"keyids">> => Wallets }, + SignAsOpts + ))#{ <<"path">> => <<"/~secret@1.0/export">> }, + ?event({sync, {export_req, ExportRequest}}), + case hb_http:get(Node, ExportRequest, SignAsOpts) of + {ok, ExportResponse} -> + ExportedWallets = export_response_to_list(ExportResponse, #{}), + ?event({sync, {received_wallets, ExportedWallets}}), + % Import each wallet. Ignore wallet imports that fail. + lists:filtermap( + fun(Wallet) -> + ?event({sync, {importing, {wallet, Wallet}}}), + case persist_registered_wallet(Wallet, SignAsOpts) of + {ok, #{ <<"body">> := Address }} -> + ?event({sync, {imported, Address}}), + {true, Address}; + {error, Reason} -> + ?event({sync, {process_import_error, Reason}}), + false + end + end, + ExportedWallets + ), + {ok, ExportedWallets}; + {error, Reason} -> + ?event({sync, {error, Reason}}), + {error, Reason} + end + end. + +%%% Helper functions + +%% @doc Convert a key to a wallet reference. +secrets_to_keyids(Secrets) when is_list(Secrets) -> + [ hd(secrets_to_keyids(Secret)) || Secret <- Secrets ]; +secrets_to_keyids(Secret) when is_binary(Secret) -> + ?event({secrets_to_keyids, {secret, Secret}}), + KeyID = dev_codec_httpsig_keyid:secret_key_to_committer(Secret), + [ {secret, <<"secret:", KeyID/binary>>, Secret} ]. + +%% @doc Parse the exportable setting for a wallet and return a list of addresses +%% which are allowed to export the wallet. +parse_controllers(default, Opts) -> + case hb_opts:get(wallet_admin, undefined, Opts) of + undefined -> + case hb_opts:get(operator, undefined, Opts) of + undefined -> + [hb_util:human_id(hb_opts:get(priv_wallet, undefined, Opts))]; + Op -> [hb_util:human_id(Op)] + end; + Admin -> [Admin] + end; +parse_controllers(true, Opts) -> parse_controllers(default, Opts); +parse_controllers(false, _Opts) -> []; +parse_controllers(Addresses, _Opts) when is_list(Addresses) -> Addresses; +parse_controllers(Address, _Opts) when is_binary(Address) -> [Address]. + +%% @doc Store a wallet in the appropriate location. +store_wallet(in_memory, KeyID, Details, Opts) -> + % Get existing wallets + CurrentWallets = hb_opts:get(priv_wallet_hosted, #{}, Opts), + % Add new wallet + UpdatedWallets = CurrentWallets#{ KeyID => Details }, + ?event({wallet_store, {updated_wallets, UpdatedWallets}}), + % Update the node's options with the new wallets. + hb_http_server:set_opts(Opts#{ priv_wallet_hosted => UpdatedWallets }), + ok; +store_wallet(non_volatile, KeyID, Details, Opts) -> + % Find the private store of the node. + PrivOpts = priv_store_opts(Opts), + {ok, Msg} = hb_cache:write(#{ KeyID => Details }, PrivOpts), + PrivStore = hb_opts:get(priv_store, undefined, PrivOpts), + % Link the wallet to the store. + ok = hb_store:make_link(PrivStore, Msg, <<"wallet@1.0/", KeyID/binary>>). + +%% @doc Find the wallet by name or address in the node's options. +find_wallet(KeyID, Opts) -> + case find_wallet(in_memory, KeyID, Opts) of + not_found -> find_wallet(non_volatile, KeyID, Opts); + Wallet -> Wallet + end. + +%% @doc Loop over the wallets and find the reference to the wallet. +find_wallet(in_memory, KeyID, Opts) -> + Wallets = hb_opts:get(priv_wallet_hosted, #{}, Opts), + ?event({find_wallet, {keyid, KeyID}, {wallets, Wallets}}), + case hb_maps:find(KeyID, Wallets, Opts) of + {ok, Wallet} -> Wallet; + error -> not_found + end; +find_wallet(non_volatile, KeyID, Opts) -> + PrivOpts = priv_store_opts(Opts), + Store = hb_opts:get(priv_store, undefined, PrivOpts), + Resolved = hb_store:resolve(Store, <<"wallet@1.0/", KeyID/binary>>), + case hb_cache:read(Resolved, PrivOpts) of + {ok, Wallet} -> + WalletDetails = hb_maps:get(KeyID, Wallet, not_found, PrivOpts), + hb_cache:ensure_all_loaded(WalletDetails, PrivOpts); + _ -> not_found + end. + +%% @doc Generate a list of all hosted wallets. +list_wallets(Opts) -> + list_wallets(in_memory, Opts) ++ list_wallets(non_volatile, Opts). +list_wallets(in_memory, Opts) -> + hb_maps:keys(hb_opts:get(priv_wallet_hosted, #{}, Opts)); +list_wallets(non_volatile, Opts) -> + PrivOpts = priv_store_opts(Opts), + hb_cache:ensure_all_loaded(hb_cache:list(<<"wallet@1.0/">>, PrivOpts), PrivOpts). + +%% @doc Generate a new `Opts' message with the `priv_store' as the only `store' +%% option. +priv_store_opts(Opts) -> + hb_private:opts(Opts). + +%% @doc Convert an export response into a list of wallet details. This is +%% necessary because if a received result over HTTP is a list with a +%% commitment attached, it will result in a message with numbered keys but +%% also additional keys for the commitment etc. +export_response_to_list(ExportResponse, Opts) -> + hb_util:numbered_keys_to_list(ExportResponse, Opts). + +%% @doc Convert a list of addresses to a binary string. If the input is a +%% binary already, it is returned as-is. +addresses_to_binary(Addresses) when is_list(Addresses) -> + hb_util:bin(string:join( + lists:map(fun hb_util:list/1, Addresses), + ", " + )); +addresses_to_binary(Address) when is_binary(Address) -> + Address. + +%% @doc Convert a binary string to a list of addresses. If the input is a +%% list already, it is returned as-is. +binary_to_addresses(AddressesBin) when is_binary(AddressesBin) -> + binary:split(AddressesBin, <<",">>, [global]); +binary_to_addresses(Addresses) when is_list(Addresses) -> + Addresses. + + +%%% Tests + +%% @doc Helper function to test wallet generation and verification flow. +test_wallet_generate_and_verify(GeneratePath, ExpectedName, CommitParams) -> + Node = hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new() + }), + % Generate wallet with specified parameters + {ok, GenResponse} = hb_http:get(Node, GeneratePath, #{}), + % Should get wallet name in body, wallet-address, and auth cookie + ?assertMatch(#{<<"body">> := _}, GenResponse), + WalletAddr = maps:get(<<"wallet-address">>, GenResponse), + case ExpectedName of + undefined -> + % For unnamed wallets, just check it's a non-empty binary + ?assert(is_binary(WalletAddr) andalso byte_size(WalletAddr) > 0); + _ -> + % For named wallets, check exact match + ?assertEqual(ExpectedName, WalletAddr) + end, + ?assertMatch(#{ <<"priv">> := #{ <<"cookie">> := _ } }, GenResponse), + #{ <<"priv">> := Priv } = GenResponse, + % Now verify by signing a message + TestMessage = + maps:merge( + #{ + <<"device">> => <<"secret@1.0">>, + <<"path">> => <<"commit">>, + <<"body">> => <<"Test message">>, + <<"priv">> => Priv + }, + CommitParams + ), + ?event({signing_with_cookie, {test_message, TestMessage}}), + {ok, SignedMessage} = hb_http:post(Node, TestMessage, #{}), + % Should return signed message with correct signer + ?assertMatch(#{ <<"body">> := <<"Test message">> }, SignedMessage), + ?assert(hb_message:signers(SignedMessage, #{}) =:= [WalletAddr]). + +client_persist_generate_and_verify_test() -> + test_wallet_generate_and_verify( + <<"/~secret@1.0/generate?persist=client">>, + undefined, + #{} + ). + +cookie_wallet_generate_and_verify_test() -> + test_wallet_generate_and_verify( + <<"/~secret@1.0/generate?persist=in-memory">>, + undefined, + #{} + ). + +non_volatile_persist_generate_and_verify_test() -> + test_wallet_generate_and_verify( + <<"/~secret@1.0/generate?persist=non-volatile">>, + undefined, + #{} + ). + +import_wallet_with_key_test() -> + Node = hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new() + }), + % Create a test wallet key to import (in real scenario from user). + TestWallet = ar_wallet:new(), + % WalletAddress = hb_util:human_id(TestWallet), + WalletKey = hb_escape:encode_quotes(ar_wallet:to_json(TestWallet)), + WalletAddress = hb_util:human_id(ar_wallet:to_address(TestWallet)), + % Import the wallet with a specific name. + ImportUrl = + <<"/~secret@1.0/import?wallet=imported-wallet&persist=in-memory&key=", + WalletKey/binary>>, + {ok, ImportResponse} = hb_http:get(Node, ImportUrl, #{}), + ?event({resp, ImportResponse, WalletAddress}), + Imported = hb_maps:get(<<"imported">>, ImportResponse, #{}), + % Response should come from auth device with wallet name in body. + % Wallet name is the address of the wallet. + ?assertMatch([WalletAddress], Imported), + % Should include cookie setup from auth device. + ?assert(maps:is_key(<<"set-cookie">>, ImportResponse)). + +list_wallets_test() -> + Node = hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new() + }), + % Generate some wallets first. + {ok, Msg1} = + hb_http:get( + Node, + <<"/~secret@1.0/generate?persist=in-memory">>, + #{} + ), + ?event({msg1, Msg1}), + {ok, Msg2} = + hb_http:get( + Node, + <<"/~secret@1.0/generate?persist=in-memory">>, + #{} + ), + WalletAddress1 = maps:get(<<"body">>, Msg1), + WalletAddress2 = maps:get(<<"body">>, Msg2), + ?assertEqual(WalletAddress1, maps:get(<<"body">>, Msg1)), + ?assertEqual(WalletAddress2, maps:get(<<"body">>, Msg2)), + % List all wallets (no authentication required for listing). + {ok, Wallets} = hb_http:get(Node, <<"/~secret@1.0/list">>, #{}), + % Each wallet entry should be a wallet name. + ?assert( + lists:all( + fun(Wallet) -> lists:member(Wallet, hb_maps:values(Wallets)) end, + [WalletAddress1, WalletAddress2] + ) + ). + +commit_with_cookie_wallet_test() -> + Node = hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new() + }), + % Generate a client wallet to get a cookie with full wallet key. + {ok, GenResponse} = + hb_http:get(Node, <<"/~secret@1.0/generate?persist=client">>, #{}), + WalletName = maps:get(<<"wallet-address">>, GenResponse), + #{ <<"priv">> := Priv } = GenResponse, + % Use the cookie to sign a message (no wallet parameter needed). + TestMessage = #{ + <<"device">> => <<"secret@1.0">>, + <<"path">> => <<"commit">>, + <<"body">> => <<"Test data">>, + <<"priv">> => Priv + }, + {ok, SignedMessage} = hb_http:post(Node, TestMessage, #{}), + % Should return the signed message with signature attached. + ?assert(hb_message:signers(SignedMessage, #{}) =:= [WalletName]). + +export_wallet_test() -> + Node = hb_http_server:start_node(#{}), + % Generate a wallet to export. + {ok, GenResponse} = + hb_http:get( + Node, + <<"/~secret@1.0/generate?persist=in-memory">>, + #{} + ), + #{ <<"priv">> := Priv } = GenResponse, + WalletAddress = maps:get(<<"wallet-address">>, GenResponse), + % Export the wallet with authentication. + {ok, ExportResponse} = + hb_http:get( + Node, + #{ + <<"path">> => <<"/~secret@1.0/export/1">>, + <<"priv">> => Priv + }, + #{} + ), + ?event({export_test, {export_response, ExportResponse}}), + % Should return wallet details including key, auth, exportable, persist. + ?assertMatch(#{<<"wallet">> := _, <<"persist">> := <<"in-memory">>}, ExportResponse), + ?assert(maps:is_key(<<"access-control">>, ExportResponse)), + ?assert(maps:is_key(<<"controllers">>, ExportResponse)), + % Should return the correct wallet address in the response. + ?assertEqual(WalletAddress, maps:get(<<"address">>, ExportResponse)), + AccessControl = maps:get(<<"access-control">>, ExportResponse), + ?assertEqual(WalletAddress, maps:get(<<"wallet-address">>, AccessControl)). + +export_non_volatile_wallet_test() -> + Node = hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new() + }), + % Generate a wallet to export. + {ok, GenResponse} = + hb_http:get( + Node, + <<"/~secret@1.0/generate?persist=non-volatile">>, + #{} + ), + #{ <<"priv">> := Priv } = GenResponse, + % Export the wallet with authentication. + {ok, ExportResponse} = + hb_http:get( + Node, + #{ + <<"device">> => <<"secret@1.0">>, + <<"path">> => <<"export/1">>, + <<"priv">> => Priv + }, + #{} + ), + % Should return wallet details including key, auth, exportable, persist. + ?assertMatch( + #{<<"wallet">> := _, <<"persist">> := <<"non-volatile">>}, + ExportResponse + ), + ?assert(maps:is_key(<<"access-control">>, ExportResponse)), + ?assert(maps:is_key(<<"controllers">>, ExportResponse)). + +export_individual_batch_wallets_test() -> + Node = + hb_http_server:start_node( + AdminOpts = + #{ + priv_wallet => AdminWallet = ar_wallet:new() + } + ), + % Generate multiple wallets and collect auth cookies. + {ok, #{ <<"body">> := WalletKeyID1 }} = + hb_http:get( + Node, + <<"/~secret@1.0/generate?persist=in-memory&exportable=", + (hb_util:human_id(AdminWallet))/binary>>, + #{} + ), + {ok, #{ <<"body">> := WalletKeyID2 }} = + hb_http:get( + Node, + <<"/~secret@1.0/generate?persist=in-memory&exportable=", + (hb_util:human_id(AdminWallet))/binary>>, + #{} + ), + % Export all wallets. + {ok, ExportAllResponse} = + hb_http:get( + Node, + (hb_message:commit( + #{ + <<"device">> => <<"secret@1.0">>, + <<"keyids">> => [WalletKeyID1, WalletKeyID2] + }, + AdminOpts + ))#{ <<"path">> => <<"/~secret@1.0/export">> }, + #{} + ), + + % Export single wallet by address. + {ok, ExportWallet1Response} = + hb_http:get( + Node, + (hb_message:commit( + #{ + <<"device">> => <<"secret@1.0">>, + <<"keyids">> => [WalletKeyID1] + }, + AdminOpts + ))#{ <<"path">> => <<"/~secret@1.0/export">> }, + #{} + ), + + ?assert(is_map(ExportAllResponse)), + ?assert(is_map(ExportWallet1Response)), + ExportedAllWallets = + [ + << + "secret:", + (hb_maps:get(<<"committer">>, Wallet, undefined, #{}))/binary + >> + || + Wallet <- export_response_to_list(ExportAllResponse, #{}) + ], + ExportedSingleWallets = + [ + << + "secret:", + (hb_maps:get(<<"committer">>, Wallet, undefined, #{}))/binary + >> + || + Wallet <- export_response_to_list(ExportWallet1Response, #{}) + ], + ?event({exported_wallets, {exported_wallets, ExportedAllWallets}}), + ?assert(length(ExportedAllWallets) >= 2), + ?assert(length(ExportedSingleWallets) == 1), + % Each exported wallet should have the required structure. + lists:foreach( + fun(Addr) -> + ?assert(lists:member(Addr, ExportedAllWallets)) + end, + [WalletKeyID1, WalletKeyID2] + ), + ?assert(lists:member(WalletKeyID1, ExportedSingleWallets)). + + + +export_batch_all_wallets_test() -> + % Remove all previous cached wallets. + hb_store:reset(hb_opts:get(priv_store, no_wallet_store, #{})), + Node = + hb_http_server:start_node( + AdminOpts = + #{ + priv_wallet => AdminWallet = ar_wallet:new() + } + ), + % Generate multiple wallets and collect auth cookies. + {ok, #{ <<"wallet-address">> := WalletAddr1 }} = + hb_http:get( + Node, + <<"/~secret@1.0/generate?persist=in-memory&exportable=", + (hb_util:human_id(AdminWallet))/binary>>, + #{} + ), + {ok, #{ <<"wallet-address">> := WalletAddr2 }} = + hb_http:get( + Node, + <<"/~secret@1.0/generate?persist=in-memory&exportable=", + (hb_util:human_id(AdminWallet))/binary>>, + #{} + ), + % Export all wallets. + {ok, ExportResponse} = + hb_http:get( + Node, + (hb_message:commit( + #{ + <<"device">> => <<"secret@1.0">>, + <<"keyids">> => <<"all">> + }, + AdminOpts + ))#{ <<"path">> => <<"/~secret@1.0/export">> }, + #{} + ), + ?event({export_batch_test, {export_response, ExportResponse}}), + ?assert(is_map(ExportResponse)), + ExportedWallets = + [ + hb_maps:get(<<"address">>, Wallet, undefined, #{}) + || + Wallet <- export_response_to_list(ExportResponse, #{}) + ], + ?event({exported_wallets, {exported_wallets, ExportedWallets}}), + ?assert(length(ExportedWallets) >= 2), + % Each exported wallet should have the required structure. + lists:foreach( + fun(Addr) -> + ?assert(lists:member(Addr, ExportedWallets)) + end, + [WalletAddr1, WalletAddr2] + ). + +sync_wallets_test() -> + % Remove all previous cached wallets. + hb_store:reset(hb_opts:get(priv_store, no_wallet_store, #{})), + Node = + hb_http_server:start_node(#{ + priv_wallet => Node1Wallet = ar_wallet:new() + }), + % Start a second node to sync from. + Node2 = + hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new(), + wallet_admin => hb_util:human_id(Node1Wallet) + }), + % Generate a wallet on the second node. + {ok, GenResponse} = + hb_http:get( + Node2, + <<"/~secret@1.0/generate?persist=in-memory">>, + #{} + ), + WalletKeyID = maps:get(<<"body">>, GenResponse), + % Test sync to the first node from the second. + {ok, _} = + hb_http:get( + Node, + <<"/~secret@1.0/sync?node=", Node2/binary, "&wallets=all">>, + #{} + ), + % Get the wallet list from the first node. + {ok, WalletList} = hb_http:get(Node, <<"/~secret@1.0/list">>, #{}), + ?event({sync_wallets_test, {wallet_list, WalletList}}), + % Should return a map of successfully imported wallets or list of names. + ?assert(lists:member(WalletKeyID, hb_maps:values(WalletList))). + +sync_non_volatile_wallets_test() -> + % Remove all the previous cached wallets. + hb_store:reset(hb_opts:get(priv_store, no_wallet_store, #{})), + Node = + hb_http_server:start_node(#{ + priv_wallet => Node1Wallet = ar_wallet:new() + }), + % Start a second node to sync from. + Node2 = + hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new(), + wallet_admin => hb_util:human_id(Node1Wallet) + }), + % Generate a wallet on the second node. + {ok, GenResponse} = + hb_http:get( + Node2, + <<"/~secret@1.0/generate?persist=non-volatile">>, + #{} + ), + WalletName = maps:get(<<"body">>, GenResponse), + % Test sync to the first node from the second. + {ok, _} = + hb_http:get( + Node, + <<"/~secret@1.0/sync?node=", Node2/binary, "&wallets=all">>, + #{} + ), + % Get the wallet list from the first node. + {ok, WalletList} = hb_http:get(Node, <<"/~secret@1.0/list">>, #{}), + ?event({sync_wallets_test, {wallet_list, WalletList}}), + % Should return a map of successfully imported wallets or list of names. + ?assert(lists:member(WalletName, hb_maps:values(WalletList))). \ No newline at end of file diff --git a/src/dev_simple_pay.erl b/src/dev_simple_pay.erl new file mode 100644 index 000000000..f489de0b0 --- /dev/null +++ b/src/dev_simple_pay.erl @@ -0,0 +1,373 @@ +%%% @doc A simple device that allows the operator to specify a price for a +%%% request and then charge the user for it, on a per route and optionally +%%% per message basis. +%%% +%%% The device's pricing rules are as follows: +%%% +%%% 1. If the request is from the operator, the cost is 0. +%%% 2. If the request matches one of the `router_opts/offered' routes, the +%%% explicit price of the route is used. +%%% 3. Else, the price is calculated by counting the number of messages in the +%%% request, and multiplying by the `simple_pay_price' node option, plus the +%%% price of the apply subrequest if applicable. Subrequests are priced by +%%% recursively calling `estimate/3' upon them. In the case of an `apply@1.0' +%%% subrequest, the two initiating apply messages are not counted towards the +%%% message count price. +%%% +%%% The device's ledger is stored in the node message at `simple_pay_ledger', +%%% and can be topped-up by either the operator, or an external device. The +%%% price is specified in the node message at `simple_pay_price'. +%%% This device acts as both a pricing device and a ledger device, by p4's +%%% definition. +-module(dev_simple_pay). +-export([estimate/3, charge/3, balance/3, topup/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Estimate the cost of the request, using the rules outlined in the +%% moduledoc. +estimate(_Base, EstimateReq, NodeMsg) -> + Req = hb_ao:get(<<"request">>, EstimateReq, NodeMsg#{ hashpath => ignore }), + case is_operator(Req, NodeMsg) of + true -> + ?event(payment, + {estimate_preprocessing, caller_is_operator} + ), + {ok, 0}; + false -> + ?event(payment, {starting_estimate, {req, Req}}), + ReqSequence = hb_singleton:from(Req, NodeMsg), + ?event(payment, + {estimating_cost, + {singleton, Req}, + {request_sequence, ReqSequence} + } + ), + % Get the user's request to match against router registration options + case price_from_routes(Req, NodeMsg) of + no_matches -> + {ok, ApplyPrice, SeqWithoutApply} = apply_price(ReqSequence, NodeMsg), + MessageCountPrice = price_from_count(SeqWithoutApply, NodeMsg), + Price = MessageCountPrice + ApplyPrice, + ?event(payment, + {calculated_generic_route_price, + {price, Price}, + {message_count_price, MessageCountPrice}, + {apply_price, ApplyPrice} + }), + {ok, Price}; + Price -> + ?event(payment, + {calculated_specific_route_price, + {price, Price} + } + ), + {ok, Price} + end + end. + +%% @doc If the request is for the `apply@1.0' device, we should price the +%% inner request in addition to the price of the outer request. +apply_price([{as, Device, Msg} | Rest], NodeMsg) -> + apply_price([Msg#{ <<"device">> => Device } | Rest], NodeMsg); +apply_price( + [Req = #{ <<"device">> := <<"apply@1.0">> }, #{ <<"path">> := Path } | Rest], + NodeMsg + ) -> + UserPath = hb_maps:get(Path, Req, <<"">>, NodeMsg), + UserMessage = + case hb_maps:find(<<"source">>, Req, NodeMsg) of + {ok, Source} -> hb_maps:get(Source, Req, Req, NodeMsg); + error -> Req + end, + UserRequest = + hb_maps:without( + [<<"device">>], + UserMessage#{ <<"path">> => UserPath } + ), + ?event(payment, {estimating_price_of_subrequest, {req, UserRequest}}), + {ok, Price} = estimate(#{}, #{ <<"request">> => UserRequest }, NodeMsg), + ?event(payment, {price_of_apply_subrequest, {price, Price}}), + {ok, Price, Rest}; +apply_price(Seq, _) -> + {ok, 0, Seq}. + +%% @doc Calculate the price of a request based on the offered routes, if +%% applicable. +price_from_routes(UserRequest, NodeMsg) -> + RouterOpts = hb_opts:get(<<"router_opts">>, #{}, NodeMsg), + Routes = hb_maps:get(<<"offered">>, RouterOpts, [], NodeMsg), + MatchRes = + dev_router:match( + #{ <<"routes">> => Routes }, + UserRequest, + NodeMsg + ), + case MatchRes of + {ok, OfferedRoute} -> + Price = hb_maps:get(<<"price">>, OfferedRoute, 0, NodeMsg), + ?event(payment, {price_from_routes, {price, Price}}), + Price; + _ -> + no_matches + end. + +%% @doc Calculate the price of a request based on the number of messages in +%% the request, if the node is configured to do so. +price_from_count(Messages, NodeMsg) -> + Price = + hb_util:int(hb_opts:get(simple_pay_price, 1, NodeMsg)) + * length(Messages), + ?event(payment, {price_from_count, {price, Price}, {count, length(Messages)}}), + Price. + +%% @doc Preprocess a request by checking the ledger and charging the user. We +%% can charge the user at this stage because we know statically what the price +%% will be +charge(_, RawReq, NodeMsg) -> + ?event(payment, {charge, RawReq}), + Req = hb_ao:get(<<"request">>, RawReq, NodeMsg#{ hashpath => ignore }), + case hb_message:signers(Req, NodeMsg) of + [] -> + ?event(payment, {charge, {error, <<"No signers">>}}), + {ok, false}; + [Signer] -> + UserBalance = get_balance(Signer, NodeMsg), + Price = hb_ao:get(<<"quantity">>, RawReq, 0, NodeMsg), + ?event(payment, + {charge, + {user, Signer}, + {balance, UserBalance}, + {price, Price} + }), + {ok, _} = + set_balance( + Signer, + NewBalance = UserBalance - Price, + NodeMsg + ), + case NewBalance >= 0 of + true -> + {ok, true}; + false -> + ?event(payment, + {charge, + {user, Signer}, + {balance, UserBalance}, + {price, Price} + } + ), + {error, #{ + <<"status">> => 402, + <<"body">> => <<"Insufficient funds. " + "User balance before charge: ", + (hb_util:bin(UserBalance))/binary, + ". Price of request: ", + (hb_util:bin(Price))/binary, + ". New balance: ", + (hb_util:bin(NewBalance))/binary, + ".">> + }} + end; + MultipleSigners -> + ?event(payment, {charge, {error_multiple_signers, MultipleSigners}}), + {error, #{ + <<"status">> => 400, + <<"body">> => <<"Multiple signers in charge.">> + }} + end. + +%% @doc Get the balance of a user in the ledger. +balance(_, RawReq, NodeMsg) -> + Target = + case hb_ao:get(<<"request">>, RawReq, NodeMsg#{ hashpath => ignore }) of + not_found -> + case hb_message:signers(RawReq, NodeMsg) of + [] -> hb_ao:get(<<"target">>, RawReq, undefined, NodeMsg); + [Signer] -> Signer + end; + Req -> hd(hb_message:signers(Req, NodeMsg)) + end, + {ok, get_balance(Target, NodeMsg)}. + +%% @doc Adjust a user's balance, normalizing their wallet ID first. +set_balance(Signer, Amount, NodeMsg) -> + NormSigner = hb_util:human_id(Signer), + Ledger = hb_opts:get(simple_pay_ledger, #{}, NodeMsg), + ?event(payment, + {modifying_balance, + {user, NormSigner}, + {amount, Amount}, + {ledger_before, Ledger} + } + ), + hb_http_server:set_opts( + #{}, + NewMsg = NodeMsg#{ + simple_pay_ledger => + hb_ao:set( + Ledger, + NormSigner, + Amount, + NodeMsg + ) + } + ), + {ok, NewMsg}. + +%% @doc Get the balance of a user in the ledger. +get_balance(Signer, NodeMsg) -> + NormSigner = hb_util:human_id(Signer), + Ledger = hb_opts:get(simple_pay_ledger, #{}, NodeMsg), + hb_ao:get(NormSigner, Ledger, 0, NodeMsg). + +%% @doc Top up the user's balance in the ledger. +topup(_, Req, NodeMsg) -> + ?event({topup, {req, Req}, {node_msg, NodeMsg}}), + case is_operator(Req, NodeMsg) of + false -> {error, <<"Unauthorized">>}; + true -> + Amount = hb_ao:get(<<"amount">>, Req, 0, NodeMsg), + Recipient = hb_ao:get(<<"recipient">>, Req, undefined, NodeMsg), + CurrentBalance = get_balance(Recipient, NodeMsg), + ?event(payment, + {topup, + {amount, Amount}, + {recipient, Recipient}, + {balance, CurrentBalance}, + {expected_new_balance, CurrentBalance + Amount} + }), + {ok, NewNodeMsg} = + set_balance( + Recipient, + CurrentBalance + Amount, + NodeMsg + ), + % Briefly wait for the ledger to be updated. + receive after 100 -> ok end, + {ok, get_balance(Recipient, NewNodeMsg)} + end. + +%% @doc Check if the request is from the operator. +is_operator(Req, NodeMsg) -> + is_operator(Req, NodeMsg, hb_opts:get(operator, undefined, NodeMsg)). + +is_operator(Req, NodeMsg, OperatorAddr) when ?IS_ID(OperatorAddr) -> + Signers = hb_message:signers(Req, NodeMsg), + HumanOperatorAddr = hb_util:human_id(OperatorAddr), + lists:any( + fun(Signer) -> + HumanOperatorAddr =:= hb_util:human_id(Signer) + end, + Signers + ); +is_operator(_, _, _) -> + false. + +%%% Tests + +test_opts(Ledger) -> + Wallet = ar_wallet:new(), + Address = hb_util:human_id(ar_wallet:to_address(Wallet)), + ProcessorMsg = + #{ + <<"device">> => <<"p4@1.0">>, + <<"ledger-device">> => <<"simple-pay@1.0">>, + <<"pricing-device">> => <<"simple-pay@1.0">> + }, + { + Address, + Wallet, + #{ + simple_pay_ledger => Ledger, + simple_pay_price => 10, + operator => Address, + on => #{ + <<"request">> => ProcessorMsg, + <<"response">> => ProcessorMsg + } + } + }. + +get_balance_and_top_up_test() -> + ClientWallet = ar_wallet:new(), + ClientAddress = hb_util:human_id(ar_wallet:to_address(ClientWallet)), + {HostAddress, HostWallet, Opts} = test_opts(#{ ClientAddress => 100 }), + Node = hb_http_server:start_node(Opts), + ?event({host_address, HostAddress}), + ?event({client_address, ClientAddress}), + {ok, Res} = + hb_http:get( + Node, + Req = hb_message:commit( + #{<<"path">> => <<"/~simple-pay@1.0/balance">>}, + Opts#{ priv_wallet => ClientWallet } + ), + Opts + ), + ?event({req_signers, hb_message:signers(Req, Opts)}), + % Balance is given during the request, before the charge is made, so we + % should expect to see the original balance. + ?assertEqual(100, Res), + % The balance should now be 80, as the check will have charged us 20. + {ok, NewBalance} = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/~simple-pay@1.0/topup">>, + <<"amount">> => 100, + <<"recipient">> => ClientAddress + }, + Opts#{ priv_wallet => HostWallet } + ), + Opts + ), + % The balance should now be 180, as the topup will have been added and will + % not have generated a charge in itself. The top-up did not generate a charge + % because it is the operator that performed it, and not a user. + ?assertEqual(180, NewBalance), + {ok, Res2} = + hb_http:get( + Node, + hb_message:commit( + #{<<"path">> => <<"/~p4@1.0/balance">>}, + Opts#{ priv_wallet => ClientWallet } + ), + Opts + ), + ?assertEqual(180, Res2). + +apply_price_test() -> + ClientWallet = ar_wallet:new(), + ClientAddress = hb_util:human_id(ar_wallet:to_address(ClientWallet)), + ClientOpts = #{ priv_wallet => ClientWallet }, + {HostAddress, _HostWallet, Opts} = + test_opts(#{ ClientAddress => 100 }), + Node = hb_http_server:start_node(Opts), + ?event({host_address, HostAddress}), + ?event({client_address, ClientAddress}), + % The balance should now be 80, as the check will have charged us 20. + {ok, _} = + hb_http:post( + Node, + hb_message:commit( + #{ + <<"path">> => <<"/~apply@1.0/user-path">>, + <<"user-path">> => <<"/~scheduler@1.0/status/keys/1">>, + <<"user-message">> => #{ <<"a">> => 1 } + }, + ClientOpts + ), + ClientOpts + ), + {ok, Res2} = + hb_http:get( + Node, + hb_message:commit( + #{<<"path">> => <<"/~p4@1.0/balance">>}, + Opts#{ priv_wallet => ClientWallet } + ), + Opts + ), + ?assertEqual(60, Res2). \ No newline at end of file diff --git a/src/dev_snp.erl b/src/dev_snp.erl new file mode 100644 index 000000000..aea38c63f --- /dev/null +++ b/src/dev_snp.erl @@ -0,0 +1,914 @@ +%%% @doc This device provides an interface for validating and generating AMD SEV-SNP +%%% commitment reports. +%%% +%%% AMD SEV-SNP (Secure Encrypted Virtualization - Secure Nested Paging) is a +%%% hardware-based security technology that provides confidential computing +%%% capabilities. This module handles the cryptographic validation of attestation +%%% reports and the generation of commitment reports for trusted execution environments. +%%% +%%% The device supports two main operations: +%%% 1. Verification of remote node attestation reports with comprehensive validation +%%% 2. Generation of local attestation reports for proving node identity and software integrity +-module(dev_snp). +-export([generate/3, verify/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% Configuration constants +-define(COMMITTED_PARAMETERS, [vcpus, vcpu_type, vmm_type, guest_features, + firmware, kernel, initrd, append]). + +%% SNP-specific constants +-define(DEBUG_FLAG_BIT, 19). +-define(REPORT_DATA_VERSION, 1). + +%% Test configuration constants +-define(TEST_VCPUS_COUNT, 32). +-define(TEST_VCPU_TYPE, 5). +-define(TEST_VMM_TYPE, 1). +-define(TEST_GUEST_FEATURES, 1). +-define(TEST_FIRMWARE_HASH, <<"b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510">>). +-define(TEST_KERNEL_HASH, <<"69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576">>). +-define(TEST_INITRD_HASH, <<"544045560322dbcd2c454bdc50f35edf0147829ec440e6cb487b4a1503f923c1">>). +-define(TEST_APPEND_HASH, <<"95a34faced5e487991f9cc2253a41cbd26b708bf00328f98dddbbf6b3ea2892e">>). + +%% @doc Verify an AMD SEV-SNP commitment report message. +%% +%% This function validates the identity of a remote node, its ephemeral private +%% address, and the integrity of the hardware-backed attestation report. +%% The verification process performs the following checks: +%% 1. Verify the address and the node message ID are the same as the ones +%% used to generate the nonce. +%% 2. Verify the address that signed the message is the same as the one used +%% to generate the nonce. +%% 3. Verify that the debug flag is disabled. +%% 4. Verify that the firmware, kernel, and OS (VMSAs) hashes, part of the +%% measurement, are trusted. +%% 5. Verify the measurement is valid. +%% 6. Verify the report's certificate chain to hardware root of trust. +%% +%% Required configuration in NodeOpts map: +%% - snp_trusted: List of trusted software configurations +%% - snp_enforced_keys: Keys to enforce during validation (optional) +%% +%% @param M1 The previous message in the verification chain +%% @param M2 The message containing the SNP commitment report +%% @param NodeOpts A map of configuration options for verification +%% @returns `{ok, Binary}' with "true" on successful verification, or +%% `{error, Reason}' on failure with specific error details +-spec verify(M1 :: term(), M2 :: term(), NodeOpts :: map()) -> + {ok, binary()} | {error, term()}. +verify(M1, M2, NodeOpts) -> + ?event(snp_verify, verify_called), + maybe + {ok, {Msg, Address, NodeMsgID, ReportJSON, MsgWithJSONReport}} + ?= extract_and_normalize_message(M2, NodeOpts), + % Perform all validation steps + {ok, NonceResult} ?= verify_nonce(Address, NodeMsgID, Msg, NodeOpts), + {ok, SigResult} ?= + verify_signature_and_address( + MsgWithJSONReport, + Address, + NodeOpts + ), + {ok, DebugResult} ?= verify_debug_disabled(Msg), + {ok, TrustedResult} ?= verify_trusted_software(M1, Msg, NodeOpts), + {ok, MeasurementResult} ?= verify_measurement(Msg, ReportJSON, NodeOpts), + {ok, ReportResult} ?= verify_report_integrity(ReportJSON), + Valid = lists:all( + fun(Bool) -> Bool end, + [ + NonceResult, + SigResult, + DebugResult, + TrustedResult, + MeasurementResult, + ReportResult + ] + ), + ?event({final_validation_result, Valid}), + {ok, hb_util:bin(Valid)} + else + {error, Reason} -> {error, Reason} + end. + +%% @doc Generate an AMD SEV-SNP commitment report and emit it as a message. +%% +%% This function creates a hardware-backed attestation report containing all +%% necessary data to validate the node's identity and software configuration. +%% The generation process performs the following operations: +%% 1. Loads and validates the provided configuration options +%% 2. Retrieves or creates a cryptographic wallet for node identity +%% 3. Generates a unique nonce using the node's address and message ID +%% 4. Extracts trusted software configuration from local options +%% 5. Generates the hardware attestation report using the NIF interface +%% 6. Packages the report with all verification data into a message +%% +%% Required configuration in Opts map: +%% - priv_wallet: Node's cryptographic wallet (created if not provided) +%% - snp_trusted: List of trusted software configurations (represents the +%% configuration of the local node generating the report) +%% +%% @param _M1 Ignored parameter +%% @param _M2 Ignored parameter +%% @param Opts A map of configuration options for report generation +%% @returns `{ok, Map}' on success with the complete report message, or +%% `{error, Reason}' on failure with error details +-spec generate(M1 :: term(), M2 :: term(), Opts :: map()) -> + {ok, map()} | {error, term()}. +generate(_M1, _M2, Opts) -> + maybe + LoadedOpts = hb_cache:ensure_all_loaded(Opts, Opts), + ?event({generate_opts, {explicit, LoadedOpts}}), + % Validate wallet availability + {ok, ValidWallet} ?= + case hb_opts:get(priv_wallet, no_viable_wallet, LoadedOpts) of + no_viable_wallet -> {error, no_wallet_available}; + Wallet -> {ok, Wallet} + end, + % Generate address and node message components + Address = hb_util:human_id(ar_wallet:to_address(ValidWallet)), + NodeMsg = hb_private:reset(LoadedOpts), + {ok, PublicNodeMsgID} ?= dev_message:id( + NodeMsg, + #{ <<"committers">> => <<"none">> }, + LoadedOpts + ), + RawPublicNodeMsgID = hb_util:native_id(PublicNodeMsgID), + ?event({snp_node_msg, NodeMsg}), + % Generate the commitment report components + ?event({snp_address, byte_size(Address)}), + ReportData = generate_nonce(Address, RawPublicNodeMsgID), + ?event({snp_report_data, byte_size(ReportData)}), + % Extract local hashes + {ok, ValidLocalHashes} ?= + case hb_opts:get(snp_trusted, [#{}], LoadedOpts) of + [] -> {error, no_trusted_configs}; + [FirstConfig | _] -> {ok, FirstConfig}; + _ -> {error, invalid_trusted_configs_format} + end, + ?event(snp_local_hashes, {explicit, ValidLocalHashes}), + % Generate the hardware attestation report + {ok, ReportJSON} ?= case get(mock_snp_nif_enabled) of + true -> + % Return mocked response for testing + MockResponse = get(mock_snp_nif_response), + {ok, MockResponse}; + _ -> + % Call actual NIF function + dev_snp_nif:generate_attestation_report( + ReportData, + ?REPORT_DATA_VERSION + ) + end, + ?event({snp_report_json, ReportJSON}), + ?event({snp_report_generated, {nonce, ReportData}, {report, ReportJSON}}), + % Package the complete report message + ReportMsg = #{ + <<"local-hashes">> => ValidLocalHashes, + <<"nonce">> => hb_util:encode(ReportData), + <<"address">> => Address, + <<"node-message">> => NodeMsg, + <<"report">> => ReportJSON + }, + ?event({snp_report_msg, ReportMsg}), + {ok, ReportMsg} + else + {error, Reason} -> {error, Reason}; + Error -> {error, Error} + end. + +%% @doc Extract and normalize the SNP commitment message from the input. +%% +%% This function processes the raw message and extracts all necessary components +%% for verification: +%% 1. Searches for a `body' key in the message, using it as the report source +%% 2. Applies message commitment and signing filters +%% 3. Extracts and decodes the JSON report +%% 4. Normalizes the message structure by merging report data +%% 5. Extracts the node address and message ID +%% +%% @param M2 The input message containing the SNP report +%% @param NodeOpts A map of configuration options +%% @returns `{ok, {Msg, Address, NodeMsgID, ReportJSON, MsgWithJSONReport}}' +%% on success with all extracted components, or `{error, Reason}' on failure +-spec extract_and_normalize_message(M2 :: term(), NodeOpts :: map()) -> + {ok, {map(), binary(), binary(), binary(), map()}} | {error, term()}. +extract_and_normalize_message(M2, NodeOpts) -> + maybe + % Search for a `body' key in the message, and if found use it as the source + % of the report. If not found, use the message itself as the source. + ?event({node_opts, {explicit, NodeOpts}}), + RawMsg = hb_ao:get(<<"body">>, M2, M2, NodeOpts#{ hashpath => ignore }), + ?event({msg, {explicit, RawMsg}}), + MsgWithJSONReport = + hb_util:ok( + hb_message:with_only_committed( + hb_message:with_only_committers( + RawMsg, + hb_message:signers( + RawMsg, + NodeOpts + ), + NodeOpts + ), + NodeOpts + ) + ), + ?event({msg_with_json_report, {explicit, MsgWithJSONReport}}), + % Normalize the request message + ReportJSON = hb_ao:get(<<"report">>, MsgWithJSONReport, NodeOpts), + Report = hb_json:decode(ReportJSON), + Msg = + maps:merge( + maps:without([<<"report">>], MsgWithJSONReport), + Report + ), + + % Extract address and node message ID + Address = hb_ao:get(<<"address">>, Msg, NodeOpts), + ?event({snp_address, Address}), + {ok, NodeMsgID} ?= extract_node_message_id(Msg, NodeOpts), + ?event({snp_node_msg_id, NodeMsgID}), + {ok, {Msg, Address, NodeMsgID, ReportJSON, MsgWithJSONReport}} + else + {error, Reason} -> {error, Reason}; + Error -> {error, Error} + end. + + +%% @doc Extract the node message ID from the SNP message. +%% +%% This function handles the extraction of the node message ID, which can be +%% provided either directly as a field or embedded within a node message that +%% needs to be processed to generate the ID. +%% +%% @param Msg The normalized SNP message +%% @param NodeOpts A map of configuration options +%% @returns `{ok, NodeMsgID}' on success with the extracted ID, or +%% `{error, missing_node_msg_id}' if no ID can be found +-spec extract_node_message_id(Msg :: map(), NodeOpts :: map()) -> + {ok, binary()} | {error, missing_node_msg_id}. +extract_node_message_id(Msg, NodeOpts) -> + case {hb_ao:get(<<"node-message">>, Msg, NodeOpts#{ hashpath => ignore }), + hb_ao:get(<<"node-message-id">>, Msg, NodeOpts)} of + {undefined, undefined} -> + {error, missing_node_msg_id}; + {undefined, ID} -> + {ok, ID}; + {NodeMsg, _} -> + dev_message:id(NodeMsg, #{}, NodeOpts) + end. + +%% @doc Verify that the nonce in the report matches the expected value. +%% +%% This function validates that the nonce in the SNP report was generated +%% using the correct address and node message ID, ensuring the report +%% corresponds to the expected request. +%% +%% @param Address The node's address used in nonce generation +%% @param NodeMsgID The node message ID used in nonce generation +%% @param Msg The normalized SNP message containing the nonce +%% @param NodeOpts A map of configuration options +%% @returns `{ok, true}' if the nonce matches, or `{error, nonce_mismatch}' on failure +-spec verify_nonce(Address :: binary(), NodeMsgID :: binary(), + Msg :: map(), NodeOpts :: map()) -> {ok, true} | {error, nonce_mismatch}. +verify_nonce(Address, NodeMsgID, Msg, NodeOpts) -> + Nonce = hb_util:decode(hb_ao:get(<<"nonce">>, Msg, NodeOpts)), + ?event({snp_nonce, Nonce}), + NonceMatches = report_data_matches(Address, NodeMsgID, Nonce), + ?event({nonce_matches, NonceMatches}), + case NonceMatches of + true -> {ok, true}; + false -> {error, nonce_mismatch} + end. + +%% @doc Verify that the message signature and signing address are valid. +%% +%% This function validates that: +%% 1. The message signature is cryptographically valid +%% 2. The address that signed the message matches the address in the report +%% +%% @param MsgWithJSONReport The message containing the JSON report and signatures +%% @param Address The expected signing address from the report +%% @param NodeOpts A map of configuration options +%% @returns `{ok, true}' if both signature and address are valid, or +%% `{error, signature_or_address_invalid}' on failure +-spec verify_signature_and_address(MsgWithJSONReport :: map(), + Address :: binary(), NodeOpts :: map()) -> + {ok, true} | {error, signature_or_address_invalid}. +verify_signature_and_address(MsgWithJSONReport, Address, NodeOpts) -> + Signers = hb_message:signers(MsgWithJSONReport, NodeOpts), + ?event({snp_signers, {explicit, Signers}}), + SigIsValid = hb_message:verify(MsgWithJSONReport, Signers), + ?event({snp_sig_is_valid, SigIsValid}), + AddressIsValid = lists:member(Address, Signers), + ?event({address_is_valid, AddressIsValid, {signer, Signers}, {address, Address}}), + case SigIsValid andalso AddressIsValid of + true -> {ok, true}; + false -> {error, signature_or_address_invalid} + end. + +%% @doc Verify that the debug flag is disabled in the SNP policy. +%% +%% This function checks the SNP policy to ensure that debug mode is disabled, +%% which is required for production environments to maintain security guarantees. +%% +%% @param Msg The normalized SNP message containing the policy +%% @returns `{ok, true}' if debug is disabled, or `{error, debug_enabled}' if enabled +-spec verify_debug_disabled(Msg :: map()) -> {ok, true} | {error, debug_enabled}. +verify_debug_disabled(Msg) -> + DebugDisabled = not is_debug(Msg), + ?event({debug_disabled, DebugDisabled}), + case DebugDisabled of + true -> {ok, true}; + false -> {error, debug_enabled} + end. + +%% @doc Verify that the software configuration is trusted. +%% +%% This function validates that the firmware, kernel, and other system +%% components match approved configurations by delegating to the +%% software trust validation system. +%% +%% @param M1 The previous message in the verification chain +%% @param Msg The normalized SNP message containing software hashes +%% @param NodeOpts A map of configuration options including trusted software list +%% @returns `{ok, true}' if the software is trusted, or `{error, untrusted_software}' +%% on failure +-spec verify_trusted_software(M1 :: term(), Msg :: map(), NodeOpts :: map()) -> + {ok, true} | {error, untrusted_software}. +verify_trusted_software(M1, Msg, NodeOpts) -> + {ok, IsTrustedSoftware} = execute_is_trusted(M1, Msg, NodeOpts), + ?event({trusted_software, IsTrustedSoftware}), + case IsTrustedSoftware of + true -> {ok, true}; + false -> {error, untrusted_software} + end. + +%% @doc Verify that the measurement in the SNP report is valid. +%% +%% This function validates the SNP measurement by: +%% 1. Extracting committed parameters from the message +%% 2. Computing the expected launch digest using those parameters +%% 3. Comparing the computed digest with the measurement in the report +%% +%% @param Msg The normalized SNP message containing local hashes +%% @param ReportJSON The raw JSON report containing the measurement +%% @param NodeOpts A map of configuration options +%% @returns `{ok, true}' if the measurement is valid, or +%% `{error, measurement_invalid}' on failure +-spec verify_measurement(Msg :: map(), ReportJSON :: binary(), + NodeOpts :: map()) -> {ok, true} | {error, measurement_invalid}. +verify_measurement(Msg, ReportJSON, NodeOpts) -> + Args = extract_measurement_args(Msg, NodeOpts), + ?event({args, { explicit, Args}}), + {ok, Expected} = dev_snp_nif:compute_launch_digest(Args), + ExpectedBin = list_to_binary(Expected), + ?event({expected_measurement, {explicit, Expected}}), + Measurement = hb_ao:get(<<"measurement">>, Msg, NodeOpts), + ?event({measurement, {explicit,Measurement}}), + {Status, MeasurementIsValid} = + dev_snp_nif:verify_measurement( + ReportJSON, + ExpectedBin + ), + ?event({status, Status}), + ?event({measurement_is_valid, MeasurementIsValid}), + case MeasurementIsValid of + true -> {ok, true}; + false -> {error, measurement_invalid} + end. + +%% @doc Extract measurement arguments from the SNP message. +%% +%% This function extracts and formats the committed parameters needed for +%% measurement computation from the local hashes in the message. +%% +%% @param Msg The normalized SNP message containing local hashes +%% @param NodeOpts A map of configuration options +%% @returns A map of measurement arguments with atom keys +-spec extract_measurement_args(Msg :: map(), NodeOpts :: map()) -> map(). +extract_measurement_args(Msg, NodeOpts) -> + maps:from_list( + lists:map( + fun({Key, Val}) -> {binary_to_existing_atom(Key), Val} end, + maps:to_list( + maps:with( + lists:map(fun atom_to_binary/1, ?COMMITTED_PARAMETERS), + hb_cache:ensure_all_loaded( + hb_ao:get(<<"local-hashes">>, Msg, NodeOpts), + NodeOpts + ) + ) + ) + ) + ). + +%% @doc Verify the integrity of the SNP report's digital signature. +%% +%% This function validates the cryptographic signature of the SNP report +%% against the hardware root of trust to ensure the report has not been +%% tampered with and originates from genuine AMD SEV-SNP hardware. +%% +%% @param ReportJSON The raw JSON report to verify +%% @returns `{ok, true}' if the report signature is valid, or +%% `{error, report_signature_invalid}' on failure +-spec verify_report_integrity(ReportJSON :: binary()) -> + {ok, true} | {error, report_signature_invalid}. +verify_report_integrity(ReportJSON) -> + {ok, ReportIsValid} = dev_snp_nif:verify_signature(ReportJSON), + ?event({report_is_valid, ReportIsValid}), + case ReportIsValid of + true -> {ok, true}; + false -> {error, report_signature_invalid} + end. + +%% @doc Check if the node's debug policy is enabled. +%% +%% This function examines the SNP policy field to determine if debug mode +%% is enabled by checking the debug flag bit in the policy bitmask. +%% +%% @param Report The SNP report containing the policy field +%% @returns `true' if debug mode is enabled, `false' otherwise +-spec is_debug(Report :: map()) -> boolean(). +is_debug(Report) -> + (hb_ao:get(<<"policy">>, Report, #{}) band (1 bsl ?DEBUG_FLAG_BIT)) =/= 0. + + +%% @doc Validate that all software hashes match trusted configurations. +%% +%% This function ensures that the firmware, kernel, and other system components +%% in the SNP report match approved configurations. The validation process: +%% 1. Extracts local hashes from the message +%% 2. Filters hashes to only include enforced keys +%% 3. Compares filtered hashes against trusted software configurations +%% 4. Returns true only if the configuration matches a trusted entry +%% +%% Configuration options in NodeOpts map: +%% - snp_trusted: List of maps containing trusted software configurations +%% - snp_enforced_keys: Keys to enforce during validation (defaults to all +%% committed parameters) +%% +%% @param _M1 Ignored parameter +%% @param Msg The SNP message containing local software hashes +%% @param NodeOpts A map of configuration options including trusted software +%% @returns `{ok, true}' if software is trusted, `{ok, false}' otherwise +-spec execute_is_trusted(M1 :: term(), Msg :: map(), NodeOpts :: map()) -> + {ok, boolean()}. +execute_is_trusted(_M1, Msg, NodeOpts) -> + FilteredLocalHashes = get_filtered_local_hashes(Msg, NodeOpts), + TrustedSoftware = hb_opts:get(snp_trusted, [#{}], NodeOpts), + ?event({trusted_software, {explicit, TrustedSoftware}}), + IsTrusted = + is_software_trusted( + FilteredLocalHashes, + TrustedSoftware, + NodeOpts + ), + ?event({is_all_software_trusted, IsTrusted}), + {ok, IsTrusted}. + +%% @doc Extract local hashes filtered to only include enforced keys. +%% +%% This function retrieves the local software hashes from the message and +%% filters them to only include the keys that are configured for enforcement. +%% +%% @param Msg The SNP message containing local hashes +%% @param NodeOpts A map of configuration options +%% @returns A map of filtered local hashes with only enforced keys +-spec get_filtered_local_hashes(Msg :: map(), NodeOpts :: map()) -> map(). +get_filtered_local_hashes(Msg, NodeOpts) -> + LocalHashes = hb_ao:get(<<"local-hashes">>, Msg, NodeOpts), + EnforcedKeys = get_enforced_keys(NodeOpts), + ?event({enforced_keys, {explicit, EnforcedKeys}}), + FilteredLocalHashes = hb_cache:ensure_all_loaded( + maps:with(EnforcedKeys, LocalHashes), + NodeOpts + ), + ?event({filtered_local_hashes, {explicit, FilteredLocalHashes}}), + FilteredLocalHashes. + +%% @doc Get the list of enforced keys for software validation. +%% +%% This function retrieves the configuration specifying which software +%% component keys should be enforced during trust validation. +%% +%% @param NodeOpts A map of configuration options +%% @returns A list of binary keys that should be enforced +-spec get_enforced_keys(NodeOpts :: map()) -> [binary()]. +get_enforced_keys(NodeOpts) -> + lists:map( + fun atom_to_binary/1, + hb_opts:get(snp_enforced_keys, ?COMMITTED_PARAMETERS, NodeOpts) + ). + +%% @doc Check if filtered local hashes match any trusted configurations. +%% +%% This function compares the filtered local hashes against a list of +%% trusted software configurations, returning true if any configuration +%% matches exactly. It handles three cases: +%% 1. Empty list of trusted configurations (returns false) +%% 2. Valid list of trusted configurations (performs matching) +%% 3. Invalid trusted software configuration (returns false) +%% +%% @param FilteredLocalHashes The software hashes to validate +%% @param TrustedSoftware List of trusted software configurations or invalid input +%% @param NodeOpts Configuration options for matching +%% @returns `true' if hashes match a trusted configuration, `false' otherwise +-spec is_software_trusted(map(), [] | [map()] | term(), map()) -> boolean(). +is_software_trusted(_FilteredLocalHashes, [], _NodeOpts) -> + false; +is_software_trusted(FilteredLocalHashes, TrustedSoftware, NodeOpts) + when is_list(TrustedSoftware) -> + lists:any( + fun(TrustedMap) -> + Match = + hb_message:match( + FilteredLocalHashes, + TrustedMap, + primary, + NodeOpts + ), + ?event({match, {explicit, Match}}), + is_map(TrustedMap) andalso Match == true + end, + TrustedSoftware + ); +is_software_trusted(_FilteredLocalHashes, _TrustedSoftware, _NodeOpts) -> + false. + +%% @doc Validate that the report data matches the expected nonce. +%% +%% This function ensures that the nonce in the SNP report was generated +%% using the same address and node message ID that are expected for this +%% verification request. +%% +%% @param Address The node's address used in nonce generation +%% @param NodeMsgID The node message ID used in nonce generation +%% @param ReportData The actual nonce data from the SNP report +%% @returns `true' if the report data matches the expected nonce, `false' otherwise +-spec report_data_matches(Address :: binary(), NodeMsgID :: binary(), + ReportData :: binary()) -> boolean(). +report_data_matches(Address, NodeMsgID, ReportData) -> + ?event({generated_nonce, {explicit, generate_nonce(Address, NodeMsgID)}}), + ?event({expected_nonce, {explicit, ReportData}}), + generate_nonce(Address, NodeMsgID) == ReportData. + +%% @doc Generate the nonce to use in the SNP commitment report. +%% +%% This function creates a unique nonce by concatenating the node's native +%% address and message ID. This nonce is embedded in the hardware attestation +%% report to bind it to a specific verification request. +%% +%% @param RawAddress The node's raw address identifier +%% @param RawNodeMsgID The raw node message identifier +%% @returns A binary nonce formed by concatenating the native address and message ID +-spec generate_nonce(RawAddress :: binary(), RawNodeMsgID :: binary()) -> binary(). +generate_nonce(RawAddress, RawNodeMsgID) -> + Address = hb_util:native_id(RawAddress), + NodeMsgID = hb_util:native_id(RawNodeMsgID), + << Address/binary, NodeMsgID/binary >>. + +%% Test helper functions and data +get_test_hashes() -> + #{ + <<"vcpus">> => ?TEST_VCPUS_COUNT, + <<"vcpu_type">> => ?TEST_VCPU_TYPE, + <<"vmm_type">> => ?TEST_VMM_TYPE, + <<"guest_features">> => ?TEST_GUEST_FEATURES, + <<"firmware">> => ?TEST_FIRMWARE_HASH, + <<"kernel">> => ?TEST_KERNEL_HASH, + <<"initrd">> => ?TEST_INITRD_HASH, + <<"append">> => ?TEST_APPEND_HASH + }. + +%% Verification test helpers +setup_test_nodes() -> + ProxyWallet = hb:wallet(<<"test/admissible-report-wallet.json">>), + ProxyOpts = #{ + store => hb_opts:get(store), + priv_wallet => ProxyWallet + }, + _ReportNode = hb_http_server:start_node(ProxyOpts), + VerifyingNode = hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new(), + store => hb_opts:get(store), + snp_trusted => [ + #{ + <<"vcpus">> => ?TEST_VCPUS_COUNT, + <<"vcpu_type">> => ?TEST_VCPU_TYPE, + <<"vmm_type">> => ?TEST_VMM_TYPE, + <<"guest_features">> => ?TEST_GUEST_FEATURES, + <<"firmware">> => ?TEST_FIRMWARE_HASH, + <<"kernel">> => ?TEST_KERNEL_HASH, + <<"initrd">> => ?TEST_INITRD_HASH, + <<"append">> => ?TEST_APPEND_HASH + } + ], + snp_enforced_keys => [ + vcpu_type, vmm_type, guest_features, + firmware, kernel, initrd, append + ] + }), + {ProxyOpts, VerifyingNode}. + + +%% @doc Load test SNP report data from file. +%% +%% This function loads a sample SNP attestation report from a test file. +%% The test will fail if the file doesn't exist, ensuring predictable test data. +%% +%% @returns Binary containing test SNP report JSON data +%% @throws {error, {file_not_found, Filename}} if test file doesn't exist +-spec load_test_report_data() -> binary(). +load_test_report_data() -> + TestFile = <<"test/admissible-report.json">>, + case file:read_file(TestFile) of + {ok, Data} -> + Data; + {error, enoent} -> + throw({error, {file_not_found, TestFile}}); + {error, Reason} -> + throw({error, {file_read_error, TestFile, Reason}}) + end. + + +%% Individual test cases +execute_is_trusted_exact_match_should_fail_test() -> + % Test case: Exact match with trusted software should fail when vcpus differ + Msg = #{ + <<"local-hashes">> => (get_test_hashes())#{ + <<"vcpus">> => 16 + } + }, + NodeOpts = #{ + snp_trusted => [get_test_hashes()], + snp_enforced_keys => [ + vcpus, vcpu_type, vmm_type, guest_features, + firmware, kernel, initrd, append + ] + }, + {ok, Result} = execute_is_trusted(#{}, Msg, NodeOpts), + ?assertEqual(false, Result). + +execute_is_trusted_subset_match_should_pass_test() -> + % Test case: Match with subset of keys in trusted software should pass + Msg = #{ + <<"local-hashes">> => (get_test_hashes())#{ + <<"vcpus">> => 16 + } + }, + NodeOpts = #{ + snp_trusted => [get_test_hashes()], + snp_enforced_keys => [ + vcpu_type, vmm_type, guest_features, + firmware, kernel, initrd, append + ] + }, + {ok, Result} = execute_is_trusted(#{}, Msg, NodeOpts), + ?assertEqual(true, Result). + +verify_test() -> + % Note: If this test fails, it may be because the unsigned ID of the node + % message in `test/admissible-report.eterm` has changed. If the format ever + % changes, this value will need to be updated. Recalculate the unsigned ID + % of the `Request/node-message' field, decode `Request/address', concatenate + % the two, and encode. The result will be the new `Request/nonce' value. + {ProxyOpts, VerifyingNode} = setup_test_nodes(), + {ok, [Request]} = file:consult(<<"test/admissible-report.eterm">>), + {ok, Result} = hb_http:post( + VerifyingNode, + <<"/~snp@1.0/verify">>, + hb_message:commit(Request, ProxyOpts), + ProxyOpts + ), + ?event({verify_test_result, Result}), + ?assertEqual(true, hb_util:atom(Result)). + + +%% @doc Test successful report generation with valid configuration. +generate_success_test() -> + % Set up test configuration + TestWallet = ar_wallet:new(), + TestOpts = #{ + priv_wallet => TestWallet, + snp_trusted => [#{ + <<"vcpus">> => ?TEST_VCPUS_COUNT, + <<"vcpu_type">> => ?TEST_VCPU_TYPE, + <<"firmware">> => ?TEST_FIRMWARE_HASH, + <<"kernel">> => ?TEST_KERNEL_HASH + }] + }, + % Load test report data from file + TestReportJSON = load_test_report_data(), + % Mock the NIF function to return test data + ok = mock_snp_nif(TestReportJSON), + try + % Call generate function + {ok, Result} = generate(#{}, #{}, TestOpts), + % Verify the result structure + ?assert(is_map(Result)), + ?assert(maps:is_key(<<"local-hashes">>, Result)), + ?assert(maps:is_key(<<"nonce">>, Result)), + ?assert(maps:is_key(<<"address">>, Result)), + ?assert(maps:is_key(<<"node-message">>, Result)), + ?assert(maps:is_key(<<"report">>, Result)), + % Verify the report content + ?assertEqual(TestReportJSON, maps:get(<<"report">>, Result)), + % Verify local hashes match the first trusted config + ExpectedHashes = maps:get(<<"local-hashes">>, Result), + ?assertEqual(?TEST_VCPUS_COUNT, maps:get(<<"vcpus">>, ExpectedHashes)), + ?assertEqual(?TEST_VCPU_TYPE, maps:get(<<"vcpu_type">>, ExpectedHashes)), + % Verify nonce is properly encoded + Nonce = maps:get(<<"nonce">>, Result), + ?assert(is_binary(Nonce)), + ?assert(byte_size(Nonce) > 0), + % Verify address is present and properly formatted + Address = maps:get(<<"address">>, Result), + ?assert(is_binary(Address)), + ?assert(byte_size(Address) > 0) + after + % Clean up mock + unmock_snp_nif() + end. + +%% @doc Test error handling when wallet is missing. +generate_missing_wallet_test() -> + TestOpts = #{ + % No priv_wallet provided + snp_trusted => [#{ <<"firmware">> => ?TEST_FIRMWARE_HASH }] + }, + % Mock the NIF function (shouldn't be called) + ok = mock_snp_nif(<<"dummy_report">>), + try + % Call generate function - should fail + Result = generate(#{}, #{}, TestOpts), + ?assertMatch({error, no_wallet_available}, Result) + after + unmock_snp_nif() + end. + +%% @doc Test error handling when trusted configurations are missing. +generate_missing_trusted_configs_test() -> + TestWallet = ar_wallet:new(), + TestOpts = #{ + priv_wallet => TestWallet, + snp_trusted => [] % Empty trusted configs + }, + + % Mock the NIF function (shouldn't be called) + ok = mock_snp_nif(<<"dummy_report">>), + + try + % Call generate function - should fail + Result = generate(#{}, #{}, TestOpts), + ?assertMatch({error, no_trusted_configs}, Result) + after + unmock_snp_nif() + end. + +%% @doc Test successful round-trip: generate then verify with same configuration. +verify_mock_generate_success_test_() -> + { timeout, 30, fun verify_mock_generate_success/0 }. +verify_mock_generate_success() -> + % Set up test configuration + TestWallet = ar_wallet:new(), + TestTrustedConfig = #{ + <<"vcpus">> => 32, + <<"vcpu_type">> => ?TEST_VCPU_TYPE, + <<"vmm_type">> => ?TEST_VMM_TYPE, + <<"guest_features">> => ?TEST_GUEST_FEATURES, + <<"firmware">> => ?TEST_FIRMWARE_HASH, + <<"kernel">> => ?TEST_KERNEL_HASH, + <<"initrd">> => ?TEST_INITRD_HASH, + <<"append">> => ?TEST_APPEND_HASH + }, + GenerateOpts = #{ + priv_wallet => TestWallet, + snp_trusted => [TestTrustedConfig] + }, + % Load test report data and set up mock + TestReportJSON = load_test_report_data(), + ok = mock_snp_nif(TestReportJSON), + try + % Step 1: Generate a test report using mocked SNP + {ok, GeneratedMsg} = generate(#{}, #{}, GenerateOpts), + % Verify the generated message structure + ?assert(is_map(GeneratedMsg)), + ?assert(maps:is_key(<<"report">>, GeneratedMsg)), + ?assert(maps:is_key(<<"address">>, GeneratedMsg)), + ?assert(maps:is_key(<<"nonce">>, GeneratedMsg)), + % Step 2: Set up verification options with the same trusted config + VerifyOpts = #{ + snp_trusted => [TestTrustedConfig], + snp_enforced_keys => [vcpu_type, vmm_type, guest_features, + firmware, kernel, initrd, append] + }, + % Step 3: Verify the generated report + {ok, VerifyResult} = + verify( + #{}, + hb_message:commit(GeneratedMsg, GenerateOpts), + VerifyOpts + ), + % Step 4: Assert that verification succeeds + ?assertEqual(<<"true">>, VerifyResult), + % Additional validation: verify specific fields + ReportData = maps:get(<<"report">>, GeneratedMsg), + ?assertEqual(TestReportJSON, ReportData), + LocalHashes = maps:get(<<"local-hashes">>, GeneratedMsg), + ?assertEqual(TestTrustedConfig, LocalHashes) + after + % Clean up mock + unmock_snp_nif() + end. + +%% @doc Test verification failure when using wrong trusted configuration. +verify_mock_generate_wrong_config_test_() -> + { timeout, 30, fun verify_mock_generate_wrong_config/0 }. +verify_mock_generate_wrong_config() -> + % Set up test configuration for generation + TestWallet = ar_wallet:new(), + GenerateTrustedConfig = #{ + <<"vcpus">> => ?TEST_VCPUS_COUNT, + <<"vcpu_type">> => ?TEST_VCPU_TYPE, + <<"vmm_type">> => ?TEST_VMM_TYPE, + <<"guest_features">> => ?TEST_GUEST_FEATURES, + <<"firmware">> => ?TEST_FIRMWARE_HASH, + <<"kernel">> => ?TEST_KERNEL_HASH, + <<"initrd">> => ?TEST_INITRD_HASH, + <<"append">> => ?TEST_APPEND_HASH + }, + GenerateOpts = #{ + priv_wallet => TestWallet, + snp_trusted => [GenerateTrustedConfig] + }, + % Load test report data and set up mock + TestReportJSON = load_test_report_data(), + ok = mock_snp_nif(TestReportJSON), + try + % Step 1: Generate a test report + {ok, GeneratedMsg} = generate(#{}, #{}, GenerateOpts), + % Step 2: Set up verification with DIFFERENT trusted config + WrongTrustedConfig = #{ + <<"vcpus">> => 32, % Different from generation config + <<"vcpu_type">> => 3, % Different from generation config + <<"firmware">> => <<"different_firmware_hash">>, + <<"kernel">> => <<"different_kernel_hash">> + }, + VerifyOpts = #{ + snp_trusted => [WrongTrustedConfig], + snp_enforced_keys => [vcpus, vcpu_type, firmware, kernel] + }, + % Step 3: Verify the generated report with wrong config + VerifyResult = + verify( + #{}, + hb_message:commit(GeneratedMsg, GenerateOpts), + VerifyOpts + ), + ?event({verify_result, {explicit, VerifyResult}}), + % Step 4: Assert that verification fails (either as error or false result) + case VerifyResult of + {ok, <<"false">>} -> + % Verification completed but returned false (all validations ran) + ok; + {error, _Reason} -> + % Verification failed early (expected for wrong config) + ok; + Other -> + % Unexpected result - should fail the test + ?assertEqual({ok, <<"false">>}, Other) + end + after + % Clean up mock + unmock_snp_nif() + end. + +%% @doc Mock the SNP NIF function to return test data. +%% +%% This function sets up a simple mock for dev_snp_nif:generate_attestation_report +%% to return predefined test data instead of calling actual hardware. +%% Uses process dictionary for simple mocking without external dependencies. +%% +%% @param TestReportJSON The test report data to return +%% @returns ok if mocking is successful +-spec mock_snp_nif(ReportJSON :: binary()) -> ok. +mock_snp_nif(TestReportJSON) -> + % Use process dictionary for simple mocking + put(mock_snp_nif_response, TestReportJSON), + put(mock_snp_nif_enabled, true), + ok. + +%% @doc Clean up SNP NIF mocking. +%% +%% This function removes the mock setup and restores normal NIF behavior. +%% +%% @returns ok +-spec unmock_snp_nif() -> ok. +unmock_snp_nif() -> + % Clean up process dictionary mock + erase(mock_snp_nif_response), + erase(mock_snp_nif_enabled), + ok. \ No newline at end of file diff --git a/src/dev_snp_nif.erl b/src/dev_snp_nif.erl new file mode 100644 index 000000000..bacf338c5 --- /dev/null +++ b/src/dev_snp_nif.erl @@ -0,0 +1,84 @@ +-module(dev_snp_nif). +-export([generate_attestation_report/2, compute_launch_digest/1, check_snp_support/0]). +-export([verify_measurement/2, verify_signature/1]). +-include("include/cargo.hrl"). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-on_load(init/0). +-define(NOT_LOADED, not_loaded(?LINE)). + +check_snp_support() -> + ?NOT_LOADED. + +generate_attestation_report(_UniqueData, _VMPL) -> + ?NOT_LOADED. + +compute_launch_digest(_Args) -> + ?NOT_LOADED. + +verify_measurement(_Report, _Expected) -> + ?NOT_LOADED. + +verify_signature(_Report) -> + ?NOT_LOADED. + +init() -> + ?load_nif_from_crate(dev_snp_nif, 0). + +not_loaded(Line) -> + erlang:nif_error({not_loaded, [{module, ?MODULE}, {line, Line}]}). + +generate_attestation_report_test() -> + %% Call check_support() to determine if SNP is supported + case dev_snp_nif:check_snp_support() of + {ok, true} -> + %% SNP is supported, generate unique data and test commitment report + UniqueData = crypto:strong_rand_bytes(64), + VMPL = 1, + ?assertEqual( + {ok, UniqueData}, + dev_snp_nif:generate_attestation_report(UniqueData, VMPL) + ); + {ok, false} -> + %% SNP is not supported, log event and assert NIF not loaded + ?event("SNP not supported on machine, skipping test..."), + ?assertEqual(ok, ok) + end. + +compute_launch_digest_test() -> + %% Define the data structure + ArgsMap = #{ + vcpus => 32, + vcpu_type => 5, + vmm_type => 1, + guest_features => 16#1, + firmware => "b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510", + kernel => "69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576", + initrd => "02e28b6c718bf0a5260d6f34d3c8fe0d71bf5f02af13e1bc695c6bc162120da1", + append => "56e1e5190622c8c6b9daa4fe3ad83f3831c305bb736735bf795b284cb462c9e7" + }, + + ?event(ArgsMap), + + %% Call the NIF + {ok, Result} = dev_snp_nif:compute_launch_digest(ArgsMap), + %% Expected result + EncTestVector = + <<"wmSDSQYuzE2M3rQcourJnDJHgalADM8TBev3gyjM5ObRNOn8oglvVznFbaWhajU_">>, + ?assertMatch(EncTestVector, hb_util:encode(Result)). + +verify_measurement_test() -> + %% Define a mock report (JSON string) as binary + {ok, MockReport} = file:read_file("test/snp-measurement.json"), + %% Define the expected measurement (binary) + ExpectedMeasurement = <<94,87,4,197,20,11,255,129,179,197,146,104,8,212,152,248,110,11,60,246,82,254,24,55,201,47,157,229,163,82,108,66,191,138,241,229,40,144,133,170,116,109,17,62,20,241,144,119>>, + %% Call the NIF + Result = dev_snp_nif:verify_measurement(MockReport, ExpectedMeasurement), + ?assertMatch({ok, true}, Result). + +verify_signature_test() -> + %% Define a mock report (JSON string) as binary + {ok, MockAttestation} = file:read_file("test/snp-attestation.json"), + Result = dev_snp_nif:verify_signature(MockAttestation), + ?assertMatch({ok, true}, Result). diff --git a/src/dev_stack.erl b/src/dev_stack.erl index 548ae4c58..27c89816f 100644 --- a/src/dev_stack.erl +++ b/src/dev_stack.erl @@ -6,29 +6,32 @@ %%% progresses through devices. %%% %%% For example, a stack of devices as follows: -%%% ``` +%%% 
 %%% Device -> Stack
 %%% Device-Stack/1/Name -> Add-One-Device
-%%% Device-Stack/2/Name -> Add-Two-Device'''
+%%% Device-Stack/2/Name -> Add-Two-Device
+%%%
%%% %%% When called with the message: -%%% ``` -%%% #{ Path = "FuncName", binary => <<"0">> }''' +%%% 
+%%% #{ Path = "FuncName", binary => `<<"0">>' }
+%%%
%%% %%% Will produce the output: -%%% ``` -%%% #{ Path = "FuncName", binary => <<"3">> } -%%% {ok, #{ bin => <<"3">> }}''' +%%% 
+%%% #{ Path = "FuncName", binary => `<<"3">>' }
+%%% {ok, #{ bin => `<<"3">>' }}
+%%%
%%% %%% In map mode, the stack will run over all the devices in the stack, and %%% combine their results into a single message. Each of the devices' -%%% output values have a key that is the device's name in the `Device-Stack` +%%% output values have a key that is the device's name in the `Device-Stack' %%% (its number if the stack is a list). %%% -%%% You can switch between fold and map modes by setting the `Mode` key in the -%%% `Msg2` to either `Fold` or `Map`, or set it globally for the stack by -%%% setting the `Mode` key in the `Msg1` message. The key in `Msg2` takes -%%% precedence over the key in `Msg1`. +%%% You can switch between fold and map modes by setting the `Mode' key in the +%%% `Msg2' to either `Fold' or `Map', or set it globally for the stack by +%%% setting the `Mode' key in the `Msg1' message. The key in `Msg2' takes +%%% precedence over the key in `Msg1'. %%% %%% The key that is called upon the device stack is the same key that is used %%% upon the devices that are contained within it. For example, in the above @@ -78,7 +81,7 @@ %%% allows dev_stack to ensure that the message's HashPath is always correct, %%% even as it delegates calls to other devices. An example flow for a `dev_stack' %%% execution is as follows: -%%%``` +%%% 
 %%% 	/Msg1/AlicesExcitingKey ->
 %%% 		dev_stack:execute ->
 %%% 			/Msg1/Set?device=/Device-Stack/1 ->
@@ -88,25 +91,26 @@
 %%% 			... ->
 %%% 			/MsgN/Set?device=[This-Device] ->
 %%% 		returns {ok, /MsgN+1} ->
-%%% 	/MsgN+1'''
+%%% 	/MsgN+1
+%%%
%%% %%% In this example, the `device' key is mutated a number of times, but the %%% resulting HashPath remains correct and verifiable. -module(dev_stack). --export([info/1, router/4, prefix/3, input_prefix/3, output_prefix/3]). +-export([info/2, router/4, prefix/3, input_prefix/3, output_prefix/3]). %%% Test exports -export([generate_append_device/1]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). -info(Msg) -> - maps:merge( +info(Msg, Opts) -> + hb_maps:merge( #{ handler => fun router/4, - excludes => [set, keys] + excludes => [<<"set">>, <<"keys">>] }, - case maps:get(<<"Stack-Keys">>, Msg, not_found) of + case hb_maps:get(<<"stack-keys">>, Msg, not_found, Opts) of not_found -> #{}; StackKeys -> #{ exports => StackKeys } end @@ -114,34 +118,34 @@ info(Msg) -> %% @doc Return the default prefix for the stack. prefix(Msg1, _Msg2, Opts) -> - hb_converge:get(<<"Output-Prefix">>, {as, dev_message, Msg1}, <<"">>, Opts). + hb_ao:get(<<"output-prefix">>, {as, dev_message, Msg1}, <<"">>, Opts). %% @doc Return the input prefix for the stack. input_prefix(Msg1, _Msg2, Opts) -> - hb_converge:get(<<"Input-Prefix">>, {as, dev_message, Msg1}, <<"">>, Opts). + hb_ao:get(<<"input-prefix">>, {as, dev_message, Msg1}, <<"">>, Opts). %% @doc Return the output prefix for the stack. output_prefix(Msg1, _Msg2, Opts) -> - hb_converge:get(<<"Output-Prefix">>, {as, dev_message, Msg1}, <<"">>, Opts). + hb_ao:get(<<"output-prefix">>, {as, dev_message, Msg1}, <<"">>, Opts). %% @doc The device stack key router. Sends the request to `resolve_stack', %% except for `set/2' which is handled by the default implementation in %% `dev_message'. -router(keys, Message1, Message2, _Opts) -> +router(<<"keys">>, Message1, Message2, Opts) -> ?event({keys_called, {msg1, Message1}, {msg2, Message2}}), - dev_message:keys(Message1); + dev_message:keys(Message1, Opts); router(Key, Message1, Message2, Opts) -> - case hb_path:matches(Key, <<"Transform">>) of + case hb_path:matches(Key, <<"transform">>) of true -> transformer_message(Message1, Opts); false -> router(Message1, Message2, Opts) end. router(Message1, Message2, Opts) -> ?event({router_called, {msg1, Message1}, {msg2, Message2}}), Mode = - case hb_converge:get(<<"Mode">>, Message2, not_found, Opts) of + case hb_ao:get(<<"mode">>, Message2, not_found, Opts) of not_found -> - hb_converge:get( - <<"Mode">>, + hb_ao:get( + <<"mode">>, {as, dev_message, Message1}, <<"Fold">>, Opts @@ -162,23 +166,24 @@ router(Message1, Message2, Opts) -> %% keyInDevice executed on DeviceName against Msg1. transformer_message(Msg1, Opts) -> ?event({creating_transformer, {for, Msg1}}), - BaseInfo = info(Msg1), + BaseInfo = info(Msg1, Opts), {ok, Msg1#{ - device => #{ + <<"device">> => #{ info => fun() -> - maps:merge( + hb_maps:merge( BaseInfo, #{ handler => fun(Key, MsgX1) -> transform(MsgX1, Key, Opts) end - } + }, + Opts ) end, - type => <<"Stack-Transformer">> + <<"type">> => <<"stack-transformer">> } } }. @@ -190,14 +195,15 @@ transformer_message(Msg1, Opts) -> transform(Msg1, Key, Opts) -> % Get the device stack message from Msg1. ?event({transforming_stack, {key, Key}, {msg1, Msg1}, {opts, Opts}}), - case hb_converge:get(<<"Device-Stack">>, {as, dev_message, Msg1}, Opts) of + case hb_ao:get(<<"device-stack">>, {as, dev_message, Msg1}, Opts) of not_found -> throw({error, no_valid_device_stack}); StackMsg -> % Find the requested key in the device stack. % TODO: Should we use `as dev_message` here? After the first transform % of a fold (for example), the message is no longer a stack, so its - % `GET` behavior may be different. - case hb_converge:resolve(StackMsg, #{ path => Key }, Opts) of + % `GET' behavior may be different. + NormKey = hb_ao:normalize_key(Key), + case hb_ao:resolve(StackMsg, #{ <<"path">> => NormKey }, Opts) of {ok, DevMsg} -> % Set the: % - Device key to the device we found. @@ -208,38 +214,38 @@ transform(Msg1, Key, Opts) -> dev_message:set( Msg1, #{ - <<"Device">> => DevMsg, - <<"Device-Key">> => Key, - <<"Input-Prefix">> => - hb_converge:get( - [<<"Input-Prefixes">>, Key], + <<"device">> => DevMsg, + <<"device-key">> => Key, + <<"input-prefix">> => + hb_ao:get( + [<<"input-prefixes">>, Key], {as, dev_message, Msg1}, undefined, Opts ), - <<"Output-Prefix">> => - hb_converge:get( - [<<"Output-Prefixes">>, Key], + <<"output-prefix">> => + hb_ao:get( + [<<"output-prefixes">>, Key], {as, dev_message, Msg1}, undefined, Opts ), - <<"Previous-Device">> => - hb_converge:get( - <<"Device">>, + <<"previous-device">> => + hb_ao:get( + <<"device">>, {as, dev_message, Msg1}, Opts ), - <<"Previous-Input-Prefix">> => - hb_converge:get( - <<"Input-Prefix">>, + <<"previous-input-prefix">> => + hb_ao:get( + <<"input-prefix">>, {as, dev_message, Msg1}, undefined, Opts ), - <<"Previous-Output-Prefix">> => - hb_converge:get( - <<"Output-Prefix">>, + <<"previous-output-prefix">> => + hb_ao:get( + <<"output-prefix">>, {as, dev_message, Msg1}, undefined, Opts @@ -256,10 +262,10 @@ transform(Msg1, Key, Opts) -> %% @doc The main device stack execution engine. See the moduledoc for more %% information. resolve_fold(Message1, Message2, Opts) -> - {ok, InitDevMsg} = dev_message:get(<<"Device">>, Message1), + {ok, InitDevMsg} = dev_message:get(<<"device">>, Message1, Opts), StartingPassValue = - hb_converge:get(<<"Pass">>, {as, dev_message, Message1}, unset, Opts), - PreparedMessage = hb_converge:set(Message1, <<"Pass">>, 1, Opts), + hb_ao:get(<<"pass">>, {as, dev_message, Message1}, unset, Opts), + PreparedMessage = hb_ao:set(Message1, <<"pass">>, 1, Opts), case resolve_fold(PreparedMessage, Message2, 1, Opts) of {ok, Raw} when not is_map(Raw) -> {ok, Raw}; @@ -267,24 +273,24 @@ resolve_fold(Message1, Message2, Opts) -> dev_message:set( Result, #{ - device => InitDevMsg, - <<"Input-Prefix">> => - hb_converge:get( - <<"Previous-Input-Prefix">>, + <<"device">> => InitDevMsg, + <<"input-prefix">> => + hb_ao:get( + <<"previous-input-prefix">>, {as, dev_message, Result}, undefined, Opts ), - <<"Output-Prefix">> => - hb_converge:get( - <<"Previous-Output-Prefix">>, + <<"output-prefix">> => + hb_ao:get( + <<"previous-output-prefix">>, {as, dev_message, Result}, undefined, Opts ), - <<"Device-Key">> => unset, - <<"Device-Stack-Previous">> => unset, - <<"Pass">> => StartingPassValue + <<"device-key">> => unset, + <<"device-stack-previous">> => unset, + <<"pass">> => StartingPassValue }, Opts ); @@ -295,7 +301,7 @@ resolve_fold(Message1, Message2, DevNum, Opts) -> case transform(Message1, DevNum, Opts) of {ok, Message3} -> ?event({stack_execute, DevNum, {msg1, Message3}, {msg2, Message2}}), - case hb_converge:resolve(Message3, Message2, Opts) of + case hb_ao:resolve(Message3, Message2, Opts) of {ok, Message4} when is_map(Message4) -> ?event({result, ok, DevNum, Message4}), resolve_fold(Message4, Message2, DevNum + 1, Opts); @@ -340,30 +346,31 @@ resolve_fold(Message1, Message2, DevNum, Opts) -> resolve_map(Message1, Message2, Opts) -> ?event({resolving_map, {msg1, Message1}, {msg2, Message2}}), DevKeys = - hb_converge:get( - <<"Device-Stack">>, + hb_ao:get( + <<"device-stack">>, {as, dev_message, Message1}, Opts ), Res = {ok, - maps:filtermap( + hb_maps:filtermap( fun(Key, _Dev) -> {ok, OrigWithDev} = transform(Message1, Key, Opts), - case hb_converge:resolve(OrigWithDev, Message2, Opts) of + case hb_ao:resolve(OrigWithDev, Message2, Opts) of {ok, Value} -> {true, Value}; _ -> false end end, - maps:without(?CONVERGE_KEYS, hb_converge:ensure_message(DevKeys)) + hb_maps:without(?AO_CORE_KEYS, hb_ao:normalize_keys(DevKeys, Opts), Opts), + Opts ) }, Res. %% @doc Helper to increment the pass number. increment_pass(Message, Opts) -> - hb_converge:set( + hb_ao:set( Message, - #{ pass => hb_converge:get(<<"Pass">>, {as, dev_message, Message}, 1, Opts) + 1 }, + #{ <<"pass">> => hb_ao:get(<<"pass">>, {as, dev_message, Message}, 1, Opts) + 1 }, Opts ). @@ -391,12 +398,12 @@ generate_append_device(Separator) -> generate_append_device(Separator, Status) -> #{ append => - fun(M1 = #{ pass := 3 }, _) -> + fun(M1 = #{ <<"pass">> := 3 }, _) -> % Stop after 3 passes. {ok, M1}; - (M1 = #{ result := Existing }, #{ bin := New }) -> + (M1 = #{ <<"result">> := Existing }, #{ <<"bin">> := New }) -> ?event({appending, {existing, Existing}, {new, New}}), - {Status, M1#{ result => + {Status, M1#{ <<"result">> => << Existing/binary, Separator/binary, New/binary>> }} end @@ -408,17 +415,17 @@ transform_internal_call_device_test() -> AppendDev = generate_append_device(<<"_">>), Msg1 = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => AppendDev, - <<"2">> => <<"Message/1.0">> + <<"2">> => <<"message@1.0">> } }, ?assertMatch( - <<"Message/1.0">>, - hb_converge:get( - <<"Device">>, + <<"message@1.0">>, + hb_ao:get( + <<"device">>, element(2, transform(Msg1, <<"2">>, #{})) ) ). @@ -427,20 +434,21 @@ transform_internal_call_device_test() -> %% return a version of msg1 with only that device attached. transform_external_call_device_test() -> Msg1 = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ - <<"Make-Cool">> => + <<"make-cool">> => #{ info => fun() -> #{ handler => - fun(keys, MsgX1) -> - {ok, maps:keys(MsgX1)}; + fun(<<"keys">>, MsgX1) -> + ?event({test_dev_keys_called, MsgX1}), + {ok, hb_maps:keys(MsgX1, #{})}; (Key, MsgX1) -> {ok, Value} = - dev_message:get(Key, MsgX1), + dev_message:get(Key, MsgX1, #{}), dev_message:set( MsgX1, #{ Key => @@ -451,15 +459,15 @@ transform_external_call_device_test() -> end } end, - suffix => <<"-Cool">> + <<"suffix">> => <<"-Cool">> } }, - <<"Value">> => <<"Super">> + <<"value">> => <<"Super">> }, ?assertMatch( - {ok, #{ <<"Value">> := <<"Super-Cool">> }}, - hb_converge:resolve(Msg1, #{ - path => <<"/Transform/Make-Cool/Value">> + {ok, #{ <<"value">> := <<"Super-Cool">> }}, + hb_ao:resolve(Msg1, #{ + <<"path">> => <<"/transform/make-cool/value">> }, #{}) ). @@ -468,34 +476,34 @@ example_device_for_stack_test() -> % we know that an error later is actually from the stack, and not from % the example device. ?assertMatch( - {ok, #{ result := <<"1_2">> }}, - hb_converge:resolve( - #{ device => generate_append_device(<<"_">>), result => <<"1">> }, - #{ path => append, bin => <<"2">> }, + {ok, #{ <<"result">> := <<"1_2">> }}, + hb_ao:resolve( + #{ <<"device">> => generate_append_device(<<"_">>), <<"result">> => <<"1">> }, + #{ <<"path">> => <<"append">>, <<"bin">> => <<"2">> }, #{} ) ). simple_stack_execute_test() -> Msg = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => generate_append_device(<<"!D1!">>), <<"2">> => generate_append_device(<<"_D2_">>) }, - result => <<"INIT">> + <<"result">> => <<"INIT">> }, ?event({stack_executing, test, {explicit, Msg}}), ?assertMatch( - {ok, #{ result := <<"INIT!D1!2_D2_2">> }}, - hb_converge:resolve(Msg, #{ path => append, bin => <<"2">> }, #{}) + {ok, #{ <<"result">> := <<"INIT!D1!2_D2_2">> }}, + hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"2">> }, #{}) ). many_devices_test() -> Msg = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => generate_append_device(<<"+D1">>), <<"2">> => generate_append_device(<<"+D2">>), @@ -506,23 +514,23 @@ many_devices_test() -> <<"7">> => generate_append_device(<<"+D7">>), <<"8">> => generate_append_device(<<"+D8">>) }, - result => <<"INIT">> + <<"result">> => <<"INIT">> }, ?assertMatch( {ok, #{ - result := + <<"result">> := <<"INIT+D12+D22+D32+D42+D52+D62+D72+D82">> } }, - hb_converge:resolve(Msg, #{ path => append, bin => <<"2">> }, #{}) + hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"2">> }, #{}) ). benchmark_test() -> BenchTime = 0.3, Msg = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => generate_append_device(<<"+D1">>), <<"2">> => generate_append_device(<<"+D2">>), @@ -530,21 +538,29 @@ benchmark_test() -> <<"4">> => generate_append_device(<<"+D4">>), <<"5">> => generate_append_device(<<"+D5">>) }, - result => <<"INIT">> + <<"result">> => <<"INIT">> }, Iterations = - hb:benchmark( + hb_test_utils:benchmark( fun() -> - hb_converge:resolve(Msg, #{ path => append, bin => <<"2">> }, #{}), + hb_ao:resolve(Msg, + #{ + <<"path">> => <<"append">>, + <<"bin">> => <<"2">> + }, + #{} + ), {count, 5} end, BenchTime ), - hb_util:eunit_print( - "Evaluated ~p stack messages in ~p seconds (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] + hb_test_utils:benchmark_print( + <<"Stack:">>, + <<"resolutions">>, + Iterations, + BenchTime ), - ?assert(Iterations > 10). + ?assert(Iterations >= 10). test_prefix_msg() -> @@ -553,10 +569,10 @@ test_prefix_msg() -> fun(M1, M2, Opts) -> In = input_prefix(M1, M2, Opts), Out = output_prefix(M1, M2, Opts), - Key = hb_converge:get(<<"Key">>, M2, Opts), - Value = hb_converge:get(<>, M2, Opts), + Key = hb_ao:get(<<"key">>, M2, Opts), + Value = hb_ao:get(<>, M2, Opts), ?event({setting, {inp, In}, {outp, Out}, {key, Key}, {value, Value}}), - {ok, hb_converge:set( + {ok, hb_ao:set( M1, <>, Value, @@ -565,73 +581,73 @@ test_prefix_msg() -> end }, #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => #{ 1 => Dev, 2 => Dev } + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => Dev, <<"2">> => Dev } }. no_prefix_test() -> Msg2 = #{ - path => prefix_set, - key => <<"Example">>, - <<"Example">> => 1 + <<"path">> => <<"prefix_set">>, + <<"key">> => <<"example">>, + <<"example">> => 1 }, - {ok, Ex1Msg3} = hb_converge:resolve(test_prefix_msg(), Msg2, #{}), + {ok, Ex1Msg3} = hb_ao:resolve(test_prefix_msg(), Msg2, #{}), ?event({ex1, Ex1Msg3}), - ?assertMatch(1, hb_converge:get(<<"Example">>, Ex1Msg3, #{})). + ?assertMatch(1, hb_ao:get(<<"example">>, Ex1Msg3, #{})). output_prefix_test() -> Msg1 = (test_prefix_msg())#{ - <<"Output-Prefixes">> => #{ 1 => <<"Out1/">>, 2 => <<"Out2/">> } + <<"output-prefixes">> => #{ <<"1">> => <<"out1/">>, <<"2">> => <<"out2/">> } }, Msg2 = #{ - path => prefix_set, - key => <<"Example">>, - <<"Example">> => 1 + <<"path">> => <<"prefix_set">>, + <<"key">> => <<"example">>, + <<"example">> => 1 }, - {ok, Ex2Msg3} = hb_converge:resolve(Msg1, Msg2, #{}), + {ok, Ex2Msg3} = hb_ao:resolve(Msg1, Msg2, #{}), ?assertMatch(1, - hb_converge:get(<<"Out1/Example">>, {as, dev_message, Ex2Msg3}, #{})), + hb_ao:get(<<"out1/example">>, {as, dev_message, Ex2Msg3}, #{})), ?assertMatch(1, - hb_converge:get(<<"Out2/Example">>, {as, dev_message, Ex2Msg3}, #{})). + hb_ao:get(<<"out2/example">>, {as, dev_message, Ex2Msg3}, #{})). input_and_output_prefixes_test() -> Msg1 = (test_prefix_msg())#{ - <<"Input-Prefixes">> => #{ 1 => <<"In1/">>, 2 => <<"In2/">> }, - <<"Output-Prefixes">> => #{ 1 => <<"Out1/">>, 2 => <<"Out2/">> } + <<"input-prefixes">> => #{ 1 => <<"in1/">>, 2 => <<"in2/">> }, + <<"output-prefixes">> => #{ 1 => <<"out1/">>, 2 => <<"out2/">> } }, Msg2 = #{ - path => prefix_set, - key => <<"Example">>, - <<"In1">> => #{ <<"Example">> => 1 }, - <<"In2">> => #{ <<"Example">> => 2 } + <<"path">> => <<"prefix_set">>, + <<"key">> => <<"example">>, + <<"in1">> => #{ <<"example">> => 1 }, + <<"in2">> => #{ <<"example">> => 2 } }, - {ok, Msg3} = hb_converge:resolve(Msg1, Msg2, #{}), + {ok, Msg3} = hb_ao:resolve(Msg1, Msg2, #{}), ?assertMatch(1, - hb_converge:get(<<"Out1/Example">>, {as, dev_message, Msg3}, #{})), + hb_ao:get(<<"out1/example">>, {as, dev_message, Msg3}, #{})), ?assertMatch(2, - hb_converge:get(<<"Out2/Example">>, {as, dev_message, Msg3}, #{})). + hb_ao:get(<<"out2/example">>, {as, dev_message, Msg3}, #{})). input_output_prefixes_passthrough_test() -> Msg1 = (test_prefix_msg())#{ - <<"Output-Prefix">> => <<"Combined-Out/">>, - <<"Input-Prefix">> => <<"Combined-In/">> + <<"output-prefix">> => <<"combined-out/">>, + <<"input-prefix">> => <<"combined-in/">> }, Msg2 = #{ - path => prefix_set, - key => <<"Example">>, - <<"Combined-In">> => #{ <<"Example">> => 1 } + <<"path">> => <<"prefix_set">>, + <<"key">> => <<"example">>, + <<"combined-in">> => #{ <<"example">> => 1 } }, - {ok, Ex2Msg3} = hb_converge:resolve(Msg1, Msg2, #{}), + {ok, Ex2Msg3} = hb_ao:resolve(Msg1, Msg2, #{}), ?assertMatch(1, - hb_converge:get( - <<"Combined-Out/Example">>, + hb_ao:get( + <<"combined-out/example">>, {as, dev_message, Ex2Msg3}, #{} ) @@ -639,101 +655,102 @@ input_output_prefixes_passthrough_test() -> reinvocation_test() -> Msg = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => generate_append_device(<<"+D1">>), <<"2">> => generate_append_device(<<"+D2">>) }, - result => <<"INIT">> + <<"result">> => <<"INIT">> }, - Res1 = hb_converge:resolve(Msg, #{ path => append, bin => <<"2">> }, #{}), + Res1 = hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"2">> }, #{}), ?assertMatch( - {ok, #{ result := <<"INIT+D12+D22">> }}, + {ok, #{ <<"result">> := <<"INIT+D12+D22">> }}, Res1 ), {ok, Msg2} = Res1, - Res2 = hb_converge:resolve(Msg2, #{ path => append, bin => <<"3">> }, #{}), + Res2 = hb_ao:resolve(Msg2, #{ <<"path">> => <<"append">>, <<"bin">> => <<"3">> }, #{}), ?assertMatch( - {ok, #{ result := <<"INIT+D12+D22+D13+D23">> }}, + {ok, #{ <<"result">> := <<"INIT+D12+D22+D13+D23">> }}, Res2 ). skip_test() -> Msg1 = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => generate_append_device(<<"+D1">>, skip), <<"2">> => generate_append_device(<<"+D2">>) }, - result => <<"INIT">> + <<"result">> => <<"INIT">> }, ?assertMatch( - {ok, #{ result := <<"INIT+D12">> }}, - hb_converge:resolve( + {ok, #{ <<"result">> := <<"INIT+D12">> }}, + hb_ao:resolve( Msg1, - #{ path => append, bin => <<"2">> }, + #{ <<"path">> => <<"append">>, <<"bin">> => <<"2">> }, #{} ) ). pass_test() -> - % The append device will return `ok` after 2 passes, so this test - % recursively calls the device by forcing its response to be `pass` + % The append device will return `ok' after 2 passes, so this test + % recursively calls the device by forcing its response to be `pass' % until that happens. Msg = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => generate_append_device(<<"+D1">>, pass) }, - result => <<"INIT">> + <<"result">> => <<"INIT">> }, ?assertMatch( - {ok, #{ result := <<"INIT+D1_+D1_">> }}, - hb_converge:resolve(Msg, #{ path => append, bin => <<"_">> }, #{}) + {ok, #{ <<"result">> := <<"INIT+D1_+D1_">> }}, + hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"_">> }, #{}) ). not_found_test() -> % Ensure that devices not exposing a key are safely skipped. Msg = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => generate_append_device(<<"+D1">>), <<"2">> => (generate_append_device(<<"+D2">>))#{ - special => + <<"special">> => fun(M1) -> - {ok, M1#{ <<"Output">> => 1337 }} + {ok, M1#{ <<"output">> => 1337 }} end } }, - result => <<"INIT">> + <<"result">> => <<"INIT">> }, - {ok, Msg3} = hb_converge:resolve(Msg, #{ path => append, bin => <<"_">> }, #{}), + {ok, Msg3} = hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"_">> }, #{}), ?assertMatch( - #{ result := <<"INIT+D1_+D2_">> }, + #{ <<"result">> := <<"INIT+D1_+D2_">> }, Msg3 ), - ?assertEqual(1337, hb_converge:get(<<"Special/Output">>, Msg3, #{})). + ?event({ex3, Msg3}), + ?assertEqual(1337, hb_ao:get(<<"special/output">>, Msg3, #{})). simple_map_test() -> Msg = #{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => #{ <<"1">> => generate_append_device(<<"+D1">>), <<"2">> => generate_append_device(<<"+D2">>) }, - result => <<"INIT">> + <<"result">> => <<"INIT">> }, {ok, Msg3} = - hb_converge:resolve( + hb_ao:resolve( Msg, - #{ path => append, <<"Mode">> => <<"Map">>, bin => <<"/">> }, + #{ <<"path">> => <<"append">>, <<"mode">> => <<"Map">>, <<"bin">> => <<"/">> }, #{} ), - ?assertMatch(<<"INIT+D1/">>, hb_converge:get(<<"1/Result">>, Msg3, #{})), - ?assertMatch(<<"INIT+D2/">>, hb_converge:get(<<"2/Result">>, Msg3, #{})). \ No newline at end of file + ?assertMatch(<<"INIT+D1/">>, hb_ao:get(<<"1/result">>, Msg3, #{})), + ?assertMatch(<<"INIT+D2/">>, hb_ao:get(<<"2/result">>, Msg3, #{})). \ No newline at end of file diff --git a/src/dev_test.erl b/src/dev_test.erl index afbdcb48a..37912c9e0 100644 --- a/src/dev_test.erl +++ b/src/dev_test.erl @@ -1,21 +1,65 @@ -module(dev_test). +-export([info/3]). -export([info/1, test_func/1, compute/3, init/3, restore/3, snapshot/3, mul/2]). +-export([update_state/3, increment_counter/3, delay/3]). +-export([index/3, postprocess/3, load/3]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). -%%% A simple test device for Converge, so that we can test the functionality that +%%% A simple test device for AO-Core, so that we can test the functionality that %%% depends on using Erlang's module system. %%% -%%% NOTE: This device is labelled `Test-Device/1.0' to avoid conflicts with +%%% NOTE: This device is labelled `test-device/1.0' to avoid conflicts with %%% other testing functionality -- care should equally be taken to avoid %%% using the `test' key in other settings. + %% @doc Exports a default_handler function that can be used to test the %% handler resolution mechanism. info(_) -> #{ + <<"default">> => dev_message, + handlers => #{ + <<"info">> => fun info/3, + <<"update_state">> => fun update_state/3, + <<"increment_counter">> => fun increment_counter/3 + } }. +%% @doc Exports a default_handler function that can be used to test the +%% handler resolution mechanism. +info(_Msg1, _Msg2, _Opts) -> + InfoBody = #{ + <<"description">> => <<"Test device for testing the AO-Core framework">>, + <<"version">> => <<"1.0">>, + <<"paths">> => #{ + <<"info">> => <<"Get device info">>, + <<"test_func">> => <<"Test function">>, + <<"compute">> => <<"Compute function">>, + <<"init">> => <<"Initialize function">>, + <<"restore">> => <<"Restore function">>, + <<"mul">> => <<"Multiply function">>, + <<"snapshot">> => <<"Snapshot function">>, + <<"response">> => <<"Response function">>, + <<"update_state">> => <<"Update state function">> + } + }, + {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. + +%% @doc Example index handler. +index(Msg, _Req, Opts) -> + Name = hb_ao:get(<<"name">>, Msg, <<"turtles">>, Opts), + {ok, + #{ + <<"content-type">> => <<"text/html">>, + <<"body">> => <<"i like ", Name/binary, "!">> + } + }. + +%% @doc Return a message with the device set to this module. +load(Base, _, _Opts) -> + {ok, Base#{ <<"device">> => <<"test-device@1.0">> }}. + test_func(_) -> {ok, <<"GOOD_FUNCTION">>}. @@ -23,15 +67,17 @@ test_func(_) -> %% the slots that have been computed in the state message and places the new %% slot number in the results key. compute(Msg1, Msg2, Opts) -> - AssignmentSlot = hb_converge:get(<<"Assignment/Slot">>, Msg2, Opts), - Seen = hb_converge:get(<<"Already-Seen">>, Msg1, Opts), + AssignmentSlot = hb_ao:get(<<"slot">>, Msg2, Opts), + Seen = hb_ao:get(<<"already-seen">>, Msg1, Opts), + ?event({compute_called, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), {ok, - hb_converge:set( + hb_ao:set( Msg1, #{ - <<"Results">> => - #{ <<"Assignment-Slot">> => AssignmentSlot }, - <<"Already-Seen">> => [AssignmentSlot | Seen] + <<"random-key">> => <<"random-value">>, + <<"results">> => + #{ <<"assignment-slot">> => AssignmentSlot }, + <<"already-seen">> => [AssignmentSlot | Seen] }, Opts ) @@ -40,13 +86,13 @@ compute(Msg1, Msg2, Opts) -> %% @doc Example `init/3' handler. Sets the `Already-Seen' key to an empty list. init(Msg, _Msg2, Opts) -> ?event({init_called_on_dev_test, Msg}), - {ok, hb_converge:set(Msg, #{ <<"Already-Seen">> => [] }, Opts)}. + {ok, hb_ao:set(Msg, #{ <<"already-seen">> => [] }, Opts)}. -%% @doc Example `restore/3' handler. Sets the hidden key `Test/Started` to the -%% value of `Current-Slot` and checks whether the `Already-Seen` key is valid. +%% @doc Example `restore/3' handler. Sets the hidden key `Test/Started' to the +%% value of `Current-Slot' and checks whether the `Already-Seen' key is valid. restore(Msg, _Msg2, Opts) -> ?event({restore_called_on_dev_test, Msg}), - case hb_converge:get(<<"Already-Seen">>, Msg, Opts) of + case hb_ao:get(<<"already-seen">>, Msg, Opts) of not_found -> ?event({restore_not_found, Msg}), {error, <<"No viable state to restore.">>}; @@ -55,64 +101,132 @@ restore(Msg, _Msg2, Opts) -> {ok, hb_private:set( Msg, - #{ <<"Test-Key/Started-State">> => AlreadySeen }, + #{ <<"test-key/started-state">> => AlreadySeen }, Opts ) } end. -%% @doc Example implementation of an `imported` function for a WASM +%% @doc Example implementation of an `imported' function for a WASM %% executor. mul(Msg1, Msg2) -> - State = hb_converge:get(<<"State">>, Msg1, #{ hashpath => ignore }), - [Arg1, Arg2] = hb_converge:get(args, Msg2, #{ hashpath => ignore }), - {ok, #{ state => State, results => [Arg1 * Arg2] }}. + ?event(mul_called), + State = hb_ao:get(<<"state">>, Msg1, #{ hashpath => ignore }), + [Arg1, Arg2] = hb_ao:get(<<"args">>, Msg2, #{ hashpath => ignore }), + ?event({mul_called, {state, State}, {args, [Arg1, Arg2]}}), + {ok, #{ <<"state">> => State, <<"results">> => [Arg1 * Arg2] }}. %% @doc Do nothing when asked to snapshot. -snapshot(_Msg1, _Msg2, _Opts) -> +snapshot(Msg1, Msg2, _Opts) -> + ?event({snapshot_called, {msg1, Msg1}, {msg2, Msg2}}), {ok, #{}}. +%% @doc Set the `postprocessor-called' key to true in the HTTP server. +postprocess(_Msg, #{ <<"body">> := Msgs }, Opts) -> + ?event({postprocess_called, Opts}), + hb_http_server:set_opts(Opts#{ <<"postprocessor-called">> => true }), + {ok, Msgs}. + +%% @doc Find a test worker's PID and send it an update message. +update_state(_Msg, Msg2, _Opts) -> + case hb_ao:get(<<"test-id">>, Msg2) of + not_found -> + {error, <<"No test ID found in message.">>}; + ID -> + LookupResult = hb_name:lookup({<<"test">>, ID}), + case LookupResult of + undefined -> + {error, <<"No test worker found.">>}; + Pid -> + Pid ! {update, Msg2}, + {ok, Pid} + end + end. + +%% @doc Find a test worker's PID and send it an increment message. +increment_counter(_Msg1, Msg2, _Opts) -> + case hb_ao:get(<<"test-id">>, Msg2) of + not_found -> + {error, <<"No test ID found in message.">>}; + ID -> + LookupResult = hb_name:lookup({<<"test">>, ID}), + case LookupResult of + undefined -> + {error, <<"No test worker found for increment.">>}; + Pid when is_pid(Pid) -> + Pid ! {increment}, + {ok, Pid}; + _ -> % Handle case where registered value isn't a PID + {error, <<"Invalid registration found for test worker.">>} + end + end. + +%% @doc Does nothing, just sleeps `Req/duration or 750' ms and returns the +%% appropriate form in order to be used as a hook. +delay(Msg1, Req, Opts) -> + Duration = + hb_ao:get_first( + [ + {Msg1, <<"duration">>}, + {Req, <<"duration">>} + ], + 750, + Opts + ), + ?event(delay, {delay, {sleeping, Duration}}), + timer:sleep(Duration), + ?event({delay, waking}), + Return = + case hb_ao:get(<<"return">>, Msg1, Opts) of + not_found -> + hb_ao:get(<<"body">>, Req, #{ <<"result">> => <<"slept">> }, Opts); + ReturnMsgs -> + ReturnMsgs + end, + ?event(delay, {returning, Return}), + {ok, Return}. + %%% Tests %% @doc Tests the resolution of a default function. device_with_function_key_module_test() -> Msg = #{ - device => <<"Test-Device/1.0">> + <<"device">> => <<"test-device@1.0">> }, ?assertEqual( {ok, <<"GOOD_FUNCTION">>}, - hb_converge:resolve(Msg, test_func, #{}) + hb_ao:resolve(Msg, test_func, #{}) ). compute_test() -> - Msg0 = #{ device => <<"Test-Device/1.0">> }, - {ok, Msg1} = hb_converge:resolve(Msg0, init, #{}), + Msg0 = #{ <<"device">> => <<"test-device@1.0">> }, + {ok, Msg1} = hb_ao:resolve(Msg0, init, #{}), Msg2 = - hb_converge:set( - #{ path => <<"Compute">> }, + hb_ao:set( + #{ <<"path">> => <<"compute">> }, #{ - <<"Assignment/Slot">> => 1, - <<"Message/Number">> => 1337 + <<"slot">> => 1, + <<"body/number">> => 1337 }, #{} ), - {ok, Msg3} = hb_converge:resolve(Msg1, Msg2, #{}), - ?assertEqual(1, hb_converge:get(<<"Results/Assignment-Slot">>, Msg3, #{})), + {ok, Msg3} = hb_ao:resolve(Msg1, Msg2, #{}), + ?assertEqual(1, hb_ao:get(<<"results/assignment-slot">>, Msg3, #{})), Msg4 = - hb_converge:set( - #{ path => <<"Compute">> }, + hb_ao:set( + #{ <<"path">> => <<"compute">> }, #{ - <<"Assignment/Slot">> => 2, - <<"Message/Number">> => 9001 + <<"slot">> => 2, + <<"body/number">> => 9001 }, #{} ), - {ok, Msg5} = hb_converge:resolve(Msg3, Msg4, #{}), - ?assertEqual(2, hb_converge:get(<<"Results/Assignment-Slot">>, Msg5, #{})), - ?assertEqual([2, 1], hb_converge:get(<<"Already-Seen">>, Msg5, #{})). + {ok, Msg5} = hb_ao:resolve(Msg3, Msg4, #{}), + ?assertEqual(2, hb_ao:get(<<"results/assignment-slot">>, Msg5, #{})), + ?assertEqual([2, 1], hb_ao:get(<<"already-seen">>, Msg5, #{})). restore_test() -> - Msg1 = #{ device => <<"Test-Device/1.0">>, <<"Already-Seen">> => [1] }, - {ok, Msg3} = hb_converge:resolve(Msg1, restore, #{}), - ?assertEqual([1], hb_private:get(<<"Test-Key/Started-State">>, Msg3, #{})). \ No newline at end of file + Msg1 = #{ <<"device">> => <<"test-device@1.0">>, <<"already-seen">> => [1] }, + {ok, Msg3} = hb_ao:resolve(Msg1, <<"restore">>, #{}), + ?assertEqual([1], hb_private:get(<<"test-key/started-state">>, Msg3, #{})). \ No newline at end of file diff --git a/src/dev_volume.erl b/src/dev_volume.erl new file mode 100644 index 000000000..5c227eead --- /dev/null +++ b/src/dev_volume.erl @@ -0,0 +1,572 @@ +%%% @doc Secure Volume Management for HyperBEAM Nodes +%%% +%%% This module handles encrypted storage operations for HyperBEAM, +%%% providing a robust and secure approach to data persistence. It manages +%%% the complete lifecycle of encrypted volumes from detection to creation, +%%% formatting, and mounting. +%%% +%%% Key responsibilities: +%%% - Volume detection and initialization +%%% - Encrypted partition creation and formatting +%%% - Secure mounting using cryptographic keys +%%% - Store path reconfiguration to use mounted volumes +%%% - Automatic handling of various system states +%%% (new device, existing partition, etc.) +%%% +%%% The primary entry point is the `mount/3' function, which orchestrates +%%% the entire process based on the provided configuration parameters. This +%%% module works alongside `hb_volume' which provides the low-level +%%% operations for device manipulation. +%%% +%%% Security considerations: +%%% - Ensures data at rest is protected through LUKS encryption +%%% - Provides proper volume sanitization and secure mounting +%%% - IMPORTANT: This module only applies configuration set in node options +%%% and does NOT accept disk operations via HTTP requests. It cannot +%%% format arbitrary disks as all operations are safeguarded by host +%%% operating system permissions enforced upon the HyperBEAM environment. +-module(dev_volume). +-export([info/1, info/3, mount/3, public_key/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +%% @doc Exported function for getting device info, controls which functions +%% are exposed via the device API. +info(_) -> + ?event(debug_volume, {info, entry, device_info_requested}), + #{ exports => [info, mount, public_key] }. + +%% @doc HTTP info response providing information about this device +info(_Msg1, _Msg2, _Opts) -> + ?event(debug_volume, {info, http_request, starting}), + InfoBody = #{ + <<"description">> => + <<"Secure Volume Management for HyperBEAM Nodes">>, + <<"version">> => <<"1.0">>, + <<"api">> => #{ + <<"info">> => #{ + <<"description">> => <<"Get device info">> + }, + <<"mount">> => #{ + <<"description">> => <<"Mount an encrypted volume">>, + <<"required_node_opts">> => #{ + <<"priv_volume_key">> => <<"The encryption key">>, + <<"volume_device">> => <<"The base device path">>, + <<"volume_partition">> => <<"The partition path">>, + <<"volume_partition_type">> => <<"The partition type">>, + <<"volume_name">> => + <<"The name for the encrypted volume">>, + <<"volume_mount_point">> => + <<"Where to mount the volume">>, + <<"volume_store_path">> => + <<"The store path on the volume">> + } + }, + <<"public_key">> => #{ + <<"description">> => + <<"Get the node's public key for encrypted key exchange">> + } + } + }, + ?event(debug_volume, {info, http_response, success}), + {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. + +%% @doc Handles the complete process of secure encrypted volume mounting. +%% +%% This function performs the following operations depending on the state: +%% 1. Validates the encryption key is present +%% 2. Checks if the base device exists +%% 3. Checks if the partition exists on the device +%% 4. If the partition exists, attempts to mount it +%% 5. If the partition doesn't exist, creates it, formats it with +%% encryption and mounts it +%% 6. Updates the node's store configuration to use the mounted volume +%% +%% Config options in Opts map: +%% - priv_volume_key: (Required) The encryption key +%% - volume_device: Base device path +%% - volume_partition: Partition path +%% - volume_partition_type: Filesystem type +%% - volume_name: Name for encrypted volume +%% - volume_mount_point: Where to mount +%% - volume_store_path: Store path on volume +%% +%% @param M1 Base message for context. +%% @param M2 Request message with operation details. +%% @param Opts A map of configuration options for volume operations. +%% @returns `{ok, Binary}' on success with operation result message, or +%% `{error, Binary}' on failure with error message. +-spec mount(term(), term(), map()) -> + {ok, binary()} | {error, binary()}. +mount(_M1, _M2, Opts) -> + ?event(debug_volume, {mount, entry, starting}), + % Check if an encrypted key was sent in the request + EncryptedKey = hb_opts:get(priv_volume_key, not_found, Opts), + % Determine if we need to decrypt a key or use one from config + SkipDecryption = hb_opts:get(volume_skip_decryption, + <<"false">>, Opts), + Key = case SkipDecryption of + <<"true">> -> + ?event(debug_mount, {mount, skip_decryption, true}), + EncryptedKey; + _ -> + ?event(debug_volume, {decrypt_volume_key}), + case decrypt_volume_key(EncryptedKey, Opts) of + {ok, DecryptedKey} -> DecryptedKey; + {error, DecryptError} -> + ?event(debug_mount, + {mount, key_decrypt_error, DecryptError} + ), + not_found + end + end, + Device = hb_opts:get(volume_device, not_found, Opts), + Partition = hb_opts:get(volume_partition, not_found, Opts), + PartitionType = hb_opts:get(volume_partition_type, not_found, Opts), + VolumeName = hb_opts:get(volume_name, not_found, Opts), + MountPoint = hb_opts:get(volume_mount_point, not_found, Opts), + StorePath = hb_opts:get(volume_store_path, not_found, Opts), + ?event(debug_volume, + {mount, options_extracted, + { + device, Device, partition, Partition, + partition_type, PartitionType, volume_name, VolumeName, + mount_point, MountPoint, store_path, StorePath + } + } + ), + % Check for missing required node options + case hb_opts:check_required_opts([ + {<<"priv_volume_key">>, Key}, + {<<"volume_device">>, Device}, + {<<"volume_partition">>, Partition}, + {<<"volume_partition_type">>, PartitionType}, + {<<"volume_name">>, VolumeName}, + {<<"volume_mount_point">>, MountPoint}, + {<<"volume_store_path">>, StorePath} + ], Opts) of + {ok, _} -> + check_base_device( + Device, Partition, PartitionType, VolumeName, + MountPoint, StorePath, Key, Opts + ); + {error, ErrorMsg} -> + ?event(debug_volume, {mount, required_opts_error, ErrorMsg}), + {error, ErrorMsg} + end. + +%% @doc Returns the node's public key for secure key exchange. +%% +%% This function retrieves the node's wallet and extracts the public key +%% for encryption purposes. It allows users to securely exchange +%% encryption keys by first encrypting their volume key with the node's +%% public key. +%% +%% The process ensures that sensitive keys are never transmitted in +%% plaintext. The encrypted key can then be securely sent to the node, +%% which will decrypt it using its private key before using it for volume +%% encryption. +%% +%% @param _M1 Ignored parameter. +%% @param _M2 Ignored parameter. +%% @param Opts A map of configuration options. +%% @returns `{ok, Map}' containing the node's public key on success, or +%% `{error, Binary}' if the node's wallet is not available. +-spec public_key(term(), term(), map()) -> + {ok, map()} | {error, binary()}. +public_key(_M1, _M2, Opts) -> + % Retrieve the node's wallet + case hb_opts:get(priv_wallet, undefined, Opts) of + undefined -> + % Node doesn't have a wallet yet + ?event(debug_volume, + {public_key, wallet_error, no_wallet_found} + ), + {error, <<"Node wallet not available">>}; + {{_KeyType, _Priv, Pub}, _PubKey} -> + ?event(debug_volume, + {public_key, wallet_found, key_conversion_starting} + ), + % Convert to a standard RSA format (PKCS#1 or X.509) + RsaPubKey = #'RSAPublicKey'{ + publicExponent = 65537, % Common RSA exponent + modulus = crypto:bytes_to_integer(Pub) + }, + % Convert to DER format + DerEncoded = public_key:der_encode('RSAPublicKey', RsaPubKey), + % Base64 encode for transmission + Base64Key = base64:encode(DerEncoded), + ?event(debug_volume, {public_key, success, key_encoded}), + {ok, #{ + <<"status">> => 200, + <<"public_key">> => Base64Key, + <<"message">> => + <<"Use this public key to encrypt your volume key">> + }} + end. + +%% @doc Decrypts an encrypted volume key using the node's private key. +%% +%% This function takes an encrypted key (typically sent by a client who +%% encrypted it with the node's public key) and decrypts it using the +%% node's private RSA key. +%% +%% @param EncryptedKey The encrypted volume key (Base64 encoded). +%% @param Opts A map of configuration options. +%% @returns `{ok, DecryptedKey}' on successful decryption, or +%% `{error, Binary}' if decryption fails. +-spec decrypt_volume_key(binary(), map()) -> + {ok, binary()} | {error, binary()}. +decrypt_volume_key(EncryptedKeyBase64, Opts) -> + % Decode the encrypted key + try + EncryptedKey = base64:decode(EncryptedKeyBase64), + ?event(debug_volume, + {decrypt_volume_key, base64_decoded, success} + ), + % Retrieve the node's wallet with private key + case hb_opts:get(priv_wallet, undefined, Opts) of + undefined -> + ?event(debug_volume, + {decrypt_volume_key, wallet_error, no_wallet} + ), + {error, <<"Node wallet not available for decryption">>}; + {{_KeyType = {rsa, E}, Priv, Pub}, _PubKey} -> + ?event(debug_volume, + {decrypt_volume_key, wallet_found, creating_private_key} + ), + % Create RSA private key record for decryption + RsaPrivKey = #'RSAPrivateKey'{ + publicExponent = E, + modulus = crypto:bytes_to_integer(Pub), + privateExponent = crypto:bytes_to_integer(Priv) + }, + % Decrypt the key + DecryptedKey = + public_key:decrypt_private( + EncryptedKey, + RsaPrivKey + ), + ?event(debug_volume, + {decrypt_volume_key, decryption_success, key_decrypted} + ), + {ok, DecryptedKey} + end + catch + _:Error -> + ?event(debug_volume, + {decrypt_volume_key, decryption_error, Error} + ), + {error, <<"Failed to decrypt volume key">>} + end. + +%% @doc Check if the base device exists and if it does, check if the +%% partition exists. +%% @param Device The base device to check. +%% @param Partition The partition to check. +%% @param PartitionType The type of partition to check. +%% @param VolumeName The name of the volume to check. +%% @param MountPoint The mount point to check. +%% @param StorePath The store path to check. +%% @param Key The key to check. +%% @param Opts The options to check. +%% @returns `{ok, Binary}' on success with operation result message, or +%% `{error, Binary}' on failure with error message. +-spec check_base_device( + term(), term(), term(), term(), term(), term(), term(), map() +) -> {ok, binary()} | {error, binary()}. +check_base_device( + Device, Partition, PartitionType, VolumeName, MountPoint, StorePath, + Key, Opts +) -> + ?event(debug_volume, + {check_base_device, entry, {checking_device, Device}} + ), + case hb_volume:check_for_device(Device) of + false -> + % Base device doesn't exist + ?event(debug_volume, + {check_base_device, device_not_found, Device} + ), + {error, <<"Base device not found">>}; + true -> + ?event(debug_volume, + {check_base_device, device_found, + {proceeding_to_partition_check, Device} + } + ), + check_partition( + Device, Partition, PartitionType, VolumeName, + MountPoint, StorePath, Key, Opts + ) + end. + +%% @doc Check if the partition exists. If it does, attempt to mount it. +%% If it doesn't exist, create it, format it with encryption and mount it. +%% @param Device The base device to check. +%% @param Partition The partition to check. +%% @param PartitionType The type of partition to check. +%% @param VolumeName The name of the volume to check. +%% @param MountPoint The mount point to check. +%% @param StorePath The store path to check. +%% @param Key The key to check. +%% @param Opts The options to check. +%% @returns `{ok, Binary}' on success with operation result message, or +%% `{error, Binary}' on failure with error message. +-spec check_partition( + term(), term(), term(), term(), term(), term(), term(), map() +) -> {ok, binary()} | {error, binary()}. +check_partition( + Device, Partition, PartitionType, VolumeName, MountPoint, StorePath, + Key, Opts +) -> + ?event(debug_volume, + {check_partition, entry, {checking_partition, Partition}} + ), + case hb_volume:check_for_device(Partition) of + true -> + ?event(debug_volume, + {check_partition, partition_exists, + {mounting_existing, Partition} + } + ), + % Partition exists, try mounting it + mount_existing_partition( + Partition, Key, MountPoint, VolumeName, StorePath, Opts + ); + false -> + ?event(debug_volume, + {check_partition, partition_not_exists, + {creating_new, Partition} + } + ), + % Partition doesn't exist, create it + create_and_mount_partition( + Device, Partition, PartitionType, Key, + MountPoint, VolumeName, StorePath, Opts + ) + end. + +%% @doc Mount an existing partition. +%% @param Partition The partition to mount. +%% @param Key The key to mount. +%% @param MountPoint The mount point to mount. +%% @param VolumeName The name of the volume to mount. +%% @param StorePath The store path to mount. +%% @param Opts The options to mount. +%% @returns `{ok, Binary}' on success with operation result message, or +%% `{error, Binary}' on failure with error message. +-spec mount_existing_partition( + term(), term(), term(), term(), term(), map() +) -> {ok, binary()} | {error, binary()}. +mount_existing_partition( + Partition, Key, MountPoint, VolumeName, StorePath, Opts +) -> + ?event(debug_volume, + {mount_existing_partition, entry, + {attempting_mount, Partition, MountPoint} + } + ), + case hb_volume:mount_disk(Partition, Key, MountPoint, VolumeName) of + {ok, MountResult} -> + ?event(debug_volume, + {mount_existing_partition, mount_success, MountResult} + ), + update_store_path(StorePath, Opts); + {error, MountError} -> + ?event(debug_volume, + {mount_existing_partition, mount_error, + {error, MountError} + } + ), + {error, <<"Failed to mount volume">>} + end. + +%% @doc Create, format and mount a new partition. +%% @param Device The device to create the partition on. +%% @param Partition The partition to create. +%% @param PartitionType The type of partition to create. +%% @param Key The key to create the partition with. +%% @param MountPoint The mount point to mount the partition to. +%% @param VolumeName The name of the volume to mount. +%% @param StorePath The store path to mount. +%% @param Opts The options to mount. +%% @returns `{ok, Binary}' on success with operation result message, or +%% `{error, Binary}' on failure with error message. +-spec create_and_mount_partition( + term(), term(), term(), term(), term(), term(), term(), map() +) -> {ok, binary()} | {error, binary()}. +create_and_mount_partition( + Device, Partition, PartitionType, Key, + MountPoint, VolumeName, StorePath, Opts +) -> + ?event(debug_volume, + {create_and_mount_partition, entry, + {creating_partition, Device, PartitionType} + } + ), + case hb_volume:create_partition(Device, PartitionType) of + {ok, PartitionResult} -> + ?event(debug_volume, + {create_and_mount_partition, partition_created, + PartitionResult + } + ), + format_and_mount( + Partition, Key, MountPoint, VolumeName, StorePath, Opts + ); + {error, PartitionError} -> + ?event(debug_volume, + {create_and_mount_partition, partition_error, + {error, PartitionError} + } + ), + {error, <<"Failed to create partition">>} + end. + +%% @doc Format and mount a newly created partition. +%% @param Partition The partition to format and mount. +%% @param Key The key to format and mount the partition with. +%% @param MountPoint The mount point to mount the partition to. +%% @param VolumeName The name of the volume to mount. +%% @param StorePath The store path to mount. +%% @param Opts The options to mount. +%% @returns `{ok, Binary}' on success with operation result message, or +%% `{error, Binary}' on failure with error message. +-spec format_and_mount( + term(), term(), term(), term(), term(), map() +) -> {ok, binary()} | {error, binary()}. +format_and_mount( + Partition, Key, MountPoint, VolumeName, StorePath, Opts +) -> + ?event(debug_volume, + {format_and_mount, entry, {formatting_partition, Partition}} + ), + case hb_volume:format_disk(Partition, Key) of + {ok, FormatResult} -> + ?event(debug_volume, + {format_and_mount, format_success, + {result, FormatResult} + } + ), + mount_formatted_partition( + Partition, Key, MountPoint, VolumeName, StorePath, Opts + ); + {error, FormatError} -> + ?event(debug_volume, + {format_and_mount, format_error, + {error, FormatError} + } + ), + {error, <<"Failed to format disk">>} + end. + +%% @doc Mount a newly formatted partition. +%% @param Partition The partition to mount. +%% @param Key The key to mount the partition with. +%% @param MountPoint The mount point to mount the partition to. +%% @param VolumeName The name of the volume to mount. +%% @param StorePath The store path to mount. +%% @param Opts The options to mount. +%% @returns `{ok, Binary}' on success with operation result message, or +%% `{error, Binary}' on failure with error message. +-spec mount_formatted_partition( + term(), term(), term(), term(), term(), map() +) -> {ok, binary()} | {error, binary()}. +mount_formatted_partition( + Partition, Key, MountPoint, VolumeName, StorePath, Opts +) -> + ?event(debug_volume, + {mount_formatted_partition, entry, + {mounting_formatted, Partition, MountPoint} + } + ), + case hb_volume:mount_disk(Partition, Key, MountPoint, VolumeName) of + {ok, RetryMountResult} -> + ?event(debug_volume, + {mount_formatted_partition, mount_success, + {result, RetryMountResult} + } + ), + update_store_path(StorePath, Opts); + {error, RetryMountError} -> + ?event(debug_volume, + {mount_formatted_partition, mount_error, + {error, RetryMountError} + } + ), + {error, <<"Failed to mount newly formatted volume">>} + end. + +%% @doc Update the store path to use the mounted volume. +%% @param StorePath The store path to update. +%% @param Opts The options to update. +%% @returns `{ok, Binary}' on success with operation result message, or +%% `{error, Binary}' on failure with error message. +-spec update_store_path(term(), map()) -> + {ok, binary()} | {error, binary()}. +update_store_path(StorePath, Opts) -> + ?event(debug_volume, + {update_store_path, entry, {updating_store, StorePath}} + ), + CurrentStore = hb_opts:get(store, [], Opts), + ?event(debug_volume, + {update_store_path, current_store, CurrentStore} + ), + case hb_volume:change_node_store(StorePath, CurrentStore) of + {ok, #{<<"store">> := NewStore} = StoreResult} -> + ?event(debug_volume, + {update_store_path, store_change_success, + {result, StoreResult} + } + ), + update_node_config(StorePath, NewStore, Opts); + {error, StoreError} -> + ?event(debug_volume, + {update_store_path, store_change_error, + {error, StoreError} + } + ), + {error, <<"Failed to update store">>} + end. + +%% @doc Update the node's configuration with the new store. +%% @param NewStore The new store to update the node's configuration with. +%% @param Opts The options to update the node's configuration with. +%% @returns `{ok, Binary}' on success with operation result message, or +%% `{error, Binary}' on failure with error message. +-spec update_node_config(term(), term(), map()) -> + {ok, binary()} | {error, binary()}. +update_node_config(StorePath, NewStore, Opts) -> + ?event(debug_volume, + {update_node_config, entry, + {updating_config, StorePath, NewStore} + } + ), + GenesisWasmDBDir = + hb_opts:get( + genesis_wasm_db_dir, + "cache-mainnet/genesis-wasm", + Opts + ), + ?event(debug_volume, + {update_node_config, genesis_dir, GenesisWasmDBDir} + ), + BinaryGenesisWasmDBDir = list_to_binary(GenesisWasmDBDir), + FullGenesisPath = + <>, + ?event(debug_volume, + {update_node_config, full_path_created, FullGenesisPath} + ), + ok = + hb_http_server:set_opts( + Opts#{ + store => NewStore, + genesis_wasm_db_dir => FullGenesisPath + } + ), + ?event(debug_volume, + {update_node_config, config_updated, success} + ), + {ok, <<"Volume mounted and store updated successfully">>}. \ No newline at end of file diff --git a/src/dev_wasi.erl b/src/dev_wasi.erl index 86f96dbc8..cfb522c72 100644 --- a/src/dev_wasi.erl +++ b/src/dev_wasi.erl @@ -1,6 +1,6 @@ %%% @doc A virtual filesystem device. %%% Implements a file-system-as-map structure, which is traversible externally. -%%% Each file is a binary and each directory is a Converge message. +%%% Each file is a binary and each directory is an AO-Core message. %%% Additionally, this module adds a series of WASI-preview-1 compatible %%% functions for accessing the filesystem as imported functions by WASM %%% modules. @@ -22,17 +22,17 @@ -define(INIT_FDS, #{ - 0 => #{ - <<"Filename">> => <<"/dev/stdin">>, - <<"Offset">> => 0 + <<"0">> => #{ + <<"filename">> => <<"/dev/stdin">>, + <<"offset">> => 0 }, - 1 => #{ - <<"Filename">> => <<"/dev/stdout">>, - <<"Offset">> => 0 + <<"1">> => #{ + <<"filename">> => <<"/dev/stdout">>, + <<"offset">> => 0 }, - 2 => #{ - <<"Filename">> => <<"/dev/stderr">>, - <<"Offset">> => 0 + <<"2">> => #{ + <<"filename">> => <<"/dev/stderr">>, + <<"offset">> => 0 } } ). @@ -44,25 +44,25 @@ init(M1, _M2, Opts) -> ?event(running_init), MsgWithLib = - hb_converge:set( + hb_ao:set( M1, #{ - <<"WASM/stdlib/wasi_snapshot_preview1">> => - #{ device => <<"WASI/1.0">>} + <<"wasm/stdlib/wasi_snapshot_preview1">> => + #{ <<"device">> => <<"wasi@1.0">>} }, Opts ), MsgWithFDs = - hb_converge:set( + hb_ao:set( MsgWithLib, - <<"File-Descriptors">>, + <<"file-descriptors">>, ?INIT_FDS, Opts ), CompleteMsg = - hb_converge:set( + hb_ao:set( MsgWithFDs, - <<"VFS">>, + <<"vfs">>, ?INIT_VFS, Opts ), @@ -71,74 +71,52 @@ init(M1, _M2, Opts) -> compute(Msg1) -> {ok, Msg1}. -% %% @doc Encode the input message for inclusion in the VFS. -% execute(M1, M2, Opts) -> -% case hb_converge:get(<<"Pass">>, M1, Opts) of -% 1 -> -% MsgToProc = hb_converge:get(<<"Message">>, M2, Opts), -% JSON = -% ar_bundles:serialize( -% hb_message:convert(MsgToProc, tx, converge, #{}), -% json -% ), -% ?event(setting_message_vfs_key), -% {ok, -% hb_converge:set( -% M1, -% <<"vfs/message">>, -% JSON, -% Opts -% ) -% }; -% _ -> {ok, M1} -% end. - %% @doc Return the stdout buffer from a state message. stdout(M) -> - hb_converge:get(<<"vfs/dev/stdout">>, M). + hb_ao:get(<<"vfs/dev/stdout">>, M). %% @doc Adds a file descriptor to the state message. %path_open(M, Instance, [FDPtr, LookupFlag, PathPtr|_]) -> path_open(Msg1, Msg2, Opts) -> - FDs = hb_converge:get(<<"File-Descriptors">>, Msg1, Opts), - Instance = hb_private:get(<<"Instance">>, Msg1, Opts), - [FDPtr, LookupFlag, PathPtr|_] = hb_converge:get(<<"Args">>, Msg2, Opts), + FDs = hb_ao:get(<<"file-descriptors">>, Msg1, Opts), + Instance = hb_private:get(<<"instance">>, Msg1, Opts), + [FDPtr, LookupFlag, PathPtr|_] = hb_ao:get(<<"args">>, Msg2, Opts), ?event({path_open, FDPtr, LookupFlag, PathPtr}), Path = hb_beamr_io:read_string(Instance, PathPtr), ?event({path_open, Path}), FD = #{ - index := Index + <<"index">> := Index } = - case hb_converge:get(<<"vfs/", Path/binary>>, Msg1) of + case hb_ao:get(<<"vfs/", Path/binary>>, Msg1, Opts) of not_found -> #{ - index => length(hb_converge:keys(FDs)) + 1, - filename => Path, - offset => 0 + <<"index">> => length(hb_ao:keys(FDs)) + 1, + <<"filename">> => Path, + <<"offset">> => 0 }; F -> F end, { ok, #{ - state => - hb_converge:set( + <<"state">> => + hb_ao:set( Msg1, <<"vfs/", Path/binary>>, FD ), - results => [0, Index] + <<"results">> => [0, Index] } }. %% @doc WASM stdlib implementation of `fd_write', using the WASI-p1 standard %% interface. fd_write(Msg1, Msg2, Opts) -> - State = hb_converge:get(<<"State">>, Msg1, Opts), - Instance = hb_private:get(<<"WASM/Instance">>, State, Opts), - [FD, Ptr, Vecs, RetPtr|_] = hb_converge:get(<<"Args">>, Msg2, Opts), + State = hb_ao:get(<<"state">>, Msg1, Opts), + Instance = hb_private:get(<<"wasm/instance">>, State, Opts), + [FD, Ptr, Vecs, RetPtr|_] = hb_ao:get(<<"args">>, Msg2, Opts), ?event({fd_write, {fd, FD}, {ptr, Ptr}, {vecs, Vecs}, {retptr, RetPtr}}), - Signature = hb_converge:get(<<"func_sig">>, Msg2, Opts), + Signature = hb_ao:get(<<"func-sig">>, Msg2, Opts), ?event({signature, Signature}), fd_write(State, Instance, [FD, Ptr, Vecs, RetPtr], 0, Opts). @@ -148,31 +126,31 @@ fd_write(S, Instance, [_, _Ptr, 0, RetPtr], BytesWritten, _Opts) -> RetPtr, <> ), - {ok, #{ state => S, results => [0] }}; + {ok, #{ <<"state">> => S, <<"results">> => [0] }}; fd_write(S, Instance, [FDnum, Ptr, Vecs, RetPtr], BytesWritten, Opts) -> FDNumStr = integer_to_binary(FDnum), - FD = hb_converge:get(<<"File-Descriptors/", FDNumStr/binary>>, S, Opts), - Filename = hb_converge:get(<<"Filename">>, FD, Opts), - StartOffset = hb_converge:get(<<"Offset">>, FD, Opts), + FD = hb_ao:get(<<"file-descriptors/", FDNumStr/binary>>, S, Opts), + Filename = hb_ao:get(<<"filename">>, FD, Opts), + StartOffset = hb_ao:get(<<"offset">>, FD, Opts), {VecPtr, Len} = parse_iovec(Instance, Ptr), {ok, Data} = hb_beamr_io:read(Instance, VecPtr, Len), Before = binary:part( - OrigData = hb_converge:get(<<"Data">>, FD, Opts), + OrigData = hb_ao:get(<<"data">>, FD, Opts), 0, StartOffset ), After = binary:part(OrigData, StartOffset, byte_size(OrigData) - StartOffset), S1 = - hb_converge:set( + hb_ao:set( S, - <<"File-Descriptors/", FDNumStr/binary, "/Offset">>, + <<"file-descriptors/", FDNumStr/binary, "/offset">>, StartOffset + byte_size(Data), Opts ), S2 = - hb_converge:set( + hb_ao:set( S1, <<"vfs/", Filename/binary>>, <>, @@ -188,10 +166,10 @@ fd_write(S, Instance, [FDnum, Ptr, Vecs, RetPtr], BytesWritten, Opts) -> %% @doc Read from a file using the WASI-p1 standard interface. fd_read(Msg1, Msg2, Opts) -> - State = hb_converge:get(<<"State">>, Msg1, Opts), - Instance = hb_private:get(<<"WASM/Instance">>, State, Opts), - [FD, VecsPtr, NumVecs, RetPtr|_] = hb_converge:get(<<"Args">>, Msg2, Opts), - Signature = hb_converge:get(<<"func_sig">>, Msg2, Opts), + State = hb_ao:get(<<"state">>, Msg1, Opts), + Instance = hb_private:get(<<"wasm/instance">>, State, Opts), + [FD, VecsPtr, NumVecs, RetPtr|_] = hb_ao:get(<<"args">>, Msg2, Opts), + Signature = hb_ao:get(<<"func-sig">>, Msg2, Opts), ?event({signature, Signature}), fd_read(State, Instance, [FD, VecsPtr, NumVecs, RetPtr], 0, Opts). @@ -199,28 +177,28 @@ fd_read(S, Instance, [FD, _VecsPtr, 0, RetPtr], BytesRead, _Opts) -> ?event({{completed_read, FD, BytesRead}}), hb_beamr_io:write(Instance, RetPtr, <>), - {ok, #{ state => S, results => [0] }}; + {ok, #{ <<"state">> => S, <<"results">> => [0] }}; fd_read(S, Instance, [FDNum, VecsPtr, NumVecs, RetPtr], BytesRead, Opts) -> ?event({fd_read, FDNum, VecsPtr, NumVecs, RetPtr}), % Parse the request FDNumStr = integer_to_binary(FDNum), Filename = - hb_converge:get( - <<"File-Descriptors/", FDNumStr/binary, "/Filename">>, S, Opts), + hb_ao:get( + <<"file-descriptors/", FDNumStr/binary, "/filename">>, S, Opts), {VecPtr, Len} = parse_iovec(Instance, VecsPtr), % Read the bytes from the file - Data = hb_converge:get(<<"vfs/", Filename/binary>>, S, Opts), + Data = hb_ao:get(<<"vfs/", Filename/binary>>, S, Opts), Offset = - hb_converge:get( - <<"File-Descriptors/", FDNumStr/binary, "/Offset">>, S, Opts), + hb_ao:get( + <<"file-descriptors/", FDNumStr/binary, "/offset">>, S, Opts), ReadSize = min(Len, byte_size(Data) - Offset), Bin = binary:part(Data, Offset, ReadSize), % Write the bytes to the WASM Instance ok = hb_beamr_io:write(Instance, VecPtr, Bin), fd_read( - hb_converge:set( + hb_ao:set( S, - <<"File-Descriptors/", FDNumStr/binary, "/Offset">>, + <<"file-descriptors/", FDNumStr/binary, "/offset">>, Offset + ReadSize, Opts ), @@ -242,8 +220,8 @@ parse_iovec(Instance, Ptr) -> %%% Misc WASI-preview-1 handlers. clock_time_get(Msg1, _Msg2, Opts) -> ?event({clock_time_get, {returning, 1}}), - State = hb_converge:get(<<"State">>, Msg1, Opts), - {ok, #{ state => State, results => [1] }}. + State = hb_ao:get(<<"state">>, Msg1, Opts), + {ok, #{ <<"state">> => State, <<"results">> => [1] }}. %%% Tests @@ -254,44 +232,40 @@ generate_wasi_stack(File, Func, Params) -> init(), Msg0 = dev_wasm:cache_wasm_image(File), Msg1 = Msg0#{ - device => <<"Stack/1.0">>, - <<"Device-Stack">> => [<<"WASI/1.0">>, <<"WASM-64/1.0">>], - <<"Output-Prefixes">> => [<<"WASM">>, <<"WASM">>], - <<"Stack-Keys">> => [<<"Init">>, <<"Compute">>], - <<"WASM-Function">> => Func, - <<"WASM-Params">> => Params + <<"device">> => <<"stack@1.0">>, + <<"device-stack">> => [<<"wasi@1.0">>, <<"wasm-64@1.0">>], + <<"output-prefixes">> => [<<"wasm">>, <<"wasm">>], + <<"stack-keys">> => [<<"init">>, <<"compute">>], + <<"function">> => Func, + <<"params">> => Params }, - {ok, Msg2} = hb_converge:resolve(Msg1, <<"Init">>, #{}), + {ok, Msg2} = hb_ao:resolve(Msg1, <<"init">>, #{}), Msg2. vfs_is_serializable_test() -> StackMsg = generate_wasi_stack("test/test-print.wasm", <<"hello">>, []), - VFSMsg = hb_converge:get(<<"VFS">>, StackMsg), + VFSMsg = hb_ao:get(<<"vfs">>, StackMsg), VFSMsg2 = hb_message:minimize( hb_message:convert( - hb_message:convert(VFSMsg, tx, converge, #{}), - converge, - tx, + hb_message:convert(VFSMsg, <<"httpsig@1.0">>, #{}), + <<"structured@1.0">>, + <<"httpsig@1.0">>, #{}) ), ?assert(hb_message:match(VFSMsg, VFSMsg2)). wasi_stack_is_serializable_test() -> Msg = generate_wasi_stack("test/test-print.wasm", <<"hello">>, []), - Msg2 = hb_message:convert( - hb_message:convert(Msg, tx, converge, #{}), - converge, - tx, - #{}), + HTTPSigMsg = hb_message:convert(Msg, <<"httpsig@1.0">>, #{}), + Msg2 = hb_message:convert(HTTPSigMsg, <<"structured@1.0">>, <<"httpsig@1.0">>, #{}), ?assert(hb_message:match(Msg, Msg2)). - basic_aos_exec_test() -> Init = generate_wasi_stack("test/aos-2-pure-xs.wasm", <<"handle">>, []), - Msg = gen_test_aos_msg("return 1+1"), + Msg = gen_test_aos_msg("return 1 + 1"), Env = gen_test_env(), - Instance = hb_private:get(<<"WASM/Instance">>, Init, #{}), + Instance = hb_private:get(<<"wasm/instance">>, Init, #{}), {ok, Ptr1} = hb_beamr_io:malloc(Instance, byte_size(Msg)), ?assertNotEqual(0, Ptr1), hb_beamr_io:write(Instance, Ptr1, Msg), @@ -303,13 +277,13 @@ basic_aos_exec_test() -> {ok, EnvBin} = hb_beamr_io:read(Instance, Ptr2, byte_size(Env)), ?assertEqual(Env, EnvBin), ?assertEqual(Msg, MsgBin), - Ready = Init#{ <<"WASM-Params">> => [Ptr1, Ptr2] }, - {ok, StateRes} = hb_converge:resolve(Ready, <<"Compute">>, #{}), - [Ptr] = hb_converge:get(<<"Results/WASM/Output">>, StateRes), + Ready = Init#{ <<"parameters">> => [Ptr1, Ptr2] }, + {ok, StateRes} = hb_ao:resolve(Ready, <<"compute">>, #{}), + [Ptr] = hb_ao:get(<<"results/wasm/output">>, StateRes), {ok, Output} = hb_beamr_io:read_string(Instance, Ptr), ?event({got_output, Output}), #{ <<"response">> := #{ <<"Output">> := #{ <<"data">> := Data }} } - = jiffy:decode(Output, [return_maps]), + = hb_json:decode(Output), ?assertEqual(<<"2">>, Data). %%% Test Helpers diff --git a/src/dev_wasm.erl b/src/dev_wasm.erl index 33e8458ad..e4f33143d 100644 --- a/src/dev_wasm.erl +++ b/src/dev_wasm.erl @@ -1,54 +1,54 @@ %%% @doc A device that executes a WASM image on messages using the Memory-64 -%%% preview standard. In the backend, this device uses `beamr`: An Erlang wrapper +%%% preview standard. In the backend, this device uses `beamr': An Erlang wrapper %%% for WAMR, the WebAssembly Micro Runtime. %%% %%% The device has the following requirements and interface: -%%% ``` +%%% 
 %%%     M1/Init ->
 %%%         Assumes:
-%%%             M1/Process
-%%%             M1/[Prefix]/Image
+%%%             M1/process
+%%%             M1/[Prefix]/image
 %%%         Generates:
-%%%             /priv/WASM/Instance
-%%%             /priv/WASM/Import-Resolver
+%%%             /priv/[Prefix]/instance
+%%%             /priv/[Prefix]/import-resolver
 %%%         Side-effects:
 %%%             Creates a WASM executor loaded in memory of the HyperBEAM node.
 %%% 
 %%%     M1/Compute ->
 %%%         Assumes:
-%%%             M1/priv/WASM/Instance
-%%%             M1/priv/WASM/Import-Resolver
-%%%             M1/Process
-%%%             M2/Message
-%%%             M2/Message/WASM-Function OR M1/WASM-Function
-%%%             M2/Message/WASM-Params OR M1/WASM-Params
+%%%             M1/priv/[Prefix]/instance
+%%%             M1/priv/[Prefix]/import-resolver
+%%%             M1/process
+%%%             M2/message
+%%%             M2/message/function OR M1/function
+%%%             M2/message/parameters OR M1/parameters
 %%%         Generates:
-%%%             /Results/WASM/Type
-%%%             /Results/WASM/Body
+%%%             /results/[Prefix]/type
+%%%             /results/[Prefix]/output
 %%%         Side-effects:
 %%%             Calls the WASM executor with the message and process.
-%%%     M1/WASM/State ->
+%%%     M1/[Prefix]/state ->
 %%%         Assumes:
-%%%             M1/priv/WASM/Instance
+%%%             M1/priv/[Prefix]/instance
 %%%         Generates:
 %%%             Raw binary WASM state
-%%% '''
+%%%
-module(dev_wasm). -export([info/2, init/3, compute/3, import/3, terminate/3, snapshot/3, normalize/3]). %%% API for other devices: -export([instance/3]). %%% Test API: --export([cache_wasm_image/1]). +-export([cache_wasm_image/1, cache_wasm_image/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). %% @doc Export all functions aside the `instance/3' function. info(_Msg1, _Opts) -> #{ - exclude => [instance] + excludes => [instance] }. -%% @doc Boot a WASM image on the image stated in the `Process/Image' field of +%% @doc Boot a WASM image on the image stated in the `process/image' field of %% the message. init(M1, M2, Opts) -> ?event(running_init), @@ -58,46 +58,71 @@ init(M1, M2, Opts) -> Prefix = dev_stack:prefix(M1, M2, Opts), ?event({in_prefix, InPrefix}), ImageBin = - case hb_converge:get(<>, M1, Opts) of + case hb_ao:get(<>, M1, Opts) of not_found -> - throw( - { - wasm_init_error, - << - "No viable image found in ", - InPrefix/binary, - "/Image." - >>, - {msg1, M1} - } - ); + case hb_ao:get(<<"body">>, M1, Opts) of + not_found -> + throw( + { + wasm_init_error, + << + "No viable image found in ", + InPrefix/binary, + "/image." + >>, + {msg1, M1} + } + ); + Bin when is_binary(Bin) -> Bin + end; ImageID when ?IS_ID(ImageID) -> ?event({getting_wasm_image, ImageID}), {ok, ImageMsg} = hb_cache:read(ImageID, Opts), - hb_converge:get(<<"Body">>, ImageMsg, Opts); + hb_ao:get(<<"body">>, ImageMsg, Opts); ImageMsg when is_map(ImageMsg) -> ?event(wasm_image_message_directly_provided), - hb_converge:get(<<"Body">>, ImageMsg, Opts); + hb_ao:get(<<"body">>, ImageMsg, Opts); Image when is_binary(Image) -> ?event(wasm_image_binary_directly_provided), Image end, + Mode = + case hb_ao:get(<>, M1, Opts) of + not_found -> wasm; + <<"WASM">> -> wasm; + <<"AOT">> -> + case hb_opts:get(wasm_allow_aot, false, Opts) of + true -> aot; + false -> wasm + end + end, % Start the WASM executor. - {ok, Instance, _Imports, _Exports} = hb_beamr:start(ImageBin), + {ok, Instance, _Imports, _Exports} = hb_beamr:start(ImageBin, Mode), % Set the WASM Instance, handler, and standard library invokation function. ?event({setting_wasm_instance, Instance, {prefix, Prefix}}), {ok, hb_private:set(M1, #{ - <> => Instance, - <> => + <> => + fun(Binary) -> + {ok, Ptr} = hb_beamr_io:write_string(Instance, Binary), + {ok, Ptr} + end, + <> => + fun Reader([Ptr]) -> Reader(Ptr); + Reader(Ptr) -> + {ok, Binary} = hb_beamr_io:read_string(Instance, Ptr), + {ok, Binary} + end, + <> => Instance, + <> => fun default_import_resolver/3 }, Opts ) }. -%% @doc Take a BEAMR import call and resolve it using `hb_converge`. +%% @doc Take a BEAMR import call and resolve it using `hb_ao'. default_import_resolver(Msg1, Msg2, Opts) -> #{ instance := WASM, @@ -108,78 +133,104 @@ default_import_resolver(Msg1, Msg2, Opts) -> } = Msg2, Prefix = dev_stack:prefix(Msg1, Msg2, Opts), {ok, Msg3} = - hb_converge:resolve( + hb_ao:resolve( hb_private:set( Msg1, - #{ <> => WASM }, + #{ <> => WASM }, Opts ), #{ - path => import, - module => list_to_binary(Module), - func => list_to_binary(Func), - args => Args, - func_sig => list_to_binary(Signature) + <<"path">> => <<"import">>, + <<"module">> => list_to_binary(Module), + <<"func">> => list_to_binary(Func), + <<"args">> => Args, + <<"func-sig">> => list_to_binary(Signature) }, Opts ), - NextState = hb_converge:get(state, Msg3, Opts), - Response = hb_converge:get(results, Msg3, Opts), + NextState = hb_ao:get(state, Msg3, Opts), + Response = hb_ao:get(results, Msg3, Opts), {ok, Response, NextState}. %% @doc Call the WASM executor with a message that has been prepared by a prior %% pass. compute(RawM1, M2, Opts) -> - % Normalize the message to have an open WASM instance, but no literal `State`. + % Normalize the message to have an open WASM instance, but no literal `State'. % The hashpath is not updated during this process. This allows us to take % two different messages and get the same result: - % - A message with a `State' key but no WASM instance in `priv/`. - % - A message with a WASM instance in `priv/` but no `State' key. + % - A message with a `State' key but no WASM instance in `priv/'. + % - A message with a WASM instance in `priv/' but no `State' key. {ok, M1} = normalize(RawM1, M2, Opts), ?event(running_compute), Prefix = dev_stack:prefix(M1, M2, Opts), - case hb_converge:get(pass, M1, Opts) of + case hb_ao:get(pass, M1, Opts) of X when X == 1 orelse X == not_found -> % Extract the WASM Instance, func, params, and standard library % invokation from the message and apply them with the WASM executor. WASMFunction = - case hb_converge:get(<<"Message/WASM-Function">>, M2, Opts) of - not_found -> - hb_converge:get(<<"WASM-Function">>, M1, Opts); - Func -> Func - end, + hb_ao:get_first( + [ + {M2, <<"body/function">>}, + {M2, <<"function">>}, + {M1, <<"function">>} + ], + Opts + ), WASMParams = - case hb_converge:get(<<"Message/WASM-Params">>, M2, Opts) of - not_found -> - hb_converge:get(<<"WASM-Params">>, M1, Opts); - Params -> Params - end, - ?event( - { - calling_wasm_executor, - {prefix, Prefix}, - {m1, M1}, - {m2, M2}, - {priv, hb_private:from_message(M1)} - } - ), - {ResType, Res, MsgAfterExecution} = - hb_beamr:call( - instance(M1, M2, Opts), - WASMFunction, - WASMParams, - hb_private:get(<>, M1, Opts), - M1, + hb_ao:get_first( + [ + {M2, <<"body/parameters">>}, + {M2, <<"parameters">>}, + {M1, <<"parameters">>} + ], Opts ), - {ok, - hb_converge:set(MsgAfterExecution, - #{ - <<"Results/", Prefix/binary, "/Type">> => ResType, - <<"Results/", Prefix/binary, "/Output">> => Res + case WASMFunction of + not_found -> + ?event( + { + skipping_wasm_exec, + {reason, wasm_function_not_provided}, + {prefix, Prefix}, + {m1, M1}, + {m2, M2} + } + ), + {ok, M1}; + _ -> + ?event( + { + calling_wasm_executor, + {prefix, Prefix}, + {wasm_function, {explicit, WASMFunction}}, + {wasm_params, WASMParams}, + {m1, M1}, + {m2, M2}, + {priv, hb_private:from_message(M1)} + } + ), + {ResType, Res, MsgAfterExecution} = + hb_beamr:call( + instance(M1, M2, Opts), + WASMFunction, + case WASMParams of + not_found -> []; + Params -> Params + end, + hb_private:get(<>, M1, Opts), + M1, + Opts + ), + {ok, + hb_ao:set(MsgAfterExecution, + #{ + <<"results/", Prefix/binary, "/type">> => ResType, + <<"results/", Prefix/binary, "/output">> => Res + }, + Opts + ) } - ) - }; + end; _ -> {ok, M1} end. @@ -191,23 +242,23 @@ normalize(RawM1, M2, Opts) -> case instance(RawM1, M2, Opts) of not_found -> DeviceKey = - case hb_converge:get(<<"Device-Key">>, RawM1, Opts) of + case hb_ao:get(<<"device-key">>, RawM1, Opts) of not_found -> []; Key -> [Key] end, - ?event(snapshot, + ?event( {no_instance_attempting_to_get_snapshot, {msg1, RawM1}, {device_key, DeviceKey} } ), Memory = - hb_converge:get( - [<<"Snapshot">>] ++ DeviceKey ++ [<<"body">>], + hb_ao:get( + [<<"snapshot">>] ++ DeviceKey ++ [<<"body">>], {as, dev_message, RawM1}, Opts ), case Memory of - not_found -> throw({error, no_wasm_instance_or_}); + not_found -> throw({error, no_wasm_instance_or_snapshot}); State -> {ok, M1} = init(RawM1, State, Opts), Res = hb_beamr:deserialize(instance(M1, M2, Opts), State), @@ -218,7 +269,7 @@ normalize(RawM1, M2, Opts) -> ?event(wasm_instance_found_not_deserializing), RawM1 end, - dev_message:set(M3, #{ <<"Snapshot">> => unset }, Opts). + dev_message:set(M3, #{ <<"snapshot">> => unset }, Opts). %% @doc Serialize the WASM state to a binary. snapshot(M1, M2, Opts) -> @@ -227,7 +278,7 @@ snapshot(M1, M2, Opts) -> {ok, Serialized} = hb_beamr:serialize(Instance), {ok, #{ - body => Serialized + <<"body">> => Serialized } }. @@ -239,17 +290,17 @@ terminate(M1, M2, Opts) -> hb_beamr:stop(Instance), {ok, hb_private:set(M1, #{ - <> => unset + <> => unset }, Opts )}. %% @doc Get the WASM instance from the message. Note that this function is exported -%% such that other devices can use it, but it is excluded from calls from Converge +%% such that other devices can use it, but it is excluded from calls from AO-Core %% resolution directly. instance(M1, M2, Opts) -> Prefix = dev_stack:prefix(M1, M2, Opts), - Path = <>, + Path = <>, ?event({searching_for_instance, Path, M1}), hb_private:get(Path, M1, Opts#{ hashpath => ignore }). @@ -261,8 +312,8 @@ instance(M1, M2, Opts) -> %% 5. If it fails with `not_found', call the stub handler. import(Msg1, Msg2, Opts) -> % 1. Adjust the path to the stdlib. - ModName = hb_converge:get(<<"Module">>, Msg2, Opts), - FuncName = hb_converge:get(<<"Func">>, Msg2, Opts), + ModName = hb_ao:get(<<"module">>, Msg2, Opts), + FuncName = hb_ao:get(<<"func">>, Msg2, Opts), Prefix = dev_stack:prefix(Msg1, Msg2, Opts), AdjustedPath = << @@ -272,24 +323,18 @@ import(Msg1, Msg2, Opts) -> "/", FuncName/binary >>, - StatePath = - << - Prefix/binary, - "/stdlib/", - ModName/binary, - "/State" - >>, - AdjustedMsg2 = Msg2#{ path => AdjustedPath }, + StatePath = << Prefix/binary, "/stdlib/", ModName/binary, "/state" >>, + AdjustedMsg2 = Msg2#{ <<"path">> => AdjustedPath }, % 2. Add the current state to the message at the stdlib path. AdjustedMsg1 = - hb_converge:set( + hb_ao:set( Msg1, #{ StatePath => Msg1 }, Opts#{ hashpath => ignore } ), - %?event({state_added_msg1, AdjustedMsg1}), + ?event({state_added_msg1, AdjustedMsg1, AdjustedMsg2}), % 3. Resolve the adjusted path against the added state. - case hb_converge:resolve(AdjustedMsg1, AdjustedMsg2, Opts) of + case hb_ao:resolve(AdjustedMsg1, AdjustedMsg2, Opts) of {ok, Res} -> % 4. Success. Return. {ok, Res}; @@ -305,20 +350,21 @@ undefined_import_stub(Msg1, Msg2, Opts) -> ?event({unimplemented_dev_wasm_call, {msg1, Msg1}, {msg2, Msg2}}), Prefix = dev_stack:prefix(Msg1, Msg2, Opts), UndefinedCallsPath = - <<"State/Results/", Prefix/binary, "/Undefined-Calls">>, - Msg3 = hb_converge:set( + <<"state/results/", Prefix/binary, "/undefined-calls">>, + Msg3 = hb_ao:set( Msg1, #{ UndefinedCallsPath => [ Msg2 | - case hb_converge:get(UndefinedCallsPath, Msg1, Opts) of + case hb_ao:get(UndefinedCallsPath, Msg1, Opts) of not_found -> []; X -> X end ] - } + }, + Opts ), {ok, #{ state => Msg3, results => [0] }}. @@ -328,64 +374,66 @@ init() -> application:ensure_all_started(hb), hb:init(). -init_test() -> - init(), - Msg = cache_wasm_image("test/test.wasm"), - {ok, Msg1} = hb_converge:resolve(Msg, <<"Init">>, #{}), - ?event({after_init, Msg1}), - Priv = hb_private:from_message(Msg1), - ?assertMatch( - {ok, Instance} when is_pid(Instance), - hb_converge:resolve(Priv, <<"Instance">>, #{}) - ), - ?assertMatch( - {ok, Fun} when is_function(Fun), - hb_converge:resolve(Priv, <<"Import-Resolver">>, #{}) - ). - +% Pass input_prefix_test() -> init(), - #{ image := ImageID } = cache_wasm_image("test/test.wasm"), + #{ <<"image">> := ImageID } = cache_wasm_image("test/test.wasm"), Msg1 = #{ - <<"Device">> => <<"WASM-64/1.0">>, - <<"Input-Prefix">> => <<"Test-In">>, - <<"Test-In">> => #{ <<"Image">> => ImageID } + <<"device">> => <<"wasm-64@1.0">>, + <<"input-prefix">> => <<"test-in">>, + <<"test-in">> => #{ <<"image">> => ImageID } }, - {ok, Msg2} = hb_converge:resolve(Msg1, <<"Init">>, #{}), + {ok, Msg2} = hb_ao:resolve(Msg1, <<"init">>, #{}), ?event({after_init, Msg2}), Priv = hb_private:from_message(Msg2), ?assertMatch( {ok, Instance} when is_pid(Instance), - hb_converge:resolve(Priv, <<"Instance">>, #{}) + hb_ao:resolve(Priv, <<"instance">>, #{}) ), ?assertMatch( {ok, Fun} when is_function(Fun), - hb_converge:resolve(Priv, <<"Import-Resolver">>, #{}) + hb_ao:resolve(Priv, <<"import-resolver">>, #{}) ). -%% @doc Test that realistic prefixing for a `dev_process` works -- -%% including both inputs (from `Process/`) and outputs (to the +%% @doc Test that realistic prefixing for a `dev_process' works -- +%% including both inputs (from `Process/') and outputs (to the %% Device-Key) work process_prefixes_test() -> init(), Msg1 = #{ - <<"Device">> => <<"WASM-64/1.0">>, - <<"Output-Prefix">> => <<"WASM">>, - <<"Input-Prefix">> => <<"Process">>, - <<"Process">> => cache_wasm_image("test/test.wasm") + <<"device">> => <<"wasm-64@1.0">>, + <<"output-prefix">> => <<"wasm">>, + <<"input-prefix">> => <<"process">>, + <<"process">> => cache_wasm_image("test/test.wasm") }, - {ok, Msg3} = hb_converge:resolve(Msg1, <<"Init">>, #{}), + {ok, Msg3} = hb_ao:resolve(Msg1, <<"init">>, #{}), ?event({after_init, Msg3}), Priv = hb_private:from_message(Msg3), ?assertMatch( {ok, Instance} when is_pid(Instance), - hb_converge:resolve(Priv, <<"WASM/Instance">>, #{}) + hb_ao:resolve(Priv, <<"wasm/instance">>, #{}) ), ?assertMatch( {ok, Fun} when is_function(Fun), - hb_converge:resolve(Priv, <<"WASM/Import-Resolver">>, #{}) + hb_ao:resolve(Priv, <<"wasm/import-resolver">>, #{}) + ). + + +init_test() -> + init(), + Msg = cache_wasm_image("test/test.wasm"), + {ok, Msg1} = hb_ao:resolve(Msg, <<"init">>, #{}), + ?event({after_init, Msg1}), + Priv = hb_private:from_message(Msg1), + ?assertMatch( + {ok, Instance} when is_pid(Instance), + hb_ao:resolve(Priv, <<"instance">>, #{}) + ), + ?assertMatch( + {ok, Fun} when is_function(Fun), + hb_ao:resolve(Priv, <<"import-resolver">>, #{}) ). basic_execution_test() -> @@ -409,7 +457,7 @@ imported_function_test() -> [2, 5], #{ <<"stdlib/my_lib">> => - #{ device => <<"Test-Device/1.0">> } + #{ <<"device">> => <<"test-device@1.0">> } } ) ). @@ -418,29 +466,29 @@ benchmark_test() -> BenchTime = 0.5, init(), Msg0 = cache_wasm_image("test/test-64.wasm"), - {ok, Msg1} = hb_converge:resolve(Msg0, <<"Init">>, #{}), + {ok, Msg1} = hb_ao:resolve(Msg0, <<"init">>, #{}), Msg2 = - maps:merge( + hb_maps:merge( Msg1, - hb_converge:set( - #{ - <<"WASM-Function">> => <<"fac">>, - <<"WASM-Params">> => [5.0] - }, - #{ hashpath => ignore } - ) + #{ + <<"function">> => <<"fac">>, + <<"parameters">> => [5.0] + }, + #{} ), Iterations = - hb:benchmark( + hb_test_utils:benchmark( fun() -> - hb_converge:resolve(Msg2, <<"Compute">>, #{}) + hb_ao:resolve(Msg2, <<"compute">>, #{}) end, BenchTime ), ?event(benchmark, {scheduled, Iterations}), - hb_util:eunit_print( - "Evaluated ~p WASM messages through Converge in ~p seconds (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] + hb_test_utils:benchmark_print( + <<"Through AO-Core:">>, + <<"resolutions">>, + Iterations, + BenchTime ), ?assert(Iterations > 5), ok. @@ -450,62 +498,66 @@ state_export_and_restore_test() -> % Generate a WASM message. We use the pow_calculator because it has a % reasonable amount of memory to work with. Msg0 = cache_wasm_image("test/pow_calculator.wasm"), - {ok, Msg1} = hb_converge:resolve(Msg0, <<"Init">>, #{}), + {ok, Msg1} = hb_ao:resolve(Msg0, <<"init">>, #{}), Msg2 = - maps:merge( + hb_maps:merge( Msg1, Extras = #{ - <<"WASM-Function">> => <<"pow">>, - <<"WASM-Params">> => [2, 2], + <<"function">> => <<"pow">>, + <<"parameters">> => [2, 2], <<"stdlib">> => #{ <<"my_lib">> => - #{ device => <<"Test-Device/1.0">> } + #{ <<"device">> => <<"test-device@1.0">> } } - } + }, + #{} ), ?event({after_setup, Msg2}), % Compute a computation and export the state. - {ok, Msg3a} = hb_converge:resolve(Msg2, <<"Compute">>, #{}), - ?assertEqual([4], hb_converge:get(<<"Results/Output">>, Msg3a, #{})), - {ok, State} = hb_converge:resolve(Msg3a, <<"Snapshot">>, #{}), + {ok, Msg3a} = hb_ao:resolve(Msg2, <<"compute">>, #{}), + ?assertEqual([4], hb_ao:get(<<"results/output">>, Msg3a, #{})), + {ok, State} = hb_ao:resolve(Msg3a, <<"snapshot">>, #{}), ?event({state_res, State}), % Restore the state without calling Init. - NewMsg1 = maps:merge(Msg0, Extras#{ <<"Snapshot">> => State }), + NewMsg1 = hb_maps:merge(Msg0, Extras#{ <<"snapshot">> => State }, #{}), ?assertEqual( {ok, [4]}, - hb_converge:resolve(NewMsg1, <<"Compute/Results/Output">>, #{}) + hb_ao:resolve(NewMsg1, <<"compute/results/output">>, #{}) ). %%% Test helpers cache_wasm_image(Image) -> + cache_wasm_image(Image, #{}). +cache_wasm_image(Image, Opts) -> {ok, Bin} = file:read_file(Image), - Msg = #{ <<"Body">> => Bin }, - {ok, ID} = hb_cache:write(Msg, #{}), + Msg = #{ <<"body">> => Bin }, + {ok, ID} = hb_cache:write(Msg, Opts), #{ - device => <<"WASM-64/1.0">>, - image => ID + <<"device">> => <<"wasm-64@1.0">>, + <<"image">> => ID }. test_run_wasm(File, Func, Params, AdditionalMsg) -> init(), Msg0 = cache_wasm_image(File), - {ok, Msg1} = hb_converge:resolve(Msg0, <<"Init">>, #{}), + {ok, Msg1} = hb_ao:resolve(Msg0, <<"init">>, #{}), ?event({after_init, Msg1}), Msg2 = - maps:merge( + hb_maps:merge( Msg1, - hb_converge:set( + hb_ao:set( #{ - <<"WASM-Function">> => Func, - <<"WASM-Params">> => Params + <<"function">> => Func, + <<"parameters">> => Params }, AdditionalMsg, #{ hashpath => ignore } - ) + ), + #{} ), ?event({after_setup, Msg2}), - {ok, StateRes} = hb_converge:resolve(Msg2, <<"Compute">>, #{}), + {ok, StateRes} = hb_ao:resolve(Msg2, <<"compute">>, #{}), ?event({after_resolve, StateRes}), - hb_converge:resolve(StateRes, <<"Results/Output">>, #{}). \ No newline at end of file + hb_ao:resolve(StateRes, <<"results/output">>, #{}). diff --git a/src/dev_whois.erl b/src/dev_whois.erl new file mode 100644 index 000000000..61011c489 --- /dev/null +++ b/src/dev_whois.erl @@ -0,0 +1,68 @@ +%%% @doc A device for returning the IP/host information of a requester or +%%% itself. +-module(dev_whois). +%%% Device API +-export([node/3, echo/3]). +%%% Public utilities +-export([ensure_host/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Return the calculated host information for the requester. +echo(_, Req, Opts) -> + {ok, hb_maps:get(<<"ao-peer">>, Req, <<"unknown">>, Opts)}. + +%% @doc Return the host information for the node. Sets the `host' key in the +%% node message if it is not already set. +node(_, _, Opts) -> + case ensure_host(Opts) of + {ok, NewOpts} -> + {ok, hb_opts:get(host, <<"unknown">>, NewOpts)}; + Error -> + Error + end. + +%% @doc Return the node message ensuring that the host is set. If it is not, we +%% attempt to find the host information from the specified bootstrap node. +ensure_host(Opts) -> + case hb_opts:get(host, <<"unknown">>, Opts) of + <<"unknown">> -> + case bootstrap_node_echo(Opts) of + {ok, Host} -> + % Set the host information in the persisted node message. + hb_http_server:set_opts(NewOpts = Opts#{ host => Host }), + {ok, NewOpts}; + Error -> + Error + end; + _ -> + {ok, Opts} + end. + +%% @doc Find the local host information from the specified bootstrap node. +bootstrap_node_echo(Opts) -> + case hb_opts:get(host_bootstrap_node, false, Opts) of + false -> + {error, <<"No bootstrap node configured.">>}; + BootstrapNode -> + hb_http:get(BootstrapNode, <<"/~whois@1.0/echo">>, Opts) + end. + +%%% Tests + +find_self_test() -> + BoostrapNode = + hb_http_server:start_node(#{ + priv_wallet => ar_wallet:new() + }), + PeerNode = + hb_http_server:start_node(#{ + port => Port = rand:uniform(40000) + 10000, + priv_wallet => ar_wallet:new(), + host_bootstrap_node => BoostrapNode, + http_client => httpc + }), + ?event({nodes, {peer, PeerNode}, {bootstrap, BoostrapNode}}), + {ok, ReceivedPeerHost} = hb_http:get(PeerNode, <<"/~whois@1.0/node">>, #{}), + ?event({find_self_test, ReceivedPeerHost}), + ?assertEqual(<<"127.0.0.1:", (hb_util:bin(Port))/binary>>, ReceivedPeerHost). \ No newline at end of file diff --git a/src/hb.app.src b/src/hb.app.src index ed33867a3..7416d6083 100644 --- a/src/hb.app.src +++ b/src/hb.app.src @@ -8,13 +8,9 @@ stdlib, inets, ssl, - debugger, cowboy, - prometheus, - prometheus_cowboy, os_mon, - rocksdb, - quicer + gun ]}, {env, []}, {modules, []}, diff --git a/src/hb.erl b/src/hb.erl index bb960c3b4..ee8fc7964 100644 --- a/src/hb.erl +++ b/src/hb.erl @@ -1,4 +1,4 @@ -%%% @doc Hyperbeam is a decentralized node implementating the Converge Protocol +%%% @doc Hyperbeam is a decentralized node implementing the AO-Core protocol %%% on top of Arweave. %%% %%% This protocol offers a computation layer for executing arbitrary logic on @@ -17,7 +17,7 @@ %%% `Hyperbeam(Message1, Message2) => Message3' %%% %%% When Hyperbeam executes a message, it will return a new message containing -%%% the result of that execution, as well as signed attestations of its +%%% the result of that execution, as well as signed commitments of its %%% correctness. If the computation that is executed is deterministic, recipients %%% of the new message are able to verify that the computation was performed %%% correctly. The new message may be stored back to Arweave if desired, @@ -47,9 +47,9 @@ %%% `hb_http' handles making requests and responding with messages. `cowboy' %%% is used to implement the underlying HTTP server. %%% -%%% 3. `hb_converge' implements the computation logic of the node: A mechanism +%%% 3. `hb_ao' implements the computation logic of the node: A mechanism %%% for resolving messages to other messages, via the application of logic -%%% implemented in `devices'. `hb_converge' also manages the loading of Erlang +%%% implemented in `devices'. `hb_ao' also manages the loading of Erlang %%% modules for each device into the node's environment. There are many %%% different default devices implemented in the hyperbeam node, using the %%% namespace `dev_*'. Some of the critical components are: @@ -82,28 +82,157 @@ %%% modules of the hyperbeam node. -module(hb). %%% Configuration and environment: --export([init/0, now/0, build/0]). +-export([init/0, now/0, build/0, deploy_scripts/0]). +%%% Base start configurations: +-export([start_simple_pay/0, start_simple_pay/1, start_simple_pay/2]). +-export([topup/3, topup/4]). +-export([start_mainnet/0, start_mainnet/1]). %%% Debugging tools: --export([event/1, event/2, event/3, event/4, event/5, event/6, no_prod/3]). --export([read/1, read/2, debug_wait/4, profile/1, benchmark/2, benchmark/3]). +-export([no_prod/3]). +-export([read/1, read/2, debug_wait/4]). %%% Node wallet and address management: -export([address/0, wallet/0, wallet/1]). -include("include/hb.hrl"). %% @doc Initialize system-wide settings for the hyperbeam node. init() -> - pg:start(pg), + hb_name:start(), ?event({setting_debug_stack_depth, hb_opts:get(debug_stack_depth)}), Old = erlang:system_flag(backtrace_depth, hb_opts:get(debug_stack_depth)), ?event({old_system_stack_depth, Old}), ok. +%% @doc Start a mainnet server without payments. +start_mainnet() -> + start_mainnet(hb_opts:get(port)). +start_mainnet(Port) when is_integer(Port) -> + start_mainnet(#{ port => Port }); +start_mainnet(Opts) -> + application:ensure_all_started([ + kernel, + stdlib, + inets, + ssl, + ranch, + cowboy, + gun, + os_mon + ]), + Wallet = hb:wallet(hb_opts:get(priv_key_location, no_viable_wallet_path, Opts)), + BaseOpts = hb_http_server:set_default_opts(Opts), + hb_http_server:start_node( + FinalOpts = + BaseOpts#{ + store => #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-mainnet">> }, + priv_wallet => Wallet + } + ), + Address = + case hb_opts:get(address, no_address, FinalOpts) of + no_address -> <<"[ !!! no-address !!! ]">>; + Addr -> Addr + end, + io:format( + "Started mainnet node at http://localhost:~p~n" + "Operator: ~s~n", + [hb_maps:get(port, Opts, undefined, Opts), Address] + ), + <<"http://localhost:", (integer_to_binary(hb_maps:get(port, Opts, undefined, Opts)))/binary>>. + +%%% @doc Start a server with a `simple-pay@1.0' pre-processor. +start_simple_pay() -> + start_simple_pay(address()). +start_simple_pay(Addr) -> + rand:seed(default), + start_simple_pay(Addr, 10000 + rand:uniform(50000)). +start_simple_pay(Addr, Port) -> + do_start_simple_pay(#{ port => Port, operator => Addr }). + +do_start_simple_pay(Opts) -> + application:ensure_all_started([ + kernel, + stdlib, + inets, + ssl, + ranch, + cowboy, + gun, + os_mon + ]), + Port = hb_maps:get(port, Opts, undefined, Opts), + Processor = + #{ + <<"device">> => <<"p4@1.0">>, + <<"ledger-device">> => <<"simple-pay@1.0">>, + <<"pricing-device">> => <<"simple-pay@1.0">> + }, + hb_http_server:start_node( + Opts#{ + on => #{ + <<"request">> => Processor, + <<"response">> => Processor + } + } + ), + io:format( + "Started simple-pay node at http://localhost:~p~n" + "Operator: ~s~n", + [Port, address()] + ), + <<"http://localhost:", (integer_to_binary(Port))/binary>>. + +%% @doc Upload all scripts from the `scripts' directory to the node to Arweave, +%% printing their IDs. +deploy_scripts() -> + deploy_scripts("scripts/"). +deploy_scripts(Dir) -> + Files = filelib:wildcard(Dir ++ "*.lua"), + lists:foreach(fun(File) -> + {ok, Script} = file:read_file(File), + Msg = + hb_message:commit( + #{ + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"type">> => <<"module">>, + <<"content-type">> => <<"application/lua">>, + <<"name">> => hb_util:bin(File), + <<"body">> => Script + }, + wallet(), + <<"ans104@1.0">> + ), + {Status, _} = hb_client:upload(Msg, #{}, <<"ans104@1.0">>), + io:format( + "~s: ~s (upload status: ~p)~n", + [File, hb_util:id(Msg), Status] + ) + end, Files), + ok. + + +%% @doc Helper for topping up a user's balance on a simple-pay node. +topup(Node, Amount, Recipient) -> + topup(Node, Amount, Recipient, wallet()). +topup(Node, Amount, Recipient, Wallet) -> + Message = hb_message:commit( + #{ + <<"path">> => <<"/~simple-pay@1.0/topup">>, + <<"amount">> => Amount, + <<"recipient">> => Recipient + }, + Wallet + ), + hb_http:get(Node, Message, #{}). + wallet() -> - wallet(hb_opts:get(key_location)). + wallet(hb_opts:get(priv_key_location)). wallet(Location) -> + wallet(Location, #{}). +wallet(Location, Opts) -> case file:read_file_info(Location) of {ok, _} -> - ar_wallet:load_keyfile(Location); + ar_wallet:load_keyfile(Location, Opts); {error, _} -> Res = ar_wallet:new_keyfile(?DEFAULT_KEY_TYPE, Location), ?event({created_new_keyfile, Location, address(Res)}), @@ -111,44 +240,13 @@ wallet(Location) -> end. %% @doc Get the address of a wallet. Defaults to the address of the wallet -%% specified by the `key_location' configuration key. It can also take a +%% specified by the `priv_key_location' configuration key. It can also take a %% wallet tuple as an argument. address() -> address(wallet()). address(Wallet) when is_tuple(Wallet) -> hb_util:encode(ar_wallet:to_address(Wallet)); address(Location) -> address(wallet(Location)). -%% @doc Debugging event logging function. For now, it just prints to standard -%% error. -event(X) -> event(global, X). -event(Topic, X) -> event(Topic, X, ""). -event(Topic, X, Mod) -> event(Topic, X, Mod, undefined). -event(Topic, X, Mod, Func) -> event(Topic, X, Mod, Func, undefined). -event(Topic, X, Mod, Func, Line) -> event(Topic, X, Mod, Func, Line, #{}). -event(Topic, X, Mod, undefined, Line, Opts) -> event(Topic, X, Mod, "", Line, Opts); -event(Topic, X, Mod, Func, undefined, Opts) -> event(Topic, X, Mod, Func, "", Opts); -event(Topic, X, ModAtom, Func, Line, Opts) when is_atom(ModAtom) -> - % Check if the module has the `hb_debug` attribute set to `print`. - case lists:member({hb_debug, [print]}, ModAtom:module_info(attributes)) of - true -> hb_util:debug_print(X, atom_to_list(ModAtom), Func, Line); - false -> - % Check if the module has the `hb_debug` attribute set to `no_print`. - case lists:keyfind(hb_debug, 1, ModAtom:module_info(attributes)) of - {hb_debug, [no_print]} -> X; - _ -> event(Topic, X, atom_to_list(ModAtom), Func, Line, Opts) - end - end; -event(Topic, X, ModStr, Func, Line, Opts) -> - % Check if the debug_print option has the topic in it if set. - case hb_opts:get(debug_print, false, Opts) of - ModList when is_list(ModList) -> - (lists:member(ModStr, ModList) - orelse lists:member(atom_to_list(Topic), ModList)) - andalso hb_util:debug_print(X, ModStr, Func, Line); - true -> hb_util:debug_print(X, ModStr, Func, Line); - false -> X - end. - %% @doc Debugging function to read a message from the cache. %% Specify either a scope atom (local or remote) or a store tuple %% as the second argument. @@ -182,62 +280,8 @@ now() -> build() -> r3:do(compile, [{dir, "src"}]). -%% @doc Utility function to start a profiling session and run a function, -%% then analyze the results. Obviously -- do not use in production. -profile(Fun) -> - eprof:start_profiling([self()]), - try - Fun() - after - eprof:stop_profiling() - end, - eprof:analyze(total). - %% @doc Utility function to wait for a given amount of time, printing a debug %% message to the console first. debug_wait(T, Mod, Func, Line) -> ?event(wait, {debug_wait, {T, Mod, Func, Line}}), - receive after T -> ok end. - -%% @doc Run a function as many times as possible in a given amount of time. -benchmark(Fun, TLen) -> - T0 = erlang:system_time(millisecond), - until( - fun() -> erlang:system_time(millisecond) - T0 > (TLen * 1000) end, - Fun, - 0 - ). - -%% @doc Run multiple instances of a function in parallel for a given amount of time. -benchmark(Fun, TLen, Procs) -> - Parent = self(), - StartWorker = - fun(_) -> - Ref = make_ref(), - link(spawn(fun() -> - Count = benchmark(Fun, TLen), - Parent ! {work_complete, Ref, Count} - end)), - Ref - end, - CollectRes = - fun(R) -> - receive - {work_complete, R, Count} -> - Count - end - end, - Refs = lists:map(StartWorker, lists:seq(1, Procs)), - lists:sum(lists:map(CollectRes, Refs)). - -until(Condition, Fun, Count) -> - case Condition() of - false -> - case apply(Fun, hb_converge:truncate_args(Fun, [Count])) of - {count, AddToCount} -> - until(Condition, Fun, Count + AddToCount); - _ -> - until(Condition, Fun, Count + 1) - end; - true -> Count - end. + receive after T -> ok end. \ No newline at end of file diff --git a/src/hb_ao.erl b/src/hb_ao.erl new file mode 100644 index 000000000..ce51670f0 --- /dev/null +++ b/src/hb_ao.erl @@ -0,0 +1,1578 @@ +%%% @doc This module is the root of the device call logic of the +%%% AO-Core protocol in HyperBEAM. +%%% +%%% At the implementation level, every message is simply a collection of keys, +%%% dictated by its `Device', that can be resolved in order to yield their +%%% values. Each key may contain a link to another message or a raw value: +%%% +%%% `ao(BaseMessage, RequestMessage) -> {Status, Result}' +%%% +%%% Under-the-hood, `AO-Core(BaseMessage, RequestMessage)' leads to a lookup of +%%% the `device' key of the base message, followed by the evaluation of +%%% `DeviceMod:PathPart(BaseMessage, RequestMessage)', which defines the user +%%% compute to be performed. If `BaseMessage' does not specify a device, +%%% `~message@1.0' is assumed. The key to resolve is specified by the `path' +%%% field of the message. +%%% +%%% After each output, the `HashPath' is updated to include the `RequestMessage' +%%% that was executed upon it. +%%% +%%% Because each message implies a device that can resolve its keys, as well +%%% as generating a merkle tree of the computation that led to the result, +%%% you can see the AO-Core protocol as a system for cryptographically chaining +%%% the execution of `combinators'. See `docs/ao-core-protocol.md' for more +%%% information about AO-Core. +%%% +%%% The `key(BaseMessage, RequestMessage)' pattern is repeated throughout the +%%% HyperBEAM codebase, sometimes with `BaseMessage' replaced with `Msg1', `M1' +%%% or similar, and `RequestMessage' replaced with `Msg2', `M2', etc. +%%% +%%% The result of any computation can be either a new message or a raw literal +%%% value (a binary, integer, float, atom, or list of such values). +%%% +%%% Devices can be expressed as either modules or maps. They can also be +%%% referenced by an Arweave ID, which can be used to load a device from +%%% the network (depending on the value of the `load_remote_devices' and +%%% `trusted_device_signers' environment settings). +%%% +%%% HyperBEAM device implementations are defined as follows: +%%% 
+%%%     DevMod:ExportedFunc : Key resolution functions. All are assumed to be
+%%%                           device keys (thus, present in every message that
+%%%                           uses it) unless specified by `DevMod:info()'.
+%%%                           Each function takes a set of parameters
+%%%                           of the form `DevMod:KeyHandler(Msg1, Msg2, Opts)'.
+%%%                           Each of these arguments can be ommitted if not
+%%%                           needed. Non-exported functions are not assumed
+%%%                           to be device keys.
+%%%
+%%%     DevMod:info : Optional. Returns a map of options for the device. All 
+%%%                   options are optional and assumed to be the defaults if 
+%%%                   not specified. This function can accept a `Message1' as 
+%%%                   an argument, allowing it to specify its functionality 
+%%%                   based on a specific message if appropriate.
+%%% 
+%%%     info/exports : Overrides the export list of the Erlang module, such that
+%%%                   only the functions in this list are assumed to be device
+%%%                   keys. Defaults to all of the functions that DevMod 
+%%%                   exports in the Erlang environment.
+%%%
+%%%     info/excludes : A list of keys that should not be resolved by the device,
+%%%                     despite being present in the Erlang module exports list.
+%%% 
+%%%     info/handler : A function that should be used to handle _all_ keys for 
+%%%                    messages using the device.
+%%% 
+%%%     info/default : A function that should be used to handle all keys that
+%%%                    are not explicitly implemented by the device. Defaults to
+%%%                    the `dev_message' device, which contains general keys for 
+%%%                    interacting with messages.
+%%% 
+%%%     info/default_mod : A different device module that should be used to
+%%%                    handle all keys that are not explicitly implemented
+%%%                    by the device. Defaults to the `dev_message' device.
+%%% 
+%%%     info/grouper : A function that returns the concurrency 'group' name for
+%%%                    an execution. Executions with the same group name will
+%%%                    be executed by sending a message to the associated process
+%%%                    and waiting for a response. This allows you to control 
+%%%                    concurrency of execution and to allow executions to share
+%%%                    in-memory state as applicable. Default: A derivation of
+%%%                    Msg1+Msg2. This means that concurrent calls for the same
+%%%                    output will lead to only a single execution.
+%%% 
+%%%     info/worker : A function that should be run as the 'server' loop of
+%%%                   the executor for interactions using the device.
+%%% 
+%%% The HyperBEAM resolver also takes a number of runtime options that change
+%%% the way that the environment operates:
+%%% 
+%%% `update_hashpath':  Whether to add the `Msg2' to `HashPath' for the `Msg3'.
+%%% 					Default: true.
+%%% `add_key':          Whether to add the key to the start of the arguments.
+%%% 					Default: `'.
+%%%
+-module(hb_ao). +%%% Main AO-Core API: +-export([resolve/2, resolve/3, resolve_many/2]). +-export([normalize_key/1, normalize_key/2, normalize_keys/1, normalize_keys/2]). +-export([message_to_fun/3, message_to_device/2, load_device/2, find_exported_function/5]). +-export([force_message/2]). +%%% Shortcuts and tools: +-export([info/2, keys/1, keys/2, keys/3, truncate_args/2]). +-export([get/2, get/3, get/4, get_first/2, get_first/3]). +-export([set/3, set/4, remove/2, remove/3]). +%%% Exports for tests in hb_ao_test_vectors.erl: +-export([deep_set/4, is_exported/4]). +-include("include/hb.hrl"). + +-define(TEMP_OPTS, [add_key, force_message, cache_control, spawn_worker]). + +%% @doc Get the value of a message's key by running its associated device +%% function. Optionally, takes options that control the runtime environment. +%% This function returns the raw result of the device function call: +%% `{ok | error, NewMessage}.' +%% The resolver is composed of a series of discrete phases: +%% 1: Normalization. +%% 2: Cache lookup. +%% 3: Validation check. +%% 4: Persistent-resolver lookup. +%% 5: Device lookup. +%% 6: Execution. +%% 7: Execution of the `step' hook. +%% 8: Subresolution. +%% 9: Cryptographic linking. +%% 10: Result caching. +%% 11: Notify waiters. +%% 12: Fork worker. +%% 13: Recurse or terminate. +resolve(Path, Opts) when is_binary(Path) -> + resolve(#{ <<"path">> => Path }, Opts); +resolve(SingletonMsg, _Opts) + when is_map(SingletonMsg), not is_map_key(<<"path">>, SingletonMsg) -> + {error, <<"Attempted to resolve a message without a path.">>}; +resolve(SingletonMsg, Opts) -> + resolve_many(hb_singleton:from(SingletonMsg, Opts), Opts). + +resolve(Msg1, Path, Opts) when not is_map(Path) -> + resolve(Msg1, #{ <<"path">> => Path }, Opts); +resolve(Msg1, Msg2, Opts) -> + PathParts = hb_path:from_message(request, Msg2, Opts), + ?event(ao_core, {stage, 1, prepare_multimessage_resolution, {path_parts, PathParts}}), + MessagesToExec = [ Msg2#{ <<"path">> => Path } || Path <- PathParts ], + ?event(ao_core, {stage, 1, prepare_multimessage_resolution, {messages_to_exec, MessagesToExec}}), + resolve_many([Msg1 | MessagesToExec], Opts). + +%% @doc Resolve a list of messages in sequence. Take the output of the first +%% message as the input for the next message. Once the last message is resolved, +%% return the result. +%% A `resolve_many' call with only a single ID will attempt to read the message +%% directly from the store. No execution is performed. +resolve_many([ID], Opts) when ?IS_ID(ID) -> + % Note: This case is necessary to place specifically here for two reasons: + % 1. It is not in `do_resolve_many' because we need to handle the case + % where a result from a prior invocation is an ID itself. We should not + % attempt to resolve such IDs further. + % 2. The main AO-Core logic looks for linkages between message input + % pairs and outputs. With only a single ID, there is not a valid pairing + % to use in looking up a cached result. + ?event(ao_core, {stage, na, resolve_directly_to_id, ID, {opts, Opts}}, Opts), + try {ok, ensure_message_loaded(ID, Opts)} + catch _:_:_ -> {error, not_found} + end; +resolve_many(ListMsg, Opts) when is_map(ListMsg) -> + % We have been given a message rather than a list of messages, so we should + % convert it to a list, assuming that the message is monotonically numbered. + ListOfMessages = + try hb_util:message_to_ordered_list(ListMsg, internal_opts(Opts)) + catch + Type:Exception:Stacktrace -> + throw( + {resolve_many_error, + {given_message_not_ordered_list, ListMsg}, + {type, Type}, + {exception, Exception}, + {stacktrace, Stacktrace} + } + ) + end, + resolve_many(ListOfMessages, Opts); +resolve_many({as, DevID, Msg}, Opts) -> + subresolve(#{}, DevID, Msg, Opts); +resolve_many([{resolve, Subres}], Opts) -> + resolve_many(Subres, Opts); +resolve_many(MsgList, Opts) -> + ?event(ao_core, {resolve_many, MsgList}, Opts), + Res = do_resolve_many(MsgList, Opts), + ?event(ao_core, {resolve_many_complete, {res, Res}, {req, MsgList}}, Opts), + Res. +do_resolve_many([], _Opts) -> + {failure, <<"Attempted to resolve an empty message sequence.">>}; +do_resolve_many([Msg3], Opts) -> + ?event(ao_core, {stage, 11, resolve_complete, Msg3}), + {ok, hb_cache:ensure_loaded(Msg3, Opts)}; +do_resolve_many([Msg1, Msg2 | MsgList], Opts) -> + ?event(ao_core, {stage, 0, resolve_many, {msg1, Msg1}, {msg2, Msg2}}), + case resolve_stage(1, Msg1, Msg2, Opts) of + {ok, Msg3} -> + ?event(ao_core, + { + stage, + 13, + resolved_step, + {msg3, Msg3}, + {opts, Opts} + }, + Opts + ), + do_resolve_many([Msg3 | MsgList], Opts); + Res -> + % The result is not a resolvable message. Return it. + ?event(ao_core, {stage, 13, resolve_many_terminating_early, Res}), + Res + end. + +resolve_stage(1, Link, Msg2, Opts) when ?IS_LINK(Link) -> + % If the first message is a link, we should load the message and + % continue with the resolution. + ?event(ao_core, {stage, 1, resolve_base_link, {link, Link}}, Opts), + resolve_stage(1, hb_cache:ensure_loaded(Link, Opts), Msg2, Opts); +resolve_stage(1, Msg1, Link, Opts) when ?IS_LINK(Link) -> + % If the second message is a link, we should load the message and + % continue with the resolution. + ?event(ao_core, {stage, 1, resolve_req_link, {link, Link}}, Opts), + resolve_stage(1, Msg1, hb_cache:ensure_loaded(Link, Opts), Opts); +resolve_stage(1, {as, DevID, Ref}, Msg2, Opts) when ?IS_ID(Ref) orelse ?IS_LINK(Ref) -> + % Normalize `as' requests with a raw ID or link as the path. Links will be + % loaded in following stages. + resolve_stage(1, {as, DevID, #{ <<"path">> => Ref }}, Msg2, Opts); +resolve_stage(1, {as, DevID, Link}, Msg2, Opts) when ?IS_LINK(Link) -> + % If the first message is an `as' with a link, we should load the message and + % continue with the resolution. + ?event(ao_core, {stage, 1, resolve_base_as_link, {link, Link}}, Opts), + resolve_stage(1, {as, DevID, hb_cache:ensure_loaded(Link, Opts)}, Msg2, Opts); +resolve_stage(1, {as, DevID, Raw = #{ <<"path">> := ID }}, Msg2, Opts) when ?IS_ID(ID) -> + % If the first message is an `as' with an ID, we should load the message and + % apply the non-path elements of the sub-request to it. + ?event(ao_core, {stage, 1, subresolving_with_load, {dev, DevID}, {id, ID}}, Opts), + RemMsg1 = hb_maps:without([<<"path">>], Raw, Opts), + ?event(subresolution, {loading_message, {id, ID}, {params, RemMsg1}}, Opts), + Msg1b = ensure_message_loaded(ID, Opts), + ?event(subresolution, {loaded_message, {msg, Msg1b}}, Opts), + Msg1c = hb_maps:merge(Msg1b, RemMsg1, Opts), + ?event(subresolution, {merged_message, {msg, Msg1c}}, Opts), + Msg1d = set(Msg1c, <<"device">>, DevID, Opts), + ?event(subresolution, {loaded_parameterized_message, {msg, Msg1d}}, Opts), + resolve_stage(1, Msg1d, Msg2, Opts); +resolve_stage(1, Raw = {as, DevID, SubReq}, Msg2, Opts) -> + % Set the device of the message to the specified one and resolve the sub-path. + % As this is the first message, we will then continue to execute the request + % on the result. + ?event(ao_core, {stage, 1, subresolving_base, {dev, DevID}, {subreq, SubReq}}, Opts), + ?event(subresolution, {as, {dev, DevID}, {subreq, SubReq}, {msg2, Msg2}}), + case subresolve(SubReq, DevID, SubReq, Opts) of + {ok, SubRes} -> + % The subresolution has returned a new message. Continue with it. + ?event(subresolution, + {continuing_with_subresolved_message, {msg1, SubRes}} + ), + resolve_stage(1, SubRes, Msg2, Opts); + OtherRes -> + % The subresolution has returned an error. Return it. + ?event(subresolution, + {subresolution_error, {msg1, Raw}, {res, OtherRes}} + ), + OtherRes + end; +resolve_stage(1, RawMsg1, Msg2Outer = #{ <<"path">> := {as, DevID, Msg2Inner} }, Opts) -> + % Set the device to the specified `DevID' and resolve the message. Merging + % the `Msg2Inner' into the `Msg2Outer' message first. We return the result + % of the sub-resolution directly. + ?event(ao_core, {stage, 1, subresolving_from_request, {dev, DevID}}, Opts), + LoadedInner = ensure_message_loaded(Msg2Inner, Opts), + Msg2 = + hb_maps:merge( + set(Msg2Outer, <<"path">>, unset, Opts), + if is_binary(LoadedInner) -> #{ <<"path">> => LoadedInner }; + true -> LoadedInner + end, + Opts + ), + ?event(subresolution, + {subresolving_request_before_execution, + {dev, DevID}, + {msg2, Msg2} + } + ), + subresolve(RawMsg1, DevID, Msg2, Opts); +resolve_stage(1, {resolve, Subres}, Msg2, Opts) -> + % If the first message is a `{resolve, Subres}' tuple, we should execute it + % directly, then apply the request to the result. + ?event(ao_core, {stage, 1, subresolving_base_message, {subres, Subres}}, Opts), + % Unlike the `request' case for pre-subresolutions, we do not need to unset + % the `force_message' option, because the result should be a message, anyway. + % If it is not, it is more helpful to have the message placed into the `body' + % of a result, which can then be executed upon. + case resolve_many(Subres, Opts) of + {ok, Msg1} -> + ?event(ao_core, {stage, 1, subresolve_success, {new_base, Msg1}}, Opts), + resolve_stage(1, Msg1, Msg2, Opts); + OtherRes -> + ?event(ao_core, + {stage, + 1, + subresolve_failed, + {subres, Subres}, + {res, OtherRes}}, + Opts + ), + OtherRes + end; +resolve_stage(1, Msg1, {resolve, Subres}, Opts) -> + % If the second message is a `{resolve, Subresolution}' tuple, we should + % execute the subresolution directly to gain the underlying `Msg2' for + % our execution. We assume that the subresolution is already in a normalized, + % executable form, so we pass it to `resolve_many' for execution. + ?event(ao_core, {stage, 1, subresolving_request_message, {subres, Subres}}, Opts), + % We make sure to unset the `force_message' option so that if the subresolution + % returns a literal, the rest of `resolve' will normalize it to a path. + case resolve_many(Subres, maps:without([force_message], Opts)) of + {ok, Msg2} -> + ?event( + ao_core, + {stage, 1, request_subresolve_success, {msg2, Msg2}}, + Opts + ), + resolve_stage(1, Msg1, Msg2, Opts); + OtherRes -> + ?event( + ao_core, + { + stage, + 1, + request_subresolve_failed, + {subres, Subres}, + {res, OtherRes} + }, + Opts + ), + OtherRes + end; +resolve_stage(1, Msg1, Msg2, Opts) when is_list(Msg1) -> + % Normalize lists to numbered maps (base=1) if necessary. + ?event(ao_core, {stage, 1, list_normalize}, Opts), + resolve_stage(1, + normalize_keys(Msg1, Opts), + Msg2, + Opts + ); +resolve_stage(1, Msg1, NonMapMsg2, Opts) when not is_map(NonMapMsg2) -> + ?event(ao_core, {stage, 1, path_normalize}), + resolve_stage(1, Msg1, #{ <<"path">> => NonMapMsg2 }, Opts); +resolve_stage(1, RawMsg1, RawMsg2, Opts) -> + % Normalize the path to a private key containing the list of remaining + % keys to resolve. + ?event(ao_core, {stage, 1, normalize}, Opts), + Msg1 = normalize_keys(RawMsg1, Opts), + Msg2 = normalize_keys(RawMsg2, Opts), + resolve_stage(2, Msg1, Msg2, Opts); +resolve_stage(2, Msg1, Msg2, Opts) -> + ?event(ao_core, {stage, 2, cache_lookup}, Opts), + % Lookup request in the cache. If we find a result, return it. + % If we do not find a result, we continue to the next stage, + % unless the cache lookup returns `halt' (the user has requested that we + % only return a result if it is already in the cache). + case hb_cache_control:maybe_lookup(Msg1, Msg2, Opts) of + {ok, Msg3} -> + ?event(ao_core, {stage, 2, cache_hit, {msg3, Msg3}, {opts, Opts}}, Opts), + {ok, Msg3}; + {continue, NewMsg1, NewMsg2} -> + resolve_stage(3, NewMsg1, NewMsg2, Opts); + {error, CacheResp} -> {error, CacheResp} + end; +resolve_stage(3, Msg1, Msg2, Opts) when not is_map(Msg1) or not is_map(Msg2) -> + % Validation check: If the messages are not maps, we cannot find a key + % in them, so return not_found. + ?event(ao_core, {stage, 3, validation_check_type_error}, Opts), + {error, not_found}; +resolve_stage(3, Msg1, Msg2, Opts) -> + ?event(ao_core, {stage, 3, validation_check}, Opts), + % Validation check: Check if the message is valid. + %Msg1Valid = (hb_message:signers(Msg1, Opts) == []) orelse hb_message:verify(Msg1, Opts), + %Msg2Valid = (hb_message:signers(Msg2, Opts) == []) orelse hb_message:verify(Msg2, Opts), + ?no_prod("Enable message validity checks!"), + case {true, true} of + _ -> resolve_stage(4, Msg1, Msg2, Opts); + _ -> error_invalid_message(Msg1, Msg2, Opts) + end; +resolve_stage(4, Msg1, Msg2, Opts) -> + ?event(ao_core, {stage, 4, persistent_resolver_lookup}, Opts), + % Persistent-resolver lookup: Search for local (or Distributed + % Erlang cluster) processes that are already performing the execution. + % Before we search for a live executor, we check if the device specifies + % a function that tailors the 'group' name of the execution. For example, + % the `dev_process' device 'groups' all calls to the same process onto + % calls to a single executor. By default, `{Msg1, Msg2}' is used as the + % group name. + case hb_persistent:find_or_register(Msg1, Msg2, hb_maps:without(?TEMP_OPTS, Opts, Opts)) of + {leader, ExecName} -> + % We are the leader for this resolution. Continue to the next stage. + case hb_opts:get(spawn_worker, false, Opts) of + true -> ?event(worker_spawns, {will_become, ExecName}); + _ -> ok + end, + resolve_stage(5, Msg1, Msg2, ExecName, Opts); + {wait, Leader} -> + % There is another executor of this resolution in-flight. + % Bail execution, register to receive the response, then + % wait. + case hb_persistent:await(Leader, Msg1, Msg2, Opts) of + {error, leader_died} -> + ?event( + ao_core, + {leader_died_during_wait, + {leader, Leader}, + {msg1, Msg1}, + {msg2, Msg2}, + {opts, Opts} + }, + Opts + ), + % Re-try again if the group leader has died. + resolve_stage(4, Msg1, Msg2, Opts); + Res -> + % Now that we have the result, we can skip right to potential + % recursion (step 11) in the outer-wrapper. + Res + end; + {infinite_recursion, GroupName} -> + % We are the leader for this resolution, but we executing the + % computation again. This may plausibly be OK in _some_ cases, + % but in general it is the sign of a bug. + ?event( + ao_core, + {infinite_recursion, + {exec_group, GroupName}, + {msg1, Msg1}, + {msg2, Msg2}, + {opts, Opts} + }, + Opts + ), + case hb_opts:get(allow_infinite, false, Opts) of + true -> + % We are OK with infinite loops, so we just continue. + resolve_stage(5, Msg1, Msg2, GroupName, Opts); + false -> + % We are not OK with infinite loops, so we raise an error. + error_infinite(Msg1, Msg2, Opts) + end + end. +resolve_stage(5, Msg1, Msg2, ExecName, Opts) -> + ?event(ao_core, {stage, 5, device_lookup}, Opts), + % Device lookup: Find the Erlang function that should be utilized to + % execute Msg2 on Msg1. + {ResolvedFunc, NewOpts} = + try + UserOpts = hb_maps:without(?TEMP_OPTS, Opts, Opts), + Key = hb_path:hd(Msg2, UserOpts), + % Try to load the device and get the function to call. + ?event( + { + resolving_key, + {key, Key}, + {msg1, Msg1}, + {msg2, Msg2}, + {opts, Opts} + } + ), + {Status, _Mod, Func} = message_to_fun(Msg1, Key, UserOpts), + ?event( + {found_func_for_exec, + {key, Key}, + {func, Func}, + {msg1, Msg1}, + {msg2, Msg2}, + {opts, Opts} + } + ), + % Next, add an option to the Opts map to indicate if we should + % add the key to the start of the arguments. + { + Func, + Opts#{ + add_key => + case Status of + add_key -> Key; + _ -> false + end + } + } + catch + Class:Exception:Stacktrace -> + ?event( + ao_result, + { + load_device_failed, + {msg1, Msg1}, + {msg2, Msg2}, + {exec_name, ExecName}, + {exec_class, Class}, + {exec_exception, Exception}, + {exec_stacktrace, Stacktrace}, + {opts, Opts} + }, + Opts + ), + % If the device cannot be loaded, we alert the caller. + error_execution( + ExecName, + Msg2, + loading_device, + {Class, Exception, Stacktrace}, + Opts + ) + end, + resolve_stage(6, ResolvedFunc, Msg1, Msg2, ExecName, NewOpts). +resolve_stage(6, Func, Msg1, Msg2, ExecName, Opts) -> + ?event(ao_core, {stage, 6, ExecName, execution}, Opts), + % Execution. + % First, determine the arguments to pass to the function. + % While calculating the arguments we unset the add_key option. + UserOpts1 = hb_maps:remove(trace, hb_maps:without(?TEMP_OPTS, Opts, Opts), Opts), + % Unless the user has explicitly requested recursive spawning, we + % unset the spawn_worker option so that we do not spawn a new worker + % for every resulting execution. + UserOpts2 = + case hb_maps:get(spawn_worker, UserOpts1, false, Opts) of + recursive -> UserOpts1; + _ -> hb_maps:remove(spawn_worker, UserOpts1, Opts) + end, + Args = + case hb_maps:get(add_key, Opts, false, Opts) of + false -> [Msg1, Msg2, UserOpts2]; + Key -> [Key, Msg1, Msg2, UserOpts2] + end, + % Try to execute the function. + Res = + try + TruncatedArgs = truncate_args(Func, Args), + MsgRes = + maybe_force_message( + maybe_profiled_apply(Func, TruncatedArgs, Msg1, Msg2, Opts), + Opts + ), + ?event( + ao_result, + { + ao_result, + {exec_name, ExecName}, + {msg1, Msg1}, + {msg2, Msg2}, + {msg3, MsgRes} + }, + Opts + ), + MsgRes + catch + ExecClass:ExecException:ExecStacktrace -> + ?event( + ao_core, + {device_call_failed, ExecName, {func, Func}}, + Opts + ), + ?event( + ao_result, + { + exec_failed, + {msg1, Msg1}, + {msg2, Msg2}, + {exec_name, ExecName}, + {func, Func}, + {exec_class, ExecClass}, + {exec_exception, ExecException}, + {exec_stacktrace, erlang:process_info(self(), backtrace)}, + {opts, Opts} + }, + Opts + ), + % If the function call fails, we raise an error in the manner + % indicated by caller's `#Opts'. + error_execution( + ExecName, + Msg2, + device_call, + {ExecClass, ExecException, ExecStacktrace}, + Opts + ) + end, + resolve_stage(7, Msg1, Msg2, Res, ExecName, Opts); +resolve_stage(7, Msg1, Msg2, {St, Res}, ExecName, Opts = #{ on := On = #{ <<"step">> := _ }}) -> + ?event(ao_core, {stage, 7, ExecName, executing_step_hook, {on, On}}, Opts), + % If the `step' hook is defined, we execute it. Note: This function clause + % matches directly on the `on' key of the `Opts' map. This is in order to + % remove the expensive lookup check that would otherwise be performed on every + % execution. + HookReq = #{ + <<"base">> => Msg1, + <<"request">> => Msg2, + <<"status">> => St, + <<"body">> => Res + }, + case dev_hook:on(<<"step">>, HookReq, Opts) of + {ok, #{ <<"status">> := NewStatus, <<"body">> := NewRes }} -> + resolve_stage(8, Msg1, Msg2, {NewStatus, NewRes}, ExecName, Opts); + Error -> + ?event( + ao_core, + {step_hook_error, + {error, Error}, + {hook_req, HookReq} + }, + Opts + ), + Error + end; +resolve_stage(7, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 7, ExecName, no_step_hook}, Opts), + resolve_stage(8, Msg1, Msg2, Res, ExecName, Opts); +resolve_stage(8, Msg1, Msg2, {ok, {resolve, Sublist}}, ExecName, Opts) -> + ?event(ao_core, {stage, 8, ExecName, subresolve_result}, Opts), + % If the result is a `{resolve, Sublist}' tuple, we need to execute it + % as a sub-resolution. + resolve_stage(9, Msg1, Msg2, resolve_many(Sublist, Opts), ExecName, Opts); +resolve_stage(8, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), + resolve_stage(9, Msg1, Msg2, Res, ExecName, Opts); +resolve_stage(9, Msg1, Msg2, {ok, Msg3}, ExecName, Opts) when is_map(Msg3) -> + ?event(ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), + % Cryptographic linking. Now that we have generated the result, we + % need to cryptographically link the output to its input via a hashpath. + resolve_stage(10, Msg1, Msg2, + case hb_opts:get(hashpath, update, Opts#{ only => local }) of + update -> + NormMsg3 = Msg3, + Priv = hb_private:from_message(NormMsg3), + HP = hb_path:hashpath(Msg1, Msg2, Opts), + if not is_binary(HP) or not is_map(Priv) -> + throw({invalid_hashpath, {hp, HP}, {msg3, NormMsg3}}); + true -> + {ok, NormMsg3#{ <<"priv">> => Priv#{ <<"hashpath">> => HP } }} + end; + reset -> + Priv = hb_private:from_message(Msg3), + {ok, Msg3#{ <<"priv">> => hb_maps:without([<<"hashpath">>], Priv, Opts) }}; + ignore -> + Priv = hb_private:from_message(Msg3), + if not is_map(Priv) -> + throw({invalid_private_message, {msg3, Msg3}}); + true -> + {ok, Msg3} + end + end, + ExecName, + Opts + ); +resolve_stage(9, Msg1, Msg2, {Status, Msg3}, ExecName, Opts) when is_map(Msg3) -> + ?event(ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), + ?event(hashpath, {resetting_hashpath_msg3, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), + % Skip cryptographic linking and reset the hashpath if the result is abnormal. + Priv = hb_private:from_message(Msg3), + resolve_stage( + 10, Msg1, Msg2, + {Status, Msg3#{ <<"priv">> => maps:without([<<"hashpath">>], Priv) }}, + ExecName, Opts); +resolve_stage(9, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), + % Skip cryptographic linking and continue if we don't have a map that can have + % a hashpath at all. + resolve_stage(10, Msg1, Msg2, Res, ExecName, Opts); +resolve_stage(10, Msg1, Msg2, {ok, Msg3}, ExecName, Opts) -> + ?event(ao_core, {stage, 10, ExecName, result_caching}, Opts), + % Result caching: Optionally, cache the result of the computation locally. + hb_cache_control:maybe_store(Msg1, Msg2, Msg3, Opts), + resolve_stage(11, Msg1, Msg2, {ok, Msg3}, ExecName, Opts); +resolve_stage(10, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), + % Skip result caching if the result is abnormal. + resolve_stage(11, Msg1, Msg2, Res, ExecName, Opts); +resolve_stage(11, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 11, ExecName}, Opts), + % Notify processes that requested the resolution while we were executing and + % unregister ourselves from the group. + hb_persistent:unregister_notify(ExecName, Msg2, Res, Opts), + resolve_stage(12, Msg1, Msg2, Res, ExecName, Opts); +resolve_stage(12, _Msg1, _Msg2, {ok, Msg3} = Res, ExecName, Opts) -> + ?event(ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), + % Check if we should spawn a worker for the current execution + case {is_map(Msg3), hb_opts:get(spawn_worker, false, Opts#{ prefer => local })} of + {A, B} when (A == false) or (B == false) -> + Res; + {_, _} -> + % Spawn a worker for the current execution + WorkerPID = hb_persistent:start_worker(ExecName, Msg3, Opts), + hb_persistent:forward_work(WorkerPID, Opts), + Res + end; +resolve_stage(12, _Msg1, _Msg2, OtherRes, ExecName, Opts) -> + ?event(ao_core, {stage, 12, ExecName, abnormal_status_skip_spawning}, Opts), + OtherRes. + +%% @doc Execute a sub-resolution. +subresolve(RawMsg1, DevID, ReqPath, Opts) when is_binary(ReqPath) -> + % If the request is a binary, we assume that it is a path. + subresolve(RawMsg1, DevID, #{ <<"path">> => ReqPath }, Opts); +subresolve(RawMsg1, DevID, Req, Opts) -> + % First, ensure that the message is loaded from the cache. + Msg1 = ensure_message_loaded(RawMsg1, Opts), + ?event(subresolution, + {subresolving, {msg1, Msg1}, {dev, DevID}, {req, Req}} + ), + % Next, set the device ID if it is given. + Msg1b = + case DevID of + undefined -> Msg1; + _ -> set(Msg1, <<"device">>, DevID, hb_maps:without(?TEMP_OPTS, Opts, Opts)) + end, + % If there is no path but there are elements to the request, we set these on + % the base message. If there is a path, we do not modify the base message + % and instead apply the request message directly. + case hb_path:from_message(request, Req, Opts) of + undefined -> + Msg1c = + case map_size(hb_maps:without([<<"path">>], Req, Opts)) of + 0 -> Msg1b; + _ -> + set( + Msg1b, + set(Req, <<"path">>, unset, Opts), + Opts#{ force_message => false } + ) + end, + ?event(subresolution, + {subresolve_modified_base, Msg1c}, + Opts + ), + {ok, Msg1c}; + Path -> + ?event(subresolution, + {exec_subrequest_on_base, + {mod_base, Msg1b}, + {req, Path}, + {req, Req} + } + ), + Res = resolve(Msg1b, Req, Opts), + ?event(subresolution, {subresolved_with_new_device, {res, Res}}), + Res + end. + +%% @doc If the `AO_PROFILING' macro is defined (set by building/launching with +%% `rebar3 as ao_profiling') we record statistics about the execution of the +%% function. This is a costly operation, so if it is not defined, we simply +%% apply the function and return the result. +-ifndef(AO_PROFILING). +maybe_profiled_apply(Func, Args, _Msg1, _Msg2, _Opts) -> + apply(Func, Args). +-else. +maybe_profiled_apply(Func, Args, Msg1, Msg2, Opts) -> + CallStack = erlang:get(ao_stack), + ?event(ao_trace, + {profiling_apply, + {func, Func}, + {args, Args}, + {call_stack, CallStack} + } + ), + Key = + case hb_maps:get(<<"device">>, Msg1, undefined, Opts) of + undefined -> + hb_util:bin(erlang:fun_to_list(Func)); + Device -> + case hb_maps:get(<<"path">>, Msg2, undefined, Opts) of + undefined -> + hb_util:bin(erlang:fun_to_list(Func)); + Path -> + MethodStr = + case hb_maps:get(<<"method">>, Msg2, undefined, Opts) of + undefined -> <<"">>; + <<"GET">> -> <<"">>; + Method -> <<"<", Method/binary, ">">> + end, + << + (hb_util:bin(Device))/binary, + "/", + MethodStr/binary, + (hb_util:bin(Path))/binary + >> + end + end, + put( + ao_stack, + case CallStack of + undefined -> [Key]; + Stack -> [Key | Stack] + end + ), + {ExecMicroSecs, Res} = timer:tc(fun() -> apply(Func, Args) end), + put(ao_stack, CallStack), + hb_event:increment(<<"ao-call-counts">>, Key, Opts), + hb_event:increment(<<"ao-total-durations">>, Key, Opts, ExecMicroSecs), + case CallStack of + undefined -> ok; + [Caller|_] -> + hb_event:increment( + <<"ao-callers:", Key/binary>>, + hb_util:bin( + [ + <<"duration:">>, + Caller + ] + ), + Opts, + ExecMicroSecs + ), + hb_event:increment( + <<"ao-callers:", Key/binary>>, + hb_util:bin( + [ + <<"calls:">>, + Caller + ]), + Opts + ) + end, + Res. +-endif. + +%% @doc Ensure that a message is loaded from the cache if it is an ID, or +%% a link, such that it is ready for execution. +ensure_message_loaded(MsgID, Opts) when ?IS_ID(MsgID) -> + case hb_cache:read(MsgID, Opts) of + {ok, LoadedMsg} -> + LoadedMsg; + not_found -> + throw({necessary_message_not_found, <<"/">>, MsgID}) + end; +ensure_message_loaded(MsgLink, Opts) when ?IS_LINK(MsgLink) -> + hb_cache:ensure_loaded(MsgLink, Opts); +ensure_message_loaded(Msg, _Opts) -> + Msg. + +%% @doc Catch all return if the message is invalid. +error_invalid_message(Msg1, Msg2, Opts) -> + ?event( + ao_core, + {error, {type, invalid_message}, + {msg1, Msg1}, + {msg2, Msg2}, + {opts, Opts} + }, + Opts + ), + { + error, + #{ + <<"status">> => 400, + <<"body">> => <<"Request contains non-verifiable message.">> + } + }. + +%% @doc Catch all return if we are in an infinite loop. +error_infinite(Msg1, Msg2, Opts) -> + ?event( + ao_core, + {error, {type, infinite_recursion}, + {msg1, Msg1}, + {msg2, Msg2}, + {opts, Opts} + }, + Opts + ), + ?trace(), + { + error, + #{ + <<"status">> => 508, + <<"body">> => <<"Request creates infinite recursion.">> + } + }. + +error_invalid_intermediate_status(Msg1, Msg2, Msg3, RemainingPath, Opts) -> + ?event( + ao_core, + {error, {type, invalid_intermediate_status}, + {msg2, Msg2}, + {msg3, Msg3}, + {remaining_path, RemainingPath}, + {opts, Opts} + }, + Opts + ), + ?event(ao_result, + {intermediate_failure, {msg1, Msg1}, + {msg2, Msg2}, {msg3, Msg3}, + {remaining_path, RemainingPath}, {opts, Opts}}), + { + error, + #{ + <<"status">> => 422, + <<"body">> => Msg3, + <<"key">> => hb_maps:get(<<"path">>, Msg2, <<"Key unknown.">>, Opts), + <<"remaining-path">> => RemainingPath + } + }. + +%% @doc Handle an error in a device call. +error_execution(ExecGroup, Msg2, Whence, {Class, Exception, Stacktrace}, Opts) -> + Error = {error, Whence, {Class, Exception, Stacktrace}}, + hb_persistent:unregister_notify(ExecGroup, Msg2, Error, Opts), + ?event(ao_core, {handle_error, Error, {opts, Opts}}, Opts), + case hb_opts:get(error_strategy, throw, Opts) of + throw -> erlang:raise(Class, Exception, Stacktrace); + _ -> Error + end. + +%% @doc Force the result of a device call into a message if the result is not +%% requested by the `Opts'. If the result is a literal, we wrap it in a message +%% and signal the location of the result inside. We also similarly handle ao-result +%% when the result is a single value and an explicit status code. +maybe_force_message({Status, Res}, Opts) -> + case hb_opts:get(force_message, false, Opts) of + true -> force_message({Status, Res}, Opts); + false -> {Status, Res} + end; +maybe_force_message(Res, Opts) -> + maybe_force_message({ok, Res}, Opts). + +force_message({Status, Res}, Opts) when is_list(Res) -> + force_message({Status, normalize_keys(Res, Opts)}, Opts); +force_message({Status, Subres = {resolve, _}}, _Opts) -> + {Status, Subres}; +force_message({Status, Literal}, _Opts) when not is_map(Literal) -> + ?event({force_message_from_literal, Literal}), + {Status, #{ <<"ao-result">> => <<"body">>, <<"body">> => Literal }}; +force_message({Status, M = #{ <<"status">> := Status, <<"body">> := Body }}, _Opts) + when map_size(M) == 2 -> + ?event({force_message_from_literal_with_status, M}), + {Status, #{ + <<"status">> => Status, + <<"ao-result">> => <<"body">>, + <<"body">> => Body + }}; +force_message({Status, Map}, _Opts) -> + ?event({force_message_from_map, Map}), + {Status, Map}. + +%% @doc Shortcut for resolving a key in a message without its status if it is +%% `ok'. This makes it easier to write complex logic on top of messages while +%% maintaining a functional style. +%% +%% Additionally, this function supports the `{as, Device, Msg}' syntax, which +%% allows the key to be resolved using another device to resolve the key, +%% while maintaining the tracability of the `HashPath' of the output message. +%% +%% Returns the value of the key if it is found, otherwise returns the default +%% provided by the user, or `not_found' if no default is provided. +get(Path, Msg) -> + get(Path, Msg, #{}). +get(Path, Msg, Opts) -> + get(Path, Msg, not_found, Opts). +get(Path, {as, Device, Msg}, Default, Opts) -> + get( + Path, + set( + Msg, + #{ <<"device">> => Device }, + internal_opts(Opts) + ), + Default, + Opts + ); +get(Path, Msg, Default, Opts) -> + case resolve(Msg, #{ <<"path">> => Path }, Opts#{ spawn_worker => false }) of + {ok, Value} -> Value; + {error, _} -> Default + end. + +%% @doc take a sequence of base messages and paths, then return the value of the +%% first message that can be resolved using a path. +get_first(Paths, Opts) -> get_first(Paths, not_found, Opts). +get_first([], Default, _Opts) -> Default; +get_first([{Base, Path}|Msgs], Default, Opts) -> + case get(Path, Base, Opts) of + not_found -> get_first(Msgs, Default, Opts); + Value -> Value + end. + +%% @doc Shortcut to get the list of keys from a message. +keys(Msg) -> keys(Msg, #{}). +keys(Msg, Opts) -> keys(Msg, Opts, keep). +keys(Msg, Opts, keep) -> + % There is quite a lot of AO-Core-specific machinery here. We: + % 1. `get' the keys from the message, via AO-Core in order to trigger the + % `keys' function on its device. + % 2. Ensure that the result is normalized to a message (not just a list) + % with `normalize_keys'. + % 3. Now we have a map of the original keys, so we can use `hb_maps:values' to + % get a list of them. + % 4. Normalize each of those keys in turn. + try + lists:map( + fun normalize_key/1, + hb_maps:values( + normalize_keys( + hb_private:reset(get(<<"keys">>, Msg, Opts)) + ), + Opts + ) + ) + catch + A:B:St -> + throw( + {cannot_get_keys, + {msg, Msg}, + {opts, Opts}, + {error, {A, B}}, + {stacktrace, St} + } + ) + end; +keys(Msg, Opts, remove) -> + lists:filter( + fun(Key) -> not lists:member(Key, ?AO_CORE_KEYS) end, + keys(Msg, Opts, keep) + ). + +%% @doc Shortcut for setting a key in the message using its underlying device. +%% Like the `get/3' function, this function honors the `error_strategy' option. +%% `set' works with maps and recursive paths while maintaining the appropriate +%% `HashPath' for each step. +set(RawMsg1, RawMsg2, Opts) when is_map(RawMsg2) -> + Msg1 = normalize_keys(RawMsg1, Opts), + Msg2 = hb_maps:without([<<"hashpath">>, <<"priv">>], normalize_keys(RawMsg2, Opts), Opts), + ?event(ao_internal, {set_called, {msg1, Msg1}, {msg2, Msg2}}, Opts), + % Get the next key to set. + case keys(Msg2, internal_opts(Opts)) of + [] -> Msg1; + [Key|_] -> + % Get the value to set. Use AO-Core by default, but fall back to + % getting via `maps' if it is not found. + Val = + case get(Key, Msg2, internal_opts(Opts)) of + not_found -> hb_maps:get(Key, Msg2, undefined, Opts); + Body -> Body + end, + ?event({got_val_to_set, {key, Key}, {val, Val}, {msg2, Msg2}}), + % Next, set the key and recurse, removing the key from the Msg2. + set( + set(Msg1, Key, Val, internal_opts(Opts)), + remove(Msg2, Key, internal_opts(Opts)), + Opts + ) + end. +set(Msg1, Key, Value, Opts) -> + % For an individual key, we run deep_set with the key as the path. + % This handles both the case that the key is a path as well as the case + % that it is a single key. + Path = hb_path:term_to_path_parts(Key, Opts), + % ?event( + % {setting_individual_key, + % {msg1, Msg1}, + % {key, Key}, + % {path, Path}, + % {value, Value} + % } + % ), + deep_set(Msg1, Path, Value, Opts). + +%% @doc Recursively search a map, resolving keys, and set the value of the key +%% at the given path. This function has special cases for handling `set' calls +%% where the path is an empty list (`/'). In this case, if the value is an +%% immediate, non-complex term, we can set it directly. Otherwise, we use the +%% device's `set' function to set the value. +deep_set(Msg, [], Value, Opts) when is_map(Msg) or is_list(Msg) -> + device_set(Msg, <<"/">>, Value, Opts); +deep_set(_Msg, [], Value, _Opts) -> + Value; +deep_set(Msg, [Key], Value, Opts) -> + device_set(Msg, Key, Value, Opts); +deep_set(Msg, [Key|Rest], Value, Opts) -> + case resolve(Msg, Key, Opts) of + {ok, SubMsg} -> + ?event( + {traversing_deeper_to_set, + {current_key, Key}, + {current_value, SubMsg}, + {rest, Rest} + } + ), + Res = device_set(Msg, Key, deep_set(SubMsg, Rest, Value, Opts), <<"explicit">>, Opts), + ?event({deep_set_result, {msg, Msg}, {key, Key}, {res, Res}}), + Res; + _ -> + ?event( + {creating_new_map, + {current_key, Key}, + {rest, Rest} + } + ), + Msg#{ Key => deep_set(#{}, Rest, Value, Opts) } + end. + +%% @doc Call the device's `set' function. +device_set(Msg, Key, Value, Opts) -> + device_set(Msg, Key, Value, <<"deep">>, Opts). +device_set(Msg, Key, Value, Mode, Opts) -> + ReqWithoutMode = + case Key of + <<"path">> -> + #{ <<"path">> => <<"set_path">>, <<"value">> => Value }; + <<"/">> when is_map(Value) -> + % The value is a map and it is to be `set' at the root of the + % message. Subsequently, we call the device's `set' function + % with all of the keys found in the message, leading it to be + % merged into the message. + Value#{ <<"path">> => <<"set">> }; + _ -> + #{ <<"path">> => <<"set">>, Key => Value } + end, + Req = + case Mode of + <<"deep">> -> ReqWithoutMode; + <<"explicit">> -> ReqWithoutMode#{ <<"set-mode">> => Mode } + end, + ?event( + ao_internal, + { + calling_device_set, + {msg, Msg}, + {applying_set, Req} + }, + Opts + ), + Res = + hb_util:ok( + resolve( + Msg, + Req, + internal_opts(Opts) + ), + internal_opts(Opts) + ), + ?event( + ao_internal, + {device_set_result, Res}, + Opts + ), + Res. + +%% @doc Remove a key from a message, using its underlying device. +remove(Msg, Key) -> remove(Msg, Key, #{}). +remove(Msg, Key, Opts) -> + hb_util:ok( + resolve( + Msg, + #{ <<"path">> => <<"remove">>, <<"item">> => Key }, + internal_opts(Opts) + ), + Opts + ). + +%% @doc Truncate the arguments of a function to the number of arguments it +%% actually takes. +truncate_args(Fun, Args) -> + {arity, Arity} = erlang:fun_info(Fun, arity), + lists:sublist(Args, Arity). + +%% @doc Calculate the Erlang function that should be called to get a value for +%% a given key from a device. +%% +%% This comes in 7 forms: +%% 1. The message does not specify a device, so we use the default device. +%% 2. The device has a `handler' key in its `Dev:info()' map, which is a +%% function that takes a key and returns a function to handle that key. We pass +%% the key as an additional argument to this function. +%% 3. The device has a function of the name `Key', which should be called +%% directly. +%% 4. The device does not implement the key, but does have a default handler +%% for us to call. We pass it the key as an additional argument. +%% 5. The device does not implement the key, and has no default handler. We use +%% the default device to handle the key. +%% Error: If the device is specified, but not loadable, we raise an error. +%% +%% Returns {ok | add_key, Fun} where Fun is the function to call, and add_key +%% indicates that the key should be added to the start of the call's arguments. +message_to_fun(Msg, Key, Opts) -> + % Get the device module from the message. + Dev = message_to_device(Msg, Opts), + Info = info(Dev, Msg, Opts), + % Is the key exported by the device? + Exported = is_exported(Info, Key, Opts), + ?event( + ao_devices, + {message_to_fun, + {dev, Dev}, + {key, Key}, + {is_exported, Exported}, + {opts, Opts} + }, + Opts + ), + % Does the device have an explicit handler function? + case {hb_maps:find(handler, Info, Opts), Exported} of + {{ok, Handler}, true} -> + % Case 2: The device has an explicit handler function. + ?event( + ao_devices, + {handler_found, {dev, Dev}, {key, Key}, {handler, Handler}} + ), + {Status, Func} = info_handler_to_fun(Handler, Msg, Key, Opts), + {Status, Dev, Func}; + _ -> + ?event(ao_devices, {no_override_handler, {dev, Dev}, {key, Key}}), + case {find_exported_function(Msg, Dev, Key, 3, Opts), Exported} of + {{ok, Func}, true} -> + % Case 3: The device has a function of the name `Key'. + {ok, Dev, Func}; + _ -> + case {hb_maps:find(default, Info, Opts), Exported} of + {{ok, DefaultFunc}, true} when is_function(DefaultFunc) -> + % Case 4: The device has a default handler. + ?event({found_default_handler, {func, DefaultFunc}}), + {add_key, Dev, DefaultFunc}; + {{ok, DefaultMod}, true} when is_atom(DefaultMod) -> + ?event({found_default_handler, {mod, DefaultMod}}), + {Status, Func} = + message_to_fun( + Msg#{ <<"device">> => DefaultMod }, Key, Opts + ), + {Status, Dev, Func}; + _ -> + % Case 5: The device has no default handler. + % We use the default device to handle the key. + case default_module() of + Dev -> + % We are already using the default device, + % so we cannot resolve the key. This should + % never actually happen in practice, but it + % resolves an infinite loop that can occur + % during development. + throw({ + error, + default_device_could_not_resolve_key, + {key, Key} + }); + DefaultDev -> + ?event( + { + using_default_device, + {dev, DefaultDev} + }), + message_to_fun( + Msg#{ <<"device">> => DefaultDev }, + Key, + Opts + ) + end + end + end + end. + +%% @doc Extract the device module from a message. +message_to_device(Msg, Opts) -> + case dev_message:get(<<"device">>, Msg, Opts) of + {error, not_found} -> + % The message does not specify a device, so we use the default device. + default_module(); + {ok, DevID} -> + case load_device(DevID, Opts) of + {error, Reason} -> + % Error case: A device is specified, but it is not loadable. + throw({error, {device_not_loadable, DevID, Reason}}); + {ok, DevMod} -> DevMod + end + end. + +%% @doc Parse a handler key given by a device's `info'. +info_handler_to_fun(Handler, _Msg, _Key, _Opts) when is_function(Handler) -> + {add_key, Handler}; +info_handler_to_fun(HandlerMap, Msg, Key, Opts) -> + case hb_maps:find(excludes, HandlerMap, Opts) of + {ok, Exclude} -> + case lists:member(Key, Exclude) of + true -> + {ok, MsgWithoutDevice} = + dev_message:remove(Msg, #{ item => device }, Opts), + message_to_fun( + MsgWithoutDevice#{ <<"device">> => default_module() }, + Key, + Opts + ); + false -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} + end; + error -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} + end. + +%% @doc Find the function with the highest arity that has the given name, if it +%% exists. +%% +%% If the device is a module, we look for a function with the given name. +%% +%% If the device is a map, we look for a key in the map. First we try to find +%% the key using its literal value. If that fails, we cast the key to an atom +%% and try again. +find_exported_function(Msg, Dev, Key, MaxArity, Opts) when is_map(Dev) -> + case hb_maps:get(normalize_key(Key), normalize_keys(Dev, Opts), not_found, Opts) of + not_found -> not_found; + Fun when is_function(Fun) -> + case erlang:fun_info(Fun, arity) of + {arity, Arity} when Arity =< MaxArity -> + case is_exported(Msg, Dev, Key, Opts) of + true -> {ok, Fun}; + false -> not_found + end; + _ -> not_found + end + end; +find_exported_function(_Msg, _Mod, _Key, Arity, _Opts) when Arity < 0 -> + not_found; +find_exported_function(Msg, Mod, Key, Arity, Opts) when not is_atom(Key) -> + try hb_util:key_to_atom(Key, false) of + KeyAtom -> find_exported_function(Msg, Mod, KeyAtom, Arity, Opts) + catch _:_ -> not_found + end; +find_exported_function(Msg, Mod, Key, Arity, Opts) -> + case erlang:function_exported(Mod, Key, Arity) of + true -> + case is_exported(Msg, Mod, Key, Opts) of + true -> {ok, fun Mod:Key/Arity}; + false -> not_found + end; + false -> + find_exported_function(Msg, Mod, Key, Arity - 1, Opts) + end. + +%% @doc Check if a device is guarding a key via its `exports' list. Defaults to +%% true if the device does not specify an `exports' list. The `info' function is +%% always exported, if it exists. Elements of the `exludes' list are not +%% exported. Note that we check for info _twice_ -- once when the device is +%% given but the info result is not, and once when the info result is given. +%% The reason for this is that `info/3' calls other functions that may need to +%% check if a key is exported, so we must avoid infinite loops. We must, however, +%% also return a consistent result in the case that only the info result is +%% given, so we check for it in both cases. +is_exported(_Msg, _Dev, info, _Opts) -> true; +is_exported(Msg, Dev, Key, Opts) -> + is_exported(info(Dev, Msg, Opts), Key, Opts). +is_exported(_, info, _Opts) -> true; +is_exported(Info = #{ excludes := Excludes }, Key, Opts) -> + case lists:member(normalize_key(Key), lists:map(fun normalize_key/1, Excludes)) of + true -> false; + false -> is_exported(hb_maps:remove(excludes, Info, Opts), Key, Opts) + end; +is_exported(#{ exports := Exports }, Key, _Opts) -> + lists:member(normalize_key(Key), lists:map(fun normalize_key/1, Exports)); +is_exported(_Info, _Key, _Opts) -> true. + +%% @doc Convert a key to a binary in normalized form. +normalize_key(Key) -> normalize_key(Key, #{}). +normalize_key(Key, _Opts) when is_binary(Key) -> Key; +normalize_key(Key, _Opts) when is_atom(Key) -> atom_to_binary(Key); +normalize_key(Key, _Opts) when is_integer(Key) -> integer_to_binary(Key); +normalize_key(Key, _Opts) when is_list(Key) -> + case hb_util:is_string_list(Key) of + true -> normalize_key(list_to_binary(Key)); + false -> + iolist_to_binary( + lists:join( + <<"/">>, + lists:map(fun normalize_key/1, Key) + ) + ) + end. + +%% @doc Ensure that a message is processable by the AO-Core resolver: No lists. +normalize_keys(Msg) -> normalize_keys(Msg, #{}). +normalize_keys(Msg1, Opts) when is_list(Msg1) -> + normalize_keys( + hb_maps:from_list( + lists:zip( + lists:seq(1, length(Msg1)), + Msg1 + ) + ), + Opts + ); + +normalize_keys(Map, Opts) when is_map(Map) -> + hb_maps:from_list( + lists:map( + fun({Key, Value}) when is_map(Value) -> + {hb_ao:normalize_key(Key), Value}; + ({Key, Value}) -> + {hb_ao:normalize_key(Key), Value} + end, + hb_maps:to_list(Map, Opts) + ) + ); +normalize_keys(Other, _Opts) -> Other. + +%% @doc Load a device module from its name or a message ID. +%% Returns {ok, Executable} where Executable is the device module. On error, +%% a tuple of the form {error, Reason} is returned. +load_device(Map, _Opts) when is_map(Map) -> {ok, Map}; +load_device(ID, _Opts) when is_atom(ID) -> + try ID:module_info(), {ok, ID} + catch _:_ -> {error, not_loadable} + end; +load_device(ID, Opts) when ?IS_ID(ID) -> + ?event(device_load, {requested_load, {id, ID}}, Opts), + case hb_opts:get(load_remote_devices, false, Opts) of + false -> + {error, remote_devices_disabled}; + true -> + ?event(device_load, {loading_from_cache, {id, ID}}, Opts), + {ok, Msg} = hb_cache:read(ID, Opts), + ?event(device_load, {received_device, {id, ID}, {msg, Msg}}, Opts), + TrustedSigners = hb_opts:get(trusted_device_signers, [], Opts), + Trusted = + lists:any( + fun(Signer) -> + lists:member(Signer, TrustedSigners) + end, + hb_message:signers(Msg, Opts) + ), + ?event(device_load, + {verifying_device_trust, + {id, ID}, + {trusted, Trusted}, + {signers, hb_message:signers(Msg, Opts)} + }, + Opts + ), + case Trusted of + false -> {error, device_signer_not_trusted}; + true -> + ?event(device_load, {loading_device, {id, ID}}, Opts), + case hb_maps:get(<<"content-type">>, Msg, undefined, Opts) of + <<"application/beam">> -> + case verify_device_compatibility(Msg, Opts) of + ok -> + ModName = + hb_util:key_to_atom( + hb_maps:get( + <<"module-name">>, + Msg, + undefined, + Opts + ), + new_atoms + ), + LoadRes = + erlang:load_module( + ModName, + hb_maps:get( + <<"body">>, + Msg, + undefined, + Opts + ) + ), + case LoadRes of + {module, _} -> + {ok, ModName}; + {error, Reason} -> + {error, {device_load_failed, Reason}} + end; + {error, Reason} -> + {error, {device_load_failed, Reason}} + end; + Other -> + {error, + {device_load_failed, + {incompatible_content_type, Other}, + {expected, <<"application/beam">>}, + {found, Other} + } + } + end + end + end; +load_device(ID, Opts) -> + NormKey = + case is_atom(ID) of + true -> ID; + false -> normalize_key(ID) + end, + case lists:search( + fun (#{ <<"name">> := Name }) -> Name =:= NormKey end, + Preloaded = hb_opts:get(preloaded_devices, [], Opts) + ) of + false -> {error, {module_not_admissable, NormKey, Preloaded}}; + {value, #{ <<"module">> := Mod }} -> load_device(Mod, Opts) + end. + +%% @doc Verify that a device is compatible with the current machine. +verify_device_compatibility(Msg, Opts) -> + ?event(device_load, {verifying_device_compatibility, {msg, Msg}}, Opts), + Required = + lists:filtermap( + fun({<<"requires-", Key/binary>>, Value}) -> + {true, + { + hb_util:key_to_atom( + hb_ao:normalize_key(Key), + new_atoms + ), + hb_cache:ensure_loaded(Value, Opts) + } + }; + (_) -> false + end, + hb_maps:to_list(Msg, Opts) + ), + ?event(device_load, + {discerned_requirements, + {required, Required}, + {msg, Msg} + }, + Opts + ), + FailedToMatch = + lists:filtermap( + fun({Property, Value}) -> + % The values of these properties are _not_ 'keys', but we normalize + % them as such in order to make them comparable. + SystemValue = erlang:system_info(Property), + Res = normalize_key(SystemValue) == normalize_key(Value), + % If the property matched, we remove it from the list of required + % properties. If it doesn't we return it with the found value, such + % that the caller knows which properties were not satisfied. + case Res of + true -> false; + false -> {true, {Property, Value}} + end + end, + Required + ), + case FailedToMatch of + [] -> ok; + _ -> {error, {failed_requirements, FailedToMatch}} + end. + +%% @doc Get the info map for a device, optionally giving it a message if the +%% device's info function is parameterized by one. +info(Msg, Opts) -> + info(message_to_device(Msg, Opts), Msg, Opts). +info(DevMod, Msg, Opts) -> + %?event({calculating_info, {dev, DevMod}, {msg, Msg}}), + case find_exported_function(Msg, DevMod, info, 2, Opts) of + {ok, Fun} -> + Res = apply(Fun, truncate_args(Fun, [Msg, Opts])), + % ?event({ + % info_result, + % {dev, DevMod}, + % {args, truncate_args(Fun, [Msg])}, + % {result, Res} + % }), + Res; + not_found -> #{} + end. + +%% @doc The default device is the identity device, which simply returns the +%% value associated with any key as it exists in its Erlang map. It should also +%% implement the `set' key, which returns a `Message3' with the values changed +%% according to the `Message2' passed to it. +default_module() -> dev_message. + +%% @doc The execution options that are used internally by this module +%% when calling itself. +internal_opts(Opts) -> + hb_maps:merge(Opts, #{ + topic => hb_opts:get(topic, ao_internal, Opts), + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>], + spawn_worker => false, + await_inprogress => false + }). \ No newline at end of file diff --git a/src/hb_ao_test_vectors.erl b/src/hb_ao_test_vectors.erl new file mode 100644 index 000000000..29f80cde3 --- /dev/null +++ b/src/hb_ao_test_vectors.erl @@ -0,0 +1,994 @@ +%%% @doc Uses a series of different `Opts' values to test the resolution engine's +%%% execution under different circumstances. +-module(hb_ao_test_vectors). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("include/hb.hrl"). + +%% The time to run the benchmarks for in seconds. +-define(BENCHMARK_TIME, 0.25). +%% The number of iterations to run each benchmark for. +-define(BENCHMARK_ITERATIONS, 1_000). + +%% @doc Easy hook to make a test executable via the command line: +%% `rebar3 eunit --test hb_ao_test_vectors:run_test' +%% Comment/uncomment out as necessary. +run_test() -> + multiple_as_subresolutions_test(#{}). + +%% @doc Run each test in the file with each set of options. Start and reset +%% the store for each test. +suite_test_() -> + hb_test_utils:suite_with_opts(test_suite(), test_opts()). + +benchmark_test_() -> + hb_test_utils:suite_with_opts(benchmark_suite(), test_opts()). + +test_suite() -> + [ + {resolve_simple, "resolve simple", + fun resolve_simple_test/1}, + {resolve_id, "resolve id", + fun resolve_id_test/1}, + {start_as, "start as", + fun start_as_test/1}, + {start_as_with_parameters, "start as with parameters", + fun start_as_with_parameters_test/1}, + {load_as, "load as", + fun load_as_test/1}, + {as_path, "as path", + fun as_path_test/1}, + {continue_as, "continue as", + fun continue_as_test/1}, + {multiple_as_subresolutions, "multiple as subresolutions", + fun multiple_as_subresolutions_test/1}, + {resolve_key_twice, "resolve key twice", + fun resolve_key_twice_test/1}, + {resolve_from_multiple_keys, "resolve from multiple keys", + fun resolve_from_multiple_keys_test/1}, + {resolve_path_element, "resolve path element", + fun resolve_path_element_test/1}, + {resolve_binary_key, "resolve binary key", + fun resolve_binary_key_test/1}, + {key_to_binary, "key to binary", + fun key_to_binary_test/1}, + {key_from_id_device_with_args, "key from id device with args", + fun key_from_id_device_with_args_test/1}, + {device_with_handler_function, "device with handler function", + fun device_with_handler_function_test/1}, + {device_with_default_handler_function, + "device with default handler function", + fun device_with_default_handler_function_test/1}, + {basic_get, "basic get", + fun basic_get_test/1}, + {recursive_get, "recursive get", + fun recursive_get_test/1}, + {deep_recursive_get, "deep recursive get", + fun deep_recursive_get_test/1}, + {basic_set, "basic set", + fun basic_set_test/1}, + {get_with_device, "get with device", + fun get_with_device_test/1}, + {get_as_with_device, "get as with device", + fun get_as_with_device_test/1}, + {set_with_device, "set with device", + fun set_with_device_test/1}, + {deep_set, "deep set", + fun deep_set_test/1}, + {deep_set_with_device, "deep set with device", + fun deep_set_with_device_test/1}, + {device_exports, "device exports", + fun device_exports_test/1}, + {device_excludes, "device excludes", + fun device_excludes_test/1}, + {denormalized_device_key, "denormalized device key", + fun denormalized_device_key_test/1}, + {list_transform, "list transform", + fun list_transform_test/1}, + {step_hook, "step hook", + fun step_hook_test/1} + ]. + +benchmark_suite() -> + [ + {benchmark_simple, "simple resolution benchmark", + fun benchmark_simple_test/1}, + {benchmark_multistep, "multistep resolution benchmark", + fun benchmark_multistep_test/1}, + {benchmark_get, "get benchmark", + fun benchmark_get_test/1}, + {benchmark_set, "single value set benchmark", + fun benchmark_set_test/1}, + {benchmark_set_multiple, "set two keys benchmark", + fun benchmark_set_multiple_test/1}, + {benchmark_set_multiple_deep, "set two keys deep benchmark", + fun benchmark_set_multiple_deep_test/1} + ]. + +test_opts() -> + [ + #{ + name => normal, + desc => "Default opts", + opts => #{}, + skip => [] + }, + #{ + name => without_hashpath, + desc => "Default without hashpath", + opts => #{ + hashpath => ignore + }, + skip => [] + }, + #{ + name => no_cache, + desc => "No cache read or write", + opts => #{ + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>], + spawn_worker => false, + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/fs">> + } + }, + skip => [load_as] + }, + #{ + name => only_store, + desc => "Store, don't read", + opts => #{ + hashpath => update, + cache_control => [<<"no-cache">>], + spawn_worker => false, + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/fs">> + } + }, + skip => [ + denormalized_device_key, + deep_set_with_device, + load_as + ], + reset => false + }, + #{ + name => only_if_cached, + desc => "Only read, don't exec", + opts => #{ + hashpath => ignore, + cache_control => [<<"only-if-cached">>], + spawn_worker => false, + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/fs">> + } + }, + skip => [ + % Exclude tests that return a list on its own for now, as raw + % lists cannot be cached yet. + set_new_messages, + resolve_from_multiple_keys, + resolve_path_element, + denormalized_device_key, + % Skip test with locally defined device + deep_set_with_device, + as + % Skip tests that call hb_ao utils (which have their own + % cache settings). + ] + } + ]. + +%%% Standalone test vectors + +%% @doc Ensure that we can read a device from the cache then execute it. By +%% extension, this will also allow us to load a device from Arweave due to the +%% remote store implementations. +exec_dummy_device(SigningWallet, Opts) -> + % Compile the test device and store it in an accessible cache to the execution + % environment. + {ok, ModName, Bin} = compile:file("test/dev_dummy.erl", [binary]), + DevMsg = + hb_message:commit( + hb_ao:normalize_keys( + #{ + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"content-type">> => <<"application/beam">>, + <<"module-name">> => ModName, + <<"requires-otp-release">> => + hb_util:bin(erlang:system_info(otp_release)), + <<"body">> => Bin + }, + Opts + ), + Opts + ), + {ok, ID} = hb_cache:write(DevMsg, Opts), + % Ensure that we can read the device message from the cache and that it matches + % the original message. + {ok, ReadMsg} = hb_cache:read(ID, Opts), + ?assertEqual(DevMsg, hb_cache:ensure_all_loaded(ReadMsg, Opts)), + % Create a base message with the device ID, then request a dummy path from + % it. + hb_ao:resolve( + #{ <<"device">> => ID }, + #{ <<"path">> => <<"echo/param">>, <<"param">> => <<"example">> }, + Opts + ). + +load_device_test() -> + % Establish an execution environment which trusts the device author. + Wallet = ar_wallet:new(), + Opts = #{ + load_remote_devices => true, + trusted_device_signers => [hb_util:human_id(ar_wallet:to_address(Wallet))], + store => Store = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/fs">> + }, + priv_wallet => Wallet + }, + hb_store:reset(Store), + ?assertEqual({ok, <<"example">>}, exec_dummy_device(Wallet, Opts)). + +untrusted_load_device_test() -> + % Establish an execution environment which does not trust the device author. + UntrustedWallet = ar_wallet:new(), + TrustedWallet = ar_wallet:new(), + Opts = #{ + load_remote_devices => true, + trusted_device_signers => [hb_util:human_id(ar_wallet:to_address(TrustedWallet))], + store => Store = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/fs">> + }, + priv_wallet => UntrustedWallet + }, + hb_store:reset(Store), + ?assertThrow( + {error, {device_not_loadable, _, device_signer_not_trusted}}, + exec_dummy_device(UntrustedWallet, Opts) + ). + +%%% Test vector suite + +resolve_simple_test(Opts) -> + Res = hb_ao:resolve(#{ <<"a">> => <<"RESULT">> }, <<"a">>, Opts), + ?assertEqual({ok, <<"RESULT">>}, Res). + +resolve_id_test(Opts) -> + ?assertMatch( + ID when byte_size(ID) == 43, + hb_ao:get(id, #{ test_key => <<"1">> }, Opts) + ). + +resolve_key_twice_test(Opts) -> + % Ensure that the same message can be resolved again. + % This is not as trivial as it may seem, because resolutions are cached and + % de-duplicated. + ?assertEqual({ok, <<"1">>}, hb_ao:resolve(#{ <<"a">> => <<"1">> }, <<"a">>, Opts)), + ?assertEqual({ok, <<"1">>}, hb_ao:resolve(#{ <<"a">> => <<"1">> }, <<"a">>, Opts)). + +resolve_from_multiple_keys_test(Opts) -> + ?assertEqual( + {ok, [<<"a">>]}, + hb_ao:resolve(#{ <<"a">> => <<"1">>, <<"priv_a">> => <<"2">> }, <<"keys">>, Opts) + ). + +resolve_path_element_test(Opts) -> + ?assertEqual( + {ok, [<<"test_path">>]}, + hb_ao:resolve(#{ <<"path">> => [<<"test_path">>] }, <<"path">>, Opts) + ), + ?assertEqual( + {ok, [<<"a">>]}, + hb_ao:resolve(#{ <<"Path">> => [<<"a">>] }, <<"Path">>, Opts) + ). + +key_to_binary_test(Opts) -> + ?assertEqual(<<"a">>, hb_ao:normalize_key(a, Opts)), + ?assertEqual(<<"a">>, hb_ao:normalize_key(<<"a">>, Opts)), + ?assertEqual(<<"a">>, hb_ao:normalize_key("a", Opts)). + +resolve_binary_key_test(Opts) -> + ?assertEqual( + {ok, <<"RESULT">>}, + hb_ao:resolve(#{ a => <<"RESULT">> }, <<"a">>, Opts) + ), + ?assertEqual( + {ok, <<"1">>}, + hb_ao:resolve( + #{ + <<"Test-Header">> => <<"1">> + }, + <<"Test-Header">>, + Opts + ) + ). + +%% @doc Generates a test device with three keys, each of which uses +%% progressively more of the arguments that can be passed to a device key. +generate_device_with_keys_using_args() -> + #{ + key_using_only_state => + fun(State) -> + {ok, + <<(hb_maps:get(<<"state_key">>, State))/binary>> + } + end, + key_using_state_and_msg => + fun(State, Msg) -> + {ok, + << + (hb_maps:get(<<"state_key">>, State))/binary, + (hb_maps:get(<<"msg_key">>, Msg))/binary + >> + } + end, + key_using_all => + fun(State, Msg, Opts) -> + {ok, + << + (hb_maps:get(<<"state_key">>, State, undefined, Opts))/binary, + (hb_maps:get(<<"msg_key">>, Msg, undefined, Opts))/binary, + (hb_maps:get(<<"opts_key">>, Opts, undefined, Opts))/binary + >> + } + end + }. + +%% @doc Create a simple test device that implements the default handler. +gen_default_device() -> + #{ + info => + fun() -> + #{ + default => + fun(_, _State) -> + {ok, <<"DEFAULT">>} + end + } + end, + <<"state_key">> => + fun(_) -> + {ok, <<"STATE">>} + end + }. + +%% @doc Create a simple test device that implements the handler key. +gen_handler_device() -> + #{ + info => + fun() -> + #{ + handler => + fun(<<"set">>, M1, M2, Opts) -> + dev_message:set(M1, M2, Opts); + (_, _, _, _) -> + {ok, <<"HANDLER VALUE">>} + end + } + end + }. + +%% @doc Test that arguments are passed to a device key as expected. +%% Particularly, we need to ensure that the key function in the device can +%% specify any arity (1 through 3) and the call is handled correctly. +key_from_id_device_with_args_test(Opts) -> + Msg = + #{ + device => generate_device_with_keys_using_args(), + state_key => <<"1">> + }, + ?assertEqual( + {ok, <<"1">>}, + hb_ao:resolve( + Msg, + #{ + <<"path">> => <<"key_using_only_state">>, + <<"msg_key">> => <<"2">> % Param message, which is ignored + }, + Opts + ) + ), + ?assertEqual( + {ok, <<"13">>}, + hb_ao:resolve( + Msg, + #{ + <<"path">> => <<"key_using_state_and_msg">>, + <<"msg_key">> => <<"3">> % Param message, with value to add + }, + Opts + ) + ), + ?assertEqual( + {ok, <<"1337">>}, + hb_ao:resolve( + Msg, + #{ + <<"path">> => <<"key_using_all">>, + <<"msg_key">> => <<"3">> % Param message + }, + Opts#{ + <<"opts_key">> => <<"37">>, + <<"cache_control">> => [<<"no-cache">>, <<"no-store">>] + } + ) + ). + +device_with_handler_function_test(Opts) -> + Msg = + #{ + device => gen_handler_device(), + test_key => <<"BAD">> + }, + ?assertEqual( + {ok, <<"HANDLER VALUE">>}, + hb_ao:resolve(Msg, <<"test_key">>, Opts) + ). + +device_with_default_handler_function_test(Opts) -> + Msg = + #{ + device => gen_default_device() + }, + ?assertEqual( + {ok, <<"STATE">>}, + hb_ao:resolve(Msg, <<"state_key">>, Opts) + ), + ?assertEqual( + {ok, <<"DEFAULT">>}, + hb_ao:resolve(Msg, <<"any_random_key">>, Opts) + ). + +basic_get_test(Opts) -> + Msg = #{ <<"key1">> => <<"value1">>, <<"key2">> => <<"value2">> }, + ?assertEqual(<<"value1">>, hb_ao:get(<<"key1">>, Msg, Opts)), + ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), + ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), + ?assertEqual(<<"value2">>, hb_ao:get([<<"key2">>], Msg, Opts)). + +recursive_get_test(Opts) -> + Msg = #{ + <<"key1">> => <<"value1">>, + <<"key2">> => #{ + <<"key3">> => <<"value3">>, + <<"key4">> => #{ + <<"key5">> => <<"value5">>, + <<"key6">> => #{ + <<"key7">> => <<"value7">> + } + } + } + }, + ?assertEqual( + {ok, <<"value1">>}, + hb_ao:resolve(Msg, #{ <<"path">> => <<"key1">> }, Opts) + ), + ?assertEqual(<<"value1">>, hb_ao:get(<<"key1">>, Msg, Opts)), + ?assertEqual( + {ok, <<"value3">>}, + hb_ao:resolve(Msg, #{ <<"path">> => [<<"key2">>, <<"key3">>] }, Opts) + ), + ?assertEqual(<<"value3">>, hb_ao:get([<<"key2">>, <<"key3">>], Msg, Opts)), + ?assertEqual(<<"value3">>, hb_ao:get(<<"key2/key3">>, Msg, Opts)). + +deep_recursive_get_test(Opts) -> + Msg = #{ + <<"key1">> => <<"value1">>, + <<"key2">> => #{ + <<"key3">> => <<"value3">>, + <<"key4">> => #{ + <<"key5">> => <<"value5">>, + <<"key6">> => #{ + <<"key7">> => <<"value7">> + } + } + } + }, + ?assertEqual(<<"value7">>, hb_ao:get(<<"key2/key4/key6/key7">>, Msg, Opts)). + +basic_set_test(Opts) -> + Msg = #{ <<"key1">> => <<"value1">>, <<"key2">> => <<"value2">> }, + UpdatedMsg = hb_ao:set(Msg, #{ <<"key1">> => <<"new_value1">> }, Opts), + ?event({set_key_complete, {key, <<"key1">>}, {value, <<"new_value1">>}}), + ?assertEqual(<<"new_value1">>, hb_ao:get(<<"key1">>, UpdatedMsg, Opts)), + ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, UpdatedMsg, Opts)). + +get_with_device_test(Opts) -> + Msg = + #{ + <<"device">> => generate_device_with_keys_using_args(), + <<"state_key">> => <<"STATE">> + }, + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), + ?assertEqual(<<"STATE">>, hb_ao:get(<<"key_using_only_state">>, Msg, Opts)). + +get_as_with_device_test(Opts) -> + Msg = + #{ + <<"device">> => gen_handler_device(), + <<"test_key">> => <<"ACTUAL VALUE">> + }, + ?assertEqual( + <<"HANDLER VALUE">>, + hb_ao:get(test_key, Msg, Opts) + ), + ?assertEqual( + <<"ACTUAL VALUE">>, + hb_ao:get(test_key, {as, dev_message, Msg}, Opts) + ). + +set_with_device_test(Opts) -> + Msg = + #{ + <<"device">> => + #{ + <<"set">> => + fun(State, _Msg) -> + Acc = hb_maps:get(<<"set_count">>, State, <<"">>, Opts), + {ok, + State#{ + <<"set_count">> => << Acc/binary, "." >> + } + } + end + }, + <<"state_key">> => <<"STATE">> + }, + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), + SetOnce = hb_ao:set(Msg, #{ <<"state_key">> => <<"SET_ONCE">> }, Opts), + ?assertEqual(<<".">>, hb_ao:get(<<"set_count">>, SetOnce, Opts)), + SetTwice = hb_ao:set(SetOnce, #{ <<"state_key">> => <<"SET_TWICE">> }, Opts), + ?assertEqual(<<"..">>, hb_ao:get(<<"set_count">>, SetTwice, Opts)), + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, SetTwice, Opts)). + +deep_set_test(Opts) -> + % First validate second layer changes are handled correctly. + Msg0 = #{ <<"a">> => #{ <<"b">> => <<"RESULT">> } }, + ?assertMatch(#{ <<"a">> := #{ <<"b">> := <<"RESULT2">> } }, + hb_ao:set(Msg0, <<"a/b">>, <<"RESULT2">>, Opts)), + ?assertMatch(#{ <<"a">> := #{ <<"b">> := <<"RESULT2">> } }, + hb_ao:set(Msg0, [<<"a">>, <<"b">>], <<"RESULT2">>, Opts)), + % Now validate deeper layer changes are handled correctly. + Msg = #{ <<"a">> => #{ <<"b">> => #{ <<"c">> => <<"1">> } } }, + ?assertMatch(#{ <<"a">> := #{ <<"b">> := #{ <<"c">> := <<"2">> } } }, + hb_ao:set(Msg, [<<"a">>, <<"b">>, <<"c">>], <<"2">>, Opts)). + +deep_set_new_messages_test() -> + Opts = hb_maps:get(opts, hd(test_opts())), + % Test that new messages are created when the path does not exist. + Msg0 = #{ <<"a">> => #{ <<"b">> => #{ <<"c">> => <<"1">> } } }, + Msg1 = hb_ao:set(Msg0, <<"d/e">>, <<"3">>, Opts), + Msg2 = hb_ao:set(Msg1, <<"d/f">>, <<"4">>, Opts), + ?assert( + hb_message:match( + Msg2, + #{ + <<"a">> => + #{ + <<"b">> => + #{ <<"c">> => <<"1">> } + }, + <<"d">> => + #{ + <<"e">> => <<"3">>, + <<"f">> => <<"4">> } + } + ) + ), + Msg3 = hb_ao:set( + Msg2, + #{ + <<"z/a">> => <<"0">>, + <<"z/b">> => <<"1">>, + <<"z/y/x">> => <<"2">> + }, + Opts + ), + ?assert( + hb_message:match( + Msg3, + #{ + <<"a">> => #{ <<"b">> => #{ <<"c">> => <<"1">> } }, + <<"d">> => #{ <<"e">> => <<"3">>, <<"f">> => <<"4">> }, + <<"z">> => + #{ + <<"a">> => <<"0">>, + <<"b">> => <<"1">>, + <<"y">> => #{ <<"x">> => <<"2">> } + } + } + ) + ). + +deep_set_with_device_test(Opts) -> + Device = #{ + set => + fun(Msg1, Msg2) -> + % A device where the set function modifies the key + % and adds a modified flag. + {Key, Val} = + hd(hb_maps:to_list(hb_maps:without([<<"path">>, <<"priv">>], Msg2, Opts), Opts)), + {ok, Msg1#{ Key => Val, <<"modified">> => true }} + end + }, + % A message with an interspersed custom device: A and C have it, + % B does not. A and C will have the modified flag set to true. + Msg = #{ + <<"device">> => Device, + <<"a">> => + #{ + <<"b">> => + #{ + <<"device">> => Device, + <<"c">> => <<"1">>, + <<"modified">> => false + }, + <<"modified">> => false + }, + <<"modified">> => false + }, + Outer = hb_ao:set(Msg, <<"a/b/c">>, <<"2">>, Opts), + A = hb_ao:get(<<"a">>, Outer, Opts), + B = hb_ao:get(<<"b">>, A, Opts), + C = hb_ao:get(<<"c">>, B, Opts), + ?assertEqual(<<"2">>, C), + ?assertEqual(true, hb_ao:get(<<"modified">>, Outer)), + ?assertEqual(false, hb_ao:get(<<"modified">>, A)), + ?assertEqual(true, hb_ao:get(<<"modified">>, B)). + +device_exports_test(Opts) -> + Msg = #{ <<"device">> => dev_message }, + ?assert(hb_ao:is_exported(Msg, dev_message, info, Opts)), + ?assert(hb_ao:is_exported(Msg, dev_message, set, Opts)), + ?assert( + hb_ao:is_exported( + Msg, + dev_message, + not_explicitly_exported, + Opts + ) + ), + Dev = #{ + info => fun() -> #{ exports => [set] } end, + set => fun(_, _) -> {ok, <<"SET">>} end + }, + Msg2 = #{ <<"device">> => Dev }, + ?assert(hb_ao:is_exported(Msg2, Dev, info, Opts)), + ?assert(hb_ao:is_exported(Msg2, Dev, set, Opts)), + ?assert(not hb_ao:is_exported(Msg2, Dev, not_exported, Opts)), + Dev2 = #{ + info => + fun() -> + #{ + exports => [test1, <<"test2">>], + handler => + fun() -> + {ok, <<"Handler-Value">>} + end + } + end + }, + Msg3 = #{ <<"device">> => Dev2, <<"test1">> => <<"BAD1">>, <<"test3">> => <<"GOOD3">> }, + ?assertEqual(<<"Handler-Value">>, hb_ao:get(<<"test1">>, Msg3, Opts)), + ?assertEqual(<<"Handler-Value">>, hb_ao:get(<<"test2">>, Msg3, Opts)), + ?assertEqual(<<"GOOD3">>, hb_ao:get(<<"test3">>, Msg3, Opts)), + ?assertEqual(<<"GOOD4">>, + hb_ao:get( + <<"test4">>, + hb_ao:set(Msg3, <<"test4">>, <<"GOOD4">>, Opts) + ) + ), + ?assertEqual(not_found, hb_ao:get(<<"test5">>, Msg3, Opts)). + +device_excludes_test(Opts) -> + % Create a device that returns an identifiable message for any key, but also + % sets excludes to [set], such that the message can be modified using the + % default handler. + Dev = #{ + <<"info">> => + fun() -> + #{ + excludes => [set], + handler => fun() -> {ok, <<"Handler-Value">>} end + } + end + }, + Msg = #{ <<"device">> => Dev, <<"Test-Key">> => <<"Test-Value">> }, + ?assert(hb_ao:is_exported(Msg, Dev, <<"test-key2">>, Opts)), + ?assert(not hb_ao:is_exported(Msg, Dev, set, Opts)), + ?assertEqual(<<"Handler-Value">>, hb_ao:get(<<"test-key2">>, Msg, Opts)), + ?assertMatch(#{ <<"test-key2">> := <<"2">> }, + hb_ao:set(Msg, <<"test-key2">>, <<"2">>, Opts)). + +denormalized_device_key_test(Opts) -> + Msg = #{ <<"device">> => dev_test }, + ?assertEqual(dev_test, hb_ao:get(device, Msg, Opts)), + ?assertEqual(dev_test, hb_ao:get(<<"device">>, Msg, Opts)), + ?assertEqual({module, dev_test}, + erlang:fun_info( + element(3, hb_ao:message_to_fun(Msg, test_func, Opts)), + module + ) + ). + +list_transform_test(Opts) -> + Msg = [<<"A">>, <<"B">>, <<"C">>, <<"D">>, <<"E">>], + ?assertEqual(<<"A">>, hb_ao:get(1, Msg, Opts)), + ?assertEqual(<<"B">>, hb_ao:get(2, Msg, Opts)), + ?assertEqual(<<"C">>, hb_ao:get(3, Msg, Opts)), + ?assertEqual(<<"D">>, hb_ao:get(4, Msg, Opts)), + ?assertEqual(<<"E">>, hb_ao:get(5, Msg, Opts)). + +start_as_test(Opts) -> + ?assertEqual( + {ok, <<"GOOD_FUNCTION">>}, + hb_ao:resolve_many( + [ + {as, <<"test-device@1.0">>, #{ <<"path">> => <<>> }}, + #{ <<"path">> => <<"test_func">> } + ], + Opts + ) + ). +start_as_with_parameters_test(Opts) -> + % Resolve a key on a message that has its device set with `as'. + Msg = #{ + <<"device">> => <<"test-device@1.0">>, + <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + }, + ?assertEqual( + {ok, <<"GOOD_FUNCTION">>}, + hb_ao:resolve_many( + [ + {as, <<"message@1.0">>, Msg}, + #{ <<"path">> => <<"test_func">> } + ], + Opts + ) + ). + +load_as_test(Opts) -> + % Load a message as a device with the `as' keyword. + Msg = #{ + <<"device">> => <<"test-device@1.0">>, + <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + }, + {ok, ID} = hb_cache:write(Msg, Opts), + ?assertEqual( + {ok, <<"MESSAGE">>}, + hb_ao:resolve_many( + [ + {as, <<"message@1.0">>, #{ <<"path">> => <> }}, + <<"test_func">>, + <<"test_key">> + ], + Opts + ) + ). + +as_path_test(Opts) -> + % Create a message with the test device, which implements the test_func + % function. It normally returns `GOOD_FUNCTION'. + Msg = #{ + <<"device">> => <<"test-device@1.0">>, + <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + }, + ?assertEqual(<<"GOOD_FUNCTION">>, hb_ao:get(<<"test_func">>, Msg, Opts)), + % Now use the `as' keyword to subresolve a key with the message device. + ?assertMatch( + {ok, #{ <<"test_key">> := <<"MESSAGE">> }}, + hb_ao:resolve( + Msg, + {as, <<"message@1.0">>, #{ <<"path">> => <<"test_func">> }}, + Opts + ) + ). + +continue_as_test(Opts) -> + % Resolve a list of messages in sequence, swapping the device in the middle. + Msg = #{ + <<"device">> => <<"test-device@1.0">>, + <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + }, + ?assertEqual( + {ok, <<"MESSAGE">>}, + hb_ao:resolve_many( + [ + Msg, + {as, <<"message@1.0">>, <<>>}, + #{ <<"path">> => <<"test_func">> }, + #{ <<"path">> => <<"test_key">> } + ], + Opts + ) + ). + +multiple_as_subresolutions_test(Opts) -> + % Test that multiple as subresolutions in a sequence are handled correctly. + Msg = #{ + <<"device">> => <<"test-device@1.0">>, + <<"test-message">> => + #{ + <<"test-key">> => <<"MESSAGE-1">>, + <<"test-message-2">> => + #{ <<"test-key-2">> => <<"MESSAGE-2">> } + } + }, + Res = hb_ao:resolve_many( + [ + {as, <<"message@1.0">>, Msg}, + #{ <<"path">> => <<"test-message">> }, + #{ <<"path">> => <<"test-message-2">>, <<"extraneous">> => <<"1">> }, + <<"test-key-2">> + ], + Opts + ), + ?assertEqual({ok, <<"MESSAGE-2">>}, Res), + % Attempt to resolve a sequence of more complex messages. + Path = <<"/~meta@1.0/info/~hyperbuddy@1.0/format">>, + Parsed = hb_singleton:from(Path, Opts), + ?event(subresolution, {parsed_sequence, Parsed}), + Res2 = hb_ao:resolve(Path, Opts), + ?assertMatch({ok, #{ <<"body">> := Bin }} when is_binary(Bin), Res2). + +step_hook_test(InitOpts) -> + % Test that the step hook is called correctly. We do this by sending ourselves + % a message each time the hook is called. We also send a `reference', such + % that this test is uniquely identified and further/prior tests do not affect + % it. + Self = self(), + Ref = make_ref(), + Opts = + InitOpts#{ + on => + #{ + <<"step">> => + #{ + <<"device">> => + #{ + <<"step">> => + fun(_, Req, _) -> + ?event(ao_core, {step_hook, {self(), Ref}}), + Self ! {step, Ref}, + {ok, Req} + end + } + } + } + }, + Msg = #{ + <<"a">> => + #{ + <<"b">> => + #{ + <<"c">> => <<"1">> + } + } + }, + % Test that the response has completed and is correct. + ?assertMatch( + {ok, <<"1">>}, + hb_ao:resolve( + Msg, + #{ <<"path">> => <<"a/b/c">> }, + Opts + ) + ), + % Test that the step hook was called. + ?assert(receive {step, Ref} -> true after 100 -> false end). + +%%% Benchmark tests +benchmark_simple_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> hb_ao:resolve(#{ <<"a">> => I }, <<"a">>, Opts) end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Single-step resolutions:">>, + ?BENCHMARK_ITERATIONS, + Time + ). + +benchmark_multistep_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:resolve( + #{ + <<"iteration">> => I, + <<"a">> => #{ + <<"b">> => #{ <<"return">> => I } + } + }, + <<"a/b/return">>, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Multistep resolutions:">>, + ?BENCHMARK_ITERATIONS, + Time + ). + +benchmark_get_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:get( + <<"a">>, + #{ <<"a">> => <<"1">>, <<"iteration">> => I }, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Get operations:">>, + ?BENCHMARK_ITERATIONS, + Time + ). + +benchmark_set_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:set( + #{ <<"a">> => <<"1">>, <<"iteration">> => I }, + <<"a">>, + <<"2">>, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Single value set operations:">>, + ?BENCHMARK_ITERATIONS, + Time + ). + +benchmark_set_multiple_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:set( + #{ <<"a">> => <<"1">>, <<"iteration">> => I }, + #{ <<"a">> => <<"1a">>, <<"b">> => <<"2">> }, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Set two keys operations:">>, + ?BENCHMARK_ITERATIONS, + Time + ). + + +benchmark_set_multiple_deep_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:set( + #{ <<"a">> => #{ <<"b">> => <<"1">> } }, + #{ <<"a">> => #{ <<"b">> => <<"2">>, <<"c">> => I } }, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Set two keys operations:">>, + ?BENCHMARK_ITERATIONS, + Time + ). \ No newline at end of file diff --git a/src/hb_beamr.erl b/src/hb_beamr.erl index 9be12ccc3..76020500b 100644 --- a/src/hb_beamr.erl +++ b/src/hb_beamr.erl @@ -9,11 +9,11 @@ %%% %%% Because each WASM module runs as an independent async worker, if you plan %%% to run many instances in parallel, you should be sure to configure the -%%% BEAM to have enough async worker threads enabled (see `erl +A N` in the +%%% BEAM to have enough async worker threads enabled (see `erl +A N' in the %%% Erlang manuals). %%% %%% The core API is simple: -%%% ``` +%%% 
 %%%     start(WasmBinary) -> {ok, Port, Imports, Exports}
 %%%         Where:
 %%%             WasmBinary is the WASM binary to load.
@@ -32,9 +32,9 @@
 %%%     call(Port, FunName, Args[, Import, State, Opts]) -> {ok, Res, NewState}
 %%%         Where:
 %%%             ImportFun is a function that will be called upon each import.
-%%%             ImportFun must have an arity of 2: Taking an arbitrary `state`
-%%%             term, and a map containing the `port`, `module`, `func`, `args`,
-%%%             `signature`, and the `options` map of the import.
+%%%             ImportFun must have an arity of 2: Taking an arbitrary `state'
+%%%             term, and a map containing the `port', `module', `func', `args',
+%%%             `signature', and the `options' map of the import.
 %%%             It must return a tuple of the form {ok, Response, NewState}.
 %%%     serialize(Port) -> {ok, Mem}
 %%%         Where:
@@ -44,37 +44,41 @@
 %%%         Where:
 %%%             Port is the port to the LID.
 %%%             Mem is a binary output of a previous `serialize/1' call.
-%%% '''
+%%%
%%% %%% BEAMR was designed for use in the HyperBEAM project, but is suitable for %%% deployment in other Erlang applications that need to run WASM modules. PRs %%% are welcome. -module(hb_beamr). %%% Control API: --export([start/1, call/3, call/4, call/5, call/6, stop/1, wasm_send/2]). +-export([start/1, start/2, call/3, call/4, call/5, call/6, stop/1, wasm_send/2]). %%% Utility API: -export([serialize/1, deserialize/2, stub/3]). --include("src/include/hb.hrl"). +-include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). %% @doc Load the driver for the WASM executor. load_driver() -> - case erl_ddll:load("./priv", ?MODULE) of + case erl_ddll:load(code:priv_dir(hb), ?MODULE) of ok -> ok; {error, already_loaded} -> ok; {error, Error} -> {error, Error} end. %% @doc Start a WASM executor context. Yields a port to the LID, and the -%% imports and exports of the WASM module. +%% imports and exports of the WASM module. Optionally, specify a mode +%% (wasm or aot) to indicate the type of WASM module being loaded. start(WasmBinary) when is_binary(WasmBinary) -> + start(WasmBinary, wasm). +start(WasmBinary, Mode) when is_binary(WasmBinary) -> + ?event({loading_module, {bytes, byte_size(WasmBinary)}, Mode}), Self = self(), WASM = spawn( fun() -> ok = load_driver(), Port = open_port({spawn, "hb_beamr"}, []), - Port ! {self(), {command, term_to_binary({init, WasmBinary})}}, + Port ! {self(), {command, term_to_binary({init, WasmBinary, Mode})}}, ?event({waiting_for_init_from, Port}), worker(Port, Self) end @@ -129,19 +133,19 @@ stop(WASM) when is_pid(WASM) -> ok. %% @doc Call a function in the WASM executor (see moduledoc for more details). -call(Port, FunctionName, Args) -> - {ok, Res, _} = call(Port, FunctionName, Args, fun stub/3), +call(PID, FuncRef, Args) -> + {ok, Res, _} = call(PID, FuncRef, Args, fun stub/3), {ok, Res}. -call(Port, FunctionName, Args, ImportFun) -> - call(Port, FunctionName, Args, ImportFun, #{}). -call(Port, FunctionName, Args, ImportFun, StateMsg) -> - call(Port, FunctionName, Args, ImportFun, StateMsg, #{}). -call(Port, FunctionName, Args, ImportFun, StateMsg, Opts) - when is_binary(FunctionName) -> - call(Port, binary_to_list(FunctionName), Args, ImportFun, StateMsg, Opts); -call(WASM, FunctionName, Args, ImportFun, StateMsg, Opts) +call(PID, FuncRef, Args, ImportFun) -> + call(PID, FuncRef, Args, ImportFun, #{}). +call(PID, FuncRef, Args, ImportFun, StateMsg) -> + call(PID, FuncRef, Args, ImportFun, StateMsg, #{}). +call(PID, FuncRef, Args, ImportFun, StateMsg, Opts) + when is_binary(FuncRef) -> + call(PID, binary_to_list(FuncRef), Args, ImportFun, StateMsg, Opts); +call(WASM, FuncRef, Args, ImportFun, StateMsg, Opts) when is_pid(WASM) - andalso is_list(FunctionName) + andalso (is_list(FuncRef) or is_integer(FuncRef)) andalso is_list(Args) andalso is_function(ImportFun) andalso is_map(Opts) -> @@ -150,13 +154,21 @@ call(WASM, FunctionName, Args, ImportFun, StateMsg, Opts) ?event( {call_started, WASM, - FunctionName, + FuncRef, Args, ImportFun, StateMsg, Opts}), wasm_send(WASM, - {command, term_to_binary({call, FunctionName, Args})}), + {command, + term_to_binary( + case is_integer(FuncRef) of + true -> {indirect_call, FuncRef, Args}; + false -> {call, FuncRef, Args} + end + ) + } + ), ?event({waiting_for_call_result, self(), WASM}), monitor_call(WASM, ImportFun, StateMsg, Opts); false -> @@ -189,7 +201,7 @@ monitor_call(WASM, ImportFun, StateMsg, Opts) -> }, Opts ), - ?event({import_ret, Module, Func, {args, Args}, {params, Res}}), + ?event({import_ret, Module, Func, {args, Args}, {res, Res}}), dispatch_response(WASM, Res), monitor_call(WASM, ImportFun, StateMsg2, Opts) catch @@ -299,7 +311,7 @@ benchmark_test() -> BenchTime = 1, {ok, File} = file:read_file("test/test-64.wasm"), {ok, WASM, _ImportMap, _Exports} = start(File), - Iterations = hb:benchmark( + Iterations = hb_test_utils:benchmark( fun() -> {ok, [Result]} = call(WASM, "fac", [5.0]), ?assertEqual(120.0, Result) @@ -308,8 +320,10 @@ benchmark_test() -> ), ?event(benchmark, {scheduled, Iterations}), ?assert(Iterations > 1000), - hb_util:eunit_print( - "Executed ~s calls through Beamr in ~p seconds (~.2f call/s)", - [hb_util:human_int(Iterations), BenchTime, Iterations / BenchTime] + hb_test_utils:benchmark_print( + <<"Direct beamr: Executed">>, + <<"calls">>, + Iterations, + BenchTime ), ok. \ No newline at end of file diff --git a/src/hb_beamr_io.erl b/src/hb_beamr_io.erl index d2fad1766..b52239505 100644 --- a/src/hb_beamr_io.erl +++ b/src/hb_beamr_io.erl @@ -141,7 +141,7 @@ size_test() -> %% @doc Test writing memory in and out of bounds. write_test() -> - % Load the `test-print` WASM module, which has a simple print function. + % Load the `test-print' WASM module, which has a simple print function. % We do not call the function here, but instead check that we can write % to its memory. It has a single page (65,536 bytes) of memory. {ok, File} = file:read_file("test/test-print.wasm"), @@ -153,7 +153,7 @@ write_test() -> %% @doc Test reading memory in and out of bounds. read_test() -> - % Our `test-print` module is hand-written in WASM, so we know that it + % Our `test-print' module is hand-written in WASM, so we know that it % has a `Hello, World!` string at precisely offset 66. {ok, File} = file:read_file("test/test-print.wasm"), {ok, WASM, _Imports, _Exports} = hb_beamr:start(File), diff --git a/src/hb_cache.erl b/src/hb_cache.erl index 256a493e0..643016964 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -1,12 +1,12 @@ -%%% @doc A cache of Converge Protocol messages and compute results. +%%% @doc A cache of AO-Core protocol messages and compute results. %%% %%% HyperBEAM stores all paths in key value stores, abstracted by the `hb_store' %%% module. Each store has its own storage backend, but each works with simple %%% key-value pairs. Each store can write binary keys at paths, and link between %%% paths. -%%% +%%% %%% There are three layers to HyperBEAMs internal data representation on-disk: -%%% +%%% %%% 1. The raw binary data, written to the store at the hash of the content. %%% Storing binary paths in this way effectively deduplicates the data. %%% 2. The hashpath-graph of all content, stored as a set of links between @@ -14,260 +14,603 @@ %%% all messages to share the same hashpath space, such that all requests %%% from users additively fill-in the hashpath space, minimizing duplicated %%% compute. -%%% 3. Messages, referrable by their IDs (attested or unsigned). These are -%%% stored as a set of links between keys and the hashes of the underlying -%%% data. We store this set of links in addition to the hashpath-graph, such -%%% that we can clearly delimit where the messages end, while the graph -%%% continues to be used for additional shared compute. -%%% -%%% Before writing a message to the store, we convert it to Type-Annotated +%%% 3. Messages, referrable by their IDs (committed or uncommitted). These are +%%% stored as a set of links commitment IDs and the uncommitted message. +%%% +%%% Before writing a message to the store, we convert it to Type-Annotated %%% Binary Messages (TABMs), such that each of the keys in the message is %%% either a map or a direct binary. %%% -%%% For example, imagine we have a computation result (`Msg3') which contains -%%% the following keys: -%%% ``` -%%% /Result/Signature -%%% /Result/Owner -%%% /Result/WASM-Output -%%% /Usage-Report/CPU-Time -%%% /Usage-Report/Memory-Usage -%%% ''' +%%% Nested keys are lazily loaded from the stores, such that large deeply +%%% nested messages where only a small part of the data is actually used are +%%% not loaded into memory unnecessarily. In order to ensure that a message is +%%% loaded from the cache after a `read', we can use the `ensure_loaded/1' and +%%% `ensure_all_loaded/1' functions. Ensure loaded will load the exact value +%%% that has been requested, while ensure all loaded will load the entire +%%% structure of the message into memory. %%% -%%% This module will first write raw binaries to the store using their hashes -%%% as keys, then right link trees for the hash path ('hashpath([Msg1, Msg2, Result, -%%% ...])', then write link trees for each of the unsigned and signed messages. +%%% Lazily loadable `links' are expressed as a tuple of the following form: +%%% `{link, ID, LinkOpts}', where `ID' is the path to the data in the store, +%%% and `LinkOpts' is a map of suggested options to use when loading the data. +%%% In particular, this module ensures to stash the `store' option in `LinkOpts', +%%% such that the `read' function can use the correct store without having to +%%% search unnecessarily. By providing an `Opts' argument to `ensure_loaded' or +%%% `ensure_all_loaded', the caller can specify additional options to use when +%%% loading the data -- overriding the suggested options in the link. -module(hb_cache). --export([read/2, read_output/3, write/2, write_binary/3]). --export([list/2, list_numbered/2, link/3]). +-export([ensure_loaded/1, ensure_loaded/2, ensure_all_loaded/1, ensure_all_loaded/2]). +-export([read/2, read_resolved/3, write/2, write_binary/3, write_hashpath/2, link/3]). +-export([match/2, list/2, list_numbered/2]). -export([test_unsigned/1, test_signed/1]). --include("src/include/hb.hrl"). +-include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% List all items in a directory, assuming they are numbered. +%% @doc Ensure that a value is loaded from the cache if it is an ID or a link. +%% If it is not loadable we raise an error. If the value is a message, we will +%% load only the first `layer' of it: Representing all nested messages inside +%% the result as links. If the value has an associated `type' key in the extra +%% options, we apply it to the read value, 'lazily' recreating a `structured@1.0' +%% form. +ensure_loaded(Msg) -> + ensure_loaded(Msg, #{}). +ensure_loaded(Msg, Opts) -> + ensure_loaded([], Msg, Opts). +ensure_loaded(Ref, {Status, Msg}, Opts) when Status == ok; Status == error -> + {Status, ensure_loaded(Ref, Msg, Opts)}; +ensure_loaded(Ref, + Lk = {link, ID, LkOpts = #{ <<"type">> := <<"link">>, <<"lazy">> := Lazy }}, + RawOpts) -> + % The link is to a submessage; either in lazy (unresolved) form, or direct + % form. + UnscopedOpts = hb_util:deep_merge(RawOpts, LkOpts, RawOpts), + Opts = hb_store:scope(UnscopedOpts, hb_opts:get(scope, local, LkOpts)), + Store = hb_opts:get(store, no_viable_store, Opts), + ?event(debug_cache, + {loading_multi_link, + {link, ID}, + {link_opts, LkOpts}, + {store, Store} + } + ), + case hb_cache:read(ID, hb_util:deep_merge(Opts, LkOpts, Opts)) of + {ok, Next} -> + ?event(debug_cache, + {loaded, + {link, ID}, + {store, Store} + }), + case Lazy of + true -> + % We have resolved the ID of the submessage, so we continue + % to load the submessage itself. + ensure_loaded( + {link, + Next, + #{ + <<"type">> => <<"link">>, + <<"lazy">> => false + } + }, + Opts + ); + false -> + % The already had the ID of the submessage, so now we have + % the data, we simply return it. + Next + end; + not_found -> + report_ensure_loaded_not_found(Ref, Lk, Opts) + end; +ensure_loaded(Ref, Link = {link, ID, LinkOpts = #{ <<"lazy">> := true }}, RawOpts) -> + % If the user provided their own options, we merge them and _overwrite_ + % the options that are already set in the link. + UnscopedOpts = hb_util:deep_merge(RawOpts, LinkOpts, RawOpts), + Opts = hb_store:scope(UnscopedOpts, hb_opts:get(scope, local, LinkOpts)), + case hb_cache:read(ID, Opts) of + {ok, LoadedMsg} -> + ?event(caching, + {lazy_loaded, + {link, ID}, + {msg, LoadedMsg}, + {link_opts, LinkOpts} + } + ), + case hb_maps:get(<<"type">>, LinkOpts, undefined, Opts) of + undefined -> LoadedMsg; + Type -> dev_codec_structured:decode_value(Type, LoadedMsg) + end; + not_found -> + report_ensure_loaded_not_found(Ref, Link, Opts) + end; +ensure_loaded(Ref, {link, ID, LinkOpts}, Opts) -> + ensure_loaded(Ref, {link, ID, LinkOpts#{ <<"lazy">> => true}}, Opts); +ensure_loaded(_Ref, Msg, _Opts) when not ?IS_LINK(Msg) -> + Msg. + +%% @doc Report that a value was not found in the cache. If a key is provided, +%% we report that the key was not found, otherwise we report that the link was +%% not found. +report_ensure_loaded_not_found(Ref, Lk, Opts) -> + ?event(link_error, {link_not_resolvable, {ref, Ref}, {link, Lk}, {opts, Opts}}), + throw( + {necessary_message_not_found, + hb_path:to_binary(lists:reverse(Ref)), + hb_link:format_unresolved(Lk, Opts, 0) + } + ). + +%% @doc Ensure that all of the components of a message (whether a map, list, +%% or immediate value) are recursively fully loaded from the stores into memory. +%% This is a catch-all function that is useful in situations where ensuring a +%% message contains no links is important, but it carries potentially extreme +%% performance costs. +ensure_all_loaded(Msg) -> + ensure_all_loaded(Msg, #{}). +ensure_all_loaded(Msg, Opts) -> + ensure_all_loaded([], Msg, Opts). +ensure_all_loaded(Ref, Link, Opts) when ?IS_LINK(Link) -> + ensure_all_loaded(Ref, ensure_loaded(Ref, Link, Opts), Opts); +ensure_all_loaded(Ref, Msg, Opts) when is_map(Msg) -> + maps:map(fun(K, V) -> ensure_all_loaded([K|Ref], V, Opts) end, Msg); +ensure_all_loaded(Ref, Msg, Opts) when is_list(Msg) -> + lists:map( + fun({N, V}) -> ensure_all_loaded([N|Ref], V, Opts) end, + hb_util:number(Msg) + ); +ensure_all_loaded(Ref, Msg, Opts) -> + ensure_loaded(Ref, Msg, Opts). + +%% @doc List all items in a directory, assuming they are numbered. list_numbered(Path, Opts) -> SlotDir = hb_store:path(hb_opts:get(store, no_viable_store, Opts), Path), - [ list_to_integer(Name) || Name <- list(SlotDir, Opts) ]. + [ hb_util:int(Name) || Name <- list(SlotDir, Opts) ]. %% @doc List all items under a given path. -list(Path, Opts) -> - case hb_store:list(hb_opts:get(store, no_viable_store, Opts), Path) of +list(Path, Opts) when is_map(Opts) and not is_map_key(<<"store-module">>, Opts) -> + case hb_opts:get(store, no_viable_store, Opts) of + not_found -> []; + Store -> + list(Path, Store) + end; +list(Path, Store) -> + ResolvedPath = hb_store:resolve(Store, Path), + case hb_store:list(Store, ResolvedPath) of {ok, Names} -> Names; - {error, _} -> [] + {error, _} -> []; + not_found -> [] end. -%% @doc Write a message to the cache. For raw binaries, we write the data at -%% the hashpath of the data (by default the SHA2-256 hash of the data). For -%% deep messages, we link the hashpath of the keys to the underlying data and -%% recurse. Additionally, we make sets of links such that signed messages can -%% be recalled from the underlying dataset, without associated keys that were -%% written to the hashpath-graph by other messages. -write(RawMsg, Opts) -> - Msg = hb_message:convert(RawMsg, tabm, converge, Opts), - write_message(Msg, hb_opts:get(store, no_viable_store, Opts), Opts). -write_message(Bin, Store, Opts) when is_binary(Bin) -> - % Write the binary in the store at its given hash. Return the path. - % We add the `Data/` prefix for a weird reason: Arweave data items have - % their signed ID based on the hash of the signature. Subsequently, if - % we store the data items at just their sha256 hash, we get a hash - % collision between the signed ID of a message, and its signature data. - Hashpath = hb_path:hashpath(Bin, Opts), - ok = hb_store:write(Store, Path = <<"Messages/", Hashpath/binary>>, Bin), - {ok, Path}; -write_message(Msg, Store, Opts) when is_map(Msg) -> - % Precalculate the hashpath of the message. - MsgHashpath = hb_path:hashpath(Msg, Opts), - MsgHashpathAlg = hb_path:hashpath_alg(Msg), - ?event({storing_msg_with_hashpath, MsgHashpath}), - % Get the ID of the unsigned message. - {ok, UnsignedID} = dev_message:unsigned_id(Msg), - % Write the keys of the message into the graph of hashpaths, generating a map of - % keys to paths of the underlying data as we do so. - UnsignedMsgPathMap = +%% @doc Match a template message against the cache, returning a list of IDs +%% that match the template. We match on the binary representation of values, +%% rather than their types explicitly, such that 'AO-Types' keys that are +%% only partial matches do not cause the match to fail. +match(MatchSpec, Opts) -> + Spec = hb_message:convert(MatchSpec, tabm, <<"structured@1.0">>, Opts), + ConvertedMatchSpec = maps:map( - WriteSubkey = - fun(Key, Value) -> - {ok, Path} = write_message(Value, Store, Opts), - KeyHashPath = - hb_path:hashpath( - MsgHashpath, - hb_path:to_binary(Key), - MsgHashpathAlg, - Opts - ), - hb_store:make_link(Store, Path, KeyHashPath), - MessageLink = - case hb_path:term_to_path_parts(KeyHashPath) of - [MsgHashpath, _] -> unnecessary; - [NewRoot|_] -> - hb_store:make_link(Store, NewRoot, MsgHashpath) - end, - ?event( - { - {link, KeyHashPath}, - {data_path, Path}, - {message_link, MessageLink} - } - ), - Path - end, - hb_message:unsigned(Msg) + fun(_, Value) -> + generate_binary_path(Value, Opts) + end, + maps:without([<<"ao-types">>], hb_ao:normalize_keys(Spec, Opts)) ), - % Write links for each key in the unsigned message to the underlying data. - write_link_tree(UnsignedID, UnsignedMsgPathMap, Store, Opts), - % If the message is signed, we need to write links for the attestations. - case dev_message:signed_id(Msg) of - {error, not_signed} -> - {ok, UnsignedID}; - {ok, SignedID} -> - % Write each attestation-related key in the message to the store. - AttestedMsgPathMap = - maps:map( - WriteSubkey, - hb_message:attestations(Msg) - ), - % Merge the unsigned and attested message path maps, such that we have - % a complete map of all keys in the message and their paths. - CompleteMsgPathMap = maps:merge(UnsignedMsgPathMap, AttestedMsgPathMap), - % Generate links for the signed message. - write_link_tree( - SignedID, - CompleteMsgPathMap, - Store, + case hb_store:match(hb_opts:get(store, no_viable_store, Opts), ConvertedMatchSpec) of + {ok, Matches} -> {ok, Matches}; + _ -> not_found + end. + +%% @doc Generate the path at which a binary value should be stored. +generate_binary_path(Bin, Opts) -> + Hashpath = hb_path:hashpath(Bin, Opts), + <<"data/", Hashpath/binary>>. + +%% @doc Write a message to the cache. For raw binaries, we write the data at +%% the hashpath of the data (by default the SHA2-256 hash of the data). We link +%% the unattended ID's hashpath for the keys (including `/commitments') on the +%% message to the underlying data and recurse. We then link each commitment ID +%% to the uncommitted message, such that any of the committed or uncommitted IDs +%% can be read, and once in memory all of the commitments are available. For +%% deep messages, the commitments will also be read, such that the ID of the +%% outer message (which does not include its commitments) will be built upon +%% the commitments of the inner messages. We do not, however, store the IDs from +%% commitments on signed _inner_ messages. We may wish to revisit this. +write(RawMsg, Opts) when is_map(RawMsg) -> + {ok, Msg} = hb_message:with_only_committed(RawMsg, Opts), + TABM = hb_message:convert(Msg, tabm, <<"structured@1.0">>, Opts), + ?event(debug_cache, {writing_full_message, {msg, TABM}}), + try + do_write_message( + TABM, + hb_opts:get(store, no_viable_store, Opts), + Opts + ) + catch + Type:Reason:Stacktrace -> + ?event(error, + {cache_write_error, + {type, Type}, + {reason, Reason}, + {stacktrace, {trace, Stacktrace}} + }, Opts ), - {ok, SignedID} - end. + erlang:raise(Type, Reason, Stacktrace) + end; +write(List, Opts) when is_list(List) -> + write(hb_message:convert(List, tabm, <<"structured@1.0">>, Opts), Opts); +write(Bin, Opts) when is_binary(Bin) -> + do_write_message(Bin, hb_opts:get(store, no_viable_store, Opts), Opts). + +do_write_message(Bin, Store, Opts) when is_binary(Bin) -> + % Write the binary in the store at its calculated content-hash. + % Return the path. + ok = hb_store:write(Store, Path = generate_binary_path(Bin, Opts), Bin), + %lists:map(fun(ID) -> hb_store:make_link(Store, Path, ID) end, AllIDs), + {ok, Path}; +do_write_message(List, Store, Opts) when is_list(List) -> + do_write_message( + hb_message:convert(List, tabm, <<"structured@1.0">>, Opts), + Store, + Opts + ); +do_write_message(Msg, Store, Opts) when is_map(Msg) -> + ?event(debug_cache, {writing_message, Msg}), + % Calculate the IDs of the message. + UncommittedID = hb_message:id(Msg, none, Opts#{ linkify_mode => discard }), + AltIDs = calculate_all_ids(Msg, Opts) -- [UncommittedID], + MsgHashpathAlg = hb_path:hashpath_alg(Msg, Opts), + ?event(debug_cache, {writing_message, {id, UncommittedID}, {alt_ids, AltIDs}, {original, Msg}}), + % Write all of the keys of the message into the store. + hb_store:make_group(Store, UncommittedID), + maps:map( + fun(Key, Value) -> + write_key(UncommittedID, Key, MsgHashpathAlg, Value, Store, Opts) + end, + maps:without([<<"priv">>], Msg) + ), + % Write the commitments to the store, linking each commitment ID to the + % uncommitted message. + lists:map( + fun(AltID) -> + ?event(debug_cache, + {linking_commitment, + {uncommitted_id, UncommittedID}, + {committed_id, AltID} + }), + hb_store:make_link(Store, UncommittedID, AltID) + end, + AltIDs + ), + {ok, UncommittedID}. -%% @doc Recursively make links to underlying data, in the form of a pathmap. -%% This allows us to have the hashpath compute space be shared across all -%% messages from users, while also allowing us to delimit where the signature/ -%% message ends. For example, if we had the following hashpath-space: -%% -%% Hashpath1/Compute/Results/1 -%% Hashpath1/Compute/Results/2/Compute/Results/1 -%% Hashpath1/Compute/Results/3 -%% -%% ...but a message that only contains the first layer of Compute results, we would -%% create the following linked structure: -%% -%% ID1/Compute/Results/1 -> Hashpath1/Compute/Results/1 -%% ID1/Compute/Results/2 -> Hashpath1/Compute/Results/2 -%% ID1/Compute/Results/3 -> Hashpath1/Compute/Results/3 -write_link_tree(RootPath, PathMap, Store, Opts) -> - ?event({write_link_tree, {root, RootPath}, {store, Store}}), +%% @doc Write a single key for a message into the store. +write_key(Base, <<"commitments">>, _HPAlg, RawCommitments, Store, Opts) -> + % The commitments are a special case: We calculate the single-part hashpath + % for the `baseID/commitments` key, then write each commitment to the store + % and link it to `baseCommHP/commitmentID`. + Commitments = prepare_commitments(RawCommitments, Opts), + CommitmentsBase = commitment_path(Base, Opts), + ok = hb_store:make_group(Store, CommitmentsBase), + ?event( + {writing_commitments, + {base, Base}, + {commitments_message, Commitments}, + {commitments_base, CommitmentsBase} + } + ), maps:map( - fun(Key, Path) when is_map(Path) -> - % The key is a map of subkeys and paths, so we adjust our root ID to - % additionally include the key and recurse. - write_link_tree( - hb_path:to_binary([RootPath, Key]), - Path, + fun(BaseCommID, Commitment) -> + ?event(debug_cache, {writing_commitment, {commitment, Commitment}}), + {ok, CommMsgID} = do_write_message(Commitment, Store, Opts), + hb_store:make_link( Store, - Opts - ); - (Key, Path) when not is_map(Path) -> - % The key is a simple binary, so we link ID/key to the path. - MsgKey = hb_path:to_binary([RootPath, Key]), - BinPath = hb_path:to_binary(Path), - ?event({linking, {msg_key, MsgKey}, {path, BinPath}}), - hb_store:make_link(Store, BinPath, MsgKey) + CommMsgID, + << CommitmentsBase/binary, "/", BaseCommID/binary >> + ) end, - PathMap + Commitments + ), + % Link the commitments base to `base/commitments`. + hb_store:make_link(Store, CommitmentsBase, <>); +write_key(Base, Key, HPAlg, Value, Store, Opts) -> + KeyHashPath = + hb_path:hashpath( + Base, + hb_path:to_binary(Key), + HPAlg, + Opts + ), + {ok, Path} = do_write_message(Value, Store, Opts), + hb_store:make_link(Store, Path, KeyHashPath), + {ok, Path}. + +%% @doc The `structured@1.0` encoder does not typically encode `commitments`, +%% subsequently, when we encounter a commitments message we prepare its contents +%% separately, then write each to the store. +prepare_commitments(RawCommitments, Opts) -> + Commitments = ensure_all_loaded(RawCommitments, Opts), + maps:map( + fun(_, StructuredCommitment) -> + hb_message:convert(StructuredCommitment, tabm, Opts) + end, + Commitments ). +%% @doc Generate the commitment path for a given base path. +commitment_path(Base, Opts) -> + hb_path:hashpath(<>, Opts). + +%% @doc Calculate the IDs for a message. +calculate_all_ids(Bin, _Opts) when is_binary(Bin) -> []; +calculate_all_ids(Msg, Opts) -> + Commitments = + hb_maps:without( + [<<"priv">>], + hb_maps:get(<<"commitments">>, Msg, #{}, Opts), + Opts + ), + CommIDs = hb_maps:keys(Commitments, Opts), + ?event({calculating_ids, {msg, Msg}, {commitments, Commitments}, {comm_ids, CommIDs}}), + All = hb_message:id(Msg, all, Opts#{ linkify_mode => discard }), + case lists:member(All, CommIDs) of + true -> CommIDs; + false -> [All | CommIDs] + end. + +%% @doc Write a hashpath and its message to the store and link it. +write_hashpath(Msg = #{ <<"priv">> := #{ <<"hashpath">> := HP } }, Opts) -> + write_hashpath(HP, Msg, Opts); +write_hashpath(MsgWithoutHP, Opts) -> + write(MsgWithoutHP, Opts). +write_hashpath(HP, Msg, Opts) when is_binary(HP) or is_list(HP) -> + Store = hb_opts:get(store, no_viable_store, Opts), + ?event({writing_hashpath, {hashpath, HP}, {msg, Msg}, {store, Store}}), + {ok, Path} = write(Msg, Opts), + hb_store:make_link(Store, Path, HP), + {ok, Path}. + %% @doc Write a raw binary keys into the store and link it at a given hashpath. write_binary(Hashpath, Bin, Opts) -> write_binary(Hashpath, Bin, hb_opts:get(store, no_viable_store, Opts), Opts). write_binary(Hashpath, Bin, Store, Opts) -> - {ok, Path} = write_message(Bin, Store, Opts), + ?event({writing_binary, {hashpath, Hashpath}, {bin, Bin}, {store, Store}}), + {ok, Path} = do_write_message(Bin, Store, Opts), hb_store:make_link(Store, Path, Hashpath), {ok, Path}. -%% @doc Read the message at a path. Returns in Converge's format: Either a rich -%% map or a direct binary. Messages are written in the stores as flat maps, so -%% we convert them to the rich format here after reading. +%% @doc Read the message at a path. Returns in `structured@1.0' format: Either a +%% richly typed map or a direct binary. read(Path, Opts) -> case store_read(Path, hb_opts:get(store, no_viable_store, Opts), Opts) of not_found -> not_found; - {ok, FlatMsg} when is_map(FlatMsg) -> - {ok, hb_message:convert(FlatMsg, converge, flat, Opts)}; - {ok, Res} -> {ok, Res} + {ok, Res} -> + %?event({applying_types_to_read_message, Res}), + %Structured = dev_codec_structured:to(Res), + %?event({finished_read, Structured}), + {ok, Res} end. - -%% @doc List all of the subpaths of a given path, read each in turn, returning a -%% flat map. + +%% @doc List all of the subpaths of a given path and return a map of keys and +%% links to the subpaths, including their types. +store_read(_Path, no_viable_store, _) -> + not_found; store_read(Path, Store, Opts) -> - ResolvedFullPath = hb_store:resolve(Store, PathToBin = hb_path:to_binary(Path)), - ?event( - {reading, - {path, PathToBin}, - {resolved, ResolvedFullPath} - } - ), + ResolvedFullPath = hb_store:resolve(Store, PathBin = hb_path:to_binary(Path)), + ?event({read_resolved, + {original_path, {string, PathBin}}, + {resolved_path, ResolvedFullPath}, + {store, Store} + }), case hb_store:type(Store, ResolvedFullPath) of + not_found -> not_found; simple -> - {ok, Binary} = hb_store:read(Store, ResolvedFullPath), - % Try to get the key type, if it exists in the cache. - case hb_path:term_to_path_parts(Path) of - [BasePath, Key] when not ?IS_ID(Key) and is_binary(Key) -> - case hb_store:read(Store, <>) of - no_viable_store -> - {ok, Binary}; - {ok, Type} -> - {ok, hb_codec_converge:decode_value(Type, Binary)} - end; - _ -> - {ok, Binary} + ?event({reading_data, ResolvedFullPath}), + case hb_store:read(Store, ResolvedFullPath) of + {ok, Bin} -> {ok, Bin}; + not_found -> not_found end; - _ -> + composite -> + ?event({reading_composite, ResolvedFullPath}), case hb_store:list(Store, ResolvedFullPath) of - {ok, Subpaths} -> + {ok, RawSubpaths} -> + Subpaths = + lists:map(fun hb_util:bin/1, RawSubpaths), ?event( {listed, {original_path, Path}, - {subpaths, Subpaths} + {subpaths, {explicit, Subpaths}} } ), - FlatMap = - maps:from_list( - lists:map( - fun(Subpath) -> - ResolvedSubpath = - hb_store:resolve(Store, - hb_path:to_binary([ - ResolvedFullPath, - Subpath - ]) - ), - ?event( - {reading_subpath, Subpath, {resolved, ResolvedSubpath}} - ), - { - Subpath, - hb_util:ok(store_read(ResolvedSubpath, Store, Opts)) - } - end, - Subpaths - ) + % Generate links for all subpaths except `commitments' and + % `ao-types'. `commitments' is always read in its entirety, + % such that all messages have their IDs and signatures + % locally available. + Msg = prepare_links(ResolvedFullPath, Subpaths, Store, Opts), + ?event( + {completed_read, + {resolved_path, ResolvedFullPath}, + {explicit, Msg} + } ), - {ok, FlatMap}; - _ -> not_found + {ok, Msg}; + _ -> + ?event({empty_composite_message, ResolvedFullPath}), + {ok, #{}} end end. -%% @doc Read the output of a computation, given Msg1, Msg2, and some options. -read_output(MsgID1, MsgID2, Opts) when ?IS_ID(MsgID1) and ?IS_ID(MsgID2) -> +%% @doc Prepare a set of links from a listing of subpaths. +prepare_links(RootPath, Subpaths, Store, Opts) -> + {ok, Implicit, Types} = read_ao_types(RootPath, Subpaths, Store, Opts), + Res = + maps:from_list(lists:filtermap( + fun(<<"ao-types">>) -> false; + (<<"commitments">>) -> + % List the commitments for this message, and load them into + % memory. If there no commitments at the path, we exclude + % commitments from the list of links. + CommPath = + hb_store:resolve( + Store, + hb_store:path(Store, [RootPath, <<"commitments">>]) + ), + ?event( + {reading_commitments, + {root_path, RootPath}, + {commitments_path, CommPath} + } + ), + case hb_store:list(Store, CommPath) of + {ok, CommitmentIDs} -> + ?event( + {found_commitments, + {path, CommPath}, + {ids, CommitmentIDs} + } + ), + % We have commitments, so we read each commitment + % into memory, and return it as part of the message. + { + true, + { + <<"commitments">>, + maps:from_list(lists:map( + fun(CommitmentID) -> + {ok, Commitment} = + read( + << + CommPath/binary, + "/", + CommitmentID/binary + >>, + Opts + ), + { + CommitmentID, + ensure_all_loaded( + Commitment, + Opts + ) + } + end, + CommitmentIDs + )) + } + }; + _ -> + false + end; + (Subpath) -> + ?event( + {returning_link, + {subpath, Subpath} + } + ), + SubkeyPath = hb_store:path(Store, [RootPath, Subpath]), + case hb_link:is_link_key(Subpath) of + false -> + % The key is a literal value, not a nested composite + % message. Subsequently, we return a resolvable link + % to the subpath, leaving the key as-is. + {true, + { + Subpath, + {link, + SubkeyPath, + (case Types of + #{ Subpath := Type } -> + % We have an `ao-types' entry for the + % subpath, so we return a link to the + % subpath with `lazy' set to `true' + % because we need to resolve the link + % to get the final value. + #{ + <<"type">> => Type, + <<"lazy">> => true + }; + _ -> + % We do not have an `ao-types' entry for the + % subpath, so we return a link to the + % subpath with `lazy' set to `true', + % because the subpath is a literal + % value. + #{ + <<"lazy">> => true + } + end)#{ store => Store } + } + } + }; + true -> + % The key is an encoded link, so we create a resolvable + % link to the underlying link. This requires that we + % dereference the link twice in order to get the final + % value. Returning the data this way avoids having to + % read each of the link keys themselves, which may be + % a large quantity. + {true, + { + binary:part(Subpath, 0, byte_size(Subpath) - 5), + {link, SubkeyPath, #{ + <<"type">> => <<"link">>, + <<"lazy">> => true + }} + } + } + end + end, + Subpaths + )), + Merged = maps:merge(Res, Implicit), + % Convert the message to an ordered list if the ao-types indicate that it + % should be so. + case dev_codec_structured:is_list_from_ao_types(Types, Opts) of + true -> + hb_util:message_to_ordered_list(Merged, Opts); + false -> + Merged + end. + +%% @doc Read and parse the ao-types for a given path if it is in the supplied +%% list of subpaths, returning a map of keys and their types. +read_ao_types(Path, Subpaths, Store, Opts) -> + ?event({reading_ao_types, {path, Path}, {subpaths, {explicit, Subpaths}}}), + case lists:member(<<"ao-types">>, Subpaths) of + true -> + {ok, TypesBin} = + hb_store:read( + Store, + hb_store:path(Store, [Path, <<"ao-types">>]) + ), + Types = dev_codec_structured:decode_ao_types(TypesBin, Opts), + ?event({parsed_ao_types, {types, Types}}), + {ok, types_to_implicit(Types), Types}; + false -> + ?event({no_ao_types_key_found, {path, Path}, {subpaths, Subpaths}}), + {ok, #{}, #{}} + end. + +%% @doc Convert a map of ao-types to an implicit map of types. +types_to_implicit(Types) -> + maps:filtermap( + fun(_K, <<"empty-message">>) -> {true, #{}}; + (_K, <<"empty-list">>) -> {true, []}; + (_K, <<"empty-binary">>) -> {true, <<>>}; + (_, _) -> false + end, + Types + ). + +%% @doc Read the output of a prior computation, given Msg1, Msg2, and some +%% options. +read_resolved(MsgID1, MsgID2, Opts) when ?IS_ID(MsgID1) and ?IS_ID(MsgID2) -> ?event({cache_lookup, {msg1, MsgID1}, {msg2, MsgID2}, {opts, Opts}}), read(<>, Opts); -read_output(MsgID1, Msg2, Opts) when ?IS_ID(MsgID1) and is_map(Msg2) -> - {ok, MsgID2} = dev_message:id(Msg2), +read_resolved(MsgID1, Msg2, Opts) when ?IS_ID(MsgID1) and is_map(Msg2) -> + {ok, MsgID2} = dev_message:id(Msg2, #{ <<"committers">> => <<"all">> }, Opts), read(<>, Opts); -read_output(Msg1, Msg2, Opts) when is_map(Msg1) and is_map(Msg2) -> +read_resolved(Msg1, Msg2, Opts) when is_map(Msg1) and is_map(Msg2) -> read(hb_path:hashpath(Msg1, Msg2, Opts), Opts); -read_output(_, _, _) -> not_found. - -%%------------------------------------------------------------------------------ +read_resolved(_, _, _) -> not_found. %% @doc Make a link from one path to another in the store. %% Note: Argument order is `link(Src, Dst, Opts)'. @@ -282,100 +625,290 @@ link(Existing, New, Opts) -> test_unsigned(Data) -> #{ - <<"Base-Test-Key">> => <<"Base-Test-Value">>, - <<"Data">> => Data + <<"base-test-key">> => <<"base-test-value">>, + <<"other-test-key">> => Data }. %% Helper function to create signed #tx items. -test_signed(Data) -> - hb_message:sign(test_unsigned(Data), ar_wallet:new()). +test_signed(Data) -> test_signed(Data, ar_wallet:new()). +test_signed(Data, Wallet) -> + hb_message:commit(test_unsigned(Data), Wallet). -test_store_binary(Opts) -> +test_store_binary(Store) -> Bin = <<"Simple unsigned data item">>, + ?event(debug_store_test, {store, Store}), + Opts = #{ store => Store }, {ok, ID} = write(Bin, Opts), {ok, RetrievedBin} = read(ID, Opts), ?assertEqual(Bin, RetrievedBin). +test_store_unsigned_empty_message(Store) -> + ?event(debug_store_test, {store, Store}), + hb_store:reset(Store), + Item = #{}, + Opts = #{ store => Store }, + {ok, Path} = write(Item, Opts), + {ok, RetrievedItem} = read(Path, Opts), + ?event( + {retrieved_item, + {path, {string, Path}}, + {expected, Item}, + {got, RetrievedItem} + } + ), + MatchRes = hb_message:match(Item, RetrievedItem, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes). + +test_store_unsigned_nested_empty_message(Store) -> + ?event(debug_store_test, {store, Store}), + hb_store:reset(Store), + Item = + #{ <<"layer1">> => + #{ <<"layer2">> => + #{ <<"layer3">> => + #{ <<"a">> => <<"b">>} + }, + <<"layer3b">> => #{ <<"c">> => <<"d">>}, + <<"layer3c">> => #{} + } + }, + Opts = #{ store => Store }, + {ok, Path} = write(Item, Opts), + {ok, RetrievedItem} = read(Path, Opts), + ?assert(hb_message:match(Item, RetrievedItem, strict, Opts)). + %% @doc Test storing and retrieving a simple unsigned item -test_store_simple_unsigned_item(Opts) -> +test_store_simple_unsigned_message(Store) -> Item = test_unsigned(<<"Simple unsigned data item">>), + ?event(debug_store_test, {store, Store}), + Opts = #{ store => Store }, %% Write the simple unsigned item {ok, _Path} = write(Item, Opts), %% Read the item back - ID = hb_util:human_id(hb_converge:get(id, Item)), + ID = hb_util:human_id(hb_ao:get(id, Item)), {ok, RetrievedItem} = read(ID, Opts), - ?assert(hb_message:match(Item, RetrievedItem)). + ?assert(hb_message:match(Item, RetrievedItem, strict, Opts)), + ok. + +test_store_ans104_message(Store) -> + ?event(debug_store_test, {store, Store}), + hb_store:reset(Store), + Opts = #{ store => Store }, + Item = #{ <<"type">> => <<"ANS104">>, <<"content">> => <<"Hello, world!">> }, + Committed = hb_message:commit(Item, hb:wallet()), + {ok, _Path} = write(Committed, Opts), + CommittedID = hb_util:human_id(hb_message:id(Committed, all)), + UncommittedID = hb_util:human_id(hb_message:id(Committed, none)), + ?event({test_message_ids, {uncommitted, UncommittedID}, {committed, CommittedID}}), + {ok, RetrievedItem} = read(CommittedID, Opts), + {ok, RetrievedItemU} = read(UncommittedID, Opts), + ?assert(hb_message:match(Committed, RetrievedItem, strict, Opts)), + ?assert(hb_message:match(Committed, RetrievedItemU, strict, Opts)), + ok. %% @doc Test storing and retrieving a simple unsigned item -test_store_simple_signed_item(Opts) -> - Item = test_signed(#{ <<"L2-Test-Key">> => <<"L2-Test-Value">> }), +test_store_simple_signed_message(Store) -> + ?event(debug_store_test, {store, Store}), + Opts = #{ store => Store }, + hb_store:reset(Store), + Wallet = ar_wallet:new(), + Address = hb_util:human_id(ar_wallet:to_address(Wallet)), + Item = test_signed(<<"Simple signed data item">>, Wallet), + ?event({writing_test_message, Item}), %% Write the simple unsigned item {ok, _Path} = write(Item, Opts), - %% Read the item back - UID = hb_util:human_id(hb_converge:get(unsigned_id, Item)), - SID = hb_util:human_id(hb_converge:get(signed_id, Item)), - {ok, RetrievedItemU} = read(UID, Opts), - ?assert(hb_message:match(Item, RetrievedItemU)), - {ok, RetrievedItemS} = read(SID, Opts), - ?assert(hb_message:match(Item, RetrievedItemS)). + % %% Read the item back + % {ok, UID} = dev_message:id(Item, #{ <<"committers">> => <<"none">> }, Opts), + % {ok, RetrievedItemUnsig} = read(UID, Opts), + % ?event({retreived_unsigned_message, {expected, Item}, {got, RetrievedItemUnsig}}), + % MatchRes = hb_message:match(Item, RetrievedItemUnsig, strict, Opts), + % ?event({match_result, MatchRes}), + % ?assert(MatchRes), + {ok, CommittedID} = dev_message:id(Item, #{ <<"committers">> => [Address] }, Opts), + {ok, RetrievedItemSigned} = read(CommittedID, Opts), + ?event({retreived_signed_message, {expected, Item}, {got, RetrievedItemSigned}}), + MatchResSigned = hb_message:match(Item, RetrievedItemSigned, strict, Opts), + ?event({match_result_signed, MatchResSigned}), + ?assert(MatchResSigned), + ok. %% @doc Test deeply nested item storage and retrieval -test_deeply_nested_complex_item(Opts) -> +test_deeply_nested_complex_message(Store) -> + ?event(debug_store_test, {store, Store}), + hb_store:reset(Store), + Wallet = ar_wallet:new(), + Opts = #{ store => Store, priv_wallet => Wallet }, %% Create nested data - DeepValueMsg = test_signed([1,2,3]), + Level3SignedSubmessage = test_signed([1,2,3], Opts#{priv_wallet => Wallet}), Outer = - #{ - <<"Level1">> => - hb_message:sign( - #{ - <<"Level2">> => - #{ - <<"Level3">> => DeepValueMsg, - <<"E">> => <<"F">>, - <<"Z">> => [1,2,3] - }, - <<"C">> => <<"D">>, - <<"G">> => [<<"H">>, <<"I">>], - <<"J">> => 1337 - }, - ar_wallet:new() - ), - <<"A">> => <<"B">> - }, + hb_message:commit( + #{ + <<"level1">> => + InnerSigned = hb_message:commit( + #{ + <<"level2">> => + #{ + <<"level3">> => Level3SignedSubmessage, + <<"e">> => <<"f">>, + <<"z">> => [1,2,3] + }, + <<"c">> => <<"d">>, + <<"g">> => [<<"h">>, <<"i">>], + <<"j">> => 1337 + }, + Opts + ), + <<"a">> => <<"b">> + }, + Opts + ), + UID = hb_message:id(Outer, none, Opts), + ?event({string, <<"================================================">>}), + CommittedID = hb_message:id(Outer, signed, Opts), + ?event({string, <<"================================================">>}), + ?event({test_message_ids, {uncommitted, UID}, {committed, CommittedID}}), %% Write the nested item {ok, _} = write(Outer, Opts), %% Read the deep value back using subpath - {ok, DeepMsg} = - read( - [ - OuterID = hb_util:human_id(hb_converge:get(unsigned_id, Outer)), - <<"Level1">>, - <<"Level2">>, - <<"Level3">> - ], - Opts - ), + OuterID = hb_util:human_id(UID), + {ok, OuterMsg} = read(OuterID, Opts), + EnsuredLoadedOuter = hb_cache:ensure_all_loaded(OuterMsg, Opts), + ?event({deep_message, {explicit, EnsuredLoadedOuter}}), %% Assert that the retrieved item matches the original deep value - ?assertEqual([1,2,3], hb_converge:get(data, DeepMsg)), ?assertEqual( - hb_converge:get(unsigned_id, DeepValueMsg), - hb_converge:get(unsigned_id, DeepMsg) + [1,2,3], + hb_ao:get( + <<"level1/level2/level3/other-test-key">>, + EnsuredLoadedOuter, + Opts + ) ), - {ok, OuterMsg} = read(OuterID, Opts), - ?assertEqual(OuterID, hb_converge:get(unsigned_id, OuterMsg)). + ?event( + {deep_message_match, + {read, EnsuredLoadedOuter}, + {write, Level3SignedSubmessage} + } + ), + ?event({reading_committed_outer, {id, CommittedID}, {expect, Outer}}), + {ok, CommittedMsg} = read(hb_util:human_id(CommittedID), Opts), + EnsuredLoadedCommitted = hb_cache:ensure_all_loaded(CommittedMsg, Opts), + ?assertEqual( + [1,2,3], + hb_ao:get( + <<"level1/level2/level3/other-test-key">>, + EnsuredLoadedCommitted, + Opts + ) + ). -test_message_with_list(Opts) -> +test_message_with_list(Store) -> + hb_store:reset(Store), + Opts = #{ store => Store }, Msg = test_unsigned([<<"a">>, <<"b">>, <<"c">>]), ?event({writing_message, Msg}), {ok, Path} = write(Msg, Opts), {ok, RetrievedItem} = read(Path, Opts), - ?assert(hb_message:match(Msg, RetrievedItem)). + ?assert(hb_message:match(Msg, RetrievedItem, strict, Opts)). + +test_match_message(Store) when map_get(<<"store-module">>, Store) =/= hb_store_lmdb -> + skip; +test_match_message(Store) -> + hb_store:reset(Store), + Opts = #{ store => Store }, + % Write two messages that match the template, and a third that does not. + {ok, ID1} = hb_cache:write(#{ <<"x">> => <<"1">> }, Opts), + {ok, ID2} = hb_cache:write(#{ <<"y">> => <<"2">>, <<"z">> => <<"3">> }, Opts), + {ok, ID2b} = hb_cache:write(#{ <<"x">> => <<"4">>, <<"z">> => <<"3">> }, Opts), + {ok, ID3} = hb_cache:write(#{ <<"z">> => <<"5">>, <<"c">> => <<"d">> }, Opts), + % Match the template, and ensure that we get two matches. + {ok, MatchedItems} = match(#{ <<"z">> => <<"3">> }, Opts), + ?assertEqual(2, length(MatchedItems)), + ?assert( + lists:all( + fun(ID) -> + {ok, Msg} = read(ID, Opts), + hb_maps:get(<<"z">>, Msg, Opts) =:= <<"3">> andalso + lists:member(ID, [ID2, ID2b]) + end, + MatchedItems + ) + ), + {ok, MatchedItems2} = match(#{ <<"x">> => <<"4">> }, Opts), + ?assertEqual(1, length(MatchedItems2)), + ?assertEqual([ID2b], MatchedItems2). + +test_match_linked_message(Store) when map_get(<<"store-module">>, Store) =/= hb_store_lmdb -> + skip; +test_match_linked_message(Store) -> + hb_store:reset(Store), + Opts = #{ store => Store }, + Msg = #{ <<"a">> => Inner = #{ <<"b">> => <<"c">>, <<"d">> => <<"e">> } }, + {ok, _ID} = write(Msg, Opts), + {ok, [MatchedID]} = match(#{ <<"b">> => <<"c">> }, Opts), + {ok, Read1} = read(MatchedID, Opts), + ?assertEqual( + #{ <<"b">> => <<"c">>, <<"d">> => <<"e">> }, + hb_cache:ensure_all_loaded(Read1, Opts) + ), + {ok, [MatchedID2]} = match(#{ <<"a">> => Inner }, Opts), + {ok, Read2} = read(MatchedID2, Opts), + ?assertEqual(#{ <<"a">> => Inner }, ensure_all_loaded(Read2, Opts)). + +test_match_typed_message(Store) when map_get(<<"store-module">>, Store) =/= hb_store_lmdb -> + skip; +test_match_typed_message(Store) -> + hb_store:reset(Store), + Opts = #{ store => Store }, + % Add some messages that should not match the template, as well as the main + % message that should match the template. + write(#{ <<"atom-value">> => atom, <<"wrong">> => <<"wrong">> }, Opts), + write(#{ <<"integer-value">> => 1337, <<"wrong">> => <<"wrong-2">> }, Opts), + Msg = + #{ + <<"int-key">> => 1337, + <<"other-key">> => <<"other-test-value">>, + <<"atom-key">> => atom + }, + {ok, _ID} = write(Msg, Opts), + {ok, [MatchedID]} = match(#{ <<"int-key">> => 1337 }, Opts), + {ok, Read1} = read(MatchedID, Opts), + ?assertEqual(Msg, ensure_all_loaded(Read1, Opts)), + {ok, [MatchedID2]} = match(#{ <<"atom-key">> => atom }, Opts), + {ok, Read2} = read(MatchedID2, Opts), + ?assertEqual(Msg, ensure_all_loaded(Read2, Opts)). cache_suite_test_() -> hb_store:generate_test_suite([ + {"store unsigned empty message", + fun test_store_unsigned_empty_message/1}, {"store binary", fun test_store_binary/1}, - {"store simple unsigned item", fun test_store_simple_unsigned_item/1}, - {"store simple signed item", fun test_store_simple_signed_item/1}, - {"deeply nested complex item", fun test_deeply_nested_complex_item/1}, - {"message with list", fun test_message_with_list/1} - ]). \ No newline at end of file + {"store unsigned nested empty message", + fun test_store_unsigned_nested_empty_message/1}, + {"store simple unsigned message", fun test_store_simple_unsigned_message/1}, + {"store simple signed message", fun test_store_simple_signed_message/1}, + {"deeply nested complex message", fun test_deeply_nested_complex_message/1}, + {"message with list", fun test_message_with_list/1}, + {"match message", fun test_match_message/1}, + {"match linked message", fun test_match_linked_message/1}, + {"match typed message", fun test_match_typed_message/1} + ]). + +%% @doc Test that message whose device is `#{}' cannot be written. If it were to +%% be written, it would cause an infinite loop. +test_device_map_cannot_be_written_test() -> + try + Opts = #{ store => StoreOpts = + [#{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-TEST">> }] }, + hb_store:reset(StoreOpts), + Danger = #{ <<"device">> => #{}}, + write(Danger, Opts), + ?assert(false) + catch + _:_:_ -> ?assert(true) + end. + +%% @doc Run a specific test with a given store module. +run_test() -> + Store = hb_test_utils:test_store(hb_store_lmdb), + test_match_message(Store). \ No newline at end of file diff --git a/src/hb_cache_control.erl b/src/hb_cache_control.erl index f9a457132..a58489f30 100644 --- a/src/hb_cache_control.erl +++ b/src/hb_cache_control.erl @@ -1,4 +1,4 @@ -%%% @doc Cache control logic for the Converge resolver. It derives cache settings +%%% @doc Cache control logic for the AO-Core resolver. It derives cache settings %%% from request, response, execution-local node Opts, as well as the global %%% node Opts. It applies these settings when asked to maybe store/lookup in %%% response to a request. @@ -7,18 +7,23 @@ -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). +%%% When other cache control settings are not specified, we default to the +%%% following settings. +-define(DEFAULT_STORE_OPT, false). +-define(DEFAULT_LOOKUP_OPT, true). + %%% Public API -%% @doc Write a resulting M3 message to the cache if requested. The precidence +%% @doc Write a resulting M3 message to the cache if requested. The precedence %% order of cache control sources is as follows: -%% 1. The `Opts` map (letting the node operator have the final say). -%% 2. The `Msg3` results message (granted by Msg1's device). -%% 3. The `Msg2` message (the user's request). +%% 1. The `Opts' map (letting the node operator have the final say). +%% 2. The `Msg3' results message (granted by Msg1's device). +%% 3. The `Msg2' message (the user's request). %% Msg1 is not used, such that it can specify cache control information about %% itself, without affecting its outputs. maybe_store(Msg1, Msg2, Msg3, Opts) -> case derive_cache_settings([Msg3, Msg2], Opts) of - #{ store := true } -> + #{ <<"store">> := true } -> ?event(caching, {caching_result, {msg1, Msg1}, {msg2, Msg2}, {msg3, Msg3}}), dispatch_cache_write(Msg1, Msg2, Msg3, Opts); _ -> @@ -26,12 +31,12 @@ maybe_store(Msg1, Msg2, Msg3, Opts) -> end. %% @doc Handles cache lookup, modulated by the caching options requested by -%% the user. Honors the following `Opts` cache keys: -%% `only_if_cached`: If set and we do not find a result in the cache, -%% return an error with a `Cache-Status` of `miss` and -%% a 504 `Status`. -%% `no_cache`: If set, the cached values are never used. Returns -%% `continue` to the caller. +%% the user. Honors the following `Opts' cache keys: +%% `only_if_cached': If set and we do not find a result in the cache, +%% return an error with a `Cache-Status' of `miss' and +%% a 504 `Status'. +%% `no_cache': If set, the cached values are never used. Returns +%% `continue' to the caller. maybe_lookup(Msg1, Msg2, Opts) -> case exec_likely_faster_heuristic(Msg1, Msg2, Opts) of true -> @@ -42,9 +47,16 @@ maybe_lookup(Msg1, Msg2, Opts) -> lookup(Msg1, Msg2, Opts) -> case derive_cache_settings([Msg1, Msg2], Opts) of - #{ lookup := false } -> {continue, Msg1, Msg2}; - Settings = #{ lookup := true } -> - case hb_cache:read_output(Msg1, Msg2, Opts) of + #{ <<"lookup">> := false } -> + ?event({skip_cache_check, lookup_disabled}), + {continue, Msg1, Msg2}; + Settings = #{ <<"lookup">> := true } -> + OutputScopedOpts = + hb_store:scope( + Opts, + hb_opts:get(store_scope_resolved, local, Opts) + ), + case hb_cache:read_resolved(Msg1, Msg2, OutputScopedOpts) of {ok, Msg3} -> ?event(caching, {cache_hit, @@ -59,21 +71,20 @@ lookup(Msg1, Msg2, Opts) -> ), {ok, Msg3}; not_found -> - ?event(caching, {cache_miss, Msg1, Msg2}), + ?event(caching, {result_cache_miss, Msg1, Msg2}), case Settings of - #{ only_if_cached := true } -> - only_if_cached_not_found_error(Msg1, Msg2, Opts); - _ -> - case ?IS_ID(Msg1) of + #{ <<"only-if-cached">> := true } -> + only_if_cached_not_found_error(Msg1, Msg2, Opts); + _ -> + case ?IS_ID(Msg1) of false -> {continue, Msg1, Msg2}; true -> case hb_cache:read(Msg1, Opts) of {ok, FullMsg1} -> - ?event( - {message_cache_hit, - {msg1, Msg1}, - {msg2, Msg2}, - {msg3, FullMsg1} + ?event(load_message, + {cache_hit_base_message_load, + {base_id, Msg1}, + {base_loaded, FullMsg1} } ), {continue, FullMsg1, Msg2}; @@ -94,24 +105,52 @@ lookup(Msg1, Msg2, Opts) -> %% @doc Dispatch the cache write to a worker process if requested. %% Invoke the appropriate cache write function based on the type of the message. dispatch_cache_write(Msg1, Msg2, Msg3, Opts) -> - Dispatch = - fun() -> - case is_binary(Msg3) of - true -> - hb_cache:write_binary( - hb_path:hashpath(Msg1, Msg2, Opts), - Msg3, - Opts - ); - false -> hb_cache:write(Msg3, Opts) - end - end, case hb_opts:get(async_cache, false, Opts) of - true -> spawn(Dispatch); - false -> Dispatch() + true -> + find_or_spawn_async_writer(Opts) ! {write, Msg1, Msg2, Msg3, Opts}, + ok; + false -> + perform_cache_write(Msg1, Msg2, Msg3, Opts) end. -%% @doc Generate a message to return when `only_if_cached` was specified, and +%% @doc Find our async cacher process, or spawn one if none exists. +find_or_spawn_async_writer(_Opts) -> + case erlang:get({hb_cache_control, async_writer}) of + undefined -> + PID = spawn(fun() -> async_writer() end), + erlang:put({hb_cache_control, async_writer}, PID), + PID; + PID -> + PID + end. + +%% @doc Optional worker process to write messages to the cache. +async_writer() -> + receive + {write, Msg1, Msg2, Msg3, Opts} -> + perform_cache_write(Msg1, Msg2, Msg3, Opts); + stop -> ok + end. + +%% @doc Internal function to write a compute result to the cache. +perform_cache_write(Msg1, Msg2, Msg3, Opts) -> + hb_cache:write(Msg1, Opts), + hb_cache:write(Msg2, Opts), + case Msg3 of + <<_/binary>> -> + hb_cache:write_binary( + hb_path:hashpath(Msg1, Msg2, Opts), + Msg3, + Opts + ); + Map when is_map(Map) -> + hb_cache:write(Msg3, Opts); + _ -> + ?event({cannot_write_result, Msg3}), + skip_caching + end. + +%% @doc Generate a message to return when `only_if_cached' was specified, and %% we don't have a cached result. only_if_cached_not_found_error(Msg1, Msg2, Opts) -> ?event( @@ -121,9 +160,8 @@ only_if_cached_not_found_error(Msg1, Msg2, Opts) -> ), {error, #{ - <<"Status">> => <<"Gateway Timeout">>, - <<"Status-Code">> => 504, - <<"Cache-Status">> => <<"miss">>, + <<"status">> => 504, + <<"cache-status">> => <<"miss">>, <<"body">> => <<"Computed result not available in cache.">> } @@ -133,14 +171,13 @@ only_if_cached_not_found_error(Msg1, Msg2, Opts) -> %% cache lookup are not found in the cache. necessary_messages_not_found_error(Msg1, Msg2, Opts) -> ?event( - caching, + load_message, {necessary_messages_not_found, {msg1, Msg1}, {msg2, Msg2}}, Opts ), {error, #{ - <<"Status">> => <<"Not Found">>, - <<"Status-Code">> => 404, + <<"status">> => 404, <<"body">> => <<"Necessary messages not found in cache.">> } @@ -148,13 +185,26 @@ necessary_messages_not_found_error(Msg1, Msg2, Opts) -> %% @doc Determine whether we are likely to be faster looking up the result in %% our cache (hoping we have it), or executing it directly. -exec_likely_faster_heuristic(Msg1, #{ path := Key }, Opts) -> +exec_likely_faster_heuristic(M1, _M2, _) when (not ?IS_ID(M1)) -> + true; +exec_likely_faster_heuristic({as, _, Msg1}, Msg2, Opts) -> + exec_likely_faster_heuristic(Msg1, Msg2, Opts); +exec_likely_faster_heuristic(Msg1, Msg2, Opts) -> + case hb_opts:get(cache_lookup_hueristics, true, Opts) of + false -> false; + true -> + case ?IS_ID(Msg1) of + true -> false; + false -> is_explicit_lookup(Msg1, Msg2, Opts) + end + end. +is_explicit_lookup(Msg1, #{ <<"path">> := Key }, Opts) -> % For now, just check whether the key is explicitly in the map. That is % a good signal that we will likely be asked by the device to grab it. - % If we have `only-if-cached` in the opts, we always force lookup, too. - case specifiers_to_cache_settings(maps:get(cache_control, Opts, [])) of - #{ only_if_cached := true } -> false; - _ -> is_map(Msg1) andalso maps:is_key(Key, Msg1) + % If we have `only-if-cached' in the opts, we always force lookup, too. + case specifiers_to_cache_settings(hb_opts:get(cache_control, [], Opts)) of + #{ <<"only-if-cached">> := true } -> false; + _ -> is_map(Msg1) andalso hb_maps:is_key(Key, Msg1, Opts) end. %% @doc Derive cache settings from a series of option sources and the opts, @@ -162,45 +212,45 @@ exec_likely_faster_heuristic(Msg1, #{ path := Key }, Opts) -> %% map with `store' and `lookup' keys, each of which is a boolean. %% %% For example, if the last source has a `no_store', the first expresses no -%% preference, but the Opts has `cache_control => [always]`, then the result +%% preference, but the Opts has `cache_control => [always]', then the result %% will contain a `store => true' entry. derive_cache_settings(SourceList, Opts) -> lists:foldr( fun(Source, Acc) -> - maybe_set(Acc, cache_source_to_cache_settings(Source)) + maybe_set(Acc, cache_source_to_cache_settings(Source, Opts), Opts) end, - #{ store => true, lookup => true }, + #{ <<"store">> => ?DEFAULT_STORE_OPT, <<"lookup">> => ?DEFAULT_LOOKUP_OPT }, [{opts, Opts}|lists:filter(fun erlang:is_map/1, SourceList)] ). %% @doc Takes a key and two maps, returning the first map with the key set to %% the value of the second map _if_ the value is not undefined. -maybe_set(Map1, Map2) -> +maybe_set(Map1, Map2, Opts) -> lists:foldl( fun(Key, AccMap) -> - case maps:get(Key, Map2) of + case hb_maps:get(Key, Map2, undefined, Opts) of undefined -> AccMap; - Value -> maps:put(Key, Value, AccMap) + Value -> hb_maps:put(Key, Value, AccMap, Opts) end end, Map1, - maps:keys(Map2) + hb_maps:keys(Map2, Opts) ). %% @doc Convert a cache source to a cache setting. The setting _must_ always be -%% directly in the source, not a Converge-derivable value. The -%% `to_cache_control_map` function is used as the source of settings in all -%% cases, except where an `Opts` specifies that hashpaths should not be updated, +%% directly in the source, not an AO-Core-derivable value. The +%% `to_cache_control_map' function is used as the source of settings in all +%% cases, except where an `Opts' specifies that hashpaths should not be updated, %% which leads to the result not being cached (as it may be stored with an %% incorrect hashpath). -cache_source_to_cache_settings({opts, Opts}) -> +cache_source_to_cache_settings({opts, Opts}, _) -> CCMap = specifiers_to_cache_settings(hb_opts:get(cache_control, [], Opts)), case hb_opts:get(hashpath, update, Opts) of - ignore -> CCMap#{ store => false }; + ignore -> CCMap#{ <<"store">> => false }; _ -> CCMap end; -cache_source_to_cache_settings(Msg) -> - case dev_message:get(<<"Cache-Control">>, Msg) of +cache_source_to_cache_settings(Msg, Opts) -> + case dev_message:get(<<"cache-control">>, Msg, Opts) of {ok, CC} -> specifiers_to_cache_settings(CC); {error, not_found} -> #{} end. @@ -209,9 +259,10 @@ cache_source_to_cache_settings(Msg) -> %% normalized map of simply whether we should store and/or lookup the result. specifiers_to_cache_settings(CCSpecifier) when not is_list(CCSpecifier) -> specifiers_to_cache_settings([CCSpecifier]); -specifiers_to_cache_settings(CCList) -> +specifiers_to_cache_settings(RawCCList) -> + CCList = lists:map(fun hb_ao:normalize_key/1, RawCCList), #{ - store => + <<"store">> => case lists:member(<<"always">>, CCList) of true -> true; false -> @@ -224,7 +275,7 @@ specifiers_to_cache_settings(CCList) -> end end end, - lookup => + <<"lookup">> => case lists:member(<<"always">>, CCList) of true -> true; false -> @@ -237,7 +288,7 @@ specifiers_to_cache_settings(CCList) -> end end end, - only_if_cached => + <<"only-if-cached">> => case lists:member(<<"only-if-cached">>, CCList) of true -> true; false -> undefined @@ -247,7 +298,7 @@ specifiers_to_cache_settings(CCList) -> %%% Tests %% Helpers to create a message with Cache-Control header -msg_with_cc(CC) -> #{ <<"Cache-Control">> => CC }. +msg_with_cc(CC) -> #{ <<"cache-control">> => CC }. opts_with_cc(CC) -> #{ cache_control => CC }. %% Test precedence order (Opts > Msg3 > Msg2) @@ -256,30 +307,34 @@ opts_override_message_settings_test() -> Msg3 = msg_with_cc([<<"no-cache">>]), Opts = opts_with_cc([<<"always">>]), Result = derive_cache_settings([Msg3, Msg2], Opts), - ?assertEqual(#{store => true, lookup => true}, Result). + ?assertEqual(#{<<"store">> => true, <<"lookup">> => true}, Result). msg_precidence_overrides_test() -> Msg2 = msg_with_cc([<<"always">>]), Msg3 = msg_with_cc([<<"no-store">>]), % No restrictions Result = derive_cache_settings([Msg3, Msg2], opts_with_cc([])), - ?assertEqual(#{store => false, lookup => true}, Result). + ?assertEqual(#{<<"store">> => false, <<"lookup">> => true}, Result). %% Test specific directives no_store_directive_test() -> Msg = msg_with_cc([<<"no-store">>]), Result = derive_cache_settings([Msg], opts_with_cc([])), - ?assertEqual(#{store => false, lookup => true}, Result). + ?assertEqual(#{<<"store">> => false, <<"lookup">> => ?DEFAULT_LOOKUP_OPT}, Result). no_cache_directive_test() -> Msg = msg_with_cc([<<"no-cache">>]), Result = derive_cache_settings([Msg], opts_with_cc([])), - ?assertEqual(#{store => true, lookup => false}, Result). + ?assertEqual(#{<<"store">> => ?DEFAULT_STORE_OPT, <<"lookup">> => false}, Result). only_if_cached_directive_test() -> Msg = msg_with_cc([<<"only-if-cached">>]), Result = derive_cache_settings([Msg], opts_with_cc([])), ?assertEqual( - #{store => true, lookup => true, only_if_cached => true}, + #{ + <<"store">> => ?DEFAULT_STORE_OPT, + <<"lookup">> => ?DEFAULT_LOOKUP_OPT, + <<"only-if-cached">> => true + }, Result ). @@ -287,71 +342,76 @@ only_if_cached_directive_test() -> hashpath_ignore_prevents_storage_test() -> Opts = (opts_with_cc([]))#{hashpath => ignore}, Result = derive_cache_settings([], Opts), - ?assertEqual(#{store => false, lookup => true}, Result). + ?assertEqual(#{<<"store">> => ?DEFAULT_STORE_OPT, <<"lookup">> => ?DEFAULT_LOOKUP_OPT}, Result). %% Test multiple directives multiple_directives_test() -> Msg = msg_with_cc([<<"no-store">>, <<"no-cache">>, <<"only-if-cached">>]), Result = derive_cache_settings([Msg], opts_with_cc([])), ?assertEqual( - #{store => false, lookup => false, only_if_cached => true}, + #{ + <<"store">> => false, + <<"lookup">> => false, + <<"only-if-cached">> => true + }, Result ). %% Test empty/missing cases empty_message_list_test() -> Result = derive_cache_settings([], opts_with_cc([])), - ?assertEqual(#{store => true, lookup => true}, Result). + ?assertEqual(#{<<"store">> => ?DEFAULT_STORE_OPT, <<"lookup">> => ?DEFAULT_LOOKUP_OPT}, Result). message_without_cache_control_test() -> Result = derive_cache_settings([#{}], opts_with_cc([])), - ?assertEqual(#{store => true, lookup => true}, Result). + ?assertEqual(#{<<"store">> => ?DEFAULT_STORE_OPT, <<"lookup">> => ?DEFAULT_LOOKUP_OPT}, Result). %% Test the cache_source_to_cache_setting function directly opts_source_cache_control_test() -> Result = cache_source_to_cache_settings( - {opts, opts_with_cc([<<"no-store">>])} + {opts, opts_with_cc([<<"no-store">>])}, + #{} ), ?assertEqual(#{ - store => false, - lookup => undefined, - only_if_cached => undefined + <<"store">> => false, + <<"lookup">> => undefined, + <<"only-if-cached">> => undefined }, Result). message_source_cache_control_test() -> Msg = msg_with_cc([<<"no-cache">>]), - Result = cache_source_to_cache_settings(Msg), + Result = cache_source_to_cache_settings(Msg, #{}), ?assertEqual(#{ - store => undefined, - lookup => false, - only_if_cached => undefined + <<"store">> => undefined, + <<"lookup">> => false, + <<"only-if-cached">> => undefined }, Result). -%%% Basic cached Converge resolution tests +%%% Basic cached AO-Core resolution tests cache_binary_result_test() -> - CachedMsg = <<"Test-Message">>, - Msg1 = #{ <<"Test-Key">> => CachedMsg }, - Msg2 = <<"Test-Key">>, - {ok, Res} = hb_converge:resolve(Msg1, Msg2, #{ cache_control => [<<"always">>] }), + CachedMsg = <<"test-message">>, + Msg1 = #{ <<"test-key">> => CachedMsg }, + Msg2 = <<"test-key">>, + {ok, Res} = hb_ao:resolve(Msg1, Msg2, #{ cache_control => [<<"always">>] }), ?assertEqual(CachedMsg, Res), - {ok, Res2} = hb_converge:resolve(Msg1, Msg2, #{ cache_control => [<<"only-if-cached">>] }), - {ok, Res3} = hb_converge:resolve(Msg1, Msg2, #{ cache_control => [<<"only-if-cached">>] }), + {ok, Res2} = hb_ao:resolve(Msg1, Msg2, #{ cache_control => [<<"only-if-cached">>] }), + {ok, Res3} = hb_ao:resolve(Msg1, Msg2, #{ cache_control => [<<"only-if-cached">>] }), ?assertEqual(CachedMsg, Res2), ?assertEqual(Res2, Res3). cache_message_result_test() -> - hb_store:reset(hb_opts:get(store)), CachedMsg = #{ - <<"Purpose">> => <<"Test-Message">>, - <<"Aux">> => #{ <<"Aux-Message">> => <<"Aux-Message-Value">> } + <<"purpose">> => <<"Test-Message">>, + <<"aux">> => #{ <<"aux-message">> => <<"Aux-Message-Value">> }, + <<"test-key">> => rand:uniform(1000000) }, - Msg1 = #{ <<"Test-Key">> => CachedMsg, <<"Local">> => <<"Binary">> }, - Msg2 = <<"Test-Key">>, + Msg1 = #{ <<"test-key">> => CachedMsg, <<"local">> => <<"Binary">> }, + Msg2 = <<"test-key">>, {ok, Res} = - hb_converge:resolve( + hb_ao:resolve( Msg1, Msg2, #{ @@ -360,9 +420,9 @@ cache_message_result_test() -> ), ?event({res1, Res}), ?event(reading_from_cache), - {ok, Res2} = hb_converge:resolve(Msg1, Msg2, #{ cache_control => [<<"only-if-cached">>] }), + {ok, Res2} = hb_ao:resolve(Msg1, Msg2, #{ cache_control => [<<"only-if-cached">>] }), ?event(reading_from_cache_again), - {ok, Res3} = hb_converge:resolve(Msg1, Msg2, #{ cache_control => [<<"only-if-cached">>] }), + {ok, Res3} = hb_ao:resolve(Msg1, Msg2, #{ cache_control => [<<"only-if-cached">>] }), ?event({res2, Res2}), ?event({res3, Res3}), ?assertEqual(Res2, Res3). \ No newline at end of file diff --git a/src/hb_cache_render.erl b/src/hb_cache_render.erl new file mode 100644 index 000000000..aa5a191f2 --- /dev/null +++ b/src/hb_cache_render.erl @@ -0,0 +1,372 @@ +%%% @doc A module that helps to render given Key graphs into the .dot files +-module(hb_cache_render). +-export([render/1, render/2, cache_path_to_dot/2, cache_path_to_dot/3, dot_to_svg/1]). +-export([get_graph_data/3, cache_path_to_graph/3]). +% Preparing data for testing +-export([prepare_unsigned_data/0, prepare_signed_data/0, + prepare_deeply_nested_complex_message/0]). +-include("include/hb.hrl"). + +%% @doc Render the given Key into svg +render(StoreOrOpts) -> + render(all, StoreOrOpts). +render(ToRender, StoreOrOpts) -> + % Collect graph elements (nodes and arcs) by traversing the store + % Generate and view the graph visualization + % Write SVG to file and open it + file:write_file("new_render_diagram.svg", + dot_to_svg(cache_path_to_dot(ToRender, StoreOrOpts))), + os:cmd("open new_render_diagram.svg"), + ok. + +%% @doc Generate a dot file from a cache path and options/store +cache_path_to_dot(ToRender, StoreOrOpts) -> + cache_path_to_dot(ToRender, #{}, StoreOrOpts). +cache_path_to_dot(ToRender, RenderOpts, StoreOrOpts) -> + graph_to_dot(cache_path_to_graph(ToRender, RenderOpts, StoreOrOpts), StoreOrOpts). + +%% @doc Main function to collect graph elements +cache_path_to_graph(ToRender, GraphOpts, StoreOrOpts) when is_map(StoreOrOpts) -> + Store = hb_opts:get(store, no_viable_store, StoreOrOpts), + ?event({store, Store}), + cache_path_to_graph(ToRender, GraphOpts, Store, StoreOrOpts). +cache_path_to_graph(all, GraphOpts, Store, Opts) -> + Keys = + case hb_store:list(Store, <<"/">>) of + {ok, KeyList} -> KeyList; + not_found -> [] + end, + ?event({all_keys, Keys}), + cache_path_to_graph(Store, GraphOpts, Keys, Opts); +cache_path_to_graph(InitPath, GraphOpts, Store, Opts) when is_binary(InitPath) -> + cache_path_to_graph(Store, GraphOpts, [InitPath], Opts); +cache_path_to_graph(Store, GraphOpts, RootKeys, Opts) -> + % Use a map to track nodes, arcs and visited paths (to avoid cycles) + EmptyGraph = GraphOpts#{ nodes => #{}, arcs => #{}, visited => #{} }, + % Process all root keys and get the final graph + lists:foldl( + fun(Key, Acc) -> traverse_store(Store, Key, undefined, Acc, Opts) end, + EmptyGraph, + RootKeys + ). + +%% @doc Traverse the store recursively to build the graph +traverse_store(Store, Path, Parent, Graph, Opts) -> + % Get the path and check if we've already visited it + JoinedPath = hb_store:join(Path), + ResolvedPath = + case hb_link:is_link_key(JoinedPath) of + true -> + ?event({is_link_key, {path, Path}, {res_path, JoinedPath}}), + {ok, Link} = hb_store:read(Store, hb_store:resolve(Store, JoinedPath)), + ?event({resolved_link, {read, Link}}), + hb_store:resolve(Store, Link); + false -> hb_store:resolve(Store, Path) + end, + ?event({traverse_store, {path, Path}, {joined_path, JoinedPath}, {resolved_path, ResolvedPath}, {parent, Parent}}), + % Skip if we've already processed this node + case hb_maps:get(visited, Graph, #{}, Opts) of + #{ JoinedPath := _ } -> Graph; + _ -> + % Mark as visited to avoid cycles + Graph1 = Graph#{visited => hb_maps:put(JoinedPath, true, hb_maps:get(visited, Graph, #{}, Opts), Opts)}, + % ?event({traverse_store, {key, Key}, {graph1, Graph1}}), + % Process node based on its type + case hb_store:type(Store, ResolvedPath) of + simple -> + process_simple_node(Store, Path, Parent, ResolvedPath, JoinedPath, Graph1, Opts); + composite -> + process_composite_node(Store, Path, Parent, ResolvedPath, JoinedPath, Graph1, Opts); + _ -> + ?event({unknown_node_type, {path, Path}, {type, hb_store:type(Store, Path)}}), + Graph1 + end + end. + +%% @doc Process a simple (leaf) node +process_simple_node(_Store, _Key, Parent, ResolvedPath, JoinedPath, Graph, Opts) -> + % ?event({process_simple_node, {key, Key}, {resolved_path, ResolvedPath}}), + % Add the node to the graph + case hb_maps:get(render_data, Graph, true, Opts) of + false -> Graph; + true -> + Graph1 = add_node(Graph, ResolvedPath, "lightblue", Opts), + % If we have a parent, add an arc from parent to this node + case Parent of + undefined -> Graph1; + ParentPath -> + Label = extract_label(JoinedPath), + add_arc(Graph1, ParentPath, ResolvedPath, Label, Opts) + end + end. + +%% @doc Process a composite (directory) node +process_composite_node(_Store, <<"data">>, _Parent, _ResolvedPath, _JoinedPath, Graph, _Opts) -> + % Data is a special case: It contains every binary item in the store. + % We don't need to render it. + Graph; +process_composite_node(Store, _Key, Parent, ResolvedPath, JoinedPath, Graph, Opts) -> + % Add the node to the graph + Graph1 = add_node(Graph, ResolvedPath, "lightcoral", Opts), + % If we have a parent, add an arc from parent to this node + Graph2 = case Parent of + undefined -> Graph1; + ParentPath -> + Label = extract_label(JoinedPath), + add_arc(Graph1, ParentPath, ResolvedPath, Label, Opts) + end, + % Process children recursively + case hb_store:list(Store, ResolvedPath) of + {ok, SubItems} -> + lists:foldl( + fun(SubItem, Acc) -> + ChildKey = [ResolvedPath, SubItem], + traverse_store(Store, ChildKey, ResolvedPath, Acc, Opts) + end, + Graph2, + SubItems + ); + _ -> Graph2 + end. + +%% @doc Add a node to the graph +add_node(Graph, ID, Color, Opts) -> + Nodes = hb_maps:get(nodes, Graph, #{}, Opts), + Graph#{nodes => hb_maps:put(ID, {ID, Color}, Nodes, Opts)}. + +%% @doc Add an arc to the graph +add_arc(Graph, From, To, Label, Opts) -> + ?event({insert_arc, {id1, From}, {id2, To}, {label, Label}}), + Arcs = hb_maps:get(arcs, Graph, #{}, Opts), + Graph#{arcs => hb_maps:put({From, To, Label}, true, Arcs, Opts)}. + +%% @doc Extract a label from a path +extract_label(Path) -> + case binary:split(Path, <<"/">>, [global]) of + [] -> Path; + Parts -> + FilteredParts = [P || P <- Parts, P /= <<>>], + case FilteredParts of + [] -> Path; + _ -> lists:last(FilteredParts) + end + end. + +%% @doc Generate the DOT file from the graph +graph_to_dot(Graph, Opts) -> + % Create graph header + Header = [ + <<"digraph filesystem {\n">>, + <<" node [shape=circle];\n">> + ], + % Create nodes section + Nodes = hb_maps:fold( + fun(ID, {Label, Color}, Acc) -> + [ + Acc, + io_lib:format( + <<" \"~s\" [label=\"~s\", color=~s, style=filled];~n">>, + [ID, hb_format:short_id(hb_util:bin(Label)), Color] + ) + ] + end, + [], + hb_maps:get(nodes, Graph, #{}, Opts), + Opts + ), + % Create arcs section + Arcs = hb_maps:fold( + fun({From, To, Label}, _, Acc) -> + [ + Acc, + io_lib:format( + <<" \"~s\" -> \"~s\" [label=\"~s\"];~n">>, + [From, To, hb_format:short_id(hb_util:bin(Label))] + ) + ] + end, + [], + hb_maps:get(arcs, Graph, #{}, Opts), + Opts + ), + % Create graph footer + Footer = <<"}\n">>, + % Combine all parts and convert to binary + iolist_to_binary([Header, Nodes, Arcs, Footer]). + +%% @doc Convert a dot graph to SVG format +dot_to_svg(DotInput) -> + % Create a port to the dot command + Port = open_port({spawn, "dot -Tsvg"}, [binary, use_stdio, stderr_to_stdout]), + % Send the dot content to the process + true = port_command(Port, iolist_to_binary(DotInput)), + % Get the SVG output + collect_output(Port, []). + +%% @doc Helper function to collect output from port +collect_output(Port, Acc) -> + receive + {Port, {data, Data}} -> + case binary:part(Data, byte_size(Data) - 7, 7) of + <<"\n">> -> + port_close(Port), + iolist_to_binary(lists:reverse([Data | Acc])); + _ -> collect_output(Port, [Data | Acc]) + end; + {Port, eof} -> + port_close(Port), + iolist_to_binary(lists:reverse(Acc)) + after 10000 -> + {error, timeout} + end. + +%% @doc Get graph data for the Three.js visualization +get_graph_data(Base, MaxSize, Opts) -> + % Try to generate graph using hb_cache_render + Graph = + try + % Use hb_cache_render to build the graph + cache_path_to_graph(Base, #{}, Opts) + catch + Error:Reason:Stack -> + ?event({hyperbuddy_graph_error, Error, Reason, Stack}), + #{nodes => #{}, arcs => #{}, visited => #{}} + end, + % Extract nodes and links for the visualization + NodesMap = maps:get(nodes, Graph, #{}), + ArcsMap = maps:get(arcs, Graph, #{}), + % Limit to top `MaxSize` nodes if there are too many + NodesList = + case maps:size(NodesMap) > MaxSize of + true -> + % Take a subset of nodes + {ReducedNodes, _} = lists:split( + MaxSize, + maps:to_list(NodesMap) + ), + ReducedNodes; + false -> + maps:to_list(NodesMap) + end, + % Get node IDs for filtering links + NodeIds = [ID || {ID, _} <- NodesList], + % Convert to JSON format for web visualization + Nodes = + [ + #{ + <<"id">> => ID, + <<"label">> => get_label(hb_util:bin(ID)), + <<"type">> => get_node_type(Color) + } + || + {ID, {_, Color}} <- NodesList + ], + % Filter links to only include those between nodes we're showing + FilteredLinks = + [ + {From, To, Label} + || + {From, To, Label} <- maps:keys(ArcsMap), + lists:member(From, NodeIds) + andalso lists:member(To, NodeIds) + ], + Links = + [ + #{ + <<"source">> => From, + <<"target">> => To, + <<"label">> => Label + } + || + {From, To, Label} <- FilteredLinks + ], + % Return the JSON data + JsonData = hb_json:encode(#{ <<"nodes">> => Nodes, <<"links">> => Links }), + {ok, #{ + <<"body">> => JsonData, + <<"content-type">> => <<"application/json">> + }}. + +%% @doc Convert node color from hb_cache_render to node type for visualization +get_node_type(Color) -> + case Color of + "lightblue" -> <<"simple">>; + "lightcoral" -> <<"composite">>; + _ -> <<"unknown">> + end. + +%% @doc Extract a readable label from a path +get_label(Path) -> + case binary:split(Path, <<"/">>, [global]) of + [] -> Path; + Parts -> + FilteredParts = [P || P <- Parts, P /= <<>>], + case FilteredParts of + [] -> Path; + _ -> lists:last(FilteredParts) + end + end. + +% Test data preparation functions +prepare_unsigned_data() -> + Opts = #{ + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/render-fs">> + } + }, + Item = test_unsigned(#{ <<"key">> => <<"Simple unsigned data item">> }), + {ok, _Path} = hb_cache:write(Item, Opts). + +prepare_signed_data() -> + Opts = #{ + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/render-fs">> + } + }, + Wallet = ar_wallet:new(), + Item = test_signed(#{ <<"l2-test-key">> => <<"l2-test-value">> }, Wallet), + %% Write the simple unsigned item + {ok, _Path} = hb_cache:write(Item, Opts). + +prepare_deeply_nested_complex_message() -> + Opts = #{ + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/render-fs">> + } + }, + Wallet = ar_wallet:new(), + %% Create nested data + Level3SignedSubmessage = test_signed([1,2,3], Wallet), + Outer = + #{ + <<"level1">> => + hb_message:commit( + #{ + <<"level2">> => + #{ + <<"level3">> => Level3SignedSubmessage, + <<"e">> => <<"f">>, + <<"z">> => [1,2,3] + }, + <<"c">> => <<"d">>, + <<"g">> => [<<"h">>, <<"i">>], + <<"j">> => 1337 + }, + ar_wallet:new() + ), + <<"a">> => <<"b">> + }, + %% Write the nested item + {ok, _} = hb_cache:write(Outer, Opts). + +test_unsigned(Data) -> + #{ + <<"base-test-key">> => <<"base-test-value">>, + <<"data">> => Data + }. + +test_signed(Data, Wallet) -> + hb_message:commit(test_unsigned(Data), Wallet). \ No newline at end of file diff --git a/src/hb_client.erl b/src/hb_client.erl index b4d392355..51428aca9 100644 --- a/src/hb_client.erl +++ b/src/hb_client.erl @@ -1,14 +1,15 @@ -module(hb_client). -%% Converge API and HyperBEAM Built-In Devices +%% AO-Core API and HyperBEAM Built-In Devices -export([resolve/4, routes/2, add_route/3]). %% Arweave node API -export([arweave_timestamp/0]). %% Arweave bundling and data access API --export([upload/1, download/1]). - +-export([upload/2, upload/3]). +%% Tests +-include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). -%%% Converge API and HyperBEAM Built-In Devices +%%% AO-Core API and HyperBEAM Built-In Devices %% @doc Resolve a message pair on a remote node. %% The message pair is first transformed into a singleton request, by @@ -16,37 +17,38 @@ %% and then adjusting the "Path" field from the second message. resolve(Node, Msg1, Msg2, Opts) -> TABM2 = - hb_converge:set( + hb_ao:set( #{ - <<"Path">> => hb_converge:get(<<"Path">>, Msg2, <<"/">>, Opts), - <<"2.Path">> => unset + <<"path">> => hb_ao:get(<<"path">>, Msg2, <<"/">>, Opts), + <<"2.path">> => unset }, prefix_keys(<<"2.">>, Msg2, Opts), Opts#{ hashpath => ignore } ), hb_http:post( Node, - maps:merge(prefix_keys(<<"1.">>, Msg1, Opts), TABM2), + hb_maps:merge(prefix_keys(<<"1.">>, Msg1, Opts), TABM2, Opts), Opts ). prefix_keys(Prefix, Message, Opts) -> - maps:fold( + hb_maps:fold( fun(Key, Val, Acc) -> - maps:put(<>, Val, Acc) + hb_maps:put(<>, Val, Acc, Opts) end, #{}, - hb_message:convert(Message, tabm, Opts) + hb_message:convert(Message, tabm, Opts), + Opts ). routes(Node, Opts) -> resolve(Node, #{ - device => <<"Router/1.0">> + <<"device">> => <<"Router@1.0">> }, #{ - path => <<"Routes">>, - method => <<"GET">> + <<"path">> => <<"routes">>, + <<"method">> => <<"GET">> }, Opts ). @@ -54,11 +56,11 @@ routes(Node, Opts) -> add_route(Node, Route, Opts) -> resolve(Node, Route#{ - device => <<"Router/1.0">> + <<"device">> => <<"Router@1.0">> }, #{ - path => <<"Routes">>, - method => <<"POST">> + <<"path">> => <<"routes">>, + <<"method">> => <<"POST">> }, Opts ). @@ -69,109 +71,116 @@ add_route(Node, Route, Opts) -> %% @doc Grab the latest block information from the Arweave gateway node. arweave_timestamp() -> case hb_opts:get(mode) of - debug -> {0, 0, <<0:256>>}; + debug -> {0, 0, hb_util:human_id(<<0:256>>)}; prod -> {ok, {{_, 200, _}, _, Body}} = httpc:request( <<(hb_opts:get(gateway))/binary, "/block/current">> ), - {Fields} = jiffy:decode(Body), - {_, Timestamp} = lists:keyfind(<<"timestamp">>, 1, Fields), - {_, Hash} = lists:keyfind(<<"indep_hash">>, 1, Fields), - {_, Height} = lists:keyfind(<<"height">>, 1, Fields), + Fields = hb_json:decode(hb_util:bin(Body)), + Timestamp = hb_maps:get(<<"timestamp">>, Fields), + Hash = hb_maps:get(<<"indep_hash">>, Fields), + Height = hb_maps:get(<<"height">>, Fields), {Timestamp, Height, Hash} end. %%% Bundling and data access API -%% @doc Download the data associated with a given ID. See TODO below. -download(ID) -> - % TODO: Need to recreate full data items, not just data... - case httpc:request(hb_opts:get(gateway) ++ "/" ++ ID) of - {ok, {{_, 200, _}, _, Body}} -> #tx{data = Body}; - _Rest -> throw({id_get_failed, ID}) - end. - %% @doc Upload a data item to the bundler node. -upload(Message) when is_map(Message) -> - upload(hb_message:convert(Message, tx, converge, #{})); -upload(Item) -> - ?event({uploading_item, Item}), - case - httpc:request( - post, - { - <<(hb_opts:get(bundler))/binary, "/tx">>, - [], - "application/octet-stream", - ar_bundles:serialize(Item) - }, - [], - [] - ) - of - {ok, {{_, 200, _}, _, Body}} -> - ?event(upload_success), - {ok, jiffy:decode(Body, [return_maps])}; - Response -> - ?event(upload_error), - {error, bundler_http_error, Response} - end. +%% Note: Uploads once per commitment device. Callers should filter the +%% commitments to only include the ones they are interested in, if this is not +%% the desired behavior. +upload(Msg, Opts) -> + UploadResults = + lists:map( + fun(Device) -> + upload(Msg, Opts, Device) + end, + hb_message:commitment_devices(Msg, Opts) + ), + {ok, UploadResults}. +upload(Msg, Opts, <<"httpsig@1.0">>) -> + case hb_opts:get(bundler_httpsig, not_found, Opts) of + not_found -> + {error, no_httpsig_bundler}; + Bundler -> + ?event({uploading_item, Msg}), + hb_http:post(Bundler, <<"/tx">>, Msg, Opts) + end; +upload(Msg, Opts, <<"ans104@1.0">>) when is_map(Msg) -> + ?event({msg_to_convert, Msg}), + Converted = hb_message:convert(Msg, <<"ans104@1.0">>, Opts), + ?event({msg_to_tx_res, {converted, Converted}}), + Serialized = ar_bundles:serialize(Converted), + ?event({converted_msg_to_tx, Serialized}), + upload(Serialized, Opts, <<"ans104@1.0">>); +upload(Serialized, Opts, <<"ans104@1.0">>) when is_binary(Serialized) -> + ?event({uploading_item, Serialized}), + hb_http:post( + hb_opts:get(bundler_ans104, not_found, Opts), + #{ + <<"path">> => <<"/tx">>, + <<"content-type">> => <<"application/octet-stream">>, + <<"body">> => Serialized + }, + Opts#{ + http_client => + hb_opts:get(bundler_ans104_http_client, httpc, Opts) + } + ). -%%% Utility functions +%%% Tests -%% @doc Convert a map of parameters into a query string, starting with the -%% given separator. -path_opts(EmptyMap, _Sep) when map_size(EmptyMap) == 0 -> ""; -path_opts(Opts, Sep) -> - PathParts = tl(lists:flatten(lists:map( - fun({Key, Val}) -> - "&" ++ format_path_opt(Key) ++ "=" ++ format_path_opt(Val) - end, - maps:to_list(Opts) - ))), - Sep ++ PathParts. - -format_path_opt(Val) when is_atom(Val) -> - atom_to_list(Val); -format_path_opt(Val) when is_binary(Val) -> - binary_to_list(Val); -format_path_opt(Val) when is_integer(Val) -> - integer_to_list(Val). - -parse_result_set(Body) -> - {JSONStruct} = jiffy:decode(Body), - {_, {PageInfoStruct}} = lists:keyfind(<<"pageInfo">>, 1, JSONStruct), - {_, HasNextPage} = lists:keyfind(<<"hasNextPage">>, 1, PageInfoStruct), - {_, EdgesStruct} = lists:keyfind(<<"edges">>, 1, JSONStruct), - {HasNextPage, lists:map(fun json_struct_to_result/1, EdgesStruct)}. - -%% Parse a CU result into a #result record. If the result is in the form of a -%% stream, then the cursor is returned in the #result record as well. -json_struct_to_result(NodeStruct) -> - json_struct_to_result(NodeStruct, #result{}). -json_struct_to_result({NodeStruct}, Res) -> - json_struct_to_result(NodeStruct, Res); -json_struct_to_result(Struct, Res) -> - case lists:keyfind(<<"node">>, 1, Struct) of - false -> - Res#result{ - messages = lists:map( - fun ar_bundles:json_struct_to_item/1, - hb_util:find_value(<<"Messages">>, Struct, []) - ), - assignments = hb_util:find_value(<<"Assignments">>, Struct, []), - spawns = lists:map( - fun ar_bundles:json_struct_to_item/1, - hb_util:find_value(<<"Spawns">>, Struct, []) - ), - output = hb_util:find_value(<<"Output">>, Struct, []) - }; - {_, {NodeStruct}} -> - json_struct_to_result( - NodeStruct, - Res#result{ - cursor = hb_util:find_value(<<"cursor">>, Struct, undefined) - } - ) - end. \ No newline at end of file +upload_empty_raw_ans104_test() -> + Serialized = ar_bundles:serialize( + ar_bundles:sign_item(#tx{ + data = <<"TEST">> + }, hb:wallet()) + ), + ?event({uploading_item, Serialized}), + Result = upload(Serialized, #{}, <<"ans104@1.0">>), + ?event({upload_result, Result}), + ?assertMatch({ok, _}, Result). + +upload_raw_ans104_test() -> + Serialized = ar_bundles:serialize( + ar_bundles:sign_item(#tx{ + data = <<"TEST">>, + tags = [{<<"test-tag">>, <<"test-value">>}] + }, hb:wallet()) + ), + ?event({uploading_item, Serialized}), + Result = upload(Serialized, #{}, <<"ans104@1.0">>), + ?event({upload_result, Result}), + ?assertMatch({ok, _}, Result). + +upload_raw_ans104_with_anchor_test() -> + Serialized = ar_bundles:serialize( + ar_bundles:sign_item(#tx{ + data = <<"TEST">>, + anchor = crypto:strong_rand_bytes(32), + tags = [{<<"test-tag">>, <<"test-value">>}] + }, hb:wallet()) + ), + ?event({uploading_item, Serialized}), + Result = upload(Serialized, #{}, <<"ans104@1.0">>), + ?event({upload_result, Result}), + ?assertMatch({ok, _}, Result). + +upload_empty_message_test() -> + Msg = #{ <<"data">> => <<"TEST">> }, + Committed = hb_message:commit(Msg, hb:wallet(), <<"ans104@1.0">>), + Result = upload(Committed, #{}, <<"ans104@1.0">>), + ?event({upload_result, Result}), + ?assertMatch({ok, _}, Result). + +upload_single_layer_message_test() -> + Msg = #{ + <<"data">> => <<"TEST">>, + <<"basic">> => <<"value">>, + <<"integer">> => 1 + }, + Committed = hb_message:commit(Msg, hb:wallet(), <<"ans104@1.0">>), + Result = upload(Committed, #{}, <<"ans104@1.0">>), + ?event({upload_result, Result}), + ?assertMatch({ok, _}, Result). \ No newline at end of file diff --git a/src/hb_codec_converge.erl b/src/hb_codec_converge.erl deleted file mode 100644 index d76373f95..000000000 --- a/src/hb_codec_converge.erl +++ /dev/null @@ -1,193 +0,0 @@ -%%% @doc A codec for the that marshals TABM encoded messages to and from the -%%% main Converge format, which features rich types with deterministic encoding -%%% built around the HTTP Structured Fields (RFC-9651) specification. --module(hb_codec_converge). --export([to/1, from/1]). --export([decode_value/2, encode_value/1]). --include("include/hb.hrl"). --include_lib("eunit/include/eunit.hrl"). - -%% @doc Convert a rich message into a 'Type-Annotated-Binary-Message' (TABM). -from(Bin) when is_binary(Bin) -> Bin; -from(Msg) when is_map(Msg) -> - maps:from_list(lists:flatten( - lists:map( - fun(Key) -> - case maps:find(Key, Msg) of - {ok, <<>>} -> - BinKey = hb_converge:key_to_binary(Key), - {<<"Converge-Type:", BinKey/binary>>, <<"Empty-Binary">>}; - {ok, Value} when is_binary(Value) -> - {Key, Value}; - {ok, Map} when is_map(Map) -> - {Key, from(Map)}; - {ok, []} -> - BinKey = hb_converge:key_to_binary(Key), - {<<"Converge-Type:", BinKey/binary>>, <<"Empty-List">>}; - {ok, Value} when - is_atom(Value) or is_integer(Value) - or is_list(Value) -> - ItemKey = hb_converge:key_to_binary(Key), - {Type, BinaryValue} = encode_value(Value), - [ - {<<"Converge-Type:", ItemKey/binary>>, Type}, - {ItemKey, BinaryValue} - ]; - {ok, _} -> [] - end - end, - lists:filter( - fun(Key) -> - % Filter keys that the user could set directly, but - % should be regenerated when moving msg -> TX, as well - % as private keys. - not lists:member(Key, ?REGEN_KEYS) andalso - not hb_private:is_private(Key) - end, - maps:keys(Msg) - ) - ) - )); -from(Other) -> hb_path:to_binary(Other). - -%% @doc Convert a TABM into a native HyperBEAM message. -to(Bin) when is_binary(Bin) -> Bin; -to(TABM0) -> - % First, handle special cases of empty items, which `ar_bundles` cannot - % handle. Needs to be transformed into a list (unfortunately) so that we - % can also remove the "Converge-Type:" prefix from the key. - TABM1 = - maps:from_list( - lists:map( - fun({<<"Converge-Type:", Key/binary>>, <<"Empty-Binary">>}) -> - {Key, <<>>}; - ({<<"Converge-Type:", Key/binary>>, <<"Empty-List">>}) -> - {Key, []}; - ({Key, Value}) -> - {Key, Value} - end, - maps:to_list(TABM0) - ) - ), - % 1. Remove any keys from output that have a "Converge-Type:" prefix; - % 2. Decode any binary values that have a "Converge-Type:" prefix; - % 3. Recursively decode any maps that we encounter; - % 4. Return the remaining keys and values as a map. - hb_message:filter_default_keys(maps:filtermap( - fun(<<"Converge-Type:", _/binary>>, _) -> - % Remove any keys from output that have a "Converge-Type:" prefix. - false; - (RawKey, BinaryValue) when is_binary(BinaryValue) -> - Key = hb_converge:key_to_binary(RawKey), - case maps:find(<<"Converge-Type:", Key/binary>>, TABM1) of - error -> {true, BinaryValue}; - {ok, Type} -> - {true, decode_value(Type, BinaryValue)} - end; - (_Key, ChildTABM) when is_map(ChildTABM) -> - {true, to(ChildTABM)}; - (_Key, Value) -> - % We encountered a key that already has a converted type. - % We can just return it as is. - {true, Value} - end, - TABM1 - )). - -%% @doc Convert a term to a binary representation, emitting its type for -%% serialization as a separate tag. -encode_value(Value) when is_integer(Value) -> - [Encoded, _] = hb_http_structured_fields:item({item, Value, []}), - {<<"Integer">>, Encoded}; -encode_value(Value) when is_float(Value) -> - ?no_prod("Must use structured field representation for floats!"), - {<<"Float">>, float_to_binary(Value)}; -encode_value(Value) when is_atom(Value) -> - [EncodedIOList, _] = - hb_http_structured_fields:item( - {item, {string, atom_to_binary(Value, latin1)}, []}), - Encoded = list_to_binary(EncodedIOList), - {<<"Atom">>, Encoded}; -encode_value(Values) when is_list(Values) -> - EncodedValues = - lists:map( - fun(Bin) when is_binary(Bin) -> {item, {string, Bin}, []}; - (Item) -> - {RawType, Encoded} = encode_value(Item), - Type = hb_converge:key_to_binary(RawType), - { - item, - { - string, - << - "(Converge-Type: ", Type/binary, ") ", - Encoded/binary - >> - }, - [] - } - end, - Values - ), - EncodedList = hb_http_structured_fields:list(EncodedValues), - {<<"List">>, iolist_to_binary(EncodedList)}; -encode_value(Value) when is_binary(Value) -> - {<<"Binary">>, Value}; -encode_value(Value) -> - Value. - -%% @doc Convert non-binary values to binary for serialization. -decode_value(Type, Value) when is_list(Type) -> - decode_value(list_to_binary(Type), Value); -decode_value(Type, Value) when is_binary(Type) -> - decode_value( - binary_to_existing_atom( - list_to_binary(string:to_lower(binary_to_list(Type))), - latin1 - ), - Value - ); -decode_value(integer, Value) -> - {item, Number, _} = hb_http_structured_fields:parse_item(Value), - Number; -decode_value(float, Value) -> - binary_to_float(Value); -decode_value(atom, Value) -> - {item, {string, AtomString}, _} = - hb_http_structured_fields:parse_item(Value), - binary_to_existing_atom(AtomString, latin1); -decode_value(list, Value) -> - lists:map( - fun({item, {string, <<"(Converge-Type: ", Rest/binary>>}, _}) -> - [Type, Item] = binary:split(Rest, <<") ">>), - decode_value(Type, Item); - ({item, {string, Binary}, _}) -> Binary - end, - hb_http_structured_fields:parse_list(iolist_to_binary(Value)) - ); -decode_value(BinType, Value) when is_binary(BinType) -> - decode_value( - list_to_existing_atom( - string:to_lower( - binary_to_list(BinType) - ) - ), - Value - ); -decode_value(OtherType, Value) -> - ?event({unexpected_type, OtherType, Value}), - throw({unexpected_type, OtherType, Value}). - -%%% Tests - -list_encoding_test() -> - % Test that we can encode and decode a list of integers. - {<<"List">>, Encoded} = encode_value(List1 = [1, 2, 3]), - Decoded = decode_value(list, Encoded), - ?assertEqual(List1, Decoded), - % Test that we can encode and decode a list of binaries. - {<<"List">>, Encoded2} = encode_value(List2 = [<<"1">>, <<"2">>, <<"3">>]), - ?assertEqual(List2, decode_value(list, Encoded2)), - % Test that we can encode and decode a mixed list. - {<<"List">>, Encoded3} = encode_value(List3 = [1, <<"2">>, 3]), - ?assertEqual(List3, decode_value(list, Encoded3)). \ No newline at end of file diff --git a/src/hb_codec_flat.erl b/src/hb_codec_flat.erl deleted file mode 100644 index 279de7513..000000000 --- a/src/hb_codec_flat.erl +++ /dev/null @@ -1,99 +0,0 @@ -%%% A codec for turning TABMs into/from flat Erlang maps that have (potentially -%%% multi-layer) paths as their keys, and a normal TABM binary as their value. --module(hb_codec_flat). --export([from/1, to/1]). --include_lib("eunit/include/eunit.hrl"). --include("include/hb.hrl"). - -%% @doc Convert a flat map to a TABM. -from(Bin) when is_binary(Bin) -> Bin; -from(Map) when is_map(Map) -> - maps:fold( - fun(Path, Value, Acc) -> - inject_at_path(hb_path:term_to_path_parts(Path), from(Value), Acc) - end, - #{}, - Map - ). - -%% Helper function to inject a value at a specific path in a nested map -inject_at_path([Key], Value, Map) -> - maps:put(Key, Value, Map); -inject_at_path([Key|Rest], Value, Map) -> - SubMap = maps:get(Key, Map, #{}), - maps:put(Key, inject_at_path(Rest, Value, SubMap), Map). - -%% @doc Convert a TABM to a flat map. -to(Bin) when is_binary(Bin) -> Bin; -to(Map) when is_map(Map) -> - maps:fold( - fun(Key, Value, Acc) -> - case to(Value) of - SubMap when is_map(SubMap) -> - maps:fold( - fun(SubKey, SubValue, InnerAcc) -> - maps:put( - hb_path:to_binary([Key, SubKey]), - SubValue, - InnerAcc - ) - end, - Acc, - SubMap - ); - SimpleValue -> - maps:put(hb_path:to_binary([Key]), SimpleValue, Acc) - end - end, - #{}, - Map - ). - -%%% Tests - -simple_conversion_test() -> - Flat = #{[<<"a">>] => <<"value">>}, - Nested = #{<<"a">> => <<"value">>}, - ?assert(hb_message:match(Nested, hb_codec_flat:from(Flat))), - ?assert(hb_message:match(Flat, hb_codec_flat:to(Nested))). - -nested_conversion_test() -> - Flat = #{<<"a/b">> => <<"value">>}, - Nested = #{<<"a">> => #{<<"b">> => <<"value">>}}, - Unflattened = hb_codec_flat:from(Flat), - Flattened = hb_codec_flat:to(Nested), - ?assert(hb_message:match(Nested, Unflattened)), - ?assert(hb_message:match(Flat, Flattened)). - -multiple_paths_test() -> - Flat = #{ - <<"x/y">> => <<"1">>, - <<"x/z">> => <<"2">>, - <<"a">> => <<"3">> - }, - Nested = #{ - <<"x">> => #{ - <<"y">> => <<"1">>, - <<"z">> => <<"2">> - }, - <<"a">> => <<"3">> - }, - ?assert(hb_message:match(Nested, hb_codec_flat:from(Flat))), - ?assert(hb_message:match(Flat, hb_codec_flat:to(Nested))). - -binary_passthrough_test() -> - Bin = <<"raw binary">>, - ?assertEqual(Bin, hb_codec_flat:from(Bin)), - ?assertEqual(Bin, hb_codec_flat:to(Bin)). - -deep_nesting_test() -> - Flat = #{<<"a/b/c/d">> => <<"deep">>}, - Nested = #{<<"a">> => #{<<"b">> => #{<<"c">> => #{<<"d">> => <<"deep">>}}}}, - Unflattened = hb_codec_flat:from(Flat), - Flattened = hb_codec_flat:to(Nested), - ?assert(hb_message:match(Nested, Unflattened)), - ?assert(hb_message:match(Flat, Flattened)). - -empty_map_test() -> - ?assertEqual(#{}, hb_codec_flat:from(#{})), - ?assertEqual(#{}, hb_codec_flat:to(#{})). \ No newline at end of file diff --git a/src/hb_codec_http.erl b/src/hb_codec_http.erl deleted file mode 100644 index 5a4c41c3f..000000000 --- a/src/hb_codec_http.erl +++ /dev/null @@ -1,342 +0,0 @@ - -%%% @doc A codec for the that marshals TABM encoded messages to and from the -%%% "HTTP" message structure. -%%% -%%% The HTTP Message is an Erlang Map with the following shape: -%%% #{ -%%% headers => [ -%%% {<<"Example-Header">>, <<"Value">>} -%%% ], -%%% body: <<"Some body">> -%%% } -%%% -%%% Every HTTP message is an HTTP multipart message. -%%% See https://datatracker.ietf.org/doc/html/rfc7578 -%%% -%%% For each TABM Key: -%%% -%%% The TABM Key will be ignored if: -%%% - The field is private (according to hb_private:is_private/1) -%%% - The field is one of ?REGEN_KEYS -%%% -%%% The Key/Value Pair will be encoded according to the following rules: -%%% "signatures" -> {SignatureInput, Signature} header Tuples, each encoded -%%% as a Structured Field Dictionary -%%% "body" -> -%%% - if a map, then recursively encode as its own HyperBEAM message -%%% - otherwise encode as a normal field -%%% _ -> encode as a normal field -%%% -%%% Each field will be mapped to the HTTP Message according to the following -%%% rules: -%%% "body" -> always encoded part of the body as with Content-Disposition -%%% type of "inline" -%%% _ -> -%%% - If the byte size of the value is less than the ?MAX_TAG_VALUE, -%%% then encode as a header, also attempting to encode as a -%%% structured field. -%%% - Otherwise encode the value as a part in the multipart response -%%% --module(hb_codec_http). --export([to/1, from/1]). --include("include/hb.hrl"). --include_lib("eunit/include/eunit.hrl"). - -% The max header length is 4KB --define(MAX_HEADER_LENGTH, 4096). -% https://datatracker.ietf.org/doc/html/rfc7231#section-3.1.1.4 --define(CRLF, <<"\r\n">>). --define(DOUBLE_CRLF, <>). - -%% @doc Convert an HTTP Message into a TABM. -%% HTTP Structured Field is encoded into it's equivalent TABM encoding. -from(Bin) when is_binary(Bin) -> Bin; -from(#{ headers := Headers, body := Body }) when is_map(Headers) -> - from(#{ headers => maps:to_list(Headers), body => Body }); -from(#{ headers := Headers, body := Body }) -> - % First, parse all headers and add as key-value pairs to the TABM - Map = from_headers(#{}, Headers), - % Next, we need to potentially parse the body and add to the TABM - % potentially as additional key-binary value pairs, or as sub-TABMs - {_, ContentType} = find_header(Headers, <<"Content-Type">>), - maps:remove(<<"Content-Type">>, from_body(Map, ContentType, Body)). - -from_headers(Map, Headers) -> from_headers(Map, Headers, Headers). -from_headers(Map, [], _) -> Map; -from_headers(Map, [{Name, Value} | Rest], Headers) -> - NewMap = case Name of - % Handled as part of "Signature" so simply skip it - <<"Signature-Input">> -> Map; - <<"signature-input">> -> Map; - <<"Signature">> -> - {_, SigInput} = find_header(Headers, <<"Signature-Input">>), - from_signature(Map, Value, SigInput); - <<"signature">> -> - {_, SigInput} = find_header(Headers, <<"Signature-Input">>), - from_signature(Map, Value, SigInput); - % Decode the header as normal - N -> maps:put(N, Value, Map) - end, - from_headers(NewMap, Rest, Headers). - -from_signature(Map, RawSig, RawSigInput) -> - SfSigs = hb_http_structured_fields:parse_dictionary(RawSig), - SfInputs = hb_http_structured_fields:parse_dictionary(RawSigInput), - % Build a Map for Signatures by gathering each Signature - % with its corresponding Inputs. - % - % Inputs are merged as fields on the Signature Map - Signatures = maps:fold( - fun (SigName, {item, {_, Sig}, _}, Sigs) -> - {list, SfInputItems, SfInputParams} = lists:keyfind(SigName, 1, SfInputs), - % [<<"foo">>, <<"bar">>] - Inputs = lists:map(fun({item, {_, Input}, _}) -> Input end, SfInputItems), - - SigMap = lists:foldl( - fun({PName, PBareItem}, PAcc) -> - maps:put(PName, PBareItem, PAcc) - end, - #{ <<"signature">> => Sig, <<"inputs">> => Inputs }, - % Signature parameters are converted into top-level keys on the signature Map - SfInputParams - ), - % #{ [SigName/binary] => #{ <<"signature">> => <<>>, <<"inputs">> => , ... } } - maps:put(SigName, SigMap, Sigs) - end, - #{}, - SfSigs - ), - % Finally place the Signatures as a top-level Map on the parent Map - maps:put(<<"Signatures">>, Signatures, Map). - -find_header(Headers, Name) -> - find_header(Headers, Name, []). -find_header(Headers, Name, Opts) when is_list(Headers) -> - Matcher = case lists:member(strict, Opts) of - true -> fun ({N, _Value}) -> N =:= Name end; - _ -> - fun ({N, _Value}) -> - hb_util:to_lower(N) =:= hb_util:to_lower(Name) - end - end, - case lists:filter(Matcher, Headers) of - [] -> {undefined, undefined}; - Found -> case lists:member(global, Opts) of - true -> Found; - _ -> - [First | _] = Found, - First - end - end. - -from_body(TABM, _ContentType, <<>>) -> TABM; -from_body(TABM, ContentType, Body) -> - {item, {_, _BodyType}, Params} = - hb_http_structured_fields:parse_item(ContentType), - case lists:keyfind(<<"boundary">>, 1, Params) of - % The body is not a multipart, so just set as is to the Body key on the TABM - false -> - maps:put(<<"Body">>, Body, TABM); - % We need to manually parse the multipart body into key/values on the TABM - {_, {_Type, Boundary}} -> - % Find the sub-part of the body within the boundary - BegPat = <<"--", Boundary/binary>>, - EndPat = <<"--", Boundary/binary, "--">>, - {Start, SL} = binary:match(Body, BegPat), - {End, _} = binary:match(Body, EndPat), - BodyPart = binary:part(Body, Start + SL, End - (Start + SL)), - Parts = binary:split(BodyPart, [<<"--", Boundary/binary>>], [global]), - % Finally, for each body part, we need to parse it into its - % own HTTP Message, then recursively convert into a TABM - TABM1 = lists:foldl( - fun - (Part, CurTABM) -> - {ok, NewTABM} = append_body_part(CurTABM, Part), - NewTABM - end, - TABM, - Parts - ), - TABM1 - end. - -append_body_part(TABM, Part) -> - % Extract the Headers block and Body. Only split on the FIRST double CRLF - [RawHeadersBlock, RawBody] = case binary:split(Part, [?DOUBLE_CRLF], []) of - % no body - [RHB] -> [RHB, <<>>]; - [RHB, RB] -> [RHB, RB] - end, - % Extract individual headers - RawHeaders = binary:split(RawHeadersBlock, ?CRLF, [global]), - % Now we parse each header, splitting into {Key, Value} - Headers = lists:filtermap( - fun - % Skip empty headers that are missing in splitting - (<<>>) -> false; - (RawHeader) -> - case binary:split(RawHeader, [<<": ">>]) of - [Name, Value] -> {true, {Name, Value}}; - % skip lines that aren't properly formatted headers - _ -> false - end - end, - RawHeaders - ), - % The Content-Disposition is from the parent message, - % so we separate off from the rest of the headers - {AllContentDisposition, RestHeaders} = lists:partition( - fun - ({Str, _}) -> hb_util:to_lower(Str) =:= <<"content-disposition">>; - (_) -> false - end, - Headers - ), - ContentDisposition = case AllContentDisposition of - [] -> undefined; - [{_, CD} | _Rest] -> CD - end, - case ContentDisposition of - undefined -> no_content_disposition_header_found; - RawDisposition when is_binary(RawDisposition) -> - {item, {_, _Disposition}, Params} = hb_http_structured_fields:parse_item(RawDisposition), - PartName = case lists:keyfind(<<"name">>, 1, Params) of - false -> <<"Body">>; - {_, {_type, PN}} -> PN - end, - SubTABM = from(#{ headers => RestHeaders, body => RawBody }), - {ok, maps:put(PartName, SubTABM, TABM)} - end. - -%%% @doc Convert a TABM into an HTTP Message. The HTTP Message is a simple Erlang Map -%%% that can translated to a given web server Response API -to(Bin) when is_binary(Bin) -> Bin; -to(TABM) when is_map(TABM) -> - % PublicMsg = hb_private:reset(TABM), - % MinimizedMsg = hb_message:minimize(PublicMsg), - Http = maps:fold( - fun(RawKey, Value, Http) -> - Key = hb_converge:key_to_binary(RawKey), - case hb_util:to_lower(Key) of - <<"body">> -> body_to_http(Http, Value); - <<"signature">> -> signatures_to_http(Http, [Value]); - <<"signatures">> -> signatures_to_http(Http, Value); - _ -> field_to_http(Http, {Key, Value}, #{}) - end - end, - #{ headers => [], body => #{} }, - TABM - ), - Body = maps:get(body, Http), - NewHttp = case maps:size(Body) of - 0 -> maps:put(body, <<>>, Http); - _ -> - {ok, RawBoundary} = dev_message:id(TABM), - Boundary = hb_util:encode(RawBoundary), - % Transform body into a binary, delimiting each part with the Boundary - BodyList = maps:fold( - fun (_, BodyPart, Acc) -> - [<<"--", Boundary/binary, ?CRLF/binary, BodyPart/binary>> | Acc] - end, - [], - Body - ), - BodyBin = iolist_to_binary(lists:join(?CRLF, lists:reverse(BodyList))), - #{ - headers => [ - { - <<"Content-Type">>, - <<"multipart/form-data; boundary=", "\"" , Boundary/binary, "\"">> - } - | maps:get(headers, Http) - ], - % End the body with a final terminating Boundary - body => <> - } - end, - NewHttp. - -encode_http_msg (_Http = #{ headers := SubHeaders, body := SubBody }) -> - % Serialize the headers, to be included in the part of the multipart response - HeaderList = lists:foldl( - fun ({HeaderName, HeaderValue}, Acc) -> - [<> | Acc] - end, - [], - SubHeaders - ), - EncodedHeaders = iolist_to_binary(lists:join(?CRLF, lists:reverse(HeaderList))), - case SubBody of - <<>> -> EncodedHeaders; - % Some-Headers: some-value - % Content-Type: image/png - % - % - _ -> <> - end. - -signatures_to_http(Http, Signatures) when is_map(Signatures) -> - signatures_to_http(Http, maps:to_list(Signatures)); -signatures_to_http(Http, Signatures) when is_list(Signatures) -> - {SfSigInputs, SfSigs} = lists:foldl( - fun ({SigName, SignatureMap = #{ <<"inputs">> := Inputs, <<"signature">> := Signature }}, {SfSigInputs, SfSigs}) -> - NextSigInput = hb_http_signature:sf_signature_params(Inputs, SignatureMap), - NextSig = hb_http_signature:sf_signature(Signature), - NextName = hb_converge:key_to_binary(SigName), - { - [{NextName, NextSigInput} | SfSigInputs], - [{NextName, NextSig} | SfSigs] - } - end, - % Start with empty Structured Field Dictionaries - {[], []}, - Signatures - ), - % Signature and Signature-Input are always encoded as Structured Field dictionaries, and then - % each transmitted either as a header, or as a part in the multi-part body - WithSig = field_to_http(Http, {<<"Signature">>, hb_http_structured_fields:dictionary(SfSigs)}, #{}), - WithSigAndInput = field_to_http(WithSig, {<<"Signature-Input">>, hb_http_structured_fields:dictionary(SfSigInputs)}, #{}), - WithSigAndInput. - -body_to_http(Http, Body) when is_map(Body) -> - Disposition = <<"Content-Disposition: inline">>, - SubHttp = to(Body), - EncodedBody = encode_http_msg(SubHttp), - field_to_http(Http, {<<"body">>, EncodedBody}, #{ disposition => Disposition, where => body }); -body_to_http(Http, Body) when is_binary(Body) -> - Disposition = <<"Content-Disposition: inline">>, - field_to_http(Http, {<<"body">>, Body}, #{ disposition => Disposition, where => body }). - -field_to_http(Http, {Name, Value}, Opts) when is_map(Value) -> - SubHttp = to(Value), - EncodedHttpMap = encode_http_msg(SubHttp), - field_to_http(Http, {Name, EncodedHttpMap}, maps:put(where, body, Opts)); -field_to_http(Http, {Name, Value}, Opts) when is_binary(Value) -> - NormalizedName = hb_converge:key_to_binary(Name), - % The default location where the value is encoded within the HTTP - % message depends on its size. - % - % So we check whether the size of the value is within the threshold - % to encode as a header, and other default to encoding in the body - DefaultWhere = case byte_size(Value) of - Fits when Fits =< ?MAX_HEADER_LENGTH -> headers; - _ -> maps:get(where, Opts, headers) - end, - case maps:get(where, Opts, DefaultWhere) of - headers -> - Headers = maps:get(headers, Http), - NewHeaders = lists:append(Headers, [{NormalizedName, Value}]), - maps:put(headers, NewHeaders, Http); - % Append the value as a part of the multipart body - % - % We'll need to prepend a Content-Disposition header to the part, using - % the field name as the form part name. (see https://www.rfc-editor.org/rfc/rfc7578#section-4.2). - % We allow the caller to provide a Content-Disposition in Opts, but default - % to appending as a field on the form-data - body -> - Body = maps:get(body, Http), - Disposition = maps:get(disposition, Opts, <<"Content-Disposition: form-data;name=", NormalizedName/binary>>), - BodyPart = <>, - NewBody = maps:put(NormalizedName, BodyPart, Body), - maps:put(body, NewBody, Http) - end. diff --git a/src/hb_codec_tx.erl b/src/hb_codec_tx.erl deleted file mode 100644 index 153c9991b..000000000 --- a/src/hb_codec_tx.erl +++ /dev/null @@ -1,190 +0,0 @@ -%%% @doc Codec for managing transformations from `ar_bundles`-style Arweave TX -%%% records to and from TABMs. --module(hb_codec_tx). --export([to/1, from/1]). --include("include/hb.hrl"). - -%% The size at which a value should be made into a body item, instead of a -%% tag. --define(MAX_TAG_VAL, 128). -%% The list of TX fields that users can set directly. --define(TX_KEYS, - [id, unsigned_id, last_tx, owner, target, signature]). --define(FILTERED_TAGS, - [ - <<"Bundle-Format">>, - <<"Bundle-Map">>, - <<"Bundle-Version">> - ] -). - -%% @doc Convert a #tx record into a message map recursively. -from(Binary) when is_binary(Binary) -> Binary; -from(TX) when is_record(TX, tx) -> - case lists:keyfind(<<"Converge-Type">>, 1, TX#tx.tags) of - false -> - do_from(TX); - {<<"Converge-Type">>, <<"Binary">>} -> - TX#tx.data - end. -do_from(RawTX) -> - % Ensure the TX is fully deserialized. - TX = ar_bundles:deserialize(ar_bundles:normalize(RawTX)), % <- Is norm necessary? - % Get the raw fields and values of the tx record and pair them. Then convert - % the list of key-value pairs into a map, removing irrelevant fields. - TXKeysMap = - maps:with(?TX_KEYS, - maps:from_list( - lists:zip( - record_info(fields, tx), - tl(tuple_to_list(TX)) - ) - ) - ), - % Generate a TABM from the tags. - MapWithoutData = maps:merge(TXKeysMap, maps:from_list(TX#tx.tags)), - DataMap = - case TX#tx.data of - Data when is_map(Data) -> - % If the data is a map, we need to recursively turn its children - % into messages from their tx representations. - maps:merge( - MapWithoutData, - maps:map( - fun(_, InnerValue) -> from(InnerValue) end, - Data - ) - ); - Data when Data == ?DEFAULT_DATA -> - MapWithoutData; - Data when is_binary(Data) -> - MapWithoutData#{ data => Data }; - Data -> - ?event({unexpected_data_type, {explicit, Data}}), - ?event({was_processing, {explicit, TX}}), - throw(invalid_tx) - end, - % Merge the data map with the rest of the TX map and remove any keys that - % are not part of the message. - maps:without(?FILTERED_TAGS, maps:merge(DataMap, MapWithoutData)). - -%% @doc Internal helper to translate a message to its #tx record representation, -%% which can then be used by ar_bundles to serialize the message. We call the -%% message's device in order to get the keys that we will be checkpointing. We -%% do this recursively to handle nested messages. The base case is that we hit -%% a binary, which we return as is. -to(Binary) when is_binary(Binary) -> - % ar_bundles cannot serialize just a simple binary or get an ID for it, so - % we turn it into a TX record with a special tag, tx_to_message will - % identify this tag and extract just the binary. - #tx{ - tags= [{<<"Converge-Type">>, <<"Binary">>}], - data = Binary - }; -to(TX) when is_record(TX, tx) -> TX; -to(TABM) when is_map(TABM) -> - % The path is a special case so we normalized it first. It may have been - % modified by `hb_converge` in order to set it to the current key that is - % being executed. We should check whether the path is in the - % `priv/Converge/Original-Path` field, and if so, use that instead of the - % stated path. This normalizes the path, such that the signed message will - % continue to validate correctly. - M = - case {maps:find(path, TABM), hb_private:from_message(TABM)} of - {{ok, _}, #{ <<"Converge">> := #{ <<"Original-Path">> := Path } }} -> - maps:put(path, Path, TABM); - _ -> TABM - end, - % Translate the keys into a binary map. If a key has a value that is a map, - % we recursively turn its children into messages. Notably, we do not simply - % call message_to_tx/1 on the inner map because that would lead to adding - % an extra layer of nesting to the data. - %?event({message_to_tx, {keys, Keys}, {map, M}}), - MsgKeyMap = - maps:map( - fun(_Key, Msg) when is_map(Msg) -> to(Msg); - (_Key, Value) -> Value - end, - M - ), - NormalizedMsgKeyMap = hb_message:normalize_keys(MsgKeyMap), - % Iterate through the default fields, replacing them with the values from - % the message map if they are present. - {RemainingMap, BaseTXList} = - lists:foldl( - fun({Field, Default}, {RemMap, Acc}) -> - NormKey = hb_converge:key_to_binary(Field), - case maps:find(NormKey, NormalizedMsgKeyMap) of - error -> {RemMap, [Default | Acc]}; - {ok, Value} when is_binary(Default) andalso ?IS_ID(Value) -> - { - maps:remove(NormKey, RemMap), - [hb_util:native_id(Value)|Acc] - }; - {ok, Value} -> - { - maps:remove(NormKey, RemMap), - [Value|Acc] - } - end - end, - {NormalizedMsgKeyMap, []}, - hb_message:default_tx_list() - ), - % Rebuild the tx record from the new list of fields and values. - TXWithoutTags = list_to_tuple([tx | lists:reverse(BaseTXList)]), - % Calculate which set of the remaining keys will be used as tags. - {Tags, RawDataItems} = - lists:partition( - fun({_Key, Value}) when is_binary(Value) -> - case unicode:characters_to_binary(Value) of - {error, _, _} -> false; - _ -> byte_size(Value) =< ?MAX_TAG_VAL - end; - (_) -> false - end, - [ - {Key, maps:get(Key, RemainingMap)} - || - Key <- maps:keys(RemainingMap) - ] - ), - % We don't let the user set the tags directly, but they can instead set any - % number of keys to short binary values, which will be included as tags. - TX = TXWithoutTags#tx { tags = Tags }, - % Recursively turn the remaining data items into tx records. - DataItems = maps:from_list(lists:map( - fun({Key, Value}) -> - {Key, to(Value)} - end, - RawDataItems - )), - % Set the data based on the remaining keys. - TXWithData = - case {TX#tx.data, maps:size(DataItems)} of - {Binary, 0} when is_binary(Binary) -> - TX; - {?DEFAULT_DATA, _} -> - TX#tx { data = DataItems }; - {Data, _} when is_map(Data) -> - TX#tx { data = maps:merge(Data, DataItems) }; - {Data, _} when is_record(Data, tx) -> - TX#tx { data = DataItems#{ data => Data } }; - {Data, _} when is_binary(Data) -> - TX#tx { data = DataItems#{ data => to(Data) } } - end, - % ar_bundles:reset_ids(ar_bundles:normalize(TXWithData)); - Res = try ar_bundles:reset_ids(ar_bundles:normalize(TXWithData)) - catch - _:Error -> - ?event({{reset_ids_error, Error}, {tx_without_data, TX}}), - ?event({prepared_tx_before_ids, - {tags, {explicit, TXWithData#tx.tags}}, - {data, TXWithData#tx.data} - }), - throw(Error) - end, - %?event({result, {explicit, Res}}), - Res; -to(Other) -> - throw(invalid_tx). \ No newline at end of file diff --git a/src/hb_converge.erl b/src/hb_converge.erl deleted file mode 100644 index 4a3871424..000000000 --- a/src/hb_converge.erl +++ /dev/null @@ -1,1058 +0,0 @@ -%%% @doc This module is the root of the device call logic of the -%%% Converge Protocol in HyperBEAM. -%%% -%%% At the implementation level, every message is simply a collection of keys, -%%% dictated by its `Device', that can be resolved in order to yield their -%%% values. Each key may return another message or a raw value: -%%% -%%% `converge(Message1, Message2) -> {Status, Message3}' -%%% -%%% Under-the-hood, `Converge(Message1, Message2)' leads to the evaluation of -%%% `DeviceMod:PathPart(Message1, Message2)', which defines the user compute -%%% to be performed. If `Message1' does not specify a device, `dev_message' is -%%% assumed. The key to resolve is specified by the `Path' field of the message. -%%% -%%% After each output, the `HashPath' is updated to include the `Message2' -%%% that was executed upon it. -%%% -%%% Because each message implies a device that can resolve its keys, as well -%%% as generating a merkle tree of the computation that led to the result, -%%% you can see Converge Protocol as a system for cryptographically chaining -%%% the execution of `combinators'. See `docs/converge-protocol.md' for more -%%% information about Converge. -%%% -%%% The `Fun(Message1, Message2)' pattern is repeated throughout the HyperBEAM -%%% codebase, sometimes with `MessageX' replaced with `MX' or `MsgX' for brevity. -%%% -%%% Message3 can be either a new message or a raw output value (a binary, integer, -%%% float, atom, or list of such values). -%%% -%%% Devices can be expressed as either modules or maps. They can also be -%%% referenced by an Arweave ID, which can be used to load a device from -%%% the network (depending on the value of the `load_remote_devices' and -%%% `trusted_device_signers' environment settings). -%%% -%%% HyperBEAM device implementations are defined as follows: -%%% ``` -%%% DevMod:ExportedFunc : Key resolution functions. All are assumed to be -%%% device keys (thus, present in every message that -%%% uses it) unless specified by `DevMod:info()`. -%%% Each function takes a set of parameters -%%% of the form `DevMod:KeyHandler(Msg1, Msg2, Opts)`. -%%% Each of these arguments can be ommitted if not -%%% needed. Non-exported functions are not assumed -%%% to be device keys. -%%% -%%% DevMod:info : Optional. Returns a map of options for the device. All -%%% options are optional and assumed to be the defaults if -%%% not specified. This function can accept a `Message1` as -%%% an argument, allowing it to specify its functionality -%%% based on a specific message if appropriate. -%%% -%%% info/exports : Overrides the export list of the Erlang module, such that -%%% only the functions in this list are assumed to be device -%%% keys. Defaults to all of the functions that DevMod -%%% exports in the Erlang environment. -%%% -%%% info/excludes : A list of keys that should not be resolved by the device, -%%% despite being present in the Erlang module exports list. -%%% -%%% info/handler : A function that should be used to handle _all_ keys for -%%% messages using the device. -%%% -%%% info/default : A function that should be used to handle all keys that -%%% are not explicitly implemented by the device. Defaults to -%%% the `dev_message` device, which contains general keys for -%%% interacting with messages. -%%% -%%% info/default_mod : A different device module that should be used to -%%% handle all keys that are not explicitly implemented -%%% by the device. Defaults to the `dev_message` device. -%%% -%%% info/grouper : A function that returns the concurrency 'group' name for -%%% an execution. Executions with the same group name will -%%% be executed by sending a message to the associated process -%%% and waiting for a response. This allows you to control -%%% concurrency of execution and to allow executions to share -%%% in-memory state as applicable. Default: A derivation of -%%% Msg1+Msg2. This means that concurrent calls for the same -%%% output will lead to only a single execution. -%%% -%%% info/worker : A function that should be run as the 'server' loop of -%%% the executor for interactions using the device. -%%% -%%% The HyperBEAM resolver also takes a number of runtime options that change -%%% the way that the environment operates: -%%% -%%% `update_hashpath': Whether to add the `Msg2' to `HashPath' for the `Msg3'. -%%% Default: true. -%%% `cache_results': Whether to cache the resolved `Msg3'. -%%% Default: true. -%%% `add_key': Whether to add the key to the start of the arguments. -%%% Default: `'. --module(hb_converge). -%%% Main Converge API: --export([resolve/2, resolve/3, load_device/2, message_to_device/2]). --export([to_key/1, to_key/2, key_to_binary/1, key_to_binary/2, ensure_message/1]). -%%% Shortcuts and tools: --export([info/2, keys/1, keys/2, keys/3, truncate_args/2]). --export([get/2, get/3, get/4, set/2, set/3, set/4, remove/2, remove/3]). -%%% Exports for tests in hb_converge_tests.erl: --export([deep_set/4, is_exported/4, message_to_fun/3]). --include("include/hb.hrl"). - -%% @doc Takes a singleton message and parse Msg1 and Msg2 from it, then invoke -%% `resolve'. -resolve(Msg, Opts) -> - Path = - hb_path:term_to_path_parts( - hb_converge:get( - path, - Msg, - #{ hashpath => ignore } - ), - Opts - ), - case Path of - [ Msg1ID | _Rest ] when ?IS_ID(Msg1ID) -> - ?event({normalizing_single_message_message_path, Msg}), - {ok, Msg1} = hb_cache:read(<<"Messages/", Msg1ID/binary>>, Opts), - resolve( - Msg1, - hb_path:tl(Msg, Opts), - Opts - ); - SingletonPath -> - resolve(Msg, #{ path => SingletonPath }, Opts) - end. - -%% @doc Get the value of a message's key by running its associated device -%% function. Optionally, takes options that control the runtime environment. -%% This function returns the raw result of the device function call: -%% `{ok | error, NewMessage}.' -%% The resolver is composed of a series of discrete phases: -%% 1: Normalization. -%% 2: Cache lookup. -%% 3: Validation check. -%% 4: Persistent-resolver lookup. -%% 5: Device lookup. -%% 6: Execution. -%% 7: Cryptographic linking. -%% 8: Result caching. -%% 9: Notify waiters. -%% 10: Fork worker. -%% 11: Recurse, fork, or terminate. -resolve(Msg1, Msg2, Opts) -> resolve_stage(1, Msg1, Msg2, Opts). -resolve_stage(1, Msg1, Msg2, Opts) when is_list(Msg1) -> - % Normalize lists to numbered maps (base=1) if necessary. - ?event(converge_core, {stage, 1, list_normalize}, Opts), - resolve_stage(1, - ensure_message(Msg1), - Msg2, - Opts - ); -resolve_stage(1, Msg1, Path, Opts) when not is_map(Path) -> - ?event(converge_core, {stage, 1, normalize_raw_path_to_message}, Opts), - % If we have been given a Path rather than a full Msg2, construct the - % message around it and recurse. - resolve_stage(1, Msg1, #{ path => Path }, Opts); -resolve_stage(1, Msg1, Msg2, Opts) -> - ?event(converge_core, {stage, 1, normalize_path}, Opts), - % Path normalization: Ensure that the path is requesting a single key. - % Stash remaining path elements in `priv/Converge/Remaining-Path'. - % Stash the original path in `priv/Converge/Original-Path', if it - % is not already there from a previous resolution. - InitialPriv = hb_private:from_message(Msg1), - OriginalPath = - case InitialPriv of - #{ <<"Converge">> := #{ <<"Original-Path">> := XPath } } -> - XPath; - _ -> hb_path:from_message(request, Msg2) - end, - Head = hb_path:hd(Msg2, Opts), - FullPath = hb_path:from_message(request, Msg2), - case FullPath of - undefined -> - throw({error, {invalid_path, FullPath, Msg2}}); - _ -> ok - end, - Msg2UpdatedPriv = - hb_path:priv_store_original( - Msg2, - OriginalPath, - hb_path:tl(FullPath, Opts) - ), - resolve_stage(2, Msg1, Msg2UpdatedPriv#{ path => Head }, Opts); -resolve_stage(2, Msg1, Msg2, Opts) -> - ?event(converge_core, {stage, 2, cache_lookup}, Opts), - % Lookup request in the cache. If we find a result, return it. - % If we do not find a result, we continue to the next stage, - % unless the cache lookup returns `halt` (the user has requested that we - % only return a result if it is already in the cache). - case hb_cache_control:maybe_lookup(Msg1, Msg2, Opts) of - {ok, Msg3} -> - resolve_stage(11, Msg1, Msg2, {ok, Msg3}, no_exec_cache_hit, Opts); - {continue, NewMsg1, NewMsg2} -> - resolve_stage(3, NewMsg1, NewMsg2, Opts); - {error, CacheResp} -> {error, CacheResp} - end; -resolve_stage(3, Msg1, Msg2, Opts) when not is_map(Msg1) or not is_map(Msg2) -> - % Validation check: If the messages are not maps, we cannot find a key - % in them, so return not_found. - ?event(converge_core, {stage, 3, validation_check_type_error}, Opts), - {error, not_found}; -resolve_stage(3, Msg1, Msg2, Opts) -> - ?event(converge_core, {stage, 3, validation_check}, Opts), - % Validation check: Check if the message is valid. - %Msg1Valid = (hb_message:signers(Msg1) == []) orelse hb_message:verify(Msg1), - %Msg2Valid = (hb_message:signers(Msg2) == []) orelse hb_message:verify(Msg2), - ?no_prod("Enable message validity checks!"), - case {true, true} of - _ -> resolve_stage(4, Msg1, Msg2, Opts); - _ -> error_invalid_message(Msg1, Msg2, Opts) - end; -resolve_stage(4, Msg1, Msg2, Opts) -> - ?event(converge_core, {stage, 4, persistent_resolver_lookup}, Opts), - % Persistent-resolver lookup: Search for local (or Distributed - % Erlang cluster) processes that are already performing the execution. - % Before we search for a live executor, we check if the device specifies - % a function that tailors the 'group' name of the execution. For example, - % the `dev_process` device 'groups' all calls to the same process onto - % calls to a single executor. By default, `{Msg1, Msg2}` is used as the - % group name. - case hb_persistent:find_or_register(Msg1, Msg2, Opts) of - {leader, ExecName} -> - % We are the leader for this resolution. Continue to the next stage. - case hb_opts:get(spawn_worker, false, Opts) of - true -> ?event(worker_spawns, {will_become, ExecName}); - _ -> ok - end, - resolve_stage(5, Msg1, Msg2, ExecName, Opts); - {wait, Leader} -> - % There is another executor of this resolution in-flight. - % Bail execution, register to receive the response, then - % wait. - case hb_persistent:await(Leader, Msg1, Msg2, Opts) of - {error, leader_died} -> - ?event( - converge_core, - {leader_died_during_wait, - {leader, Leader}, - {msg1, Msg1}, - {msg2, Msg2}, - {opts, Opts} - }, - Opts - ), - % Re-try again if the group leader already finished it's work - resolve_stage(4, Msg1, Msg2, Opts); - Res -> - % Now that we have the result, we can skip right to potential - % recursion (step 11). - resolve_stage(11, Msg1, Msg2, Res, Leader, - Opts#{ spawn_worker => false }) - end; - {infinite_recursion, GroupName} -> - % We are the leader for this resolution, but we executing the - % computation again. This may plausibly be OK in _some_ cases, - % but in general it is the sign of a bug. - ?event( - converge_core, - {infinite_recursion, - {exec_group, GroupName}, - {msg1, Msg1}, - {msg2, Msg2}, - {opts, Opts} - }, - Opts - ), - case hb_opts:get(allow_infinite, false, Opts) of - true -> - % We are OK with infinite loops, so we just continue. - resolve_stage(5, Msg1, Msg2, GroupName, Opts); - false -> - % We are not OK with infinite loops, so we raise an error. - error_infinite(Msg1, Msg2, Opts) - end - end. -resolve_stage(5, Msg1, Msg2, ExecName, Opts) -> - ?event(converge_core, {stage, 5, device_lookup}), - % Device lookup: Find the Erlang function that should be utilized to - % execute Msg2 on Msg1. - {ResolvedFunc, NewOpts} = - try - Key = hb_path:hd(Msg2, Opts), - % Try to load the device and get the function to call. - ?event( - { - resolving_key, - {key, Key}, - {msg1, Msg1}, - {msg2, Msg2}, - {opts, Opts} - } - ), - {Status, _Mod, Func} = message_to_fun(Msg1, Key, Opts), - ?event( - {found_func_for_exec, - {key, Key}, - {func, Func}, - {msg1, Msg1}, - {msg2, Msg2}, - {opts, Opts} - } - ), - % Next, add an option to the Opts map to indicate if we should - % add the key to the start of the arguments. - { - Func, - Opts#{ - add_key => - case Status of - add_key -> Key; - _ -> false - end - } - } - catch - Class:Exception:Stacktrace -> - ?event( - converge_result, - { - load_device_failed, - {msg1, Msg1}, - {msg2, Msg2}, - {exec_name, ExecName}, - {exec_class, Class}, - {exec_exception, Exception}, - {exec_stacktrace, Stacktrace}, - {opts, Opts} - } - ), - % If the device cannot be loaded, we alert the caller. - error_execution( - ExecName, - Msg2, - loading_device, - {Class, Exception, Stacktrace}, - Opts - ) - end, - resolve_stage(6, ResolvedFunc, Msg1, Msg2, ExecName, NewOpts). -resolve_stage(6, Func, Msg1, Msg2, ExecName, Opts) -> - ?event(converge_core, {stage, 6, ExecName, execution}, Opts), - % Execution. - % First, determine the arguments to pass to the function. - % While calculating the arguments we unset the add_key option. - UserOpts1 = maps:remove(add_key, Opts), - % Unless the user has explicitly requested recursive spawning, we - % unset the spawn_worker option so that we do not spawn a new worker - % for every resulting execution. - UserOpts2 = - case maps:get(spawn_worker, UserOpts1, false) of - recursive -> UserOpts1; - _ -> maps:remove(spawn_worker, UserOpts1) - end, - Args = - case maps:get(add_key, Opts, false) of - false -> [Msg1, Msg2, UserOpts2]; - Key -> [Key, Msg1, Msg2, UserOpts2] - end, - % Try to execute the function. - Res = - try - MsgRes = apply(Func, truncate_args(Func, Args)), - ?event( - converge_result, - { - result, - {exec_name, ExecName}, - {msg1, Msg1}, - {msg2, Msg2}, - {msg3, MsgRes}, - {opts, Opts} - }, - Opts - ), - MsgRes - catch - ExecClass:ExecException:ExecStacktrace -> - ?event( - converge_core, - {device_call_failed, ExecName, {func, Func}}, - Opts - ), - ?event( - converge_result, - { - exec_failed, - {msg1, Msg1}, - {msg2, Msg2}, - {exec_name, ExecName}, - {func, Func}, - {exec_class, ExecClass}, - {exec_exception, ExecException}, - {exec_stacktrace, erlang:process_info(self(), backtrace)}, - {opts, Opts} - } - ), - % If the function call fails, we raise an error in the manner - % indicated by caller's `#Opts`. - error_execution( - ExecName, - Msg2, - device_call, - {ExecClass, ExecException, ExecStacktrace}, - Opts - ) - end, - resolve_stage(7, Msg1, Msg2, Res, ExecName, Opts); -resolve_stage(7, Msg1, Msg2, {ok, Msg3}, ExecName, Opts) when is_map(Msg3) -> - ?event(converge_core, {stage, 7, ExecName, generate_hashpath}, Opts), - % Cryptographic linking. Now that we have generated the result, we - % need to cryptographically link the output to its input via a hashpath. - resolve_stage(8, Msg1, Msg2, - case hb_opts:get(hashpath, update, Opts#{ only => local }) of - update -> - ?event({setting_hashpath_msg3, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), - {ok, - maps:without(?REGEN_KEYS, - Msg3#{ hashpath => hb_path:hashpath(Msg1, Msg2, Opts) } - ) - }; - ignore -> - {ok, maps:without([hashpath] ++ ?REGEN_KEYS, Msg3)} - end, - ExecName, - Opts - ); -resolve_stage(7, Msg1, Msg2, {Status, Msg3}, ExecName, Opts) when is_map(Msg3) -> - ?event(converge_core, {stage, 7, ExecName, abnormal_status_reset_hashpath}, Opts), - % Skip cryptographic linking and reset the hashpath if the result is abnormal. - resolve_stage( - 8, Msg1, Msg2, - {Status, maps:without([hashpath] ++ ?REGEN_KEYS, Msg3)}, - ExecName, Opts); -resolve_stage(7, Msg1, Msg2, Res, ExecName, Opts) -> - ?event(converge_core, {stage, 7, ExecName, non_map_result_skipping_hash_path}, Opts), - % Skip cryptographic linking and continue if we don't have a map that can have - % a hashpath at all. - resolve_stage(8, Msg1, Msg2, Res, ExecName, Opts); -resolve_stage(8, Msg1, Msg2, {ok, Msg3}, ExecName, Opts) -> - ?event(converge_core, {stage, 8, ExecName, result_caching}, Opts), - % Result caching: Optionally, cache the result of the computation locally. - hb_cache_control:maybe_store(Msg1, Msg2, Msg3, Opts), - resolve_stage(9, Msg1, Msg2, {ok, Msg3}, ExecName, Opts); -resolve_stage(8, Msg1, Msg2, Res, ExecName, Opts) -> - ?event(converge_core, {stage, 8, ExecName, abnormal_status_skip_caching}, Opts), - % Skip result caching if the result is abnormal. - resolve_stage(9, Msg1, Msg2, Res, ExecName, Opts); -resolve_stage(9, Msg1, Msg2, Res, ExecName, Opts) -> - ?event(converge_core, {stage, 9, ExecName}, Opts), - % Notify processes that requested the resolution while we were executing and - % unregister ourselves from the group. - hb_persistent:unregister_notify(ExecName, Msg2, Res, Opts), - resolve_stage(10, Msg1, Msg2, Res, ExecName, Opts); -resolve_stage(10, Msg1, Msg2, {ok, Msg3} = Res, ExecName, Opts) -> - ?event(converge_core, {stage, 10, ExecName, maybe_spawn_worker}, Opts), - % Check if we should spawn a worker for the current execution - case {is_map(Msg3), hb_opts:get(spawn_worker, false, Opts#{ prefer => local })} of - {A, B} when (A == false) or (B == false) -> - resolve_stage(11, Msg1, Msg2, Res, ExecName, Opts#{ spawn_worker => false }); - {_, _} -> - % Spawn a worker for the current execution - WorkerPID = hb_persistent:start_worker(ExecName, Msg3, Opts), - hb_persistent:forward_work(WorkerPID, Opts), - resolve_stage(11, Msg1, Msg2, Res, ExecName, Opts#{ spawn_worker => false }) - end; -resolve_stage(10, Msg1, Msg2, OtherRes, ExecName, Opts) -> - ?event(converge_core, {stage, 10, ExecName, abnormal_status_skip_spawning}, Opts), - resolve_stage(11, Msg1, Msg2, OtherRes, ExecName, Opts); -resolve_stage(11, Msg1, Msg2, {Status, Msg3}, ExecName, Opts) -> - ?event(converge_core, {stage, 11, ExecName, recursing_or_returning}, Opts), - % Recurse, or terminate. - RemainingPath = hb_path:priv_remaining(Msg2, Opts), - case RemainingPath of - undefined -> - % Terminate: We have reached the end of the path. - {Status, Msg3}; - _ when Status == ok -> - % There are more elements in the path, so we recurse. - ?event( - converge_core, - {resolution_recursing, - {remaining_path, RemainingPath} - } - ), - resolve(Msg3, Msg2#{ path => RemainingPath }, Opts); - _ -> - error_invalid_intermediate_status( - Msg1, Msg2, Msg3, RemainingPath, Opts) - end. - -%% @doc Catch all return if the message is invalid. -error_invalid_message(Msg1, Msg2, Opts) -> - ?event( - converge_core, - {error, {type, invalid_message}, - {msg1, Msg1}, - {msg2, Msg2}, - {opts, Opts} - }, - Opts - ), - { - error, - #{ - <<"Status">> => <<"Forbidden">>, - <<"body">> => <<"Request contains non-verifiable message.">> - } - }. - -%% @doc Catch all return if we are in an infinite loop. -error_infinite(Msg1, Msg2, Opts) -> - ?event( - converge_core, - {error, {type, infinite_recursion}, - {msg1, Msg1}, - {msg2, Msg2}, - {opts, Opts} - }, - Opts - ), - { - error, - #{ - <<"Status">> => <<"Loop Detected">>, - <<"Status-Code">> => 508, - <<"body">> => <<"Request creates infinite recursion.">> - } - }. - -error_invalid_intermediate_status(_Msg1, Msg2, Msg3, RemainingPath, Opts) -> - ?event( - converge_core, - {error, {type, invalid_intermediate_status}, - {msg2, Msg2}, - {msg3, Msg3}, - {remaining_path, RemainingPath}, - {opts, Opts} - }, - Opts - ), - { - error, - #{ - <<"Status">> => <<"Unprocessable Content">>, - <<"Status-Code">> => 422, - <<"body">> => Msg3, - <<"Key">> => maps:get(path, Msg2, <<"Key unknown.">>), - <<"Remaining-Path">> => RemainingPath - } - }. - -%% @doc Handle an error in a device call. -error_execution(ExecGroup, Msg2, Whence, {Class, Exception, Stacktrace}, Opts) -> - Error = {error, Whence, {Class, Exception, Stacktrace}}, - hb_persistent:unregister_notify(ExecGroup, Msg2, Error, Opts), - ?event(converge_core, {handle_error, Error, {opts, Opts}}), - case hb_opts:get(error_strategy, throw, Opts) of - throw -> erlang:raise(Class, Exception, Stacktrace); - _ -> Error - end. - -%% @doc Shortcut for resolving a key in a message without its status if it is -%% `ok'. This makes it easier to write complex logic on top of messages while -%% maintaining a functional style. -%% -%% Additionally, this function supports the `{as, Device, Msg}' syntax, which -%% allows the key to be resolved using another device to resolve the key, -%% while maintaining the tracability of the `HashPath` of the output message. -%% -%% Returns the value of the key if it is found, otherwise returns the default -%% provided by the user, or `not_found' if no default is provided. -get(Path, Msg) -> - get(Path, Msg, #{}). -get(Path, Msg, Opts) -> - get(Path, Msg, not_found, Opts). -get(Path, {as, Device, Msg}, Default, Opts) -> - get( - Path, - set( - Msg, - #{ device => Device }, - internal_opts(Opts) - ), - Default, - Opts - ); -get(Path, Msg, Default, Opts) -> - case resolve(Msg, #{ path => Path }, Opts#{ spawn_worker => false }) of - {ok, Value} -> Value; - {error, _} -> Default - end. - -%% @doc Shortcut to get the list of keys from a message. -keys(Msg) -> keys(Msg, #{}). -keys(Msg, Opts) -> keys(Msg, Opts, keep). -keys(Msg, Opts, keep) -> - get(keys, Msg, Opts); -keys(Msg, Opts, remove) -> - lists:filter( - fun(Key) -> not lists:member(Key, ?CONVERGE_KEYS) end, - keys(Msg, Opts, keep) - ). - -%% @doc Shortcut for setting a key in the message using its underlying device. -%% Like the `get/3' function, this function honors the `error_strategy' option. -%% `set' works with maps and recursive paths while maintaining the appropriate -%% `HashPath' for each step. -set(Msg1, Msg2) -> - set(Msg1, Msg2, #{}). -set(Msg1, RawMsg2, Opts) when is_map(RawMsg2) -> - Msg2 = maps:without([hashpath, priv], RawMsg2), - ?event(converge_internal, {set_called, {msg1, Msg1}, {msg2, Msg2}}, Opts), - % Get the next key to set. - case keys(Msg2, internal_opts(Opts)) of - [] -> Msg1; - [Key|_] -> - % Get the value to set. Use Converge by default, but fall back to - % getting via `maps` if it is not found. - Val = - case get(Key, Msg2, internal_opts(Opts)) of - not_found -> maps:get(Key, Msg2); - Body -> Body - end, - ?event({got_val_to_set, {key, Key}, {val, Val}, {msg2, Msg2}}), - % Next, set the key and recurse, removing the key from the Msg2. - set( - set(Msg1, Key, Val, internal_opts(Opts)), - remove(Msg2, Key, internal_opts(Opts)), - Opts - ) - end. -set(Msg1, Key, Value, Opts) -> - % For an individual key, we run deep_set with the key as the path. - % This handles both the case that the key is a path as well as the case - % that it is a single key. - Path = hb_path:term_to_path_parts(Key), - % ?event( - % {setting_individual_key, - % {msg1, Msg1}, - % {key, Key}, - % {path, Path}, - % {value, Value} - % } - % ), - deep_set(Msg1, Path, Value, Opts). - -%% @doc Recursively search a map, resolving keys, and set the value of the key -%% at the given path. -deep_set(Msg, [Key], Value, Opts) -> - device_set(Msg, Key, Value, Opts); -deep_set(Msg, [Key|Rest], Value, Opts) -> - case resolve(Msg, Key, Opts) of - {ok, SubMsg} -> - ?event( - {traversing_deeper_to_set, - {current_key, Key}, - {current_value, SubMsg}, - {rest, Rest} - } - ), - device_set(Msg, Key, deep_set(SubMsg, Rest, Value, Opts), Opts); - _ -> - ?event( - {creating_new_map, - {current_key, Key}, - {rest, Rest} - } - ), - Msg#{ Key => deep_set(#{}, Rest, Value, Opts) } - end. - -device_set(Msg, Key, Value, Opts) -> - ?event( - converge_internal, - { - calling_device_set, - {msg, Msg}, - {applying_path, #{ path => set, Key => Value }} - } - ), - Res = hb_util:ok( - resolve( - Msg, - #{ path => set, Key => Value }, - internal_opts(Opts) - ), - internal_opts(Opts) - ), - ?event( - converge_internal, - {device_set_result, Res} - ), - Res. - -%% @doc Remove a key from a message, using its underlying device. -remove(Msg, Key) -> remove(Msg, Key, #{}). -remove(Msg, Key, Opts) -> - hb_util:ok( - resolve( - Msg, - #{ path => remove, item => Key }, - internal_opts(Opts) - ), - Opts - ). - -%% @doc Truncate the arguments of a function to the number of arguments it -%% actually takes. -truncate_args(Fun, Args) -> - {arity, Arity} = erlang:fun_info(Fun, arity), - lists:sublist(Args, Arity). - -%% @doc Calculate the Erlang function that should be called to get a value for -%% a given key from a device. -%% -%% This comes in 7 forms: -%% 1. The message does not specify a device, so we use the default device. -%% 2. The device has a `handler' key in its `Dev:info()' map, which is a -%% function that takes a key and returns a function to handle that key. We pass -%% the key as an additional argument to this function. -%% 3. The device has a function of the name `Key', which should be called -%% directly. -%% 4. The device does not implement the key, but does have a default handler -%% for us to call. We pass it the key as an additional argument. -%% 5. The device does not implement the key, and has no default handler. We use -%% the default device to handle the key. -%% Error: If the device is specified, but not loadable, we raise an error. -%% -%% Returns {ok | add_key, Fun} where Fun is the function to call, and add_key -%% indicates that the key should be added to the start of the call's arguments. -message_to_fun(Msg, Key, Opts) -> - % Get the device module from the message. - Dev = message_to_device(Msg, Opts), - Info = info(Dev, Msg, Opts), - % Is the key exported by the device? - Exported = is_exported(Info, Key), - ?event( - converge_devices, - {message_to_fun, - {dev, Dev}, - {key, Key}, - {is_exported, Exported}, - {opts, Opts} - } - ), - % Does the device have an explicit handler function? - case {maps:find(handler, Info), Exported} of - {{ok, Handler}, true} -> - % Case 2: The device has an explicit handler function. - ?event( - converge_devices, - {handler_found, {dev, Dev}, {key, Key}, {handler, Handler}} - ), - {Status, Func} = info_handler_to_fun(Handler, Msg, Key, Opts), - {Status, Dev, Func}; - _ -> - ?event(converge_devices, {handler_not_found, {dev, Dev}, {key, Key}}), - case {find_exported_function(Msg, Dev, Key, 3, Opts), Exported} of - {{ok, Func}, true} -> - % Case 3: The device has a function of the name `Key`. - {ok, Dev, Func}; - _ -> - case {maps:find(default, Info), Exported} of - {{ok, DefaultFunc}, true} when is_function(DefaultFunc) -> - % Case 4: The device has a default handler. - ?event({found_default_handler, {func, DefaultFunc}}), - {add_key, Dev, DefaultFunc}; - {{ok, DefaultMod}, true} when is_atom(DefaultMod) -> - ?event({found_default_handler, {mod, DefaultMod}}), - {Status, Func} = - message_to_fun( - Msg#{ device => DefaultMod }, Key, Opts - ), - {Status, Dev, Func}; - _ -> - % Case 5: The device has no default handler. - % We use the default device to handle the key. - case default_module() of - Dev -> - % We are already using the default device, - % so we cannot resolve the key. This should - % never actually happen in practice, but it - % resolves an infinite loop that can occur - % during development. - throw({ - error, - default_device_could_not_resolve_key, - {key, Key} - }); - DefaultDev -> - ?event( - { - using_default_device, - {dev, DefaultDev} - }), - message_to_fun( - Msg#{ device => DefaultDev }, - Key, - Opts - ) - end - end - end - end. - -%% @doc Extract the device module from a message. -message_to_device(Msg, Opts) -> - case dev_message:get(device, Msg) of - {error, not_found} -> - % The message does not specify a device, so we use the default device. - default_module(); - {ok, DevID} -> - case load_device(DevID, Opts) of - {error, _} -> - % Error case: A device is specified, but it is not loadable. - throw({error, {device_not_loadable, DevID}}); - {ok, DevMod} -> DevMod - end - end. - -%% @doc Parse a handler key given by a device's `info'. -info_handler_to_fun(Handler, _Msg, _Key, _Opts) when is_function(Handler) -> - {add_key, Handler}; -info_handler_to_fun(HandlerMap, Msg, Key, Opts) -> - case maps:find(exclude, HandlerMap) of - {ok, Exclude} -> - case lists:member(Key, Exclude) of - true -> - {ok, MsgWithoutDevice} = - dev_message:remove(Msg, #{ item => device }), - message_to_fun( - MsgWithoutDevice#{ device => default_module() }, - Key, - Opts - ); - false -> {add_key, maps:get(func, HandlerMap)} - end; - error -> {add_key, maps:get(func, HandlerMap)} - end. - -%% @doc Find the function with the highest arity that has the given name, if it -%% exists. -%% -%% If the device is a module, we look for a function with the given name. -%% -%% If the device is a map, we look for a key in the map. First we try to find -%% the key using its literal value. If that fails, we cast the key to an atom -%% and try again. -find_exported_function(Msg, Dev, Key, MaxArity, Opts) when is_map(Dev) -> - case maps:get(Key, Dev, not_found) of - not_found -> - case to_key(Key) of - undefined -> not_found; - Key -> - % The key is unchanged, so we return not_found. - not_found; - KeyAtom -> - % The key was cast to an atom, so we try again. - find_exported_function(Msg, Dev, KeyAtom, MaxArity, Opts) - end; - Fun when is_function(Fun) -> - case erlang:fun_info(Fun, arity) of - {arity, Arity} when Arity =< MaxArity -> - case is_exported(Msg, Dev, Key, Opts) of - true -> {ok, Fun}; - false -> not_found - end; - _ -> not_found - end - end; -find_exported_function(_Msg, _Mod, _Key, Arity, _Opts) when Arity < 0 -> - not_found; -find_exported_function(Msg, Mod, Key, Arity, Opts) when not is_atom(Key) -> - case to_key(Key, Opts) of - ConvertedKey when is_atom(ConvertedKey) -> - find_exported_function(Msg, Mod, ConvertedKey, Arity, Opts); - undefined -> not_found; - BinaryKey when is_binary(BinaryKey) -> - not_found - end; -find_exported_function(Msg, Mod, Key, Arity, Opts) -> - %?event({finding, {mod, Mod}, {key, Key}, {arity, Arity}}), - case erlang:function_exported(Mod, Key, Arity) of - true -> - case is_exported(Msg, Mod, Key, Opts) of - true -> - %?event({found, {ok, fun Mod:Key/Arity}}), - {ok, fun Mod:Key/Arity}; - false -> - %?event({result, not_found}), - not_found - end; - false -> - %?event( - % { - % find_exported_function_result, - % {mod, Mod}, - % {key, Key}, - % {arity, Arity}, - % {result, false} - % } - % ), - find_exported_function(Msg, Mod, Key, Arity - 1, Opts) - end. - -%% @doc Check if a device is guarding a key via its `exports' list. Defaults to -%% true if the device does not specify an `exports' list. The `info' function is -%% always exported, if it exists. Elements of the `exludes` list are not -%% exported. Note that we check for info _twice_ -- once when the device is -%% given but the info result is not, and once when the info result is given. -%% The reason for this is that `info/3` calls other functions that may need to -%% check if a key is exported, so we must avoid infinite loops. We must, however, -%% also return a consistent result in the case that only the info result is -%% given, so we check for it in both cases. -is_exported(_Msg, _Dev, info, _Opts) -> true; -is_exported(Msg, Dev, Key, Opts) -> - is_exported(info(Dev, Msg, Opts), Key). -is_exported(_, info) -> true; -is_exported(Info = #{ excludes := Excludes }, Key) -> - case lists:member(to_key(Key), lists:map(fun to_key/1, Excludes)) of - true -> false; - false -> is_exported(maps:remove(excludes, Info), Key) - end; -is_exported(#{ exports := Exports }, Key) -> - lists:member(to_key(Key), lists:map(fun to_key/1, Exports)); -is_exported(_Info, _Key) -> true. - -%% @doc Convert a key to an atom if it already exists in the Erlang atom table, -%% or to a binary otherwise. -to_key(Key) -> to_key(Key, #{ error_strategy => throw }). -to_key(Key, _Opts) when byte_size(Key) == 43 -> Key; -to_key(Key, Opts) -> - % If the `atom_keys' option is set, we try to convert the key to an atom. - % If this fails, we fall back to using the binary representation. - AtomKeys = hb_opts:get(atom_keys, true, Opts), - if AtomKeys -> - try to_atom_unsafe(Key) - catch _Type:_:_Trace -> key_to_binary(Key, Opts) - end; - true -> key_to_binary(Key, Opts) - end. - -%% @doc Convert a key to its binary representation. -key_to_binary(Key) -> key_to_binary(Key, #{}). -key_to_binary(Key, _Opts) when is_binary(Key) -> Key; -key_to_binary(Key, _Opts) when is_atom(Key) -> atom_to_binary(Key); -key_to_binary(Key, _Opts) when is_integer(Key) -> integer_to_binary(Key); -key_to_binary(Key = [ASCII | _], _Opts) when is_integer(ASCII) -> list_to_binary(Key); -key_to_binary(Key, _Opts) when is_list(Key) -> - iolist_to_binary(lists:join(<<"/">>, lists:map(fun key_to_binary/1, Key))). - -%% @doc Helper function for key_to_atom that does not check for errors. -to_atom_unsafe(Key) when is_integer(Key) -> - integer_to_binary(Key); -to_atom_unsafe(Key) when is_binary(Key) -> - binary_to_existing_atom(hb_util:to_lower(Key), utf8); -to_atom_unsafe(Key) when is_list(Key) -> - FlattenedKey = lists:flatten(Key), - list_to_existing_atom(FlattenedKey); -to_atom_unsafe(Key) when is_atom(Key) -> Key. - -%% @doc Ensure that a message is processable by the Converge resolver: No lists. -ensure_message(Msg1) when is_list(Msg1) -> - maps:from_list( - lists:zip( - [ - key_to_binary(Key) - || - Key <- lists:seq(1, length(Msg1)) - ], - Msg1 - ) - ); -ensure_message(Msg) -> Msg. - -%% @doc Load a device module from its name or a message ID. -%% Returns {ok, Executable} where Executable is the device module. On error, -%% a tuple of the form {error, Reason} is returned. -load_device(Map, _Opts) when is_map(Map) -> {ok, Map}; -load_device(ID, _Opts) when is_atom(ID) -> - try ID:module_info(), {ok, ID} - catch _:_ -> {error, not_loadable} - end; -load_device(ID, Opts) when is_binary(ID) and byte_size(ID) == 43 -> - case hb_opts:get(load_remote_devices) of - true -> - {ok, Msg} = hb_cache:read(maps:get(store, Opts), ID), - Trusted = - lists:any( - fun(Signer) -> - lists:member(Signer, hb_opts:get(trusted_device_signers)) - end, - hb_message:signers(Msg) - ), - case Trusted of - true -> - RelBin = erlang:system_info(otp_release), - case lists:keyfind(<<"Content-Type">>, 1, Msg#tx.tags) of - <<"BEAM/", RelBin/bitstring>> -> - {_, ModNameBin} = - lists:keyfind( - <<"Module-Name">>, - 1, - Msg#tx.tags - ), - ModName = list_to_atom(binary_to_list(ModNameBin)), - case erlang:load_module(ModName, Msg#tx.data) of - {module, _} -> {ok, ModName}; - {error, Reason} -> {error, Reason} - end - end; - false -> {error, device_signer_not_trusted} - end; - false -> - {error, remote_devices_disabled} - end; -load_device(ID, Opts) -> - case maps:get(ID, hb_opts:get(preloaded_devices), unsupported) of - unsupported -> {error, module_not_admissable}; - Mod -> load_device(Mod, Opts) - end. - -%% @doc Get the info map for a device, optionally giving it a message if the -%% device's info function is parameterized by one. -info(Msg, Opts) -> - info(message_to_device(Msg, Opts), Msg, Opts). -info(DevMod, Msg, Opts) -> - %?event({calculating_info, {dev, DevMod}, {msg, Msg}}), - case find_exported_function(Msg, DevMod, info, 1, Opts) of - {ok, Fun} -> - Res = apply(Fun, truncate_args(Fun, [Msg, Opts])), - % ?event({ - % info_result, - % {dev, DevMod}, - % {args, truncate_args(Fun, [Msg])}, - % {result, Res} - % }), - Res; - not_found -> #{} - end. - -%% @doc The default device is the identity device, which simply returns the -%% value associated with any key as it exists in its Erlang map. It should also -%% implement the `set' key, which returns a `Message3' with the values changed -%% according to the `Message2' passed to it. -default_module() -> dev_message. - -%% @doc The execution options that are used internally by the converge module -%% when calling itself. -internal_opts(Opts) -> - maps:merge(Opts, #{ - topic => converge_internal, - hashpath => ignore, - cache_control => [<<"no-cache">>, <<"no-store">>], - spawn_worker => false - }). diff --git a/src/hb_converge_test_vectors.erl b/src/hb_converge_test_vectors.erl deleted file mode 100644 index 6f70eb880..000000000 --- a/src/hb_converge_test_vectors.erl +++ /dev/null @@ -1,588 +0,0 @@ -%%% @doc Tests for the core Converge resolution engine. -%%% Uses a series of different `Opts` values to test the resolution engine's -%%% execution under different circumstances. --module(hb_converge_test_vectors). --include_lib("eunit/include/eunit.hrl"). --include_lib("include/hb.hrl"). --export([run_single/2]). - -%% @doc Easy hook to make a test executable via the command line: -%% `rebar3 eunit --test hb_converge_test_vectors:run_test` -%% Comment/uncomment out as necessary. -% run_test() -> -% run_single(only_store, deep_set_new_messages), -% run_single(only_if_cached, deep_set_new_messages). - -%% @doc Run a single test with a given set of opts. -run_single(OptsName, TestName) -> - {_, _, Test} = lists:keyfind(TestName, 1, test_suite()), - [Opts|_] = [ O || #{ name := OName, opts := O } <- test_opts(), OName == OptsName ], - Test(Opts). - -%% @doc Run each test in the file with each set of options. Start and reset -%% the store for each test. -run_all_test_() -> - lists:map( - fun(#{ name := _Name, opts := Opts, skip := Skip, desc := ODesc}) -> - Store = hb_opts:get(store, Opts), - {foreach, - fun() -> hb_store:start(Store) end, - fun(_) -> hb_store:reset(Store) end, - [ - {ODesc ++ ": " ++ TestDesc, fun() -> Test(Opts) end} - || - {TestAtom, TestDesc, Test} <- test_suite(), - not lists:member(TestAtom, Skip) - ] - } - end, - test_opts() - ). - -test_suite() -> - [ - {resolve_simple, "resolve simple", - fun resolve_simple_test/1}, - {resolve_id, "resolve id", - fun resolve_id_test/1}, - {resolve_key_twice, "resolve key twice", - fun resolve_key_twice_test/1}, - {resolve_from_multiple_keys, "resolve from multiple keys", - fun resolve_from_multiple_keys_test/1}, - {resolve_path_element, "resolve path element", - fun resolve_path_element_test/1}, - {resolve_binary_key, "resolve binary key", - fun resolve_binary_key_test/1}, - {key_to_binary, "key to binary", - fun key_to_binary_test/1}, - {key_from_id_device_with_args, "key from id device with args", - fun key_from_id_device_with_args_test/1}, - {device_with_handler_function, "device with handler function", - fun device_with_handler_function_test/1}, - {device_with_default_handler_function, "device with default handler function", - fun device_with_default_handler_function_test/1}, - {basic_get, "basic get", - fun basic_get_test/1}, - {recursive_get, "recursive get", - fun recursive_get_test/1}, - {basic_set, "basic set", - fun basic_set_test/1}, - {get_with_device, "get with device", - fun get_with_device_test/1}, - {get_as_with_device, "get as with device", - fun get_as_with_device_test/1}, - {set_with_device, "set with device", - fun set_with_device_test/1}, - {deep_set, "deep set", - fun deep_set_test/1}, - {deep_set_with_device, "deep set with device", - fun deep_set_with_device_test/1}, - {device_exports, "device exports", - fun device_exports_test/1}, - {device_excludes, "device excludes", - fun device_excludes_test/1}, - {denormalized_device_key, "denormalized device key", - fun denormalized_device_key_test/1}, - {list_transform, "list transform", - fun list_transform_test/1} - ]. - -test_opts() -> - [ - #{ - name => no_cache, - desc => "No cache read or write", - opts => #{ - hashpath => ignore, - cache_control => [<<"no-cache">>, <<"no-store">>], - spawn_worker => false, - store => {hb_store_fs, #{ prefix => "TEST-cache-fs" }} - }, - skip => [] - }, - #{ - name => only_store, - desc => "Store, don't read", - opts => #{ - hashpath => update, - cache_control => [<<"no-cache">>], - spawn_worker => false, - store => {hb_store_fs, #{ prefix => "TEST-cache-fs" }} - }, - skip => [] - }, - #{ - name => only_if_cached, - desc => "Only read, don't exec", - opts => #{ - hashpath => ignore, - cache_control => [<<"only-if-cached">>], - spawn_worker => false, - store => {hb_store_fs, #{ prefix => "TEST-cache-fs" }} - }, - skip => [ - % Exclude tests that return a list on its own for now, as raw - % lists cannot be cached yet. - set_new_messages, - resolve_from_multiple_keys, - resolve_path_element, - denormalized_device_key, - % Skip test with locally defined device - deep_set_with_device - % Skip tests that call hb_converge utils (which have their own - % cache settings). - ] - }, - #{ - name => normal, - desc => "Default opts", - opts => #{}, - skip => [] - } - ]. - -%%% Test vector suite - -resolve_simple_test(Opts) -> - Res = hb_converge:resolve(#{ a => <<"RESULT">> }, a, Opts), - ?assertEqual({ok, <<"RESULT">>}, Res). - -resolve_id_test(Opts) -> - ?assertMatch( - ID when byte_size(ID) == 43, - hb_converge:get(id, #{ test_key => <<"1">> }, Opts) - ). - -resolve_key_twice_test(Opts) -> - % Ensure that the same message can be resolved again. - % This is not as trivial as it may seem, because resolutions are cached and - % de-duplicated. - ?assertEqual({ok, <<"1">>}, hb_converge:resolve(#{ <<"a">> => <<"1">> }, <<"a">>, Opts)), - ?assertEqual({ok, <<"1">>}, hb_converge:resolve(#{ <<"a">> => <<"1">> }, <<"a">>, Opts)). - -resolve_from_multiple_keys_test(Opts) -> - ?assertEqual( - {ok, [a]}, - hb_converge:resolve(#{ a => <<"1">>, "priv_a" => <<"2">> }, keys, Opts) - ). - -resolve_path_element_test(Opts) -> - ?assertEqual( - {ok, [test_path]}, - hb_converge:resolve(#{ path => [test_path] }, path, Opts) - ), - ?assertEqual( - {ok, [a]}, - hb_converge:resolve(#{ <<"Path">> => [a] }, <<"Path">>, Opts) - ). - -key_to_binary_test(Opts) -> - ?assertEqual(<<"a">>, hb_converge:key_to_binary(a, Opts)), - ?assertEqual(<<"a">>, hb_converge:key_to_binary(<<"a">>, Opts)), - ?assertEqual(<<"a">>, hb_converge:key_to_binary("a", Opts)). - -resolve_binary_key_test(Opts) -> - ?assertEqual( - {ok, <<"RESULT">>}, - hb_converge:resolve(#{ a => <<"RESULT">> }, <<"a">>, Opts) - ), - ?assertEqual( - {ok, <<"1">>}, - hb_converge:resolve( - #{ - <<"Test-Header">> => <<"1">> }, - <<"Test-Header">>, - Opts - ) - ). - -%% @doc Generates a test device with three keys, each of which uses -%% progressively more of the arguments that can be passed to a device key. -generate_device_with_keys_using_args() -> - #{ - key_using_only_state => - fun(State) -> - {ok, - <<(maps:get(state_key, State))/binary>> - } - end, - key_using_state_and_msg => - fun(State, Msg) -> - {ok, - << - (maps:get(state_key, State))/binary, - (maps:get(msg_key, Msg))/binary - >> - } - end, - key_using_all => - fun(State, Msg, Opts) -> - {ok, - << - (maps:get(state_key, State))/binary, - (maps:get(msg_key, Msg))/binary, - (maps:get(opts_key, Opts))/binary - >> - } - end - }. - -%% @doc Create a simple test device that implements the default handler. -gen_default_device() -> - #{ - info => - fun() -> - #{ - default => - fun(_, _State) -> - {ok, <<"DEFAULT">>} - end - } - end, - state_key => - fun(_) -> - {ok, <<"STATE">>} - end - }. - -%% @doc Create a simple test device that implements the handler key. -gen_handler_device() -> - #{ - info => - fun() -> - #{ - handler => - fun(set, M1, M2, Opts) -> - dev_message:set(M1, M2, Opts); - (_, _, _, _) -> - {ok, <<"HANDLER VALUE">>} - end - } - end - }. - -%% @doc Test that arguments are passed to a device key as expected. -%% Particularly, we need to ensure that the key function in the device can -%% specify any arity (1 through 3) and the call is handled correctly. -key_from_id_device_with_args_test(Opts) -> - Msg = - #{ - device => generate_device_with_keys_using_args(), - state_key => <<"1">> - }, - ?assertEqual( - {ok, <<"1">>}, - hb_converge:resolve( - Msg, - #{ - path => key_using_only_state, - msg_key => <<"2">> % Param message, which is ignored - }, - Opts - ) - ), - ?assertEqual( - {ok, <<"13">>}, - hb_converge:resolve( - Msg, - #{ - path => key_using_state_and_msg, - msg_key => <<"3">> % Param message, with value to add - }, - Opts - ) - ), - ?assertEqual( - {ok, <<"1337">>}, - hb_converge:resolve( - Msg, - #{ - path => key_using_all, - msg_key => <<"3">> % Param message - }, - Opts#{ - opts_key => <<"37">>, - cache_control => [<<"no-cache">>, <<"no-store">>] - } - ) - ). - -device_with_handler_function_test(Opts) -> - Msg = - #{ - device => gen_handler_device(), - test_key => <<"BAD">> - }, - ?assertEqual( - {ok, <<"HANDLER VALUE">>}, - hb_converge:resolve(Msg, test_key, Opts) - ). - -device_with_default_handler_function_test(Opts) -> - Msg = - #{ - device => gen_default_device() - }, - ?assertEqual( - {ok, <<"STATE">>}, - hb_converge:resolve(Msg, state_key, Opts) - ), - ?assertEqual( - {ok, <<"DEFAULT">>}, - hb_converge:resolve(Msg, any_random_key, Opts) - ). - -basic_get_test(Opts) -> - Msg = #{ key1 => <<"value1">>, key2 => <<"value2">> }, - ?assertEqual(<<"value1">>, hb_converge:get(key1, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_converge:get(key2, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_converge:get(<<"key2">>, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_converge:get([<<"key2">>], Msg, Opts)). - -recursive_get_test(Opts) -> - Msg = #{ key1 => <<"value1">>, key2 => #{ key3 => <<"value3">> } }, - ?assertEqual( - {ok, <<"value1">>}, - hb_converge:resolve(Msg, #{ path => key1 }, Opts) - ), - ?assertEqual(<<"value1">>, hb_converge:get(key1, Msg, Opts)), - ?assertEqual( - {ok, <<"value3">>}, - hb_converge:resolve(Msg, #{ path => [key2, key3] }, Opts) - ), - ?assertEqual(<<"value3">>, hb_converge:get([key2, key3], Msg, Opts)), - ?assertEqual(<<"value3">>, hb_converge:get(<<"key2/key3">>, Msg, Opts)). - -basic_set_test(Opts) -> - Msg = #{ key1 => <<"value1">>, key2 => <<"value2">> }, - UpdatedMsg = hb_converge:set(Msg, #{ key1 => <<"new_value1">> }, Opts), - ?event({set_key_complete, {key, key1}, {value, <<"new_value1">>}}), - ?assertEqual(<<"new_value1">>, hb_converge:get(key1, UpdatedMsg, Opts)), - ?assertEqual(<<"value2">>, hb_converge:get(key2, UpdatedMsg, Opts)). - -get_with_device_test(Opts) -> - Msg = - #{ - device => generate_device_with_keys_using_args(), - state_key => <<"STATE">> - }, - ?assertEqual(<<"STATE">>, hb_converge:get(state_key, Msg, Opts)), - ?assertEqual(<<"STATE">>, hb_converge:get(key_using_only_state, Msg, Opts)). - -get_as_with_device_test(Opts) -> - Msg = - #{ - device => gen_handler_device(), - test_key => <<"ACTUAL VALUE">> - }, - ?assertEqual( - <<"HANDLER VALUE">>, - hb_converge:get(test_key, Msg, Opts) - ), - ?assertEqual( - <<"ACTUAL VALUE">>, - hb_converge:get(test_key, {as, dev_message, Msg}, Opts) - ). - -set_with_device_test(Opts) -> - Msg = - #{ - device => - #{ - set => - fun(State, _Msg) -> - Acc = maps:get(set_count, State, <<"">>), - {ok, - State#{ - set_count => << Acc/binary, "." >> - } - } - end - }, - state_key => <<"STATE">> - }, - ?assertEqual(<<"STATE">>, hb_converge:get(state_key, Msg, Opts)), - SetOnce = hb_converge:set(Msg, #{ state_key => <<"SET_ONCE">> }, Opts), - ?assertEqual(<<".">>, hb_converge:get(set_count, SetOnce, Opts)), - SetTwice = hb_converge:set(SetOnce, #{ state_key => <<"SET_TWICE">> }, Opts), - ?assertEqual(<<"..">>, hb_converge:get(set_count, SetTwice, Opts)), - ?assertEqual(<<"STATE">>, hb_converge:get(state_key, SetTwice, Opts)). - -deep_set_test(Opts) -> - % First validate second layer changes are handled correctly. - Msg0 = #{ a => #{ b => <<"RESULT">> } }, - ?assertMatch(#{ a := #{ b := <<"RESULT2">> } }, - hb_converge:set(Msg0, [a, b], <<"RESULT2">>, Opts)), - % Now validate deeper layer changes are handled correctly. - Msg = #{ a => #{ b => #{ c => 1 } } }, - ?assertMatch(#{ a := #{ b := #{ c := 2 } } }, - hb_converge:set(Msg, [a, b, c], 2, Opts)). - -deep_set_new_messages_test() -> - Opts = maps:get(opts, hd(test_opts())), - % Test that new messages are created when the path does not exist. - Msg0 = #{ a => #{ b => #{ c => <<"1">> } } }, - Msg1 = hb_converge:set(Msg0, <<"d/e">>, <<"3">>, Opts), - Msg2 = hb_converge:set(Msg1, <<"d/f">>, <<"4">>, Opts), - ?assert( - hb_message:match( - Msg2, - #{ - a => - #{ - b => - #{ c => <<"1">> } - }, - d => - #{ - e => <<"3">>, - f => <<"4">> } - } - ) - ), - Msg3 = hb_converge:set( - Msg2, - #{ - <<"z/a">> => <<"0">>, - <<"z/b">> => <<"1">>, - <<"z/y/x">> => <<"2">> - }, - Opts - ), - ?assert( - hb_message:match( - Msg3, - #{ - a => #{ b => #{ c => <<"1">> } }, - d => #{ e => <<"3">>, f => <<"4">> }, - z => #{ a => <<"0">>, b => <<"1">>, y => #{ x => <<"2">> } } - } - ) - ). - -deep_set_with_device_test(Opts) -> - Device = #{ - set => - fun(Msg1, Msg2) -> - % A device where the set function modifies the key - % and adds a modified flag. - {Key, Val} = - hd(maps:to_list(maps:without([path, priv], Msg2))), - {ok, Msg1#{ Key => Val, modified => true }} - end - }, - % A message with an interspersed custom device: A and C have it, - % B does not. A and C will have the modified flag set to true. - Msg = #{ - device => Device, - a => - #{ - b => - #{ - device => Device, - c => <<"1">>, - modified => false - }, - modified => false - }, - modified => false - }, - Outer = hb_converge:deep_set(Msg, [a, b, c], <<"2">>, Opts), - A = hb_converge:get(a, Outer, Opts), - B = hb_converge:get(b, A, Opts), - C = hb_converge:get(c, B, Opts), - ?assertEqual(<<"2">>, C), - ?assertEqual(true, hb_converge:get(modified, Outer)), - ?assertEqual(false, hb_converge:get(modified, A)), - ?assertEqual(true, hb_converge:get(modified, B)). - -device_exports_test(Opts) -> - Msg = #{ device => dev_message }, - ?assert(hb_converge:is_exported(Msg, dev_message, info, Opts)), - ?assert(hb_converge:is_exported(Msg, dev_message, set, Opts)), - ?assert( - hb_converge:is_exported( - Msg, - dev_message, - not_explicitly_exported, - Opts - ) - ), - Dev = #{ - info => fun() -> #{ exports => [set] } end, - set => fun(_, _) -> {ok, <<"SET">>} end - }, - Msg2 = #{ device => Dev }, - ?assert(hb_converge:is_exported(Msg2, Dev, info, Opts)), - ?assert(hb_converge:is_exported(Msg2, Dev, set, Opts)), - ?assert(not hb_converge:is_exported(Msg2, Dev, not_exported, Opts)), - Dev2 = #{ - info => - fun() -> - #{ - exports => [test1, <<"Test2">>], - handler => - fun() -> - {ok, <<"Handler-Value">>} - end - } - end - }, - Msg3 = #{ device => Dev2, <<"Test1">> => <<"BAD1">>, test3 => <<"GOOD3">> }, - ?assertEqual(<<"Handler-Value">>, hb_converge:get(test1, Msg3, Opts)), - ?assertEqual(<<"Handler-Value">>, hb_converge:get(test2, Msg3, Opts)), - ?assertEqual(<<"GOOD3">>, hb_converge:get(test3, Msg3, Opts)), - ?assertEqual(<<"GOOD4">>, - hb_converge:get( - <<"Test4">>, - hb_converge:set(Msg3, <<"Test4">>, <<"GOOD4">>, Opts) - ) - ), - ?assertEqual(not_found, hb_converge:get(test5, Msg3, Opts)). - -device_excludes_test(Opts) -> - % Create a device that returns an identifiable message for any key, but also - % sets excludes to [set], such that the message can be modified using the - % default handler. - Dev = #{ - info => - fun() -> - #{ - excludes => [set], - handler => fun() -> {ok, <<"Handler-Value">>} end - } - end - }, - Msg = #{ device => Dev, <<"Test-Key">> => <<"Test-Value">> }, - ?assert(hb_converge:is_exported(Msg, Dev, <<"Test-Key2">>, Opts)), - ?assert(not hb_converge:is_exported(Msg, Dev, set, Opts)), - ?assertEqual(<<"Handler-Value">>, hb_converge:get(<<"Test-Key2">>, Msg, Opts)), - ?assertMatch(#{ <<"Test-Key2">> := <<"2">> }, - hb_converge:set(Msg, <<"Test-Key2">>, <<"2">>, Opts)). - -denormalized_device_key_test(Opts) -> - Msg = #{ <<"Device">> => dev_test }, - ?assertEqual(dev_test, hb_converge:get(device, Msg, Opts)), - ?assertEqual(dev_test, hb_converge:get(<<"Device">>, Msg, Opts)), - ?assertEqual({module, dev_test}, - erlang:fun_info( - element(3, hb_converge:message_to_fun(Msg, test_func, Opts)), - module - ) - ). - -list_transform_test(Opts) -> - Msg = [<<"A">>, <<"B">>, <<"C">>, <<"D">>, <<"E">>], - ?assertEqual(<<"A">>, hb_converge:get(1, Msg, Opts)), - ?assertEqual(<<"B">>, hb_converge:get(2, Msg, Opts)), - ?assertEqual(<<"C">>, hb_converge:get(3, Msg, Opts)), - ?assertEqual(<<"D">>, hb_converge:get(4, Msg, Opts)), - ?assertEqual(<<"E">>, hb_converge:get(5, Msg, Opts)). - -singleton_resolve_test() -> - Msg1 = #{ - % Should be parsed out and used alone as Msg2: - path => <<"Key1">>, - <<"Key1">> => <<"Value1">> - }, - ?assertEqual({ok, <<"Value1">>}, hb_converge:resolve(Msg1, #{})). diff --git a/src/hb_crypto.erl b/src/hb_crypto.erl index 9b94e8107..d5914151d 100644 --- a/src/hb_crypto.erl +++ b/src/hb_crypto.erl @@ -12,22 +12,29 @@ %%% The accumulate algorithm is experimental and at this point only exists to %%% allow us to test multiple HashPath algorithms in HyperBEAM. -module(hb_crypto). --export([sha256/1, sha256_chain/2, accumulate/2]). +-export([sha256/1, sha256_chain/2, accumulate/1, accumulate/2]). +-export([pbkdf2/5]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). %% @doc Add a new ID to the end of a SHA-256 hash chain. sha256_chain(ID1, ID2) when ?IS_ID(ID1) -> - ?no_prod("CAUTION: Unaudited cryptographic function invoked."), sha256(<>); sha256_chain(ID1, ID2) -> throw({cannot_chain_bad_ids, ID1, ID2}). -%% @doc Accumulate two IDs into a single commitment. -%% Experimental! This is not necessarily a cryptographically-secure operation. -accumulate(ID1 = << ID1Int:256 >>, ID2) when ?IS_ID(ID1) -> - ?no_prod("CAUTION: Experimental cryptographic algorithm invoked."), - << ID2Int:256 >> = sha256_chain(ID1, ID2), +%% @doc Accumulate two IDs, or a list of IDs, into a single commitment. This +%% function requires that the IDs given are already cryptographically-secure, +%% 256-bit values. No further cryptographic operations are performed upon the +%% values, they are simply added together. +%% +%% This is useful in situations where the ordering of the IDs is not important, +%% or explicitly detrimental to the utility of the final commitment. No ordering +%% information is preserved in the final commitment. +accumulate(IDs) when is_list(IDs) -> + lists:foldl(fun accumulate/2, << 0:256 >>, IDs). +accumulate(ID1 = << ID1Int:256 >>, ID2 = << ID2Int:256 >>) + when (byte_size(ID1) =:= 32) and (byte_size(ID2) =:= 32) -> << (ID1Int + ID2Int):256 >>; accumulate(ID1, ID2) -> throw({cannot_accumulate_bad_ids, ID1, ID2}). @@ -37,6 +44,21 @@ accumulate(ID1, ID2) -> sha256(Data) -> crypto:hash(sha256, Data). +%% @doc Wrap Erlang's `crypto:pbkdf2_hmac/5' to provide a standard interface. +pbkdf2(Alg, Password, Salt, Iterations, KeyLength) -> + case crypto:pbkdf2_hmac(Alg, Password, Salt, Iterations, KeyLength) of + Key when is_binary(Key) -> {ok, Key}; + {Tag, CFileInfo, Desc} -> + ?event( + {pbkdf2_error, + {tag, Tag}, + {desc, Desc}, + {c_file_info, CFileInfo} + } + ), + {error, Desc} + end. + %%% Tests %% @doc Count the number of leading zeroes in a bitstring. diff --git a/src/hb_debugger.erl b/src/hb_debugger.erl new file mode 100644 index 000000000..0b6e0eb0e --- /dev/null +++ b/src/hb_debugger.erl @@ -0,0 +1,168 @@ +%%% @doc A module that provides bootstrapping interfaces for external debuggers +%%% to connect to HyperBEAM. +%%% +%%% The simplest way to utilize an external graphical debugger is to use the +%%% `erlang-ls' extension for VS Code, Emacs, or other Language Server Protocol +%%% (LSP) compatible editors. This repository contains a `launch.json' +%%% configuration file for VS Code that can be used to spawn a new HyperBEAM, +%%% attach the debugger to it, and execute the specified `Module:Function(Args)'. +%%% Additionally, the node can be started with `rebar3 debugging' in order to +%%% allow access to the console while also allowing the debugger to attach. +%%% +%%% Boot time is approximately 10 seconds. +-module(hb_debugger). +-export([start/0, start_and_break/2, start_and_break/3, start_and_break/4]). +-export([profile_and_stop/1]). +-export([await_breakpoint/0]). + +%% @doc Profile a function with eflame and stop the node. +profile_and_stop(Fun) -> + {ok, F} = file:open("profiling-output", [write]), + group_leader(F, self()), + io:format("profiling-output: started.~n"), + io:format("Profiling function: ~p.~n", [Fun]), + Res = + dev_profile:eval( + Fun, + #{ <<"return-mode">> => <<"open">>, <<"engine">> => <<"eflame">> }, + #{} + ), + io:format("Profiling complete. Res: ~p~n", [Res]), + init:stop(), + erlang:halt(). + +%% Wait for another node (which we assume to be the debugger) to be attached, +%% then return to the caller. +start() -> + io:format("Starting debugger...~n", []), + DebuggerRes = application:ensure_all_started(debugger), + io:format("Started debugger server. Result: ~p.~n", [DebuggerRes]), + io:format( + "Waiting for debugger. Node is: ~p. Cookie is: ~p.~n", + [node(), erlang:get_cookie()] + ), + await_debugger(). + +%% @doc Attempt to interpret a specified module to load it into the debugger. +%% `int:i/1' seems to have an issue that will cause it to fail sporadically +%% with `error:undef' on some modules. This error appears not to be catchable +%% through the normal means. Subsequently, we attempt the load in a separate +%% process and wait for it to complete. If we do not receive a response in a +%% reasonable amount of time, we assume that the module failed to load and +%% return `false'. +interpret(Module) -> + Parent = self(), + spawn(fun() -> + case int:interpretable(Module) of + true -> + try Parent ! {interpreted, Module, int:i(Module) == ok} + catch _:_ -> + io:format("Could not load module: ~p.~n", [Module]), + false + end; + Error -> + io:format( + "Could not interpret module: ~p. Error: ~p.~n", + [Module, Error] + ), + false + end + end), + receive {interpreted, Module, Res} -> Res + after 250 -> false + end. + +%% @doc Interpret modules from a list of atom prefixes. +interpret_modules(Prefixes) when is_binary(Prefixes) -> + interpret_modules(binary:split(Prefixes, <<",">>, [global, trim_all])); +interpret_modules(Prefixes) when is_list(Prefixes) -> + RelevantModules = + lists:filter( + fun(Mod) -> + ModBin = hb_util:bin(Mod), + lists:any( + fun(Prefix) -> + PrefixBin = hb_util:bin(Prefix), + binary:longest_common_prefix([ModBin, PrefixBin]) == + byte_size(PrefixBin) + end, + Prefixes + ) + end, + hb_util:all_hb_modules() + ), + io:format("Relevant modules: ~p.~n", [RelevantModules]), + lists:foreach( + fun(Mod) -> + io:format("Interpreting module: ~p.~n", [Mod]), + interpret(Mod) + end, + RelevantModules + ), + RelevantModules. + +%% @doc A bootstrapping function to wait for an external debugger to be attached, +%% then add a breakpoint on the specified `Module:Function(Args)', then call it. +start_and_break(Module, Function) -> + start_and_break(Module, Function, [], []). +start_and_break(Module, Function, Args) -> + start_and_break(Module, Function, Args, []). +start_and_break(Module, Function, Args, DebuggerScope) -> + timer:sleep(1000), + spawn(fun() -> + start(), + interpret(Module), + interpret_modules(DebuggerScope), + SetRes = int:break_in(Module, Function, length(Args)), + io:format( + "Breakpoint set. Result from `int:break_in/3`: ~p.~n", + [SetRes] + ), + io:format("Invoking function...~n", []), + apply(Module, Function, Args), + io:format("Function invoked. Terminating.~n", []), + init:stop(), + erlang:halt() + end). + +%% @doc Await a debugger to be attached to the node. +await_debugger() -> await_debugger(0). +await_debugger(N) -> + case is_debugging_node_connected() of + false -> + timer:sleep(1000), + io:format("Still waiting for debugger after ~p seconds...~n", [N]), + await_debugger(N + 1); + Node -> + io:format( + "External node connection detected. Peer: ~p.~n", + [Node] + ), + N + end. + +%% @doc Is another Distributed Erlang node connected to us? +is_debugging_node_connected() -> + case nodes() ++ nodes(hidden) of + [] -> false; + [Node | _] -> Node + end. + +%% @doc Await a new breakpoint being set by the debugger. +await_breakpoint() -> + case is_debugging_node_connected() of + false -> start(); + _ -> do_nothing + end, + await_breakpoint(0). +await_breakpoint(N) -> + io:format("Waiting for breakpoint to be set in function...~n", []), + case int:all_breaks() of + [] -> + timer:sleep(1000), + io:format("Still waiting for breakpoint after ~p seconds...~n", [N]), + await_breakpoint(N + 1); + [Breakpoint | _] -> + io:format("Breakpoint set. Info: ~p.~n", [Breakpoint]), + Breakpoint + end. \ No newline at end of file diff --git a/src/hb_escape.erl b/src/hb_escape.erl new file mode 100644 index 000000000..5dc935332 --- /dev/null +++ b/src/hb_escape.erl @@ -0,0 +1,147 @@ +%%% @doc Functions for escaping and unescaping mixed case values, for use in HTTP +%%% headers. Both percent-encoding and escaping of double-quoted strings +%%% (`"' => `\"') are supported. +%%% +%%% This is necessary for encodings of AO-Core messages for transmission in +%%% HTTP/2 and HTTP/3, because uppercase header keys are explicitly disallowed. +%%% While most map keys in HyperBEAM are normalized to lowercase, IDs are not. +%%% Subsequently, we encode all header keys to lowercase %-encoded URI-style +%%% strings because transmission. +-module(hb_escape). +-export([encode/1, decode/1, encode_keys/2, decode_keys/2]). +-export([encode_quotes/1, decode_quotes/1]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% @doc Encode a binary as a URI-encoded string. +encode(Bin) when is_binary(Bin) -> + list_to_binary(percent_escape(binary_to_list(Bin))). + +%% @doc Decode a URI-encoded string back to a binary. +decode(Bin) when is_binary(Bin) -> + list_to_binary(percent_unescape(binary_to_list(Bin))). + +%% @doc Encode a string with escaped quotes. +encode_quotes(String) when is_binary(String) -> + list_to_binary(encode_quotes(binary_to_list(String))); +encode_quotes([]) -> []; +encode_quotes([$\" | Rest]) -> [$\\, $\" | encode_quotes(Rest)]; +encode_quotes([C | Rest]) -> [C | encode_quotes(Rest)]. + +%% @doc Decode a string with escaped quotes. +decode_quotes(String) when is_binary(String) -> + list_to_binary(decode_quotes(binary_to_list(String))); +decode_quotes([]) -> []; +decode_quotes([$\\, $\" | Rest]) -> [$\" | decode_quotes(Rest)]; +decode_quotes([$\" | Rest]) -> decode_quotes(Rest); +decode_quotes([C | Rest]) -> [C | decode_quotes(Rest)]. + +%% @doc Return a message with all of its keys decoded. +decode_keys(Msg, Opts) when is_map(Msg) -> + hb_maps:from_list( + lists:map( + fun({Key, Value}) -> {decode(Key), Value} end, + hb_maps:to_list(Msg, Opts) + ) + ); +decode_keys(Other, _Opts) -> Other. + +%% @doc URI encode keys in the base layer of a message. Does not recurse. +encode_keys(Msg, Opts) when is_map(Msg) -> + hb_maps:from_list( + lists:map( + fun({Key, Value}) -> {encode(Key), Value} end, + hb_maps:to_list(Msg, Opts) + ) + ); +encode_keys(Other, _Opts) -> Other. + +%% @doc Escape a list of characters as a URI-encoded string. +percent_escape([]) -> []; +percent_escape([C | Cs]) when C >= $a, C =< $z -> [C | percent_escape(Cs)]; +percent_escape([C | Cs]) when C >= $0, C =< $9 -> [C | percent_escape(Cs)]; +percent_escape([C | Cs]) when + C == $.; C == $-; C == $_; C == $/; + C == $?; C == $& -> + [C | percent_escape(Cs)]; +percent_escape([C | Cs]) -> [escape_byte(C) | percent_escape(Cs)]. + +%% @doc Escape a single byte as a URI-encoded string. +escape_byte(C) when C >= 0, C =< 255 -> + [$%, hex_digit(C bsr 4), hex_digit(C band 15)]. + +hex_digit(N) when N >= 0, N =< 9 -> + N + $0; +hex_digit(N) when N > 9, N =< 15 -> + N + $a - 10. + +%% @doc Unescape a URI-encoded string. +percent_unescape([$%, H1, H2 | Cs]) -> + Byte = (hex_value(H1) bsl 4) + hex_value(H2), + [Byte | percent_unescape(Cs)]; +percent_unescape([C | Cs]) -> + [C | percent_unescape(Cs)]; +percent_unescape([]) -> + []. + +hex_value(C) when C >= $0, C =< $9 -> + C - $0; +hex_value(C) when C >= $a, C =< $f -> + C - $a + 10; +hex_value(C) when C >= $A, C =< $F -> + C - $A + 10. + +%%% Tests + +escape_unescape_identity_test() -> + % Test that unescape(escape(X)) == X for various inputs + TestCases = [ + <<"hello">>, + <<"hello, world!">>, + <<"hello+list">>, + <<"special@chars#here">>, + <<"UPPERCASE">>, + <<"MixedCASEstring">>, + <<"12345">>, + <<>> % Empty string + ], + ?event(parsing, + {escape_unescape_identity_test, + {test_cases, + [ + {Case, {explicit, encode(Case)}} + || + Case <- TestCases + ] + } + } + ), + lists:foreach(fun(TestCase) -> + ?assertEqual(TestCase, decode(encode(TestCase))) + end, TestCases). + +unescape_specific_test() -> + % Test specific unescape cases + ?assertEqual(<<"a">>, decode(<<"%61">>)), + ?assertEqual(<<"A">>, decode(<<"%41">>)), + ?assertEqual(<<"!">>, decode(<<"%21">>)), + ?assertEqual(<<"hello, World!">>, decode(<<"hello%2c%20%57orld%21">>)), + ?assertEqual(<<"/">>, decode(<<"%2f">>)), + ?assertEqual(<<"?">>, decode(<<"%3f">>)). + +uppercase_test() -> + % Test uppercase characters are properly escaped + ?assertEqual(<<"%41">>, encode(<<"A">>)), + ?assertEqual(<<"%42">>, encode(<<"B">>)), + ?assertEqual(<<"%5a">>, encode(<<"Z">>)), + ?assertEqual(<<"hello%20%57orld">>, encode(<<"hello World">>)), + ?assertEqual(<<"test%41%42%43">>, encode(<<"testABC">>)). + +escape_unescape_special_chars_test() -> + % Test characters that should be escaped + SpecialChars = [ + $@, $#, $", $$, $%, $&, $', $(, $), $*, $+, $,, $/, $:, $;, + $<, $=, $>, $?, $[, $\\, $], $^, $`, ${, $|, $}, $~, $\s + ], + TestString = list_to_binary(SpecialChars), + ?assertEqual(TestString, decode(encode(TestString))). \ No newline at end of file diff --git a/src/hb_event.erl b/src/hb_event.erl new file mode 100644 index 000000000..63468ca70 --- /dev/null +++ b/src/hb_event.erl @@ -0,0 +1,289 @@ +%%% @doc Wrapper for incrementing prometheus counters. +-module(hb_event). +-export([counters/0, diff/1, diff/2]). +-export([log/1, log/2, log/3, log/4, log/5, log/6]). +-export([increment/3, increment/4, increment_callers/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(OVERLOAD_QUEUE_LENGTH, 10000). + +-ifdef(NO_EVENTS). +log(_X) -> ok. +log(_Topic, _X) -> ok. +log(_Topic, _X, _Mod) -> ok. +log(_Topic, _X, _Mod, _Func) -> ok. +log(_Topic, _X, _Mod, _Func, _Line) -> ok. +log(_Topic, _X, _Mod, _Func, _Line, _Opts) -> ok. +-else. +%% @doc Debugging log logging function. For now, it just prints to standard +%% error. +log(X) -> log(global, X). +log(Topic, X) -> log(Topic, X, ""). +log(Topic, X, Mod) -> log(Topic, X, Mod, undefined). +log(Topic, X, Mod, Func) -> log(Topic, X, Mod, Func, undefined). +log(Topic, X, Mod, Func, Line) -> log(Topic, X, Mod, Func, Line, #{}). +log(Topic, X, Mod, undefined, Line, Opts) -> log(Topic, X, Mod, "", Line, Opts); +log(Topic, X, Mod, Func, undefined, Opts) -> log(Topic, X, Mod, Func, "", Opts); +log(Topic, X, Mod, Func, Line, Opts) -> + % Check if the debug_print option has the topic in it if set. + case should_print(Topic, Opts) orelse should_print(Mod, Opts) of + true -> hb_format:print(X, Mod, Func, Line, Opts); + false -> X + end, + %handle_tracer(Topic, X, Opts), + try increment(Topic, X, Opts) catch _:_ -> ok end, + % Return the logged value to the caller. This allows callers to insert + % `?event(...)' macros into the flow of other executions, without having to + % break functional style. + X. +-endif. + +%% @doc Determine if the topic should be printed. Uses a cache in the process +%% dictionary to avoid re-checking the same topic multiple times. +should_print(Topic, Opts) -> + case erlang:get({event_print, Topic}) of + {cached, X} -> X; + undefined -> + Result = + case hb_opts:get(debug_print, false, Opts) of + EventList when is_list(EventList) -> + lists:member(Topic, EventList); + true -> true; + false -> false + end, + erlang:put({event_print, Topic}, {cached, Result}), + Result + end. + +handle_tracer(Topic, X, Opts) -> + AllowedTopics = [http, ao_result], + case lists:member(Topic, AllowedTopics) of + true -> + case hb_opts:get(trace, undefined, Opts) of + undefined -> + case tuple_to_list(X) of + [_ | Rest] -> + try + Map = maps:from_list(Rest), + TopicOpts = hb_opts:get(opts, #{}, Map), + case hb_opts:get(trace, undefined, TopicOpts) of + undefined -> ok; + TracePID -> + hb_tracer:record_step(TracePID, {Topic, X}) + end + catch + _:_ -> ok + end; + _ -> + ok + end; + TracePID -> hb_tracer:record_step(TracePID, {Topic, X}) + end; + _ -> ok + end. + +%% @doc Increment the counter for the given topic and message. Registers the +%% counter if it doesn't exist. If the topic is `global', the message is ignored. +%% This means that events must specify a topic if they want to be counted, +%% filtering debug messages. +%% +%% This function uses a series of hard-coded topics to ignore explicitly in +%% order to quickly filter events that are executed so frequently that they +%% would otherwise cause heavy performance costs. +increment(Topic, Message, Opts) -> + increment(Topic, Message, Opts, 1). +increment(global, _Message, _Opts, _Count) -> ignored; +increment(ao_core, _Message, _Opts, _Count) -> ignored; +increment(ao_internal, _Message, _Opts, _Count) -> ignored; +increment(ao_devices, _Message, _Opts, _Count) -> ignored; +increment(ao_subresolution, _Message, _Opts, _Count) -> ignored; +increment(signature_base, _Message, _Opts, _Count) -> ignored; +increment(id_base, _Message, _Opts, _Count) -> ignored; +increment(parsing, _Message, _Opts, _Count) -> ignored; +increment(Topic, Message, _Opts, Count) -> + case parse_name(Message) of + <<"debug", _/binary>> -> ignored; + EventName -> + TopicBin = parse_name(Topic), + case find_event_server() of + Pid when is_pid(Pid) -> + Pid ! {increment, TopicBin, EventName, Count}; + undefined -> + PID = spawn(fun() -> server() end), + hb_name:register(?MODULE, PID), + PID ! {increment, TopicBin, EventName, Count} + end + end. + +%% @doc Increment the call paths and individual upstream calling functions of +%% the current execution. This function generates the stacktrace itself. It is +%% **extremely** expensive, so it should only be used in very specific cases. +%% Do not ship code that calls this function to prod. +increment_callers(Topic) -> + increment_callers(Topic, erlang). +increment_callers(Topic, Type) -> + BinTopic = hb_util:bin(Topic), + increment( + <>, + hb_format:trace_short(Type), + #{} + ), + lists:foreach( + fun(Caller) -> + increment(<>, Caller, #{}) + end, + hb_format:trace_to_list(hb_format:get_trace(Type)) + ). + +%% @doc Return a message containing the current counter values for all logged +%% HyperBEAM events. The result comes in a form as follows: +%% /GroupName/EventName -> Count +%% Where the `EventName` is derived from the value of the first term sent to the +%% `?event(...)' macros. +counters() -> + UnaggregatedCounts = + [ + {Group, Name, Count} + || + {{default, <<"event">>, [Group, Name], _}, Count, _} <- raw_counters() + ], + lists:foldl( + fun({Group, Name, Count}, Acc) -> + Acc#{ + Group => (maps:get(Group, Acc, #{}))#{ + Name => maps:get(Name, maps:get(Group, Acc, #{}), 0) + Count + } + } + end, + #{}, + UnaggregatedCounts + ). + +%% @doc Return the change in the event counters before and after executing the +%% given function. +diff(Fun) -> + diff(Fun, #{}). +diff(Fun, Opts) -> + EventsBefore = counters(), + Res = Fun(), + EventsAfter = counters(), + {hb_message:diff(EventsBefore, EventsAfter, Opts), Res}. + +-ifdef(NO_EVENTS). +raw_counters() -> + []. +-else. +raw_counters() -> + ets:tab2list(prometheus_counter_table). +-endif. + +%% @doc Find the event server, creating it if it doesn't exist. We cache the +%% result in the process dictionary to avoid looking it up multiple times. +find_event_server() -> + case erlang:get({event_server, ?MODULE}) of + {cached, Pid} -> Pid; + undefined -> + PID = + case hb_name:lookup(?MODULE) of + Pid when is_pid(Pid) -> Pid; + undefined -> + NewServer = spawn(fun() -> server() end), + hb_name:register(?MODULE, NewServer), + NewServer + end, + erlang:put({event_server, ?MODULE}, {cached, PID}), + PID + end. + +server() -> + await_prometheus_started(), + prometheus_counter:declare( + [ + {name, <<"event">>}, + {help, <<"AO-Core execution events">>}, + {labels, [topic, event]} + ]), + handle_events(). +handle_events() -> + receive + {increment, TopicBin, EventName, Count} -> + case erlang:process_info(self(), message_queue_len) of + {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH -> + % Print a warning, but do so less frequently the more + % overloaded the system is. + case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of + 1 -> + ?debug_print( + {warning, + prometheus_event_queue_overloading, + {queue, Len}, + {current_message, EventName} + } + ); + _ -> ignored + end; + _ -> ignored + end, + prometheus_counter:inc(<<"event">>, [TopicBin, EventName], Count), + handle_events() + end. + +%% @doc Delay the event server until prometheus is started. +await_prometheus_started() -> + receive + Msg -> + case application:get_application(prometheus) of + undefined -> await_prometheus_started(); + _ -> self() ! Msg, ok + end + end. + +parse_name(Name) when is_tuple(Name) -> + parse_name(element(1, Name)); +parse_name(Name) when is_atom(Name) -> + atom_to_binary(Name, utf8); +parse_name(Name) when is_binary(Name) -> + Name; +parse_name(Name) when is_list(Name) -> + iolist_to_binary(Name); +parse_name(_) -> no_event_name. + +%%% Benchmark tests + +%% @doc Benchmark the performance of a full log of an event. +benchmark_event_test() -> + Iterations = + hb_test_utils:benchmark( + fun() -> + log(test_module, {test, 1}) + end + ), + hb_test_utils:benchmark_print(<<"Recorded">>, <<"events">>, Iterations), + ?assert(Iterations >= 1000), + ok. + +%% @doc Benchmark the performance of looking up whether a topic and module +%% should be printed. +benchmark_print_lookup_test() -> + DefaultOpts = hb_opts:default_message_with_env(), + Iterations = + hb_test_utils:benchmark( + fun() -> + should_print(test_module, DefaultOpts) + orelse should_print(test_event, DefaultOpts) + end + ), + hb_test_utils:benchmark_print(<<"Looked-up">>, <<"topics">>, Iterations), + ?assert(Iterations >= 1000), + ok. + +%% @doc Benchmark the performance of incrementing an event. +benchmark_increment_test() -> + Iterations = + hb_test_utils:benchmark( + fun() -> increment(test_module, {test, 1}, #{}) end + ), + hb_test_utils:benchmark_print(<<"Incremented">>, <<"events">>, Iterations), + ?assert(Iterations >= 1000), + ok. \ No newline at end of file diff --git a/src/hb_examples.erl b/src/hb_examples.erl new file mode 100644 index 000000000..fb19e0477 --- /dev/null +++ b/src/hb_examples.erl @@ -0,0 +1,369 @@ +%%% @doc This module contains end-to-end tests for Hyperbeam, accessing through +%%% the HTTP interface. As well as testing the system, you can use these tests +%%% as examples of how to interact with HyperBEAM nodes. +-module(hb_examples). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("include/hb.hrl"). + +%% @doc Start a node running the simple pay meta device, and use it to relay +%% a message for a client. We must ensure: +%% 1. When the client has no balance, the relay fails. +%% 2. The operator is able to topup for the client. +%% 3. The client has the correct balance after the topup. +%% 4. The relay succeeds when the client has enough balance. +%% 5. The received message is signed by the host using http-sig and validates +%% correctly. +relay_with_payments_test_() -> + {timeout, 30, fun relay_with_payments_test/0}. +relay_with_payments_test() -> + HostWallet = ar_wallet:new(), + ClientWallet = ar_wallet:new(), + ClientAddress = hb_util:human_id(ar_wallet:to_address(ClientWallet)), + % Start a node with the simple-pay device enabled. + ProcessorMsg = + #{ + <<"device">> => <<"p4@1.0">>, + <<"ledger-device">> => <<"simple-pay@1.0">>, + <<"pricing-device">> => <<"simple-pay@1.0">> + }, + HostNode = + hb_http_server:start_node( + #{ + operator => ar_wallet:to_address(HostWallet), + on => #{ + <<"request">> => ProcessorMsg, + <<"response">> => ProcessorMsg + } + } + ), + % Create a message for the client to relay. + ClientMessage1 = + hb_message:commit( + #{<<"path">> => <<"/~relay@1.0/call?relay-path=https://www.google.com">>}, + ClientWallet + ), + % Relay the message. + Res = hb_http:get(HostNode, ClientMessage1, #{}), + ?assertMatch({error, #{ <<"body">> := <<"Insufficient funds">> }}, Res), + % Topup the client's balance. + % Note: The fields must be in the headers, for now. + TopupMessage = + hb_message:commit( + #{ + <<"path">> => <<"/~simple-pay@1.0/topup">>, + <<"recipient">> => ClientAddress, + <<"amount">> => 100 + }, + HostWallet + ), + ?assertMatch({ok, _}, hb_http:get(HostNode, TopupMessage, #{})), + % Relay the message again. + Res2 = hb_http:get(HostNode, ClientMessage1, #{}), + ?assertMatch({ok, #{ <<"body">> := Bin }} when byte_size(Bin) > 10_000, Res2), + {ok, Resp} = Res2, + ?assert(length(hb_message:signers(Resp, #{})) > 0), + ?assert(hb_message:verify(Resp, all, #{})). + +%% @doc Gain signed WASM responses from a node and verify them. +%% 1. Start the client with a small balance. +%% 2. Execute a simple WASM function on the host node. +%% 3. Verify the response is correct and signed by the host node. +%% 4. Get the balance of the client and verify it has been deducted. +paid_wasm_test_() -> + {timeout, 30, fun paid_wasm/0}. +paid_wasm() -> + HostWallet = ar_wallet:new(), + ClientWallet = ar_wallet:new(), + ClientAddress = hb_util:human_id(ar_wallet:to_address(ClientWallet)), + ProcessorMsg = + #{ + <<"device">> => <<"p4@1.0">>, + <<"ledger-device">> => <<"simple-pay@1.0">>, + <<"pricing-device">> => <<"simple-pay@1.0">> + }, + HostNode = + hb_http_server:start_node( + Opts = #{ + store => [ + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + } + ], + simple_pay_ledger => #{ ClientAddress => 100 }, + simple_pay_price => 10, + operator => ar_wallet:to_address(HostWallet), + on => #{ + <<"request">> => ProcessorMsg, + <<"response">> => ProcessorMsg + } + } + ), + % Read the WASM file from disk, post it to the host and execute it. + {ok, WASMFile} = file:read_file(<<"test/test-64.wasm">>), + ClientMessage1 = + hb_message:commit( + #{ + <<"path">> => + <<"/~wasm-64@1.0/init/compute/results?function=fac">>, + <<"body">> => WASMFile, + <<"parameters+list">> => <<"3.0">> + }, + Opts#{ priv_wallet => ClientWallet } + ), + {ok, Res} = hb_http:post(HostNode, ClientMessage1, Opts), + % Check that the message is signed by the host node. + ?assert(length(hb_message:signers(Res, Opts)) > 0), + ?assert(hb_message:verify(Res, all, Opts)), + % Now we have the results, we can verify them. + ?assertMatch(6.0, hb_ao:get(<<"output/1">>, Res, Opts)), + % Check that the client's balance has been deducted. + ClientMessage2 = + hb_message:commit( + #{<<"path">> => <<"/~p4@1.0/balance">>}, + ClientWallet + ), + {ok, Res2} = hb_http:get(HostNode, ClientMessage2, Opts), + ?assertMatch(60, Res2). + +create_schedule_aos2_test_disabled() -> + % The legacy process format, according to the ao.tn.1 spec: + % Data-Protocol The name of the Data-Protocol for this data-item 1-1 ao + % Variant The network version that this data-item is for 1-1 ao.TN.1 + % Type Indicates the shape of this Data-Protocol data-item 1-1 Process + % Module Links the process to ao module using the module's unique + % Transaction ID (TXID). 1-1 {TXID} + % Scheduler Specifies the scheduler unit by Wallet Address or Name, and can + % be referenced by a recent Scheduler-Location. 1-1 {ADDRESS} + % Cron-Interval An interval at which a particular Cron Message is recevied by the process, + % in the format X-Y, where X is a scalar value, and Y is milliseconds, + % seconds, minutes, hours, days, months, years, or blocks 0-n 1-second + % Cron-Tag-{Name} defines tags for Cron Messages at set intervals, + % specifying relevant metadata. 0-1 + % Memory-Limit Overrides maximum memory, in megabytes or gigabytes, set by + % Module, can not exceed modules setting 0-1 16-mb + % Compute-Limit Caps the compute cycles for a module per evaluation, ensuring + % efficient, controlled execution 0-1 1000 + % Pushed-For Message TXID that this Process is pushed as a result 0-1 {TXID} + % Cast Sets message handling: 'True' for do not push, 'False' for normal + % pushing 0-1 {True or False} + % Authority Defines a trusted wallet address which can send Messages to + % the Process 0-1 {ADDRESS} + % On-Boot Defines a startup script to run when the process is spawned. If + % value "Data" it uses the Data field of the Process Data Item. If it is a + % TXID it will load that TX from Arweave and execute it. 0-1 {Data or TXID} + % {Any-Tags} Custom Tags specific for the initial input of the Process 0-n + Node = + try hb_http_server:start_node(#{ priv_wallet => hb:wallet() }) + catch + _:_ -> + <<"http://localhost:8734">> + end, + ProcMsg = #{ + <<"data-protocol">> => <<"ao">>, + <<"type">> => <<"Process">>, + <<"variant">> => <<"ao.TN.1">>, + <<"type">> => <<"Process">>, + <<"module">> => <<"bkjb55i07GUCUSWROtKK4HU1mBS_X0TyH3M5jMV6aPg">>, + <<"scheduler">> => hb_util:human_id(hb:address()), + <<"memory-limit">> => <<"1024-mb">>, + <<"compute-limit">> => <<"10000000">>, + <<"authority">> => hb_util:human_id(hb:address()), + <<"scheduler-location">> => hb_util:human_id(hb:address()) + }, + Wallet = hb:wallet(), + SignedProc = hb_message:commit(ProcMsg, Wallet), + IDNone = hb_message:id(SignedProc, none), + IDAll = hb_message:id(SignedProc, all), + {ok, Res} = schedule(SignedProc, IDNone, Wallet, Node), + ?event({res, Res}), + receive after 100 -> ok end, + ?event({id, IDNone, IDAll}), + {ok, Res2} = hb_http:get( + Node, + <<"/~scheduler@1.0/slot?target=", IDNone/binary>>, + #{} + ), + ?assertMatch(Slot when Slot >= 0, hb_ao:get(<<"at-slot">>, Res2, #{})). + +schedule(ProcMsg, Target) -> + schedule(ProcMsg, Target, hb:wallet()). +schedule(ProcMsg, Target, Wallet) -> + schedule(ProcMsg, Target, Wallet, <<"http://localhost:8734">>). +schedule(ProcMsg, Target, Wallet, Node) -> + SignedReq = + hb_message:commit( + #{ + <<"path">> => <<"/~scheduler@1.0/schedule">>, + <<"target">> => Target, + <<"body">> => ProcMsg + }, + Wallet + ), + ?event({signed_req, SignedReq}), + hb_http:post(Node, SignedReq, #{}). + + +%% @doc Test that we can schedule an ANS-104 data item on a relayed node. The +%% input to the relaying server comes in the form of a serialized ANS-104 +%% data item, which should then be correctly deserialized and sent to the +%% scheduler node. +relay_schedule_ans104_test() -> + SchedulerWallet = ar_wallet:new(), + ComputeWallet = ar_wallet:new(), + RelayWallet = ar_wallet:new(), + ?event(debug_test, + {wallets, + {scheduler, hb_util:human_id(SchedulerWallet)}, + {compute, hb_util:human_id(ComputeWallet)}, + {relay, hb_util:human_id(RelayWallet)} + } + ), + Scheduler = + hb_http_server:start_node( + #{ + on => #{ + <<"start">> => #{ + <<"device">> => <<"scheduler@1.0">>, + <<"path">> => <<"location">>, + <<"method">> => <<"POST">>, + <<"target">> => <<"self">>, + <<"require-codec">> => <<"ans104@1.0">>, + <<"hook">> => #{ + <<"result">> => <<"ignore">>, + <<"commit-request">> => true + } + } + }, + store => [hb_test_utils:test_store()], + priv_wallet => SchedulerWallet + } + ), + ?event(debug_test, {scheduler, Scheduler}), + Compute = + hb_http_server:start_node( + #{ + priv_wallet => ComputeWallet, + store => + [ + ComputeStore = hb_test_utils:test_store(), + #{ + <<"store-module">> => hb_store_remote_node, + <<"name">> => <<"cache-TEST/remote-node">>, + <<"node">> => Scheduler + } + ] + } + ), + % Get the scheduler location of the scheduling node and write it to the + % compute node's store. + {ok, SchedulerLocation} = + hb_http:get( + Scheduler, + <<"/~scheduler@1.0/location">>, + #{} + ), + ?event({scheduler_location, SchedulerLocation}), + dev_scheduler_cache:write_location( + hb_maps:get(<<"body">>, SchedulerLocation, <<"NO BODY">>, #{}), + #{ store => [ComputeStore] } + ), + % Create the relaying server. + Relay = + hb_http_server:start_node(#{ + priv_wallet => RelayWallet, + relay_allow_commit_request => true, + store => [hb_test_utils:test_store()], + routes => + [ + #{ + <<"template">> => <<"^/push">>, + <<"strategy">> => <<"Nearest">>, + <<"nodes">> => [ + #{ + <<"wallet">> => hb_util:human_id(SchedulerWallet), + <<"prefix">> => Scheduler + } + ] + }, + #{ + <<"template">> => <<"^/.*">>, + <<"strategy">> => <<"Nearest">>, + <<"nodes">> => [ + #{ + <<"wallet">> => hb_util:human_id(ComputeWallet), + <<"prefix">> => Compute + } + ] + } + ], + on => #{ + <<"request">> => + #{ + <<"device">> => <<"router@1.0">>, + <<"path">> => <<"preprocess">>, + <<"commit-request">> => true + } + } + }), + ?event(debug_test, + {nodes, + {scheduler, {url, Scheduler}, {wallet, hb_util:human_id(SchedulerWallet)}}, + {compute, {url, Compute}, {wallet, hb_util:human_id(ComputeWallet)}}, + {relay, {url, Relay}, {wallet, hb_util:human_id(RelayWallet)}} + } + ), + ClientOpts = + #{ + store => [hb_test_utils:test_store()], + priv_wallet => ar_wallet:new() + }, + % Create process to schedule, then send it to the relaying server as + % a serialized ANS-104 data item. + Process = + hb_message:commit( + #{ + <<"device">> => <<"process@1.0">>, + <<"execution-device">> => <<"test-device@1.0">>, + <<"push-device">> => <<"push@1.0">>, + <<"scheduler">> => hb_util:human_id(SchedulerWallet), + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"module">> => <<"URgYpPQzvxxfYQtjrIQ116bl3YBfcImo3JEnNo8Hlrk">> + }, + ClientOpts, + #{ <<"commitment-device">> => <<"ans104@1.0">> } + ), + % Push the initial message via the scheduler node. + ScheduleRes = + hb_http:post( + Relay, + Process#{ + <<"path">> => <<"push">>, + <<"codec-device">> => <<"ans104@1.0">> + }, + ClientOpts + ), + ?event(debug_test, {post_result, ScheduleRes}), + ?assertMatch({ok, #{ <<"status">> := 200, <<"slot">> := 0 }}, ScheduleRes), + % Push another message via the compute node. + ProcID = hb_message:id(Process, all, ClientOpts), + ToPush = + hb_message:commit( + #{ + <<"test-key">> => <<"value">>, + <<"rand-key">> => hb_util:encode(crypto:strong_rand_bytes(32)) + }, + ClientOpts, + #{ <<"commitment-device">> => <<"ans104@1.0">> } + ), + PushRes = + hb_http:post( + Relay, + ToPush#{ + <<"path">> => <>, + <<"codec-device">> => <<"ans104@1.0">> + }, + ClientOpts + ), + ?event(debug_test, {post_result, PushRes}), + ?assertMatch({ok, #{ <<"status">> := 200, <<"slot">> := 1 }}, PushRes). \ No newline at end of file diff --git a/src/hb_features.erl b/src/hb_features.erl new file mode 100644 index 000000000..8cd922f37 --- /dev/null +++ b/src/hb_features.erl @@ -0,0 +1,69 @@ +%%% @doc A module that exports a list of feature flags that the node supports +%%% using the `-ifdef' macro. +%%% As a consequence, this module acts as a proxy of information between the +%%% build system and the runtime execution environment. +-module(hb_features). +%%% Public API. +-export([all/0, enabled/1]). +%%% Individual feature flags. +-export([http3/0, rocksdb/0, test/0, genesis_wasm/0, eflame/0]). + +%% @doc Returns a list of all feature flags that the node supports. +all() -> + Features = + lists:filtermap( + fun({Name, _}) -> + case lists:member(Name, [all, enabled, module_info]) of + true -> false; + false -> {true, Name} + end + end, + ?MODULE:module_info(exports) + ), + hb_maps:from_list( + lists:map( + fun(Name) -> + {Name, ?MODULE:Name()} + end, + Features + ) + ). + +%% @doc Returns true if the feature flag is enabled. +enabled(Feature) -> + hb_maps:get(Feature, all(), false). + +%%% Individual feature flags. +%%% These functions use the `-ifdef' macro to conditionally return a boolean +%%% value based on the presence of the `ENABLE_' macro during +%%% compilation. + +-ifdef(ENABLE_HTTP3). +http3() -> true. +-else. +http3() -> false. +-endif. + +-ifdef(ENABLE_ROCKSDB). +rocksdb() -> true. +-else. +rocksdb() -> false. +-endif. + +-ifdef(ENABLE_GENESIS_WASM). +genesis_wasm() -> true. +-else. +genesis_wasm() -> false. +-endif. + +-ifdef(ENABLE_EFLAME). +eflame() -> true. +-else. +eflame() -> false. +-endif. + +-ifdef(TEST). +test() -> true. +-else. +test() -> false. +-endif. diff --git a/src/hb_format.erl b/src/hb_format.erl new file mode 100644 index 000000000..b1408a542 --- /dev/null +++ b/src/hb_format.erl @@ -0,0 +1,845 @@ +%%% @doc Formatting and debugging utilities for HyperBEAM. +%%% +%%% This module provides text formatting capabilities for debugging output, +%%% message pretty-printing, stack trace formatting, and human-readable +%%% representations of binary data and cryptographic identifiers. +%%% +%%% The functions in this module are primarily used for development and +%%% debugging purposes, supporting the logging and diagnostic infrastructure +%%% throughout the HyperBEAM system. +-module(hb_format). +%%% Public API. +-export([term/1, term/2, term/3]). +-export([print/1, print/3, print/4, print/5, eunit_print/2]). +-export([message/1, message/2, message/3]). +-export([binary/1, error/2, trace/1, trace_short/0, trace_short/1]). +-export([indent/2, indent/3, indent/4, indent_lines/2, maybe_multiline/3]). +-export([remove_leading_noise/1, remove_trailing_noise/1, remove_noise/1]). +%%% Public Utility Functions. +-export([escape_format/1, short_id/1, trace_to_list/1]). +-export([get_trace/1, print_trace/4, trace_macro_helper/5, print_trace_short/4]). +-include("include/hb.hrl"). + +%%% Characters that are considered noise and should be removed from strings +%%% with the `remove_noise_[leading|trailing]' functions. +-define(NOISE_CHARS, " \t\n,"). + +%% @doc Print a message to the standard error stream, prefixed by the amount +%% of time that has elapsed since the last call to this function. +print(X) -> + print(X, <<>>, #{}). +print(X, Info, Opts) -> + io:format( + standard_error, + "=== HB DEBUG ===~s==>~n~s~n", + [Info, term(X, Opts, 0)] + ), + X. +print(X, Mod, Func, LineNum) -> + print(X, format_debug_trace(Mod, Func, LineNum, #{}), #{}). +print(X, Mod, Func, LineNum, Opts) -> + Now = erlang:system_time(millisecond), + Last = erlang:put(last_debug_print, Now), + TSDiff = case Last of undefined -> 0; _ -> Now - Last end, + Info = + hb_util:bin( + io_lib:format( + "[~pms in ~s @ ~s]", + [ + TSDiff, + case server_id() of + undefined -> hb_util:bin(io_lib:format("~p", [self()])); + ServerID -> + hb_util:bin( + io_lib:format( + "~s (~p)", + [short_id(ServerID), self()] + ) + ) + end, + format_debug_trace(Mod, Func, LineNum, Opts) + ] + ) + ), + print(X, Info, Opts). + +%% @doc Retreive the server ID of the calling process, if known. +server_id() -> + server_id(#{ server_id => undefined }). +server_id(Opts) -> + case hb_opts:get(server_id, undefined, Opts) of + undefined -> get(server_id); + ServerID -> ServerID + end. + +%% @doc Generate the appropriate level of trace for a given call. +format_debug_trace(Mod, Func, Line, Opts) -> + case hb_opts:get(debug_print_trace, false, #{}) of + short -> + Trace = + case hb_opts:get(debug_trace_type, erlang, Opts) of + erlang -> get_trace(erlang); + ao -> + % If we are printing AO-Core traces, we add the module + % and line number to the end to show exactly where in + % the handler-flow the event arose. + [ + hb_util:bin(format_trace_element({Mod, Line})) + | + get_trace(ao) + ] + end, + trace_short(Trace); + false -> + io_lib:format("~p:~w ~p", [Mod, Line, Func]) + end. + +%% @doc Convert a term to a string for debugging print purposes. +term(X) -> term(X, #{}). +term(X, Opts) -> term(X, Opts, 0). +term(X, Opts, Indent) -> + try do_debug_fmt(X, Opts, Indent) + catch A:B:C -> + Mode = hb_opts:get(mode, prod, Opts), + PrintFailPreference = hb_opts:get(debug_print_fail_mode, quiet, Opts), + case {Mode, PrintFailPreference} of + {debug, quiet} -> + indent("[!Format failed!] ~p", [X], Opts, Indent); + {debug, _} -> + indent( + "[PRINT FAIL:] ~80p~n===== PRINT ERROR WAS ~p:~p =====~n~s", + [ + X, + A, + B, + hb_util:bin( + format_trace( + C, + hb_opts:get(stack_print_prefixes, [], #{}) + ) + ) + ], + Opts, + Indent + ); + _ -> + indent("[!Format failed!]", [], Opts, Indent) + end + end. + +do_debug_fmt( + { { {rsa, _PublicExpnt1}, _Priv1, _Priv2 }, + { {rsa, _PublicExpnt2}, Pub } + }, + Opts, Indent +) -> + format_address(Pub, Opts, Indent); +do_debug_fmt( + { AtomValue, + { + { {rsa, _PublicExpnt1}, _Priv1, _Priv2 }, + { {rsa, _PublicExpnt2}, Pub } + } + }, + Opts, Indent +) -> + AddressString = format_address(Pub, Opts, Indent), + indent("~p: ~s", [AtomValue, AddressString], Opts, Indent); +do_debug_fmt({explicit, X}, Opts, Indent) -> + indent("[Explicit:] ~p", [X], Opts, Indent); +do_debug_fmt({string, X}, Opts, Indent) -> + indent("~s", [X], Opts, Indent); +do_debug_fmt({trace, Trace}, Opts, Indent) -> + indent("~n~s", [trace(Trace)], Opts, Indent); +do_debug_fmt({as, undefined, Msg}, Opts, Indent) -> + "\n" ++ indent("Subresolve => ", [], Opts, Indent) ++ + maybe_multiline(Msg, Opts, Indent + 1); +do_debug_fmt({as, DevID, Msg}, Opts, Indent) -> + "\n" ++ indent("Subresolve as ~s => ", [DevID], Opts, Indent) ++ + maybe_multiline(Msg, Opts, Indent + 1); +do_debug_fmt({X, Y}, Opts, Indent) when is_atom(X) and is_atom(Y) -> + indent("~p: ~p", [X, Y], Opts, Indent); +do_debug_fmt({X, Y}, Opts, Indent) when is_record(Y, tx) -> + indent("~p: [TX item]~n~s", + [X, ar_bundles:format(Y, Indent + 1, Opts)], + Opts, + Indent + ); +do_debug_fmt({X, Y}, Opts, Indent) when is_map(Y); is_list(Y) -> + Formatted = maybe_multiline(Y, Opts, Indent + 1), + indent( + case is_binary(X) of + true -> "~s"; + false -> "~p" + end ++ "~s", + [ + X, + case is_multiline(Formatted) of + true -> " ==>" ++ Formatted; + false -> ": " ++ Formatted + end + ], + Opts, + Indent + ); +do_debug_fmt({X, Y}, Opts, Indent) -> + indent( + "~s: ~s", + [ + remove_leading_noise(term(X, Opts, Indent)), + remove_leading_noise(term(Y, Opts, Indent)) + ], + Opts, + Indent + ); +do_debug_fmt(TX, Opts, Indent) when is_record(TX, tx) -> + indent("[TX item]~n~s", + [ar_bundles:format(TX, Indent, Opts)], + Opts, + Indent + ); +do_debug_fmt(MaybePrivMap, Opts, Indent) when is_map(MaybePrivMap) -> + Map = hb_private:reset(MaybePrivMap), + case maybe_format_short(Map, Opts, Indent) of + {ok, SimpleFmt} -> SimpleFmt; + error -> + "\n" ++ lists:flatten(message(Map, Opts, Indent)) + end; +do_debug_fmt(Tuple, Opts, Indent) when is_tuple(Tuple) -> + format_tuple(Tuple, Opts, Indent); +do_debug_fmt(X, Opts, Indent) when is_binary(X) -> + indent("~s", [binary(X)], Opts, Indent); +do_debug_fmt(Str = [X | _], Opts, Indent) when is_integer(X) andalso X >= 32 andalso X < 127 -> + indent("~s", [Str], Opts, Indent); +do_debug_fmt(MsgList, Opts, Indent) when is_list(MsgList) -> + format_list(MsgList, Opts, Indent); +do_debug_fmt(X, Opts, Indent) -> + indent("~80p", [X], Opts, Indent). + +%% @doc If the user attempts to print a wallet, format it as an address. +format_address(Wallet, Opts, Indent) -> + indent("Wallet [Addr: ~s]", + [short_id(hb_util:human_id(ar_wallet:to_address(Wallet)))], + Opts, + Indent + ). + +%% @doc Helper function to format tuples with arity greater than 2. +format_tuple(Tuple, Opts, Indent) -> + to_lines(lists:map( + fun(Elem) -> + term(Elem, Opts, Indent) + end, + tuple_to_list(Tuple) + )). + +%% @doc Format a list. Comes in three forms: all on one line, individual items +%% on their own line, or each item a multi-line string. +format_list(MsgList, Opts, Indent) -> + case maybe_format_short(MsgList, Opts, Indent) of + {ok, SimpleFmt} -> SimpleFmt; + error -> + "\n" ++ + indent("List [~w] {", [length(MsgList)], Opts, Indent) ++ + format_list_lines(MsgList, Opts, Indent) + end. + +%% @doc Format a list as a multi-line string. +format_list_lines(MsgList, Opts, Indent) -> + Numbered = hb_util:number(MsgList), + Lines = + lists:map( + fun({N, Msg}) -> + format_list_item(N, Msg, Opts, Indent) + end, + Numbered + ), + AnyLong = + lists:any( + fun({Mode, _}) -> Mode == multiline end, + Lines + ), + case AnyLong of + false -> + "\n" ++ + remove_trailing_noise( + lists:flatten( + lists:map( + fun({_, Line}) -> + Line + end, + Lines + ) + ) + ) ++ + "\n" ++ + indent("}", [], Opts, Indent); + true -> + "\n" ++ + lists:flatten(lists:map( + fun({N, Msg}) -> + {_, Line} = format_list_item(multiline, N, Msg, Opts, Indent), + Line + end, + Numbered + )) ++ indent("}", [], Opts, Indent) + end. + +%% @doc Format a single element of a list. +format_list_item(N, Msg, Opts, Indent) -> + case format_list_item(short, N, Msg, Opts, Indent) of + {short, String} -> {short, String}; + error -> format_list_item(multiline, N, Msg, Opts, Indent) + end. +format_list_item(short, N, Msg, Opts, Indent) -> + case maybe_format_short(Msg, Opts, Indent) of + {ok, SimpleFmt} -> + {short, indent("~s => ~s~n", [N, SimpleFmt], Opts, Indent + 1)}; + error -> error + end; +format_list_item(multiline, N, Msg, Opts, Indent) -> + Formatted = + case is_multiline(Base = term(Msg, Opts, Indent + 2)) of + true -> Base; + false -> remove_leading_noise(Base) + end, + { + multiline, + indent( + "~s => ~s~n", + [N, Formatted], + Opts, + Indent + 1 + ) + }. + +%% @doc Join a list of strings and remove trailing noise. +to_lines(Elems) -> + remove_trailing_noise(do_to_lines(Elems)). +do_to_lines([]) -> []; +do_to_lines(In =[RawElem | Rest]) -> + Elem = lists:flatten(RawElem), + case lists:member($\n, Elem) of + true -> lists:flatten(lists:join("\n", In)); + false -> Elem ++ ", " ++ do_to_lines(Rest) + end. + +%% @doc Remove any leading or trailing noise from a string. +remove_noise(Str) -> + remove_leading_noise(remove_trailing_noise(Str)). + +%% @doc Remove any leading whitespace from a string. +remove_leading_noise(Str) -> + remove_leading_noise(Str, ?NOISE_CHARS). +remove_leading_noise(Bin, Noise) when is_binary(Bin) -> + hb_util:bin(remove_leading_noise(hb_util:list(Bin), Noise)); +remove_leading_noise([], _) -> []; +remove_leading_noise([Char|Str], Noise) -> + case lists:member(Char, Noise) of + true -> + remove_leading_noise(Str, Noise); + false -> [Char|Str] + end. + +%% @doc Remove trailing noise characters from a string. By default, this is +%% whitespace, newlines, and `,'. +remove_trailing_noise(Str) -> + removing_trailing_noise(Str, ?NOISE_CHARS). +removing_trailing_noise(Bin, Noise) when is_binary(Bin) -> + removing_trailing_noise(binary:bin_to_list(Bin), Noise); +removing_trailing_noise(BinList, Noise) when is_list(BinList) -> + case lists:member(lists:last(BinList), Noise) of + true -> + removing_trailing_noise(lists:droplast(BinList), Noise); + false -> BinList + end. + +%% @doc Format a string with an indentation level. +indent(Str, Indent) -> indent(Str, #{}, Indent). +indent(Str, Opts, Indent) -> indent(Str, [], Opts, Indent). +indent(FmtStr, Terms, Opts, Ind) -> + IndentSpaces = hb_opts:get(debug_print_indent, Opts), + EscapedFmt = escape_format(FmtStr), + lists:droplast( + lists:flatten( + io_lib:format( + [$\s || _ <- lists:seq(1, Ind * IndentSpaces)] ++ + lists:flatten(EscapedFmt) ++ "\n", + Terms + ) + ) + ). + +%% @doc Escape a string for use as an io_lib:format specifier. +escape_format(Str) when is_list(Str) -> + re:replace( + Str, + "~([a-z\\-_]+@[0-9]+\\.[0-9]+)", "~~\\1", + [global, {return, list}] + ); +escape_format(Else) -> Else. + +%% @doc Format an error message as a string. +error(ErrorMsg, Opts) -> + Type = hb_ao:get(<<"type">>, ErrorMsg, <<"">>, Opts), + Details = hb_ao:get(<<"details">>, ErrorMsg, <<"">>, Opts), + Stacktrace = hb_ao:get(<<"stacktrace">>, ErrorMsg, <<"">>, Opts), + hb_util:bin( + [ + <<"Termination type: '">>, Type, + <<"'\n\nStacktrace:\n\n">>, Stacktrace, + <<"\n\nError details:\n\n">>, Details + ] + ). + +%% @doc Take a series of strings or a combined string and format as a +%% single string with newlines and indentation to the given level. Note: This +%% function returns a binary. +indent_lines(Strings, Indent) when is_binary(Strings) -> + indent_lines(binary:split(Strings, <<"\n">>, [global]), Indent); +indent_lines(Strings, Indent) when is_list(Strings) -> + hb_util:bin(lists:join( + "\n", + [ + indent(hb_util:list(String), #{}, Indent) + || + String <- Strings + ] + )). + +%% @doc Format a binary as a short string suitable for printing. +binary(Bin) -> + case short_id(Bin) of + undefined -> + MaxBinPrint = hb_opts:get(debug_print_binary_max), + Printable = + binary:part( + Bin, + 0, + case byte_size(Bin) of + X when X < MaxBinPrint -> X; + _ -> MaxBinPrint + end + ), + PrintSegment = + case is_human_binary(Printable) of + true -> Printable; + false -> hb_util:encode(Printable) + end, + lists:flatten( + [ + "\"", + [PrintSegment], + case Printable == Bin of + true -> "\""; + false -> + io_lib:format( + "...\" <~s bytes>", + [hb_util:human_int(byte_size(Bin))] + ) + end + ] + ); + ShortID -> + lists:flatten(io_lib:format("~s", [ShortID])) + end. + + +%% @doc Format a map as either a single line or a multi-line string depending +%% on the value of the `debug_print_map_line_threshold' runtime option. +maybe_multiline(X, Opts, Indent) -> + case maybe_format_short(X, Opts, Indent) of + {ok, SimpleFmt} -> SimpleFmt; + error -> + "\n" ++ lists:flatten(message(X, Opts, Indent)) + end. + +%% @doc Attempt to generate a short formatting of a message, using the given +%% node options. +maybe_format_short(X, Opts, _Indent) -> + MaxLen = hb_opts:get(debug_print_map_line_threshold, 100, Opts), + SimpleFmt = + case is_binary(X) of + true -> binary(X); + false -> io_lib:format("~p", [X]) + end, + case is_multiline(SimpleFmt) orelse (lists:flatlength(SimpleFmt) > MaxLen) of + true -> error; + false -> {ok, SimpleFmt} + end. + +%% @doc Is the given string a multi-line string? +is_multiline(Str) -> + lists:member($\n, Str). + +%% @doc Format and print an indented string to standard error. +eunit_print(FmtStr, FmtArgs) -> + io:format( + standard_error, + "~n~s ", + [indent(FmtStr ++ "...", FmtArgs, #{}, 4)] + ). + +%% @doc Print the trace of the current stack, up to the first non-hyperbeam +%% module. Prints each stack frame on a new line, until it finds a frame that +%% does not start with a prefix in the `stack_print_prefixes' hb_opts. +%% Optionally, you may call this function with a custom label and caller info, +%% which will be used instead of the default. +print_trace(Stack, CallMod, CallFunc, CallLine) -> + print_trace(Stack, "HB TRACE", + lists:flatten(io_lib:format("[~s:~w ~p]", + [CallMod, CallLine, CallFunc]) + )). + +print_trace(Stack, Label, CallerInfo) -> + io:format(standard_error, "=== ~s ===~s==>~n~s", + [ + Label, CallerInfo, + lists:flatten(trace(Stack)) + ]). + +%% @doc Format a stack trace as a list of strings, one for each stack frame. +%% Each stack frame is formatted if it matches the `stack_print_prefixes' +%% option. At the first frame that does not match a prefix in the +%% `stack_print_prefixes' option, the rest of the stack is not formatted. +trace(Stack) -> + format_trace(Stack, hb_opts:get(stack_print_prefixes, [], #{})). +format_trace([], _) -> []; +format_trace([Item|Rest], Prefixes) -> + case element(1, Item) of + Atom when is_atom(Atom) -> + case true of %is_hb_module(Atom, Prefixes) of + true -> + [ + format_trace(Item, Prefixes) | + format_trace(Rest, Prefixes) + ]; + false -> [] + end; + _ -> [] + end; +format_trace({Func, ArityOrTerm, Extras}, Prefixes) -> + format_trace({no_module, Func, ArityOrTerm, Extras}, Prefixes); +format_trace({Mod, Func, ArityOrTerm, Extras}, _Prefixes) -> + ExtraMap = hb_maps:from_list(Extras), + indent( + "~p:~p/~p [~s]~n", + [ + Mod, Func, ArityOrTerm, + case hb_maps:get(line, ExtraMap, undefined) of + undefined -> "No details"; + Line -> + hb_maps:get(file, ExtraMap) + ++ ":" ++ integer_to_list(Line) + end + ], + #{}, + 1 + ). + +%% @doc Print a trace to the standard error stream. +print_trace_short(Trace, Mod, Func, Line) -> + io:format(standard_error, "=== [ HB SHORT TRACE ~p:~w ~p ] ==> ~s~n", + [ + Mod, Line, Func, + trace_short(Trace) + ] + ). + +%% @doc Return a list of calling modules and lines from a trace, removing all +%% frames that do not match the `stack_print_prefixes' option. +trace_to_list(Trace) -> + Prefixes = hb_opts:get(stack_print_prefixes, [], #{}), + lists:filtermap( + fun(TraceItem) when is_binary(TraceItem) -> + {true, TraceItem}; + (TraceItem) -> + Formatted = format_trace_element(TraceItem), + case hb_util:is_hb_module(Formatted, Prefixes) of + true -> {true, Formatted}; + false -> false + end + end, + Trace + ). + +%% @doc Format a trace to a short string. +trace_short() -> trace_short(get_trace(erlang)). +trace_short(Type) when is_atom(Type) -> trace_short(get_trace(Type)); +trace_short(Trace) when is_list(Trace) -> + lists:join(" / ", lists:reverse(trace_to_list(Trace))). + +%% @doc Format a trace element in form `mod:line' or `mod:func' for Erlang +%% traces, or their raw form for others. +format_trace_element(Bin) when is_binary(Bin) -> Bin; +format_trace_element({Mod, Line}) -> + lists:flatten(io_lib:format("~p:~p", [Mod, Line])); +format_trace_element({Mod, _, _, [{file, _}, {line, Line}|_]}) -> + lists:flatten(io_lib:format("~p:~p", [Mod, Line])); +format_trace_element({Mod, Func, _ArityOrTerm, _Extras}) -> + lists:flatten(io_lib:format("~p:~p", [Mod, Func])). + +%% @doc Utility function to help macro `?trace/0' remove the first frame of the +%% stack trace. +trace_macro_helper(Fun, {_, {_, Stack}}, Mod, Func, Line) -> + Fun(Stack, Mod, Func, Line). + +%% @doc Get the trace of the current execution. If the argument is `erlang', +%% we return the Erlang stack trace. If the argument is `ao', we return the +%% AO-Core execution stack. +get_trace(erlang) -> + case catch error(debugging_print) of + {_, {_, Stack}} -> normalize_trace(Stack); + _ -> [] + end; +get_trace(ao) -> + case get(ao_stack) of + undefined -> []; + Stack -> Stack + end. + +%% @doc Remove all calls from this module from the top of a trace. +normalize_trace([]) -> []; +normalize_trace([{Mod, _, _, _}|Rest]) when Mod == ?MODULE -> + normalize_trace(Rest); +normalize_trace(Trace) -> Trace. + +%% @doc Format a message for printing, optionally taking an indentation level +%% to start from. +message(Item) -> message(Item, #{}). +message(Item, Opts) -> message(Item, Opts, 0). +message(Bin, Opts, Indent) when is_binary(Bin) -> + indent( + binary(Bin), + Opts, + Indent + ); +message(List, Opts, Indent) when is_list(List) -> + % Remove the leading newline from the formatted list, if it exists. + case term(List, Opts, Indent) of + [$\n | String] -> String; + String -> String + end; +message(RawMap, Opts, Indent) when is_map(RawMap) -> + % Should we filter out the priv key? + FilterPriv = hb_opts:get(debug_show_priv, false, Opts), + MainPriv = hb_maps:get(<<"priv">>, RawMap, #{}, Opts), + % Add private keys to the output if they are not hidden. Opt takes 3 forms: + % 1. `false' -- never show priv + % 2. `if_present' -- show priv only if there are keys inside + % 2. `always' -- always show priv + FooterKeys = + case {FilterPriv, MainPriv} of + {false, _} -> []; + {if_present, #{}} -> []; + {_, Priv} -> [{<<"!Private!">>, Priv}] + end, + Map = + case FilterPriv of + false -> RawMap; + _ -> hb_private:reset(RawMap) + end, + % Define helper functions for formatting elements of the map. + ValOrUndef = + fun(<<"hashpath">>) -> + case Map of + #{ <<"priv">> := #{ <<"hashpath">> := HashPath } } -> + short_id(HashPath); + _ -> + undefined + end; + (Key) -> + case dev_message:get(Key, Map, Opts) of + {ok, Val} -> + case short_id(Val) of + undefined -> Val; + ShortID -> ShortID + end; + {error, _} -> undefined + end + end, + FilterUndef = + fun(List) -> + lists:filter(fun({_, undefined}) -> false; (_) -> true end, List) + end, + % Prepare the metadata row for formatting. + % Note: We try to get the IDs _if_ they are *already* in the map. We do not + % force calculation of the IDs here because that may cause significant + % overhead unless the `debug_ids' option is set. + IDMetadata = + case hb_opts:get(debug_ids, false, #{}) of + false -> + [ + {<<"#P">>, ValOrUndef(<<"hashpath">>)}, + {<<"*U">>, ValOrUndef(<<"unsigned_id">>)}, + {<<"*S">>, ValOrUndef(<<"id">>)} + ]; + true -> + {ok, UID} = dev_message:id(Map, #{}, Opts), + {ok, ID} = + dev_message:id(Map, #{ <<"commitments">> => <<"all">> }, Opts), + [ + {<<"#P">>, short_id(ValOrUndef(<<"hashpath">>))}, + {<<"*U">>, short_id(UID)} + ] ++ + case ID of + UID -> []; + _ -> [{<<"*S">>, short_id(ID)}] + end + end, + CommitterMetadata = + case hb_opts:get(debug_committers, true, Opts) of + false -> []; + true -> + case dev_message:committers(Map, #{}, Opts) of + {ok, []} -> []; + {ok, [Committer]} -> + [{<<"Comm.">>, short_id(Committer)}]; + {ok, Committers} -> + [ + { + <<"Comms.">>, + string:join( + lists:map( + fun(X) -> + [short_id(X)] + end, + Committers + ), + ", " + ) + } + ] + end + end, + % Concatenate the present metadata rows. + Metadata = FilterUndef(lists:flatten([IDMetadata, CommitterMetadata])), + % Format the metadata row. + Header = + indent("Message [~s] {", + [ + string:join( + [ + io_lib:format("~s: ~s", [Lbl, Val]) + || + {Lbl, Val} <- Metadata, + Val /= undefined + ], + ", " + ) + ], + Opts, + Indent + ), + % Put the path and device rows into the output at the _top_ of the map. + PriorityKeys = + [ + {<<"device">>, ValOrUndef(<<"device">>)}, + {<<"path">>, ValOrUndef(<<"path">>)}, + {<<"commitments">>, ValOrUndef(<<"commitments">>)} + ], + % Concatenate the path and device rows with the rest of the key values. + UnsortedGeneralKeyVals = + maps:to_list( + maps:without( + [ PriorityKey || {PriorityKey, _} <- PriorityKeys ], + Map + ) + ), + KeyVals = + FilterUndef(PriorityKeys) ++ + lists:sort( + fun({K1, _}, {K2, _}) -> K1 < K2 end, + UnsortedGeneralKeyVals + ) ++ + FooterKeys, + % Format the remaining 'normal' keys and values. + Res = lists:map( + fun({Key, Val}) -> + NormKey = hb_ao:normalize_key(Key, Opts#{ error_strategy => ignore }), + KeyStr = + case NormKey of + undefined -> + io_lib:format("~p [!!! INVALID KEY !!!]", [Key]); + _ -> + hb_ao:normalize_key(Key) + end, + indent( + "~s => ~s~n", + [ + lists:flatten([KeyStr]), + case Val of + NextMap when is_map(NextMap) -> + maybe_multiline(NextMap, Opts, Indent + 2); + Next when is_list(Next); is_record(Next, tx) -> + remove_leading_noise(term(Next, Opts, Indent + 2)); + _ when (byte_size(Val) == 32) -> + Short = short_id(Val), + io_lib:format("~s [*]", [Short]); + _ when byte_size(Val) == 43 -> + short_id(Val); + _ when byte_size(Val) == 87 -> + io_lib:format("~s [#p]", [short_id(Val)]); + Bin when is_binary(Bin) -> + binary(Bin); + Link when ?IS_LINK(Link) -> + remove_leading_noise( + hb_util:bin( + hb_link:format(Link, Opts, Indent + 2) + ) + ); + Other -> + io_lib:format("~p", [Other]) + end + ], + Opts, + Indent + 1 + ) + end, + KeyVals + ), + case Res of + [] -> lists:flatten(Header ++ " [Empty] }"); + _ -> + lists:flatten( + Header ++ ["\n"] ++ Res ++ indent("}", Indent) + ) + end; +message(Item, Opts, Indent) -> + % Whatever we have is not a message map. + indent("~p", [Item], Opts, Indent). + +%%% Utility functions. + +%% @doc Return a short ID for the different types of IDs used in AO-Core. +short_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 32 -> + short_id(hb_util:human_id(Bin)); +short_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 43 -> + << FirstTag:5/binary, _:33/binary, LastTag:5/binary >> = Bin, + << FirstTag/binary, "..", LastTag/binary >>; +short_id(Bin) when byte_size(Bin) > 43 andalso byte_size(Bin) < 100 -> + case binary:split(Bin, <<"/">>, [trim_all, global]) of + [First, Second] when byte_size(Second) == 43 -> + FirstEnc = short_id(First), + SecondEnc = short_id(Second), + << FirstEnc/binary, "/", SecondEnc/binary >>; + [First, Key] -> + FirstEnc = short_id(First), + << FirstEnc/binary, "/", Key/binary >>; + _ -> + Bin + end; +short_id(<< "/", SingleElemHashpath/binary >>) -> + Enc = short_id(SingleElemHashpath), + if is_binary(Enc) -> << "/", Enc/binary >>; + true -> undefined + end; +short_id(Key) when byte_size(Key) < 43 -> Key; +short_id(_) -> undefined. + +%% @doc Determine whether a binary is human-readable. +is_human_binary(Bin) when is_binary(Bin) -> + case unicode:characters_to_binary(Bin) of + {error, _, _} -> false; + _ -> true + end. \ No newline at end of file diff --git a/src/hb_gateway_client.erl b/src/hb_gateway_client.erl new file mode 100644 index 000000000..2b6479a6c --- /dev/null +++ b/src/hb_gateway_client.erl @@ -0,0 +1,434 @@ +%%% @doc Implementation of Arweave's GraphQL API to gain access to specific +%%% items of data stored on the network. +%%% +%%% This module must be used to get full HyperBEAM `structured@1.0' form messages +%%% from data items stored on the network, as Arweave gateways do not presently +%%% expose all necessary fields to retrieve this information outside of the +%%% GraphQL API. When gateways integrate serving in `httpsig@1.0' form, this +%%% module will be deprecated. +-module(hb_gateway_client). +%% Raw access primitives: +-export([query/2, query/3, query/4, query/5]). +-export([read/2, data/2, result_to_message/2, item_spec/0]). +%% Application-specific data access functions: +-export([scheduler_location/2]). +-include_lib("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Get a data item (including data and tags) by its ID, using the node's +%% GraphQL peers. +%% It uses the following GraphQL schema: +%% type Transaction { +%% id: ID! +%% anchor: String! +%% signature: String! +%% recipient: String! +%% owner: Owner { address: String! key: String! }! +%% fee: Amount! +%% quantity: Amount! +%% data: MetaData! +%% tags: [Tag { name: String! value: String! }!]! +%% } +%% type Amount { +%% winston: String! +%% ar: String! +%% } +read(ID, Opts) -> + {Query, Variables} = case maps:is_key(<<"subindex">>, Opts) of + true -> + Tags = subindex_to_tags(maps:get(<<"subindex">>, Opts)), + { + << + "query($transactionIds: [ID!]!) { ", + "transactions(ids: $transactionIds,", + "tags: ", (Tags)/binary , ",", + "first: 1){ ", + "edges { ", (item_spec())/binary , " } ", + "} ", + "} " + >>, + #{ + <<"transactionIds">> => [hb_util:human_id(ID)] + } + }; + false -> + { + << + "query($transactionIds: [ID!]!) { ", + "transactions(ids: $transactionIds, first: 1){ ", + "edges { ", (item_spec())/binary , " } ", + "} ", + "} " + >>, + #{ + <<"transactionIds">> => [hb_util:human_id(ID)] + } + } + end, + case query(Query, Variables, Opts) of + {error, Reason} -> {error, Reason}; + {ok, GqlMsg} -> + case hb_ao:get(<<"data/transactions/edges/1/node">>, GqlMsg, Opts) of + not_found -> {error, not_found}; + Item = #{<<"id">> := ID} -> result_to_message(ID, Item, Opts) + end + end. + +%% @doc Gives the fields of a transaction that are needed to construct an +%% ANS-104 message. +item_spec() -> + <<""" + node { + id + anchor + signature + recipient + owner { key } + fee { winston } + quantity { winston } + tags { name value } + data { size } + } + cursor + """>>. + +%% @doc Get the data associated with a transaction by its ID, using the node's +%% Arweave `gateway' peers. The item is expected to be available in its +%% unmodified (by caches or other proxies) form at the following location: +%% https:///raw/ +%% where `' is the base64-url-encoded transaction ID. +data(ID, Opts) -> + Req = #{ + <<"multirequest-accept-status">> => 200, + <<"multirequest-responses">> => 1, + <<"path">> => <<"/raw/", ID/binary>>, + <<"method">> => <<"GET">> + }, + case hb_http:request(Req, Opts) of + {ok, Res} -> + ?event(gateway, + {data, + {id, ID}, + {response, Res}, + {body, hb_ao:get(<<"body">>, Res, <<>>, Opts)} + } + ), + {ok, hb_ao:get(<<"body">>, Res, <<>>, Opts)}; + Res -> + ?event(gateway, {request_error, {id, ID}, {response, Res}}), + {error, no_viable_gateway} + end. + +%% @doc Find the location of the scheduler based on its ID, through GraphQL. +scheduler_location(Address, Opts) -> + Query = + <<"query($SchedulerAddrs: [String!]!) { ", + "transactions(", + "owners: $SchedulerAddrs, ", + "tags: { name: \"Type\" values: [\"Scheduler-Location\"] }, ", + "first: 1", + "){ ", + "edges { ", + (item_spec())/binary , + " } ", + "} ", + "}">>, + Variables = #{ <<"SchedulerAddrs">> => [Address] }, + case query(Query, Variables, Opts) of + {error, Reason} -> + ?event({scheduler_location, {query, Query}, {error, Reason}}), + {error, Reason}; + {ok, GqlMsg} -> + ?event({scheduler_location_req, {query, Query}, {response, GqlMsg}}), + case hb_ao:get(<<"data/transactions/edges/1/node">>, GqlMsg, Opts) of + not_found -> + ?event(scheduler_location, + {graphql_scheduler_location_not_found, + {address, Address} + } + ), + {error, not_found}; + Item = #{ <<"id">> := ID } -> + ?event(scheduler_location, + {found_via_graphql, + {address, Address}, + {id, ID} + } + ), + result_to_message(ID, Item, Opts) + end + end. + +%% @doc Run a GraphQL request encoded as a binary. The node message may contain +%% a list of URLs to use, optionally as a tuple with an additional map of options +%% to use for the request. +query(Query, Opts) -> + query(Query, undefined, Opts). +query(Query, Variables, Opts) -> + query(Query, Variables, undefined, Opts). +query(Query, Variables, Node, Opts) -> + query(Query, Variables, Node, undefined, Opts). +query(Query, Variables, Node, Operation, Opts) -> + % Either use the given node if provided, or use the local machine's routes + % to find the GraphQL endpoint. + Path = + case Node of + undefined -> <<"/graphql">>; + _ -> << Node/binary, "/graphql">> + end, + ?event(graphql, + {request, + {path, Path}, + {query, Query}, + {variables, Variables}, + {operation, Operation} + } + ), + CombinedQuery = + maps:filter( + fun(_, V) -> V =/= undefined end, + #{ + <<"query">> => Query, + <<"variables">> => Variables, + <<"operationName">> => Operation + } + ), + % Find the routes for the GraphQL API. + Res = hb_http:request( + #{ + % Add options for the HTTP request, in case it is being made to + % many nodes. + <<"multirequest-responses">> => 1, + <<"multirequest-admissible-status">> => 200, + <<"multirequest-admissible">> => + #{ + <<"device">> => <<"query@1.0">>, + <<"path">> => <<"has-results">> + }, + % Main request fields + <<"method">> => <<"POST">>, + <<"path">> => <<"/graphql">>, + <<"content-type">> => <<"application/json">>, + <<"body">> => hb_json:encode(CombinedQuery) + }, + Opts + ), + case Res of + {ok, Msg} -> + {ok, hb_json:decode(hb_ao:get(<<"body">>, Msg, <<>>, Opts))}; + {error, Reason} -> {error, Reason} + end. + +%% @doc Takes a GraphQL item node, matches it with the appropriate data from a +%% gateway, then returns `{ok, ParsedMsg}'. +result_to_message(Item, Opts) -> + case hb_maps:get(<<"id">>, Item, not_found, Opts) of + ExpectedID when is_binary(ExpectedID) -> + result_to_message(ExpectedID, Item, Opts); + _ -> + result_to_message(undefined, Item, Opts) + end. +result_to_message(ExpectedID, Item, Opts) -> + GQLOpts = + Opts#{ + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>] + }, + % We have the headers, so we can get the data. + Data = + case hb_maps:get(<<"data">>, Item, not_found, GQLOpts) of + BinData when is_binary(BinData) -> BinData; + #{ <<"size">> := 0 } -> <<>>; + _ -> + {ok, Bytes} = data(ExpectedID, Opts), + Bytes + end, + DataSize = byte_size(Data), + ?event(gateway, {data, {id, ExpectedID}, {data, Data}, {item, Item}}, Opts), + % Convert the response to an ANS-104 message. + Tags = hb_maps:get(<<"tags">>, Item, tags_not_found, GQLOpts), + Signature = + hb_util:decode( + hb_maps:get(<<"signature">>, Item, not_found, GQLOpts) + ), + SignatureType = + case byte_size(Signature) of + 65 -> {ecdsa, 256}; + 512 -> {rsa, 65537}; + _ -> unsupported_tx_signature_type + end, + TX = + ar_bundles:reset_ids(#tx { + format = ans104, + anchor = + normalize_null(hb_maps:get(<<"anchor">>, Item, not_found, GQLOpts)), + signature = Signature, + signature_type = SignatureType, + target = + decode_or_null( + hb_ao:get_first( + [ + {Item, <<"recipient">>}, + {Item, <<"target">>} + ], + GQLOpts + ) + ), + owner = + hb_util:decode( + hb_util:deep_get(<<"owner/key">>, Item, GQLOpts) + ), + tags = + [ + {Name, Value} + || + #{<<"name">> := Name, <<"value">> := Value} <- Tags + ], + data_size = DataSize, + data = Data + }), + ?event({raw_ans104, TX}), + ?event({ans104_form_response, TX}), + TABM = hb_util:ok(dev_codec_ans104:from(TX, #{}, Opts)), + ?event({decoded_tabm, TABM}), + Structured = hb_util:ok(dev_codec_structured:to(TABM, #{}, Opts)), + % Some graphql nodes do not grant the `anchor' or `last_tx' fields, so we + % verify the data item and optionally add the explicit keys as committed + % fields _if_ the node desires it. + Embedded = + case try ar_bundles:verify_item(TX) catch _:_ -> false end of + true -> + ?event({gql_verify_succeeded, Structured}), + Structured; + _ -> + % The item does not verify on its own, but does the node choose + % to trust the GraphQL API anyway? + case hb_opts:get(ans104_trust_gql, false, Opts) of + false -> + ?event( + warning, + {gql_verify_failed, returning_unverifiable_tx} + ), + Structured; + true -> + % The node trusts the GraphQL API, so we add the explicit + % keys as committed fields. + ?event(warning, + {gql_verify_failed, + adding_trusted_fields, + {tags, Tags} + } + ), + Comms = hb_maps:get(<<"commitments">>, Structured, #{}, Opts), + AttName = hd(hb_maps:keys(Comms, Opts)), + Comm = hb_maps:get(AttName, Comms, not_found, Opts), + Structured#{ + <<"commitments">> => #{ + AttName => + Comm#{ + <<"trusted-keys">> => + hb_ao:normalize_keys( + [ + hb_ao:normalize_key(Name) + || + #{ <<"name">> := Name } <- + hb_maps:values( + hb_ao:normalize_keys( + Tags, + Opts + ), + Opts + ) + ], + Opts + ) + } + } + } + end + end, + {ok, Embedded}. + +normalize_null(null) -> <<>>; +normalize_null(not_found) -> <<>>; +normalize_null(Bin) when is_binary(Bin) -> Bin. + +decode_id_or_null(Bin) when byte_size(Bin) > 0 -> + hb_util:human_id(Bin); +decode_id_or_null(_) -> + <<>>. + +decode_or_null(Bin) when is_binary(Bin) -> + hb_util:decode(Bin); +decode_or_null(_) -> + <<>>. + +%% @doc Takes a list of messages with `name' and `value' fields, and formats +%% them as a GraphQL `tags' argument. +subindex_to_tags(Subindex) -> + Formatted = + lists:map( + fun(Spec) -> + io_lib:format( + "{ name: \"~s\", values: [\"~s\"]}", + [ + hb_ao:get(<<"name">>, Spec), + hb_ao:get(<<"value">>, Spec) + ] + ) + end, + hb_util:message_to_ordered_list(Subindex) + ), + ListInner = + hb_util:bin( + string:join([lists:flatten(E) || E <- Formatted], ", ") + ), + <<"[", ListInner/binary, "]">>. + +%%% Tests +ans104_no_data_item_test() -> + % Start a random node so that all of the services come up. + _Node = hb_http_server:start_node(#{}), + {ok, Res} = read(<<"BOogk_XAI3bvNWnxNxwxmvOfglZt17o4MOVAdPNZ_ew">>, #{}), + ?event(gateway, {get_ans104_test, Res}), + ?event(gateway, {signer, hb_message:signers(Res, #{})}), + ?assert(true). + +%% @doc Test that we can get the scheduler location. +scheduler_location_test() -> + % Start a random node so that all of the services come up. + _Node = hb_http_server:start_node(#{}), + {ok, Res} = + scheduler_location( + <<"fcoN_xJeisVsPXA-trzVAuIiqO3ydLQxM-L4XbrQKzY">>, + #{} + ), + ?event(gateway, {get_scheduler_location_test, Res}), + ?assertEqual(<<"Scheduler-Location">>, hb_ao:get(<<"Type">>, Res, #{})), + ?event(gateway, {scheduler_location, {explicit, hb_ao:get(<<"url">>, Res, #{})}}), + % Will need updating when Legacynet terminates. + ?assertEqual(<<"https://su-router.ao-testnet.xyz">>, hb_ao:get(<<"url">>, Res, #{})). + +%% @doc Test l1 message from graphql +l1_transaction_test() -> + _Node = hb_http_server:start_node(#{}), + {ok, Res} = read(<<"uJBApOt4ma3pTfY6Z4xmknz5vAasup4KcGX7FJ0Of8w">>, #{}), + ?event(gateway, {l1_transaction, Res}), + Data = maps:get(<<"data">>, Res), + ?assertEqual(<<"Hello World">>, Data). + +%% @doc Test l2 message from graphql +l2_dataitem_test() -> + _Node = hb_http_server:start_node(#{}), + {ok, Res} = read(<<"oyo3_hCczcU7uYhfByFZ3h0ELfeMMzNacT-KpRoJK6g">>, #{}), + ?event(gateway, {l2_dataitem, Res}), + Data = maps:get(<<"data">>, Res), + ?assertEqual(<<"Hello World">>, Data). + +%% @doc Test optimistic index +ao_dataitem_test() -> + _Node = hb_http_server:start_node(#{}), + {ok, Res} = read(<<"oyo3_hCczcU7uYhfByFZ3h0ELfeMMzNacT-KpRoJK6g">>, #{ }), + ?event(gateway, {l2_dataitem, Res}), + Data = maps:get(<<"data">>, Res), + ?assertEqual(<<"Hello World">>, Data). \ No newline at end of file diff --git a/src/hb_http.erl b/src/hb_http.erl index 62486b174..7ff8f8c53 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -5,12 +5,14 @@ %%% HTTP requests. -module(hb_http). -export([start/0]). --export([get/2, get/3, post/3, post/4, request/4, request/5]). --export([reply/2, reply/3]). --export([message_to_status/1, req_to_tabm_singleton/2]). +-export([get/2, get/3, post/3, post/4, request/2, request/4, request/5]). +-export([message_to_request/2, reply/4, accept_to_codec/2]). +-export([req_to_tabm_singleton/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). +-define(DEFAULT_FILTER_KEYS, [<<"content-length">>]). + start() -> httpc:set_options([{max_keep_alive_length, 0}]), ok. @@ -18,234 +20,934 @@ start() -> %% @doc Gets a URL via HTTP and returns the resulting message in deserialized %% form. get(Node, Opts) -> get(Node, <<"/">>, Opts). -get(Node, Path, Opts) -> - case request(<<"GET">>, Node, Path, #{}, Opts) of - {ok, Body} -> - {ok, - hb_message:convert( - ar_bundles:deserialize(Body), - converge, - tx, - #{} - ) - }; - Error -> Error - end. +get(Node, PathBin, Opts) when is_binary(PathBin) -> + get(Node, #{ <<"path">> => PathBin }, Opts); +get(Node, Message, Opts) -> + request( + <<"GET">>, + Node, + hb_ao:get(<<"path">>, Message, <<"/">>, Opts), + Message, + Opts + ). %% @doc Posts a message to a URL on a remote peer via HTTP. Returns the %% resulting message in deserialized form. +post(Node, Path, Opts) when is_binary(Path) -> + post(Node, #{ <<"path">> => Path }, Opts); post(Node, Message, Opts) -> post(Node, - hb_converge:get(<<"Path">>, Message, <<"/">>, #{}), + hb_ao:get( + <<"path">>, + Message, + <<"/">>, + Opts#{ topic => ao_internal } + ), Message, Opts ). post(Node, Path, Message, Opts) -> case request(<<"POST">>, Node, Path, Message, Opts) of {ok, Res} -> - {ok, - hb_message:convert( - ar_bundles:deserialize(Res), - converge, - tx, - #{} - ) - }; + ?event(http, {post_response, Res}), + {ok, Res}; Error -> Error end. %% @doc Posts a binary to a URL on a remote peer via HTTP, returning the raw %% binary body. +request(Message, Opts) -> + % Special case: We are not given a peer and a path, so we need to + % preprocess the URL to find them. + {ok, Method, Peer, Path, MessageToSend, NewOpts} = + message_to_request(Message, Opts), + request(Method, Peer, Path, MessageToSend, NewOpts). request(Method, Peer, Path, Opts) -> request(Method, Peer, Path, #{}, Opts). -request(Method, Config, Path, Message, Opts) when is_map(Config) -> - multirequest(Config, Method, Path, Message, Opts); +request(Method, Config = #{ <<"nodes">> := Nodes }, Path, Message, Opts) when is_list(Nodes) -> + % The request has a `route' (see `dev_router' for more details), so we use the + % `multirequest' functionality, rather than a single request. + hb_http_multi:request(Config, Method, Path, Message, Opts); +request(Method, #{ <<"opts">> := ReqOpts, <<"uri">> := URI }, _Path, Message, Opts) -> + % The request has a set of additional options, so we apply them to the + % request. + MergedOpts = + hb_maps:merge( + Opts, + hb_opts:mimic_default_types(ReqOpts, new_atoms, Opts), + Opts + ), + % We also recalculate the request. The order of precidence here is subtle: + % We favor the args given to the function, but the URI rules take precidence + % over that. + {ok, NewMethod, Node, NewPath, NewMsg, NewOpts} = + message_to_request( + Message#{ <<"path">> => URI, <<"method">> => Method }, + MergedOpts + ), + request(NewMethod, Node, NewPath, NewMsg, NewOpts); request(Method, Peer, Path, RawMessage, Opts) -> - Message = hb_converge:ensure_message(RawMessage), - BinPeer = if is_binary(Peer) -> Peer; true -> list_to_binary(Peer) end, - BinPath = hb_path:normalize(hb_path:to_binary(Path)), - ?event(http, {http_outbound, Method, BinPeer, BinPath, Message}), - BinMessage = - case map_size(Message) of - 0 -> <<>>; - _ -> ar_bundles:serialize(hb_message:convert(Message, tx, #{})) - end, + ?event({request, {method, Method}, {peer, Peer}, {path, Path}, {message, RawMessage}}), Req = - #{ - peer => BinPeer, - path => BinPath, - method => Method, - headers => [{<<"Content-Type">>, <<"application/x-ans-104">>}], - body => BinMessage + prepare_request( + hb_maps:get( + <<"codec-device">>, + RawMessage, + <<"httpsig@1.0">>, + Opts + ), + Method, + Peer, + Path, + RawMessage, + Opts + ), + StartTime = os:system_time(millisecond), + % Perform the HTTP request. + {_ErlStatus, Status, Headers, Body} = hb_http_client:req(Req, Opts), + % Process the response. + EndTime = os:system_time(millisecond), + ?event(http_outbound, + { + http_response, + {req, Req}, + {response, + #{ + status => Status, + headers => Headers, + body => Body + } + } }, - case ar_http:req(Req, Opts) of - {ok, Status, Headers, Body} when Status >= 200, Status < 300 -> - ?event({http_got, BinPeer, BinPath, Status, Headers, Body}), - { - case Status of - 201 -> created; - _ -> ok - end, - Body - }; - {ok, Status, _Headers, Body} when Status == 400 -> - ?event({http_got_client_error, BinPeer, BinPath}), - {error, Body}; - {ok, Status, _Headers, Body} when Status > 400 -> - ?event({http_got_server_error, BinPeer, BinPath}), - {unavailable, Body}; - Response -> - ?event({http_error, BinPeer, BinPath, Response}), - Response - end. - -%% @doc Dispatch the same HTTP request to many nodes. Can be configured to -%% await responses from all nodes or just one, and to halt all requests after -%% after it has received the required number of responses, or to leave all -%% requests running until they have all completed. Default: Race for first -%% response. -%% -%% Expects a config message of the following form: -%% /Nodes/1..n: Hostname | #{ hostname => Hostname, address => Address } -%% /Responses: Number of responses to gather -%% /Stop-After: Should we stop after the required number of responses? -%% /Parallel: Should we run the requests in parallel? -multirequest(Config, Method, Path, Message, Opts) -> - Nodes = hb_converge:get(<<"Peers">>, Config, #{}, Opts), - Responses = hb_converge:get(<<"Responses">>, Config, 1, Opts), - StopAfter = hb_converge:get(<<"Stop-After">>, Config, true, Opts), - case hb_converge:get(<<"Parallel">>, Config, false, Opts) of + Opts + ), + % Convert the set-cookie headers into a cookie message, if they are present. + % We do this by extracting the set-cookie headers and converting them into a + % cookie message if they are present. + SetCookieLines = + [ + KeyVal + || + {<<"set-cookie">>, KeyVal} <- Headers + ], + MaybeSetCookie = + case SetCookieLines of + [] -> #{}; + _ -> + ?event( + debug_cookie, + {normalizing_setcookie_headers, + {set_cookie_lines, [ {string, Line} || Line <- SetCookieLines ]} + }, + Opts + ), + {ok, MsgWithCookies} = + dev_codec_cookie:from( + #{ <<"set-cookie">> => SetCookieLines }, + #{}, + Opts + ), + ?event(debug_cookie, {msg_with_cookies, MsgWithCookies}), + MsgWithCookies + end, + % Merge the set-cookie message into the header map, which itself is + % constructed from the header key-value pair list. + HeaderMap = hb_maps:merge(hb_maps:from_list(Headers), MaybeSetCookie, Opts), + NormHeaderMap = hb_ao:normalize_keys(HeaderMap, Opts), + ?event(http_outbound, + {normalized_response_headers, {norm_header_map, NormHeaderMap}}, + Opts + ), + ?event(http_short, + {received, + {status, Status}, + {duration, EndTime - StartTime}, + {method, Method}, + {peer, Peer}, + {path, {string, Path}}, + {body_size, byte_size(Body)} + }), + ReturnAOResult = + hb_opts:get(http_only_result, true, Opts) andalso + hb_maps:get(<<"ao-result">>, NormHeaderMap, false, Opts), + case ReturnAOResult of + Key when is_binary(Key) -> + Msg = http_response_to_httpsig(Status, NormHeaderMap, Body, Opts), + ?event( + http_outbound, + {result_is_single_key, {key, Key}, {msg, Msg}}, + Opts + ), + case {Key, hb_maps:get(Key, Msg, undefined, Opts)} of + {<<"body">>, undefined} -> + {response_status_to_atom(Status), <<>>}; + {_, undefined} -> + {failure, + << + "Result key '", + Key/binary, + "' not found in response from '", + Peer/binary, + "' for path '", + Path/binary, + "': ", + Body/binary + >> + }; + {_, Value} -> + {response_status_to_atom(Status), Value} + end; false -> - serial_multirequest( - Nodes, Responses, Method, Path, Message, Opts); - true -> - parallel_multirequest( - Nodes, Responses, StopAfter, Method, Path, Message, Opts) + % Find the codec device from the headers, if set. + CodecDev = + hb_maps:get( + <<"codec-device">>, + NormHeaderMap, + <<"httpsig@1.0">>, + Opts + ), + outbound_result_to_message( + CodecDev, + Status, + NormHeaderMap, + Body, + Opts + ) end. -serial_multirequest(_Nodes, 0, _Method, _Path, _Message, _Opts) -> []; -serial_multirequest([Node | Nodes], Remaining, Method, Path, Message, Opts) -> - case request(Method, Node, Path, Message, Opts) of - {Status, Res} when Status == ok; Status == error -> - [ - {Status, Res} - | - serial_multirequest(Nodes, Remaining - 1, Method, Path, Message, Opts) - ]; - _ -> - serial_multirequest(Nodes, Remaining, Method, Path, Message, Opts) +%% @doc Convert a HTTP status code to a status atom. +response_status_to_atom(Status) -> + case Status of + 201 -> created; + X when X < 400 -> ok; + X when X < 500 -> error; + _ -> failure end. -%% @doc Dispatch the same HTTP request to many nodes in parallel. -parallel_multirequest(Nodes, Responses, StopAfter, Method, Path, Message, Opts) -> - Ref = make_ref(), - Parent = self(), - Procs = lists:map( - fun(Node) -> - spawn( - fun() -> - Res = request(Method, Node, Path, Message, Opts), - receive no_reply -> stopping - after 0 -> Parent ! {Ref, self(), Res} - end - end - ) - end, - Nodes +%% @doc Convert an HTTP response to a message. +outbound_result_to_message(<<"ans104@1.0">>, Status, Headers, Body, Opts) -> + ?event(http_outbound, + {result_is_ans104, {headers, Headers}, {body, Body}}, + Opts ), - parallel_responses([], Procs, Ref, Responses, StopAfter, Opts). - -%% @doc Collect the necessary number of responses, and stop workers if -%% configured to do so. -parallel_responses(Res, Procs, Ref, 0, false, _Opts) -> - lists:foreach(fun(P) -> P ! no_reply end, Procs), - empty_inbox(Ref), - {ok, Res}; -parallel_responses(Res, Procs, Ref, 0, true, _Opts) -> - lists:foreach(fun(P) -> exit(P, kill) end, Procs), - empty_inbox(Ref), - Res; -parallel_responses(Res, Procs, Ref, Awaiting, StopAfter, Opts) -> - receive - {Ref, Pid, {Status, NewRes}} when Status == ok; Status == error -> - parallel_responses( - [NewRes | Res], - lists:delete(Pid, Procs), - Ref, - Awaiting - 1, - StopAfter, + try ar_bundles:deserialize(Body) of + Deserialized -> + { + response_status_to_atom(Status), + hb_message:convert( + Deserialized, + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ) + } + catch + _Class:ExceptionPattern:Stacktrace -> + % The response message had a `codec-device: ans104@1.0', but we + % failed to deserialize it, so we fallback to HTTPSig. + ?event(http_outbound, + {failed_to_deserialize_ans104_attempting_httpsig, + {headers, Headers}, + {body, Body}, + {error, ExceptionPattern}, + {stacktrace, {trace, Stacktrace}} + }, + Opts + ), + outbound_result_to_message(<<"httpsig@1.0">>, Status, Headers, Body, Opts) + end; +outbound_result_to_message(<<"httpsig@1.0">>, Status, Headers, Body, Opts) -> + ?event(http_outbound, {result_is_httpsig, {body, Body}}, Opts), + { + response_status_to_atom(Status), + http_response_to_httpsig(Status, Headers, Body, Opts) + }. + +%% @doc Convert a HTTP response to a httpsig message. +http_response_to_httpsig(Status, HeaderMap, Body, Opts) -> + (hb_message:convert( + hb_maps:merge( + HeaderMap#{ <<"status">> => hb_util:bin(Status) }, + case Body of + <<>> -> #{}; + _ -> #{ <<"body">> => Body } + end, + Opts + ), + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + <<"httpsig@1.0">>, + Opts + ))#{ <<"status">> => hb_util:int(Status) }. + +%% @doc Given a message, return the information needed to make the request. +message_to_request(M, Opts) -> + % Get the route for the message + Res = route_to_request(M, RouteRes = dev_router:route(M, Opts), Opts), + ?event(debug_http, {route_res, {route_res, RouteRes}, {full_res, Res}, {msg, M}}), + Res. + +%% @doc Parse a `dev_router:route' response and return a tuple of request +%% parameters. +route_to_request(M, {ok, URI}, Opts) when is_binary(URI) -> + route_to_request(M, {ok, #{ <<"uri">> => URI, <<"opts">> => #{} }}, Opts); +route_to_request(M, {ok, #{ <<"uri">> := XPath, <<"opts">> := ReqOpts}}, Opts) -> + % The request is a direct HTTP URL, so we need to split the path into a + % host and path. + URI = uri_string:parse(XPath), + ?event(http_outbound, {parsed_uri, {uri, {explicit, URI}}}), + Method = hb_ao:get(<<"method">>, M, <<"GET">>, Opts), + % We must remove the path and host from the message, because they are not + % valid for outbound requests. The path is retrieved from the route, and + % the host should already be known to the caller. + MsgWithoutMeta = hb_maps:without([<<"path">>, <<"host">>], M, Opts), + Port = + case maps:get(port, URI, undefined) of + undefined -> + % If no port is specified, use 80 for HTTP and 443 + % for HTTPS. + case XPath of + <<"https", _/binary>> -> <<"443">>; + _ -> <<"80">> + end; + X -> integer_to_binary(X) + end, + Protocol = maps:get(scheme, URI, <<"https">>), + Host = maps:get(host, URI, <<"localhost">>), + Node = << Protocol/binary, "://", Host/binary, ":", Port/binary >>, + PathParts = [maps:get(path, URI, <<"/">>)] ++ + case maps:get(query, URI, <<>>) of + <<>> -> []; + Query -> [<<"?", Query/binary>>] + end, + Path = iolist_to_binary(PathParts), + ?event(http_outbound, {parsed_req, {node, Node}, {method, Method}, {path, Path}}), + {ok, Method, Node, Path, MsgWithoutMeta, hb_util:deep_merge(Opts, ReqOpts, Opts)}; +route_to_request(M, {ok, Routes}, Opts) -> + ?event(http_outbound, {found_routes, {req, M}, {routes, Routes}}), + % The result is a route, so we leave it to `request' to handle it. + Path = hb_ao:get(<<"path">>, M, <<"/">>, Opts), + Method = hb_ao:get(<<"method">>, M, <<"GET">>, Opts), + % We must remove the path and host from the message, because they are not + % valid for outbound requests. The path is retrieved from the route, and + % the host should already be known to the caller. + MsgWithoutMeta = hb_maps:without([<<"path">>, <<"host">>], M, Opts), + {ok, Method, Routes, Path, MsgWithoutMeta, Opts}; +route_to_request(M, {error, Reason}, _Opts) -> + {error, {no_viable_route, {reason, Reason}, {message, M}}}. + +%% @doc Turn a set of request arguments into a request message, formatted in the +%% preferred format. This function honors the `accept-bundle' option, if it is +%% already present in the message, and sets it to `true' if it is not. +prepare_request(Format, Method, Peer, Path, RawMessage, Opts) -> + Message = hb_ao:normalize_keys(RawMessage, Opts), + % Generate a `cookie' key for the message, if an unencoded cookie is + % present. + {MaybeCookie, WithoutCookie} = + case dev_codec_cookie:extract(Message, #{}, Opts) of + {ok, NoCookies} when map_size(NoCookies) == 0 -> + {#{}, Message}; + {ok, _Cookies} -> + {ok, #{ <<"cookie">> := CookieLines }} = + dev_codec_cookie:to( + Message, + #{ <<"format">> => <<"cookie">> }, + Opts + ), + {ok, CookieReset} = dev_codec_cookie:reset(Message, Opts), + ?event(http, {cookie_lines, CookieLines}), + { + #{ <<"cookie">> => CookieLines }, + CookieReset + } + end, + % Remove the private components from the message, if they are present. + WithoutPriv = hb_private:reset(WithoutCookie), + % Add the `accept-bundle: true' key to the message, if the caller has not + % set an explicit preference. + WithAcceptBundle = + case hb_maps:get(<<"accept-bundle">>, Message, not_found, Opts) of + not_found -> WithoutPriv#{ <<"accept-bundle">> => true }; + _ -> WithoutPriv + end, + % Determine the `ao-peer-port' from the message to send or the node message. + % `port_external' can be set in the node message to override the port that + % the peer node should receive. This allows users to proxy requests to their + % HB node from another port. + WithSelfPort = + WithAcceptBundle#{ + <<"ao-peer-port">> => + hb_maps:get( + <<"ao-peer-port">>, + WithAcceptBundle, + hb_opts:get( + port_external, + hb_opts:get(port, undefined, Opts), + Opts + ), + Opts + ) + }, + BinPeer = if is_binary(Peer) -> Peer; true -> list_to_binary(Peer) end, + BinPath = hb_path:normalize(hb_path:to_binary(Path)), + ReqBase = #{ peer => BinPeer, path => BinPath, method => Method }, + case Format of + <<"httpsig@1.0">> -> + FullEncoding = + hb_message:convert( + WithSelfPort, + #{ + <<"device">> => <<"httpsig@1.0">>, + <<"bundle">> => true + }, + Opts + ), + Body = hb_maps:get(<<"body">>, FullEncoding, <<>>, Opts), + Headers = hb_maps:without([<<"body">>], FullEncoding, Opts), + ?event(http, {request_headers, {explicit, {headers, Headers}}}), + ?event(http, {request_body, {explicit, {body, Body}}}), + hb_maps:merge( + ReqBase, + #{ headers => maps:merge(MaybeCookie, Headers), body => Body }, Opts ); - {Ref, Pid, _} -> - parallel_responses( - Res, - lists:delete(Pid, Procs), - Ref, - Awaiting, - StopAfter, - Opts - ) + <<"ans104@1.0">> -> + ?event(debug_accept, {request_message, {message, Message}}), + {ok, FilteredMessage} = + case hb_message:signers(Message, Opts) of + [] -> WithSelfPort; + _ -> + hb_message:with_only_committed(WithSelfPort, Opts) + end, + ReqBase#{ + headers => + MaybeCookie#{ + <<"codec-device">> => <<"ans104@1.0">>, + <<"content-type">> => <<"application/ans104">>, + <<"accept-bundle">> => + hb_util:bin( + hb_ao:get( + <<"accept-bundle">>, + WithSelfPort, + true, + Opts + ) + ) + }, + body => + ar_bundles:serialize( + hb_message:convert( + FilteredMessage, + #{ + <<"device">> => <<"ans104@1.0">>, + <<"bundle">> => true + }, + Opts + ) + ) + }; + _ -> + ReqBase#{ + headers => + maps:merge(MaybeCookie, maps:without([<<"body">>], Message)), + body => maps:get(<<"body">>, Message, <<>>) + } end. -%% @doc Empty the inbox of the current process for all messages with the given -%% reference. -empty_inbox(Ref) -> - receive - {Ref, _} -> empty_inbox(Ref) - after 0 -> ok +%% @doc Reply to the client's HTTP request with a message. +reply(Req, TABMReq, Message, Opts) -> + Status = + case hb_ao:get(<<"status">>, Message, Opts) of + not_found -> 200; + S-> S + end, + reply(Req, TABMReq, Status, Message, Opts). +reply(Req, TABMReq, BinStatus, RawMessage, Opts) when is_binary(BinStatus) -> + reply(Req, TABMReq, binary_to_integer(BinStatus), RawMessage, Opts); +reply(InitReq, TABMReq, Status, RawMessage, Opts) -> + KeyNormMessage = hb_ao:normalize_keys(RawMessage, Opts), + {ok, Req, Message} = reply_handle_cookies(InitReq, KeyNormMessage, Opts), + {ok, HeadersBeforeCors, EncodedBody} = + encode_reply( + Status, + TABMReq, + Message, + Opts + ), + % Get the CORS request headers from the message, if they exist. + ReqHdr = cowboy_req:header(<<"access-control-request-headers">>, Req, <<"">>), + HeadersWithCors = add_cors_headers(HeadersBeforeCors, ReqHdr, Opts), + EncodedHeaders = hb_private:reset(HeadersWithCors), + ?event(http, + {http_replying, + {status, {explicit, Status}}, + {path, hb_maps:get(<<"path">>, Req, undefined_path, Opts)}, + {raw_message, RawMessage}, + {enc_headers, {explicit, EncodedHeaders}}, + {enc_body, EncodedBody} + } + ), + ReqBeforeStream = Req#{ resp_headers => EncodedHeaders }, + PostStreamReq = cowboy_req:stream_reply(Status, #{}, ReqBeforeStream), + cowboy_req:stream_body(EncodedBody, nofin, PostStreamReq), + EndTime = os:system_time(millisecond), + ?event(http, {reply_headers, {explicit, PostStreamReq}}), + ?event(http_short, + {sent, + {status, Status}, + {duration, EndTime - hb_maps:get(start_time, Req, undefined, Opts)}, + {method, cowboy_req:method(Req)}, + {path, + {string, + uri_string:percent_decode( + hb_maps:get(<<"path">>, TABMReq, <<"[NO PATH]">>, Opts) + ) + } + }, + {body_size, byte_size(EncodedBody)} + } + ), + {ok, PostStreamReq, no_state}. + +%% @doc Handle replying with cookies if the message contains them. Returns the +%% new Cowboy `Req` object, and the message with the cookies removed. Both +%% `set-cookie' and `cookie' fields are treated as viable sources of cookies. +reply_handle_cookies(Req, Message, Opts) -> + {ok, Cookies} = dev_codec_cookie:extract(Message, #{}, Opts), + ?event(debug_cookie, {encoding_reply_cookies, {explicit, Cookies}}), + case Cookies of + NoCookies when map_size(NoCookies) == 0 -> {ok, Req, Message}; + _ -> + % The internal values of the `cookie' field will be stored in the + % `priv_store' by default, so we let `dev_codec_cookie:opts/1' + % reset the options. + {ok, #{ <<"set-cookie">> := SetCookieLines }} = + dev_codec_cookie:to( + Message, + #{ <<"format">> => <<"set-cookie">> }, + Opts + ), + ?event(debug_cookie, {outbound_set_cookie_lines, SetCookieLines}), + % Add the cookies to the response headers. + FinalReq = + lists:foldl( + fun(FullCookieLine, ReqAcc) -> + [CookieRef, _] = binary:split(FullCookieLine, <<"=">>), + RespCookies = maps:get(resp_cookies, ReqAcc, #{}), + % Note: Cowboy handles cookies peculiarly. The key + % given in the `resp_cookies' map is not used directly + % in the response headers. Nonetheless, we use the + % key parsed from the cookie line as the key, but do not + % be surprised if while debugging you see a different + % key created by Cowboy in the response headers. + ReqAcc#{ + resp_cookies => + RespCookies#{ CookieRef => FullCookieLine } + } + end, + Req, + SetCookieLines + ), + {ok, CookieReset} = dev_codec_cookie:reset(Message, Opts), + { + ok, + FinalReq, + CookieReset + } end. -%% @doc Reply to the client's HTTP request with a message. -reply(Req, Message) -> - reply(Req, message_to_status(Message), Message). -reply(Req, Status, RawMessage) -> - Message = hb_converge:ensure_message(RawMessage), - TX = hb_message:convert(Message, tx, converge, #{}), +%% @doc Add permissive CORS headers to a message, if the message has not already +%% specified CORS headers. +add_cors_headers(Msg, ReqHdr, Opts) -> + CorHeaders = #{ + <<"access-control-allow-origin">> => <<"*">>, + <<"access-control-allow-methods">> => <<"GET, POST, PUT, DELETE, OPTIONS">>, + <<"access-control-expose-headers">> => <<"*">> + }, + WithAllowHeaders = case ReqHdr of + <<>> -> CorHeaders; + _ -> CorHeaders#{ + <<"access-control-allow-headers">> => ReqHdr + } + end, + % Keys in the given message will overwrite the defaults listed below if + % included, due to `hb_maps:merge''s precidence order. + hb_maps:merge(WithAllowHeaders, Msg, Opts). + +%% @doc Generate the headers and body for a HTTP response message. +encode_reply(Status, TABMReq, Message, Opts) -> + Codec = accept_to_codec(TABMReq, Message, Opts), + ?event(http, {encoding_reply, {codec, Codec}, {message, Message}}), + BaseHdrs = + hb_maps:merge( + #{ + <<"codec-device">> => Codec + }, + case codec_to_content_type(Codec, Opts) of + undefined -> #{}; + CT -> #{ <<"content-type">> => CT } + end, + Opts + ), + AcceptBundle = + hb_util:atom( + hb_maps:get(<<"accept-bundle">>, TABMReq, false, Opts) + ), ?event(http, - {replying, + {encoding_reply, {status, Status}, - {path, maps:get(path, Req, undefined_path)}, - {tx, TX} + {codec, Codec}, + {should_bundle, AcceptBundle}, + {response_message, Message} } ), - Req2 = cowboy_req:stream_reply( - Status, - #{<<"content-type">> => <<"application/x-ans-104">>}, - Req + % Codecs generally do not need to specify headers outside of the content-type, + % aside the default `httpsig@1.0' codec, which expresses its form in HTTP + % documents, and subsequently must set its own headers. + case {Status, Codec, AcceptBundle} of + {500, <<"httpsig@1.0">>, false} -> + ?event(debug_accept, + {returning_500_error, + {status, Status}, + {codec, Codec}, + {bundle, AcceptBundle} + } + ), + {ok, ErrMsg} = + dev_hyperbuddy:return_error(Message, Opts), + {ok, + maps:without([<<"body">>], ErrMsg), + maps:get(<<"body">>, ErrMsg, <<>>) + }; + {404, <<"httpsig@1.0">>, false} -> + {ok, ErrMsg} = + dev_hyperbuddy:return_file( + <<"hyperbuddy@1.0">>, + <<"404.html">> + ), + {ok, + maps:without([<<"body">>], ErrMsg), + maps:get(<<"body">>, ErrMsg, <<>>) + }; + {_, <<"httpsig@1.0">>, _} -> + TABM = + hb_message:convert( + Message, + tabm, + <<"structured@1.0">>, + Opts#{ topic => ao_internal } + ), + {ok, EncMessage} = + dev_codec_httpsig:to( + TABM, + case AcceptBundle of + true -> + #{ + <<"bundle">> => true + }; + false -> + #{ + <<"index">> => + hb_opts:get(generate_index, true, Opts) + } + end, + Opts + ), + { + ok, + hb_maps:without([<<"body">>], EncMessage, Opts), + hb_maps:get(<<"body">>, EncMessage, <<>>, Opts) + }; + {_, <<"ans104@1.0">>, _} -> + % The `ans104@1.0' codec is a binary format, so we must serialize + % the message to a binary before sending it. + { + ok, + BaseHdrs, + ar_bundles:serialize( + hb_message:convert( + hb_message:with_only_committers( + Message, + hb_message:signers(Message, Opts), + Opts + ), + #{ + <<"device">> => <<"ans104@1.0">>, + <<"bundle">> => + hb_util:atom( + hb_ao:get( + <<"accept-bundle">>, + {as, <<"message@1.0">>, TABMReq}, + true, + Opts + ) + ) + }, + <<"structured@1.0">>, + Opts#{ topic => ao_internal } + ) + ) + }; + _ -> + % Other codecs are already in binary format, so we can just convert + % the message to the codec. We also include all of the top-level + % fields, except for maps and lists, in the message and return them + % as headers. + ExtraHdrs = + hb_maps:filter( + fun(Key, V) -> + not is_map(V) + andalso not is_list(V) + andalso Key =/= <<"body">> + andalso Key =/= <<"data">> + end, + Message, + Opts + ), + % Encode all header values as strings. + EncodedExtraHdrs = + maps:map( + fun(_K, V) -> hb_util:bin(V) end, + ExtraHdrs + ), + {ok, + hb_maps:merge(EncodedExtraHdrs, BaseHdrs, Opts), + hb_message:convert( + Message, + #{ <<"device">> => Codec, <<"bundle">> => AcceptBundle }, + <<"structured@1.0">>, + Opts#{ topic => ao_internal } + ) + } + end. + +%% @doc Calculate the codec name to use for a reply given the original parsed +%% singleton TABM request and the response message. The precidence +%% order for finding the codec is: +%% 1. If the `content-type' field is present in the response message, we always +%% use `httpsig@1.0', as the device is expected to have already encoded the +%% message and the `body' field. +%% 2. The `accept-codec' field in the original request. +%% 3. The `accept' field in the original request. +%% 4. The default codec +%% Options can be specified in mime-type format (`application/*') or in +%% AO device format (`device@1.0'). +accept_to_codec(OriginalReq, Opts) -> + accept_to_codec(OriginalReq, undefined, Opts). +accept_to_codec(#{ <<"require-codec">> := RequiredCodec }, _Reply, Opts) -> + mime_to_codec(RequiredCodec, Opts); +accept_to_codec(_OriginalReq, #{ <<"content-type">> := _ }, _Opts) -> + <<"httpsig@1.0">>; +accept_to_codec(OriginalReq, _, Opts) -> + Accept = hb_maps:get(<<"accept">>, OriginalReq, <<"*/*">>, Opts), + ?event(debug_accept, + {accept_to_codec, + {original_req, OriginalReq}, + {accept, Accept} + } ), - Req3 = cowboy_req:stream_body( - ar_bundles:serialize(TX), - nofin, - Req2 + mime_to_codec(Accept, Opts). + +%% @doc Find a codec name from a mime-type. +mime_to_codec(<<"application/", Mime/binary>>, Opts) -> + Name = + case binary:match(Mime, <<"@">>) of + nomatch -> << Mime/binary, "@1.0" >>; + _ -> Mime + end, + case hb_ao:load_device(Name, Opts) of + {ok, _} -> Name; + {error, _} -> + Default = default_codec(Opts), + ?event(http, + {codec_parsing_error, + {given, Name}, + {defaulting_to, Default} + } + ), + Default + end; +mime_to_codec(<<"device/", Name/binary>>, _Opts) -> Name; +mime_to_codec(Device, Opts) -> + case binary:match(Device, <<"@">>) of + nomatch -> default_codec(Opts); + _ -> Device + end. + +%% @doc Return the default codec for the given options. +default_codec(Opts) -> + hb_opts:get(default_codec, <<"httpsig@1.0">>, Opts). + +%% @doc Call the `content-type' key on a message with the given codec, using +%% a fast-path for options that are not needed for this one-time lookup. +codec_to_content_type(Codec, Opts) -> + FastOpts = + Opts#{ + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>], + cache_lookup_hueristics => false, + load_remote_devices => false, + error_strategy => continue + }, + case hb_ao:get(<<"content-type">>, #{ <<"device">> => Codec }, FastOpts) of + not_found -> undefined; + CT -> CT + end. + +%% @doc Convert a cowboy request to a normalized message. We first parse the +%% `primitive' message from the request: A message (represented as an Erlang +%% map) of binary keys and values for the request headers and query parameters. +%% We then determine the codec to use for the request, decode it, and merge it +%% overriding the keys of the `primitive' message. +req_to_tabm_singleton(Req, Body, Opts) -> + FullPath = + << + (cowboy_req:path(Req))/binary, + "?", + (cowboy_req:qs(Req))/binary + >>, + Headers = cowboy_req:headers(Req), + {ok, _Path, QueryKeys} = hb_singleton:from_path(FullPath), + PrimitiveMsg = maps:merge(Headers, QueryKeys), + Codec = + case hb_maps:find(<<"codec-device">>, PrimitiveMsg, Opts) of + {ok, ExplicitCodec} -> ExplicitCodec; + error -> + case hb_maps:find(<<"content-type">>, PrimitiveMsg, Opts) of + {ok, ContentType} -> mime_to_codec(ContentType, Opts); + error -> default_codec(Opts) + end + end, + ?event(http, + {parsing_req, + {path, FullPath}, + {query, QueryKeys}, + {headers, Headers}, + {primitive_message, PrimitiveMsg} + } ), - {ok, Req3, no_state}. - -%% @doc Get the HTTP status code from a transaction (if it exists). -message_to_status(Item) -> - case dev_message:get(<<"Status">>, Item) of - {ok, RawStatus} -> - case is_integer(RawStatus) of - true -> RawStatus; - false -> binary_to_integer(RawStatus) + ?event({req_to_tabm_singleton, {codec, Codec}}), + case Codec of + <<"httpsig@1.0">> -> + ?event( + {req_to_tabm_singleton, + {request, {explicit, Req}, + {body, {string, Body}} + }} + ), + httpsig_to_tabm_singleton(PrimitiveMsg, Req, Body, Opts); + <<"ans104@1.0">> -> + Item = ar_bundles:deserialize(Body), + ?event(debug_accept, + {deserialized_ans104, + {item, Item}, + {exact, {explicit, Item}} + } + ), + case ar_bundles:verify_item(Item) of + true -> + ?event(ans104, {valid_ans104_signature, Item}), + ANS104 = + hb_message:convert( + Item, + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ), + normalize_unsigned(PrimitiveMsg, Req, ANS104, Opts); + false -> + throw({invalid_ans104_signature, Item}) end; - _ -> 200 + Codec -> + % Assume that the codec stores the encoded message in the `body' field. + ?event(http, {decoding_body, {codec, Codec}, {body, {string, Body}}}), + Decoded = + hb_message:convert( + Body, + <<"structured@1.0">>, + Codec, + Opts + ), + ReqMessage = hb_maps:merge(PrimitiveMsg, Decoded, Opts), + ?event( + {verifying_encoded_message, + {codec, Codec}, + {body, {string, Body}}, + {decoded, ReqMessage} + } + ), + case hb_message:verify(ReqMessage, all) of + true -> + normalize_unsigned(PrimitiveMsg, Req, ReqMessage, Opts); + false -> + throw({invalid_commitment, ReqMessage}) + end end. -%% @doc Convert a cowboy request to a normalized message. -req_to_tabm_singleton(Req, Opts) -> - case cowboy_req:header(<<"content-type">>, Req) of - {ok, <<"application/x-ans-104">>} -> - {ok, Body} = read_body(Req), - hb_message:convert(ar_bundles:deserialize(Body), tabm, tx, Opts); - _ -> - http_sig_to_tabm_singleton(Req, Opts) +%% @doc HTTPSig messages are inherently mixed into the transport layer, so they +%% require special handling in order to be converted to a normalized message. +%% In particular, the signatures are verified if present and required by the +%% node configuration. Additionally, non-committed fields are removed from the +%% message if it is signed, with the exception of the `path' and `method' fields. +httpsig_to_tabm_singleton(PrimMsg, Req, Body, Opts) -> + {ok, Decoded} = + hb_message:with_only_committed( + hb_message:convert( + PrimMsg#{ <<"body">> => Body }, + <<"structured@1.0">>, + <<"httpsig@1.0">>, + Opts + ), + Opts + ), + ?event(http, {decoded, Decoded}, Opts), + ForceSignedRequests = hb_opts:get(force_signed_requests, false, Opts), + case (not ForceSignedRequests) orelse hb_message:verify(Decoded, all, Opts) of + true -> + ?event(http_verify, {verified_signature, Decoded}), + Signers = hb_message:signers(Decoded, Opts), + case Signers =/= [] andalso hb_opts:get(store_all_signed, false, Opts) of + true -> + ?event(http_verify, {storing_signed_from_wire, Decoded}), + {ok, _} = + hb_cache:write(Decoded, + Opts#{ + store => + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-http">> + } + } + ); + false -> + do_nothing + end, + normalize_unsigned(PrimMsg, Req, Decoded, Opts); + false -> + ?event(http_verify, + {invalid_signature, + {signed, Decoded}, + {force, ForceSignedRequests} + } + ), + throw({invalid_commitments, Decoded}) end. -http_sig_to_tabm_singleton(Req = #{ headers := RawHeaders }, _Opts) -> - {ok, Body} = read_body(Req), - Headers = - RawHeaders#{ - <<"relative-reference">> => +%% @doc Add the method and path to a message, if they are not already present. +%% Remove browser-added fields that are unhelpful during processing (for example, +%% `content-length'). +%% The precidence order for finding the path is: +%% 1. The path in the message +%% 2. The path in the request URI +normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> + ?event({adding_method_and_path_from_request, {explicit, Req}}), + Method = cowboy_req:method(Req), + MsgPath = + hb_maps:get( + <<"path">>, + Msg, + hb_maps:get( + <<"path">>, + RawHeaders, iolist_to_binary( cowboy_req:uri( Req, @@ -256,103 +958,261 @@ http_sig_to_tabm_singleton(Req = #{ headers := RawHeaders }, _Opts) -> } ) ), - <<"method">> => cowboy_req:method(Req) - }, - HTTPEncoded = - #{ - headers => maps:to_list(Headers), - body => Body + Opts + ), + Opts + ), + FilterKeys = hb_opts:get(http_inbound_filter_keys, ?DEFAULT_FILTER_KEYS, Opts), + FilteredMsg = hb_message:without_unless_signed(FilterKeys, Msg, Opts), + BaseMsg = + FilteredMsg#{ + <<"method">> => Method, + <<"path">> => MsgPath, + <<"accept-bundle">> => + maps:get( + <<"accept-bundle">>, + Msg, + maps:get( + <<"accept-bundle">>, + PrimMsg, + maps:get(<<"accept-bundle">>, RawHeaders, false) + ) + ), + <<"accept">> => + Accept = maps:get( + <<"accept">>, + Msg, + maps:get( + <<"accept">>, + PrimMsg, + maps:get(<<"accept">>, RawHeaders, <<"*/*">>) + ) + ) }, - hb_codec_http:from(HTTPEncoded). - -%% @doc Helper to grab the full body of a HTTP request, even if it's chunked. -read_body(Req) -> read_body(Req, <<>>). -read_body(Req0, Acc) -> - case cowboy_req:read_body(Req0) of - {ok, Data, _Req} -> {ok, << Acc/binary, Data/binary >>}; - {more, Data, Req} -> read_body(Req, << Acc/binary, Data/binary >>) + ?event(debug_accept, {normalize_unsigned, {accept, Accept}}), + % Parse and add the cookie from the request, if present. We reinstate the + % `cookie' field in the message, as it is not typically signed, yet should + % be honored by the node anyway. + {ok, WithCookie} = + case maps:get(<<"cookie">>, RawHeaders, undefined) of + undefined -> {ok, BaseMsg}; + Cookie -> + dev_codec_cookie:from( + BaseMsg#{ <<"cookie">> => Cookie }, + Req, + Opts + ) + end, + % If the body is empty and unsigned, we remove it. + NormalBody = + case hb_maps:get(<<"body">>, WithCookie, undefined, Opts) of + <<"">> -> hb_message:without_unless_signed(<<"body">>, WithCookie, Opts); + _ -> WithCookie + end, + case hb_maps:get(<<"ao-peer-port">>, NormalBody, undefined, Opts) of + undefined -> NormalBody; + P2PPort -> + % Calculate the peer address from the request. We honor the + % `x-real-ip' header if it is present. + RealIP = + case hb_maps:get(<<"x-real-ip">>, RawHeaders, undefined, Opts) of + undefined -> + {{A, B, C, D}, _} = cowboy_req:peer(Req), + hb_util:bin( + io_lib:format( + "~b.~b.~b.~b", + [A, B, C, D] + ) + ); + IP -> IP + end, + Peer = <>, + (hb_message:without_unless_signed(<<"ao-peer-port">>, NormalBody, Opts))#{ + <<"ao-peer">> => Peer + } end. %%% Tests -simple_converge_resolve_test() -> - URL = hb_http_server:start_test_node(), +simple_ao_resolve_unsigned_test() -> + URL = hb_http_server:start_node(), + TestMsg = #{ <<"path">> => <<"/key1">>, <<"key1">> => <<"Value1">> }, + ?assertEqual({ok, <<"Value1">>}, post(URL, TestMsg, #{})). + +simple_ao_resolve_signed_test() -> + URL = hb_http_server:start_node(), + TestMsg = #{ <<"path">> => <<"/key1">>, <<"key1">> => <<"Value1">> }, + Wallet = hb:wallet(), {ok, Res} = post( URL, - #{ - path => <<"Key1">>, - <<"Key1">> => - #{<<"Key2">> => + hb_message:commit(TestMsg, Wallet), + #{} + ), + ?assertEqual(<<"Value1">>, Res). + +nested_ao_resolve_test() -> + URL = hb_http_server:start_node(), + Wallet = hb:wallet(), + {ok, Res} = + post( + URL, + hb_message:commit(#{ + <<"path">> => <<"/key1/key2/key3">>, + <<"key1">> => + #{<<"key2">> => #{ - <<"Key3">> => <<"Value2">> + <<"key3">> => <<"Value2">> } } - }, + }, Wallet), #{} ), - ?assertEqual(<<"Value2">>, hb_converge:get(<<"Key2/Key3">>, Res, #{})). + ?assertEqual(<<"Value2">>, Res). wasm_compute_request(ImageFile, Func, Params) -> + wasm_compute_request(ImageFile, Func, Params, <<"">>). +wasm_compute_request(ImageFile, Func, Params, ResultPath) -> {ok, Bin} = file:read_file(ImageFile), - #{ - path => <<"Init/Compute/Results">>, - device => <<"WASM-64/1.0">>, - <<"WASM-Function">> => Func, - <<"WASM-Params">> => Params, - <<"Image">> => Bin - }. + Wallet = hb:wallet(), + hb_message:commit(#{ + <<"path">> => <<"/init/compute/results", ResultPath/binary>>, + <<"device">> => <<"wasm-64@1.0">>, + <<"function">> => Func, + <<"parameters">> => Params, + <<"body">> => Bin + }, Wallet). run_wasm_unsigned_test() -> - Node = hb_http_server:start_test_node(#{force_signed => false}), - Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), + Node = hb_http_server:start_node(#{force_signed => false}), + Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [3.0]), {ok, Res} = post(Node, Msg, #{}), - ?assertEqual(ok, hb_converge:get(<<"Type">>, Res, #{})). + ?event({res, Res}), + ?assertEqual(6.0, hb_ao:get(<<"output/1">>, Res, #{})). run_wasm_signed_test() -> - URL = hb_http_server:start_test_node(#{force_signed => true}), - Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), + Opts = #{ priv_wallet => hb:wallet() }, + URL = hb_http_server:start_node(#{force_signed => true}), + Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [3.0], <<"">>), + {ok, Res} = post(URL, hb_message:commit(Msg, Opts), Opts), + ?assertEqual(6.0, hb_ao:get(<<"output/1">>, Res, #{})). + +get_deep_unsigned_wasm_state_test() -> + URL = hb_http_server:start_node(#{force_signed => false}), + Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [3.0], <<"">>), + {ok, Res} = post(URL, Msg, #{}), + ?assertEqual(6.0, hb_ao:get(<<"/output/1">>, Res, #{})). + +get_deep_signed_wasm_state_test() -> + URL = hb_http_server:start_node(#{force_signed => true}), + Msg = + wasm_compute_request( + <<"test/test-64.wasm">>, + <<"fac">>, + [3.0], + <<"/output">> + ), {ok, Res} = post(URL, Msg, #{}), - ?assertEqual(ok, hb_converge:get(<<"Type">>, Res, #{})). - -% http_scheduling_test() -> -% % We need the rocksdb backend to run for hb_cache module to work -% application:ensure_all_started(hb), -% pg:start(pg), -% <> -% = crypto:strong_rand_bytes(12), -% rand:seed(exsplus, {I1, I2, I3}), -% URL = hb_http_server:start_test_node(#{force_signed => true}), -% Msg1 = dev_scheduler:test_process(), -% Proc = hb_converge:get(process, Msg1, #{ hashpath => ignore }), -% ProcID = hb_util:id(Proc), -% {ok, Res} = -% hb_converge:resolve( -% Msg1, -% #{ -% path => <<"Append">>, -% <<"Method">> => <<"POST">>, -% <<"Message">> => Proc -% }, -% #{} -% ), -% MsgX = #{ -% device => <<"Scheduler/1.0">>, -% path => <<"Append">>, -% <<"Process">> => Proc, -% <<"Message">> => -% #{ -% <<"Target">> => ProcID, -% <<"Type">> => <<"Message">>, -% <<"Test-Val">> => 1 -% } -% }, -% Res = post(URL, MsgX), -% ?event(debug, {post_result, Res}), -% Msg3 = #{ -% path => <<"Slot">>, -% <<"Method">> => <<"GET">>, -% <<"Process">> => ProcID -% }, -% SlotRes = post(URL, Msg3), -% ?event(debug, {slot_result, SlotRes}). + ?assertEqual(6.0, hb_ao:get(<<"1">>, Res, #{})). + +cors_get_test() -> + URL = hb_http_server:start_node(), + {ok, Res} = get(URL, <<"/~meta@1.0/info">>, #{}), + ?assertEqual( + <<"*">>, + hb_ao:get(<<"access-control-allow-origin">>, Res, #{}) + ). + +ans104_wasm_test() -> + TestStore = [hb_test_utils:test_store()], + TestOpts = + #{ + force_signed => true, + store => TestStore, + priv_wallet => ar_wallet:new() + }, + ClientStore = [hb_test_utils:test_store()], + ClientOpts = #{ store => ClientStore, priv_wallet => hb:wallet() }, + URL = hb_http_server:start_node(TestOpts), + {ok, Bin} = file:read_file(<<"test/test-64.wasm">>), + Msg = + hb_message:commit( + #{ + <<"require-codec">> => <<"ans104@1.0">>, + <<"codec-device">> => <<"ans104@1.0">>, + <<"device">> => <<"wasm-64@1.0">>, + <<"function">> => <<"fac">>, + <<"parameters">> => [3.0], + <<"body">> => Bin + }, + ClientOpts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true } + ), + ?assert(hb_message:verify(Msg, all, ClientOpts)), + ?event({msg, Msg}), + {ok, Res} = + post( + URL, + Msg#{ <<"path">> => <<"/init/compute/results">> }, + ClientOpts + ), + ?event({res, Res}), + ?assertEqual(6.0, hb_ao:get(<<"output/1">>, Res, ClientOpts)). + +send_large_signed_request_test() -> + % Note: If the signature scheme ever changes, we will need to run the + % following to get a freshly signed request. + % file:write_file( + % "test/large-message.eterm", + % hb_util:bin( + % io_lib:format( + % "~p.", + % [ + % hb_cache:ensure_all_loaded(hb_message:commit( + % hb_message:uncommitted(hd(hb_util:ok( + % file:consult(<<"test/large-message.eterm">>) + % ))), + % #{ priv_wallet => hb:wallet() } + % )) + % ] + % ) + % ) + % ). + {ok, [Req]} = file:consult(<<"test/large-message.eterm">>), + % Get the short trace length from the node message in the large, stored + % request. + ?assertMatch( + {ok, 5}, + post( + hb_http_server:start_node(), + <<"/node-message/short_trace_len">>, + Req, + #{ http_client => httpc } + ) + ). + +index_test() -> + NodeURL = hb_http_server:start_node(), + {ok, Res} = + get( + NodeURL, + #{ + <<"path">> => <<"/~test-device@1.0/load">>, + <<"accept-bundle">> => false + }, + #{} + ), + ?assertEqual(<<"i like turtles!">>, hb_ao:get(<<"body">>, Res, #{})). + +index_request_test() -> + URL = hb_http_server:start_node(), + {ok, Res} = + get( + URL, + #{ + <<"path">> => <<"/~test-device@1.0/load?name=dogs">>, + <<"accept-bundle">> => false + }, + #{} + ), + ?assertEqual(<<"i like dogs!">>, hb_ao:get(<<"body">>, Res, #{})). \ No newline at end of file diff --git a/src/hb_http_benchmark_tests.erl b/src/hb_http_benchmark_tests.erl index 9cb6b2e80..905b899a2 100644 --- a/src/hb_http_benchmark_tests.erl +++ b/src/hb_http_benchmark_tests.erl @@ -8,186 +8,186 @@ %% 1: 50% performance of Macbook Pro M2 Max -define(PERFORMANCE_DIVIDER, 1). -unsigned_resolve_benchmark_test() -> - BenchTime = 1, - URL = hb_http_server:start_test_node(#{force_signed => false}), - Iterations = hb:benchmark( - fun() -> - hb_http:post(URL, - #{ - path => <<"Key1">>, - <<"Key1">> => #{<<"Key2">> => <<"Value1">>} - }, - #{} - ) - end, - BenchTime - ), - hb_util:eunit_print( - "Resolved ~p messages through Converge via HTTP in ~p seconds (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] - ), - ?assert(Iterations > 400 / ?PERFORMANCE_DIVIDER). - -parallel_unsigned_resolve_benchmark_test() -> - BenchTime = 1, - BenchWorkers = 16, - URL = hb_http_server:start_test_node(#{force_signed => false}), - Iterations = hb:benchmark( - fun(_Count) -> - hb_http:post( - URL, - #{ - path => <<"Key1">>, - <<"Key1">> => #{<<"Key2">> => <<"Value1">>} - }, - #{} - ) - end, - BenchTime, - BenchWorkers - ), - hb_util:eunit_print( - "Resolved ~p messages via HTTP (~p workers) in ~p seconds (~.2f msg/s)", - [Iterations, BenchWorkers, BenchTime, Iterations / BenchTime] - ), - ?assert(Iterations > 1000 / ?PERFORMANCE_DIVIDER). - -wasm_compute_request(ImageFile, Func, Params) -> - {ok, Bin} = file:read_file(ImageFile), - #{ - path => <<"Init/Compute/Results">>, - device => <<"WASM-64/1.0">>, - <<"WASM-Function">> => Func, - <<"WASM-Params">> => Params, - <<"Image">> => Bin - }. +% unsigned_resolve_benchmark_test() -> +% BenchTime = 1, +% URL = hb_http_server:start_node(#{force_signed => false}), +% Iterations = hb_test_utils:benchmark( +% fun() -> +% hb_http:post(URL, +% #{ +% <<"path">> => <<"key1">>, +% <<"key1">> => #{<<"key2">> => <<"value1">>} +% }, +% #{} +% ) +% end, +% BenchTime +% ), +% hb_formatter:eunit_print( +% "Resolved ~p messages through AO-Core via HTTP in ~p seconds (~.2f msg/s)", +% [Iterations, BenchTime, Iterations / BenchTime] +% ), +% ?assert(Iterations > 400 / ?PERFORMANCE_DIVIDER). -run_wasm_unsigned_benchmark_test() -> - BenchTime = 1, - URL = hb_http_server:start_test_node(#{force_signed => false}), - Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), - Iterations = hb:benchmark( - fun(_) -> - case hb_http:post(URL, Msg, #{}) of - {ok, _} -> 1; - _ -> 0 - end - end, - BenchTime - ), - hb_util:eunit_print( - "Resolved ~p WASM invocations via HTTP in ~p seconds (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] - ), - ?assert(Iterations > 100 / ?PERFORMANCE_DIVIDER). +% parallel_unsigned_resolve_benchmark_test() -> +% BenchTime = 1, +% BenchWorkers = 16, +% URL = hb_http_server:start_node(#{force_signed => false}), +% Iterations = hb_test_utils:benchmark( +% fun(_Count) -> +% hb_http:post( +% URL, +% #{ +% <<"path">> => <<"key1">>, +% <<"key1">> => #{<<"key2">> => <<"value1">>} +% }, +% #{} +% ) +% end, +% BenchTime, +% BenchWorkers +% ), +% hb_formatter:eunit_print( +% "Resolved ~p messages via HTTP (~p workers) in ~p seconds (~.2f msg/s)", +% [Iterations, BenchWorkers, BenchTime, Iterations / BenchTime] +% ), +% ?assert(Iterations > 1000 / ?PERFORMANCE_DIVIDER). +% wasm_compute_request(ImageFile, Func, Params) -> +% {ok, Bin} = file:read_file(ImageFile), +% #{ +% <<"path">> => <<"init/compute/results">>, +% <<"device">> => <<"wasm-64@1.0">>, +% <<"function">> => Func, +% <<"parameters">> => Params, +% <<"image">> => Bin +% }. -run_wasm_signed_benchmark_test_disabled() -> - BenchTime = 1, - URL = hb_http_server:start_test_node(#{force_signed => true}), - Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), - Iterations = hb:benchmark( - fun(_) -> - case hb_http:post(URL, Msg, #{}) of - {ok, _} -> 1; - _ -> 0 - end - end, - BenchTime - ), - hb_util:eunit_print( - "Resolved ~p WASM invocations via HTTP in ~p seconds (~.2f msg/s)", - [Iterations, BenchTime, Iterations / BenchTime] - ), - ?assert(Iterations > 50 / ?PERFORMANCE_DIVIDER). +% run_wasm_unsigned_benchmark_test() -> +% BenchTime = 1, +% URL = hb_http_server:start_node(#{force_signed => false}), +% Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), +% Iterations = hb_test_utils:benchmark( +% fun(_) -> +% case hb_http:post(URL, Msg, #{}) of +% {ok, _} -> 1; +% _ -> 0 +% end +% end, +% BenchTime +% ), +% hb_formatter:eunit_print( +% "Resolved ~p WASM invocations via HTTP in ~p seconds (~.2f msg/s)", +% [Iterations, BenchTime, Iterations / BenchTime] +% ), +% ?assert(Iterations > 100 / ?PERFORMANCE_DIVIDER). -parallel_wasm_unsigned_benchmark_test_disabled() -> - BenchTime = 1, - BenchWorkers = 16, - URL = hb_http_server:start_test_node(#{force_signed => false}), - Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), - Iterations = hb:benchmark( - fun(X) -> - ?event(debug, {post_start, X}), - case hb_http:post(URL, Msg, #{}) of - {ok, _} -> - 1; - _ -> 0 - end - end, - BenchTime, - BenchWorkers - ), - hb_util:eunit_print( - "Resolved ~p WASM invocations via HTTP (~p workers) in ~p seconds (~.2f msg/s)", - [Iterations, BenchWorkers, BenchTime, Iterations / BenchTime] - ), - ?assert(Iterations > 200 / ?PERFORMANCE_DIVIDER). -parallel_wasm_signed_benchmark_test_disabled() -> - BenchTime = 1, - BenchWorkers = 16, - URL = hb_http_server:start_test_node(#{force_signed => true}), - Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), - Iterations = hb:benchmark( - fun(_) -> - case hb_http:post(URL, Msg, #{}) of - {ok, _ResMsg} -> - 1; - _ -> 0 - end - end, - BenchTime, - BenchWorkers - ), - hb_util:eunit_print( - "Resolved ~p WASM invocations via HTTP (~p workers) in ~p seconds (~.2f msg/s)", - [Iterations, BenchWorkers, BenchTime, Iterations / BenchTime] - ), - ?assert(Iterations > 100 / ?PERFORMANCE_DIVIDER). +% run_wasm_signed_benchmark_test_disabled() -> +% BenchTime = 1, +% URL = hb_http_server:start_node(#{force_signed => true}), +% Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), +% Iterations = hb_test_utils:benchmark( +% fun(_) -> +% case hb_http:post(URL, Msg, #{}) of +% {ok, _} -> 1; +% _ -> 0 +% end +% end, +% BenchTime +% ), +% hb_formatter:eunit_print( +% "Resolved ~p WASM invocations via HTTP in ~p seconds (~.2f msg/s)", +% [Iterations, BenchTime, Iterations / BenchTime] +% ), +% ?assert(Iterations > 50 / ?PERFORMANCE_DIVIDER). -% parallel_http_scheduling_benchmark_test() -> -% application:ensure_all_started(hb), -% URL = hb_http_server:start_test_node(#{force_signed => true}), -% BenchTime = 3, +% parallel_wasm_unsigned_benchmark_test_disabled() -> +% BenchTime = 1, % BenchWorkers = 16, -% Msg1 = dev_scheduler:test_process(), -% Proc = hb_converge:get(process, Msg1, #{ hashpath => ignore }), -% ProcID = hb_util:id(Proc), -% ?event({benchmark_start, ?MODULE}), -% Iterations = hb:benchmark( +% URL = hb_http_server:start_node(#{force_signed => false}), +% Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), +% Iterations = hb_test_utils:benchmark( % fun(X) -> -% MsgX = #{ -% device => <<"Scheduler/1.0">>, -% path => <<"Schedule">>, -% <<"Method">> => <<"POST">>, -% <<"Message">> => -% #{ -% <<"Type">> => <<"Message">>, -% <<"Test-Val">> => X -% } -% }, -% Res = hb_http:post(URL, MsgX), -% ?event(debug, {post_result, Res}), -% case Res of -% {ok, _} -> 1; +% ?event({post_start, X}), +% case hb_http:post(URL, Msg, #{}) of +% {ok, _} -> +% 1; % _ -> 0 % end % end, % BenchTime, % BenchWorkers % ), -% ?event(benchmark, {scheduled, Iterations}), -% Msg3 = #{ -% path => <<"Slot">>, -% <<"Method">> => <<"GET">>, -% <<"Process">> => ProcID -% }, -% Res = hb_http:post(URL, Msg3), -% ?event(debug, {slot_result, Res}), -% hb_util:eunit_print( -% "Scheduled ~p messages through Converge in ~p seconds (~.2f msg/s)", -% [Iterations, BenchTime, Iterations / BenchTime] +% hb_formatter:eunit_print( +% "Resolved ~p WASM invocations via HTTP (~p workers) in ~p seconds (~.2f msg/s)", +% [Iterations, BenchWorkers, BenchTime, Iterations / BenchTime] % ), -% ?assert(Iterations > 100). \ No newline at end of file +% ?assert(Iterations > 200 / ?PERFORMANCE_DIVIDER). + +% parallel_wasm_signed_benchmark_test_disabled() -> +% BenchTime = 1, +% BenchWorkers = 16, +% URL = hb_http_server:start_node(#{force_signed => true}), +% Msg = wasm_compute_request(<<"test/test-64.wasm">>, <<"fac">>, [10]), +% Iterations = hb_test_utils:benchmark( +% fun(_) -> +% case hb_http:post(URL, Msg, #{}) of +% {ok, _ResMsg} -> +% 1; +% _ -> 0 +% end +% end, +% BenchTime, +% BenchWorkers +% ), +% hb_formatter:eunit_print( +% "Resolved ~p WASM invocations via HTTP (~p workers) in ~p seconds (~.2f msg/s)", +% [Iterations, BenchWorkers, BenchTime, Iterations / BenchTime] +% ), +% ?assert(Iterations > 100 / ?PERFORMANCE_DIVIDER). + +% % parallel_http_scheduling_benchmark_test() -> +% % application:ensure_all_started(hb), +% % URL = hb_http_server:start_node(#{force_signed => true}), +% % BenchTime = 3, +% % BenchWorkers = 16, +% % Msg1 = dev_scheduler:test_process(), +% % Proc = hb_ao:get(process, Msg1, #{ hashpath => ignore }), +% % ProcID = hb_util:id(Proc), +% % ?event({benchmark_start, ?MODULE}), +% % Iterations = hb_test_utils:benchmark( +% % fun(X) -> +% % MsgX = #{ +% % <<"device">> => <<"Scheduler@1.0">>, +% % <<"path">> => <<"schedule">>, +% % <<"method">> => <<"POST">>, +% % <<"body">> => +% % #{ +% % <<"type">> => <<"body">>, +% % <<"test-val">> => X +% % } +% % }, +% % Res = hb_http:post(URL, MsgX), +% % ?event({post_result, Res}), +% % case Res of +% % {ok, _} -> 1; +% % _ -> 0 +% % end +% % end, +% % BenchTime, +% % BenchWorkers +% % ), +% % ?event(benchmark, {scheduled, Iterations}), +% % Msg3 = #{ +% % <<"path">> => <<"slot">>, +% % <<"method">> => <<"GET">>, +% % <<"process">> => ProcID +% % }, +% % Res = hb_http:post(URL, Msg3), +% % ?event({slot_result, Res}), +% % hb_formatter:eunit_print( +% % "Scheduled ~p messages through AO-Core in ~p seconds (~.2f msg/s)", +% % [Iterations, BenchTime, Iterations / BenchTime] +% % ), +% % ?assert(Iterations > 100). \ No newline at end of file diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl new file mode 100644 index 000000000..4df7e3ce9 --- /dev/null +++ b/src/hb_http_client.erl @@ -0,0 +1,758 @@ +%%% @doc A wrapper library for gun. This module originates from the Arweave +%%% project, and has been modified for use in HyperBEAM. +-module(hb_http_client). +-behaviour(gen_server). +-include("include/hb.hrl"). +-export([start_link/1, req/2]). +-export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]). + +-record(state, { + pid_by_peer = #{}, + status_by_pid = #{}, + opts = #{} +}). + +%%% ================================================================== +%%% Public interface. +%%% ================================================================== + +start_link(Opts) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, Opts, []). + +req(Args, Opts) -> req(Args, false, Opts). +req(Args, ReestablishedConnection, Opts) -> + case hb_opts:get(http_client, gun, Opts) of + gun -> gun_req(Args, ReestablishedConnection, Opts); + httpc -> httpc_req(Args, ReestablishedConnection, Opts) + end. + +httpc_req(Args, _, Opts) -> + #{ + peer := Peer, + path := Path, + method := RawMethod, + headers := Headers, + body := Body + } = Args, + ?event({httpc_req, Args}), + {Host, Port} = parse_peer(Peer, Opts), + Scheme = case Port of + 443 -> "https"; + _ -> "http" + end, + ?event(http_client, {httpc_req, {explicit, Args}}), + URL = binary_to_list(iolist_to_binary([Scheme, "://", Host, ":", integer_to_binary(Port), Path])), + FilteredHeaders = hb_maps:without([<<"content-type">>, <<"cookie">>], Headers, Opts), + HeaderKV = + [ + {binary_to_list(Key), binary_to_list(Value)} + || + {Key, Value} <- hb_maps:to_list(FilteredHeaders, Opts) + ] ++ + [ + {<<"cookie">>, CookieLine} + || + CookieLine <- + case hb_maps:get(<<"cookie">>, Headers, [], Opts) of + Binary when is_binary(Binary) -> + [Binary]; + List when is_list(List) -> + List + end + ], + Method = binary_to_existing_atom(hb_util:to_lower(RawMethod)), + ContentType = hb_maps:get(<<"content-type">>, Headers, <<"application/octet-stream">>, Opts), + Request = + case Method of + get -> + { + URL, + HeaderKV + }; + _ -> + { + URL, + HeaderKV, + binary_to_list(ContentType), + Body + } + end, + ?event({http_client_outbound, Method, URL, Request}), + HTTPCOpts = [{full_result, true}, {body_format, binary}], + StartTime = os:system_time(millisecond), + case httpc:request(Method, Request, [], HTTPCOpts) of + {ok, {{_, Status, _}, RawRespHeaders, RespBody}} -> + EndTime = os:system_time(millisecond), + RespHeaders = + [ + {list_to_binary(Key), list_to_binary(Value)} + || + {Key, Value} <- RawRespHeaders + ], + ?event(http_client, {httpc_resp, Status, RespHeaders, RespBody}), + record_duration(#{ + <<"request-method">> => method_to_bin(Method), + <<"request-path">> => hb_util:bin(Path), + <<"status-class">> => get_status_class(Status), + <<"duration">> => EndTime - StartTime + }, + Opts + ), + {ok, Status, RespHeaders, RespBody}; + {error, Reason} -> + ?event(http_client, {httpc_error, Reason}), + {error, Reason} + end. + +gun_req(Args, ReestablishedConnection, Opts) -> + StartTime = os:system_time(millisecond), + #{ peer := Peer, path := Path, method := Method } = Args, + Response = + case catch gen_server:call(?MODULE, {get_connection, Args, Opts}, infinity) of + {ok, PID} -> + ar_rate_limiter:throttle(Peer, Path, Opts), + case request(PID, Args, Opts) of + {error, Error} when Error == {shutdown, normal}; + Error == noproc -> + case ReestablishedConnection of + true -> + {error, client_error}; + false -> + req(Args, true, Opts) + end; + Reply -> + Reply + end; + {'EXIT', _} -> + {error, client_error}; + Error -> + Error + end, + EndTime = os:system_time(millisecond), + %% Only log the metric for the top-level call to req/2 - not the recursive call + %% that happens when the connection is reestablished. + case ReestablishedConnection of + true -> + ok; + false -> + record_duration(#{ + <<"request-method">> => method_to_bin(Method), + <<"request-path">> => hb_util:bin(Path), + <<"status-class">> => get_status_class(Response), + <<"duration">> => EndTime - StartTime + }, + Opts + ) + end, + Response. + +%% @doc Record the duration of the request in an async process. We write the +%% data to prometheus if the application is enabled, as well as invoking the +%% `http_monitor' if appropriate. +record_duration(Details, Opts) -> + spawn( + fun() -> + % First, write to prometheus if it is enabled. Prometheus works + % only with strings as lists, so we encode the data before granting + % it. + GetFormat = fun(Key) -> hb_util:list(maps:get(Key, Details)) end, + case application:get_application(prometheus) of + undefined -> ok; + _ -> + prometheus_histogram:observe( + http_request_duration_seconds, + lists:map( + GetFormat, + [ + <<"request-method">>, + <<"request-path">>, + <<"status-class">> + ] + ), + maps:get(<<"duration">>, Details) + ) + end, + maybe_invoke_monitor( + Details#{ <<"path">> => <<"duration">> }, + Opts + ) + end + ). + +%% @doc Invoke the HTTP monitor message with AO-Core, if it is set in the +%% node message key. We invoke the given message with the `body' set to a signed +%% version of the details. This allows node operators to configure their machine +%% to record duration statistics into customized data stores, computations, or +%% processes etc. Additionally, we include the `http_reference' value, if set in +%% the given `opts'. +%% +%% We use `hb_ao:get' rather than `hb_opts:get', as settings configured +%% by the `~router@1.0' route `opts' key are unable to generate atoms. +maybe_invoke_monitor(Details, Opts) -> + case hb_ao:get(<<"http_monitor">>, Opts, Opts) of + not_found -> ok; + Monitor -> + % We have a monitor message. Place the `details' into the body, set + % the `method' to "POST", add the `http_reference' (if applicable) + % and sign the request. We use the node message's wallet as the + % source of the key. + MaybeWithReference = + case hb_ao:get(<<"http_reference">>, Opts, Opts) of + not_found -> Details; + Ref -> Details#{ <<"reference">> => Ref } + end, + Req = + Monitor#{ + <<"body">> => + hb_message:commit( + MaybeWithReference#{ + <<"method">> => <<"POST">> + }, + Opts + ) + }, + % Use the singleton parse to generate the message sequence to + % execute. + ReqMsgs = hb_singleton:from(Req, Opts), + Res = hb_ao:resolve_many(ReqMsgs, Opts), + ?event(http_monitor, {resolved_monitor, Res}) + end. + +%%% ================================================================== +%%% gen_server callbacks. +%%% ================================================================== + +init(Opts) -> + case hb_opts:get(prometheus, not hb_features:test(), Opts) of + true -> + ?event({starting_prometheus_application, + {test_mode, hb_features:test()} + } + ), + try + application:ensure_all_started([prometheus, prometheus_cowboy]), + init_prometheus(Opts) + catch + Type:Reason:Stack -> + ?event(warning, + {prometheus_not_started, + {type, Type}, + {reason, Reason}, + {stack, Stack} + } + ), + {ok, #state{ opts = Opts }} + end; + false -> {ok, #state{ opts = Opts }} + end. + +init_prometheus(Opts) -> + application:ensure_all_started([prometheus, prometheus_cowboy]), + prometheus_counter:new([ + {name, gun_requests_total}, + {labels, [http_method, route, status_class]}, + { + help, + "The total number of GUN requests." + } + ]), + prometheus_gauge:new([{name, outbound_connections}, + {help, "The current number of the open outbound network connections"}]), + prometheus_histogram:new([ + {name, http_request_duration_seconds}, + {buckets, [0.01, 0.1, 0.5, 1, 5, 10, 30, 60]}, + {labels, [http_method, route, status_class]}, + { + help, + "The total duration of an hb_http_client:req call. This includes more than" + " just the GUN request itself (e.g. establishing a connection, " + "throttling, etc...)" + } + ]), + prometheus_histogram:new([ + {name, http_client_get_chunk_duration_seconds}, + {buckets, [0.1, 1, 10, 60]}, + {labels, [status_class, peer]}, + { + help, + "The total duration of an HTTP GET chunk request made to a peer." + } + ]), + prometheus_counter:new([ + {name, http_client_downloaded_bytes_total}, + {help, "The total amount of bytes requested via HTTP, per remote endpoint"}, + {labels, [route]} + ]), + prometheus_counter:new([ + {name, http_client_uploaded_bytes_total}, + {help, "The total amount of bytes posted via HTTP, per remote endpoint"}, + {labels, [route]} + ]), + ?event(started), + {ok, #state{ opts = Opts }}. + +handle_call({get_connection, Args, Opts}, From, + #state{ pid_by_peer = PIDPeer, status_by_pid = StatusByPID } = State) -> + Peer = hb_maps:get(peer, Args, undefined, Opts), + case hb_maps:get(Peer, PIDPeer, not_found, Opts) of + not_found -> + {ok, PID} = open_connection(Args, hb_maps:merge(State#state.opts, Opts, Opts)), + MonitorRef = monitor(process, PID), + PIDPeer2 = hb_maps:put(Peer, PID, PIDPeer, Opts), + StatusByPID2 = + hb_maps:put( + PID, + {{connecting, [{From, Args}]}, MonitorRef, Peer}, + StatusByPID, + Opts + ), + { + reply, + {ok, PID}, + State#state{ + pid_by_peer = PIDPeer2, + status_by_pid = StatusByPID2 + } + }; + PID -> + case hb_maps:get(PID, StatusByPID, undefined, Opts) of + {{connecting, PendingRequests}, MonitorRef, Peer} -> + StatusByPID2 = + hb_maps:put(PID, + { + {connecting, [{From, Args} | PendingRequests]}, + MonitorRef, + Peer + }, + StatusByPID, + Opts + ), + {noreply, State#state{ status_by_pid = StatusByPID2 }}; + {connected, _MonitorRef, Peer} -> + {reply, {ok, PID}, State} + end + end; + +handle_call(Request, _From, State) -> + ?event(warning, {unhandled_call, {module, ?MODULE}, {request, Request}}), + {reply, ok, State}. + +handle_cast(Cast, State) -> + ?event(warning, {unhandled_cast, {module, ?MODULE}, {cast, Cast}}), + {noreply, State}. + +handle_info({gun_up, PID, _Protocol}, #state{ status_by_pid = StatusByPID } = State) -> + case hb_maps:get(PID, StatusByPID, not_found) of + not_found -> + %% A connection timeout should have occurred. + {noreply, State}; + {{connecting, PendingRequests}, MonitorRef, Peer} -> + [gen_server:reply(ReplyTo, {ok, PID}) || {ReplyTo, _} <- PendingRequests], + StatusByPID2 = hb_maps:put(PID, {connected, MonitorRef, Peer}, StatusByPID), + inc_prometheus_gauge(outbound_connections), + {noreply, State#state{ status_by_pid = StatusByPID2 }}; + {connected, _MonitorRef, Peer} -> + ?event(warning, + {gun_up_pid_already_exists, {peer, Peer}}), + {noreply, State} + end; + +handle_info({gun_error, PID, Reason}, + #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> + case hb_maps:get(PID, StatusByPID, not_found) of + not_found -> + ?event(warning, {gun_connection_error_with_unknown_pid}), + {noreply, State}; + {Status, _MonitorRef, Peer} -> + PIDByPeer2 = hb_maps:remove(Peer, PIDByPeer), + StatusByPID2 = hb_maps:remove(PID, StatusByPID), + Reason2 = + case Reason of + timeout -> + connect_timeout; + {Type, _} -> + Type; + _ -> + Reason + end, + case Status of + {connecting, PendingRequests} -> + reply_error(PendingRequests, Reason2); + connected -> + dec_prometheus_gauge(outbound_connections), + ok + end, + gun:shutdown(PID), + ?event({connection_error, {reason, Reason}}), + {noreply, State#state{ status_by_pid = StatusByPID2, pid_by_peer = PIDByPeer2 }} + end; + +handle_info({gun_down, PID, Protocol, Reason, _KilledStreams, _UnprocessedStreams}, + #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> + case hb_maps:get(PID, StatusByPID, not_found) of + not_found -> + ?event(warning, + {gun_connection_down_with_unknown_pid, {protocol, Protocol}}), + {noreply, State}; + {Status, _MonitorRef, Peer} -> + PIDByPeer2 = hb_maps:remove(Peer, PIDByPeer), + StatusByPID2 = hb_maps:remove(PID, StatusByPID), + Reason2 = + case Reason of + {Type, _} -> + Type; + _ -> + Reason + end, + case Status of + {connecting, PendingRequests} -> + reply_error(PendingRequests, Reason2); + _ -> + dec_prometheus_gauge(outbound_connections), + ok + end, + {noreply, + State#state{ + status_by_pid = StatusByPID2, + pid_by_peer = PIDByPeer2 + } + } + end; + +handle_info({'DOWN', _Ref, process, PID, Reason}, + #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> + case hb_maps:get(PID, StatusByPID, not_found) of + not_found -> + {noreply, State}; + {Status, _MonitorRef, Peer} -> + PIDByPeer2 = hb_maps:remove(Peer, PIDByPeer), + StatusByPID2 = hb_maps:remove(PID, StatusByPID), + case Status of + {connecting, PendingRequests} -> + reply_error(PendingRequests, Reason); + _ -> + dec_prometheus_gauge(outbound_connections), + ok + end, + {noreply, + State#state{ + status_by_pid = StatusByPID2, + pid_by_peer = PIDByPeer2 + } + } + end; + +handle_info(Message, State) -> + ?event(warning, {unhandled_info, {module, ?MODULE}, {message, Message}}), + {noreply, State}. + +terminate(Reason, #state{ status_by_pid = StatusByPID }) -> + ?event(info,{http_client_terminating, {reason, Reason}}), + hb_maps:map(fun(PID, _Status) -> gun:shutdown(PID) end, StatusByPID), + ok. + +%%% ================================================================== +%%% Private functions. +%%% ================================================================== + +%% @doc Safe wrapper for prometheus_gauge:inc/2. +inc_prometheus_gauge(Name) -> + case application:get_application(prometheus) of + undefined -> ok; + _ -> + try prometheus_gauge:inc(Name) + catch _:_ -> + init_prometheus(#{}), + prometheus_gauge:inc(Name) + end + end. + +%% @doc Safe wrapper for prometheus_gauge:dec/2. +dec_prometheus_gauge(Name) -> + case application:get_application(prometheus) of + undefined -> ok; + _ -> prometheus_gauge:dec(Name) + end. + +inc_prometheus_counter(Name, Labels, Value) -> + case application:get_application(prometheus) of + undefined -> ok; + _ -> prometheus_counter:inc(Name, Labels, Value) + end. + +open_connection(#{ peer := Peer }, Opts) -> + {Host, Port} = parse_peer(Peer, Opts), + ?event(http_outbound, {parsed_peer, {peer, Peer}, {host, Host}, {port, Port}}), + BaseGunOpts = + #{ + http_opts => + #{ + keepalive => + hb_opts:get( + http_keepalive, + no_keepalive_timeout, + Opts + ) + }, + retry => 0, + connect_timeout => + hb_opts:get( + http_connect_timeout, + no_connect_timeout, + Opts + ) + }, + Transport = + case Port of + 443 -> tls; + _ -> tcp + end, + DefaultProto = + case hb_features:http3() of + true -> http3; + false -> http2 + end, + % Fallback through earlier HTTP versions if the protocol is not supported. + GunOpts = + case Proto = hb_opts:get(protocol, DefaultProto, Opts) of + http3 -> BaseGunOpts#{protocols => [http3], transport => quic}; + _ -> BaseGunOpts + end, + ?event(http_outbound, + {gun_open, + {host, Host}, + {port, Port}, + {protocol, Proto}, + {transport, Transport} + } + ), + gun:open(Host, Port, GunOpts). + +parse_peer(Peer, Opts) -> + Parsed = uri_string:parse(Peer), + case Parsed of + #{ host := Host, port := Port } -> + {hb_util:list(Host), Port}; + URI = #{ host := Host } -> + { + hb_util:list(Host), + case hb_maps:get(scheme, URI, undefined, Opts) of + <<"https">> -> 443; + _ -> hb_opts:get(port, 8734, Opts) + end + } + end. + +reply_error([], _Reason) -> + ok; +reply_error([PendingRequest | PendingRequests], Reason) -> + ReplyTo = element(1, PendingRequest), + Args = element(2, PendingRequest), + Method = hb_maps:get(method, Args), + Path = hb_maps:get(path, Args), + record_response_status(Method, Path, {error, Reason}), + gen_server:reply(ReplyTo, {error, Reason}), + reply_error(PendingRequests, Reason). + +record_response_status(Method, Path, Response) -> + inc_prometheus_counter(gun_requests_total, + [ + hb_util:list(method_to_bin(Method)), + Path, + hb_util:list(get_status_class(Response)) + ], + 1 + ). + +method_to_bin(get) -> + <<"GET">>; +method_to_bin(post) -> + <<"POST">>; +method_to_bin(put) -> + <<"PUT">>; +method_to_bin(head) -> + <<"HEAD">>; +method_to_bin(delete) -> + <<"DELETE">>; +method_to_bin(connect) -> + <<"CONNECT">>; +method_to_bin(options) -> + <<"OPTIONS">>; +method_to_bin(trace) -> + <<"TRACE">>; +method_to_bin(patch) -> + <<"PATCH">>; +method_to_bin(_) -> + <<"unknown">>. + +request(PID, Args, Opts) -> + Timer = + inet:start_timer( + hb_opts:get(http_request_send_timeout, no_request_send_timeout, Opts) + ), + Method = hb_maps:get(method, Args, undefined, Opts), + Path = hb_maps:get(path, Args, undefined, Opts), + HeaderMap = hb_maps:get(headers, Args, #{}, Opts), + % Normalize cookie header lines from the header map. We support both + % lists of cookie lines and a single cookie line. + HeadersWithoutCookie = + hb_maps:to_list( + hb_maps:without([<<"cookie">>], HeaderMap, Opts), + Opts + ), + CookieLines = + case hb_maps:get(<<"cookie">>, HeaderMap, [], Opts) of + BinCookieLine when is_binary(BinCookieLine) -> [BinCookieLine]; + CookieLinesList -> CookieLinesList + end, + CookieHeaders = [ {<<"cookie">>, CookieLine} || CookieLine <- CookieLines ], + Headers = HeadersWithoutCookie ++ CookieHeaders, + Body = hb_maps:get(body, Args, <<>>, Opts), + ?event( + http_client, + {gun_request, + {method, Method}, + {path, Path}, + {headers, {explicit, Headers}}, + {body, {explicit, {body, Body}}} + }, + Opts + ), + Ref = gun:request(PID, Method, Path, Headers, Body), + ResponseArgs = + #{ + pid => PID, stream_ref => Ref, + timer => Timer, limit => hb_maps:get(limit, Args, infinity, Opts), + counter => 0, acc => [], start => os:system_time(microsecond), + is_peer_request => hb_maps:get(is_peer_request, Args, true, Opts) + }, + Response = await_response(hb_maps:merge(Args, ResponseArgs, Opts), Opts), + record_response_status(Method, Path, Response), + inet:stop_timer(Timer), + Response. + +await_response(Args, Opts) -> + #{ pid := PID, stream_ref := Ref, timer := Timer, limit := Limit, + counter := Counter, acc := Acc, method := Method, path := Path } = Args, + case gun:await(PID, Ref, inet:timeout(Timer)) of + {response, fin, Status, Headers} -> + upload_metric(Args), + ?event(http, {gun_response, {status, Status}, {headers, Headers}, {body, none}}), + {ok, Status, Headers, <<>>}; + {response, nofin, Status, Headers} -> + await_response(Args#{ status => Status, headers => Headers }, Opts); + {data, nofin, Data} -> + case Limit of + infinity -> + await_response(Args#{ acc := [Acc | Data] }, Opts); + Limit -> + Counter2 = size(Data) + Counter, + case Limit >= Counter2 of + true -> + await_response( + Args#{ + counter := Counter2, + acc := [Acc | Data] + }, + Opts + ); + false -> + ?event(error, {http_fetched_too_much_data, Args, + <<"Fetched too much data">>, Opts}), + {error, too_much_data} + end + end; + {data, fin, Data} -> + FinData = iolist_to_binary([Acc | Data]), + download_metric(FinData, Args), + upload_metric(Args), + {ok, + hb_maps:get(status, Args, undefined, Opts), + hb_maps:get(headers, Args, undefined, Opts), + FinData + }; + {error, timeout} = Response -> + record_response_status(Method, Path, Response), + gun:cancel(PID, Ref), + log(warn, gun_await_process_down, Args, Response, Opts), + Response; + {error, Reason} = Response when is_tuple(Reason) -> + record_response_status(Method, Path, Response), + log(warn, gun_await_process_down, Args, Reason, Opts), + Response; + Response -> + record_response_status(Method, Path, Response), + log(warn, gun_await_unknown, Args, Response, Opts), + Response + end. + +log(Type, Event, #{method := Method, peer := Peer, path := Path}, Reason, Opts) -> + ?event( + http, + {gun_log, + {type, Type}, + {event, Event}, + {method, Method}, + {peer, Peer}, + {path, Path}, + {reason, Reason} + }, + Opts + ), + ok. + +download_metric(Data, #{path := Path}) -> + inc_prometheus_counter( + http_client_downloaded_bytes_total, + [Path], + byte_size(Data) + ). + +upload_metric(#{method := post, path := Path, body := Body}) -> + inc_prometheus_counter( + http_client_uploaded_bytes_total, + [Path], + byte_size(Body) + ); +upload_metric(_) -> + ok. + +% @doc Return the HTTP status class label for cowboy_requests_total and +% gun_requests_total metrics. +get_status_class({ok, {{Status, _}, _, _, _, _}}) -> + get_status_class(Status); +get_status_class({error, connection_closed}) -> + <<"connection_closed">>; +get_status_class({error, connect_timeout}) -> + <<"connect_timeout">>; +get_status_class({error, timeout}) -> + <<"timeout">>; +get_status_class({error,{shutdown,timeout}}) -> + <<"shutdown_timeout">>; +get_status_class({error, econnrefused}) -> + <<"econnrefused">>; +get_status_class({error, {shutdown,econnrefused}}) -> + <<"shutdown_econnrefused">>; +get_status_class({error, {shutdown,ehostunreach}}) -> + <<"shutdown_ehostunreach">>; +get_status_class({error, {shutdown,normal}}) -> + <<"shutdown_normal">>; +get_status_class({error, {closed,_}}) -> + <<"closed">>; +get_status_class({error, noproc}) -> + <<"noproc">>; +get_status_class(208) -> + <<"already_processed">>; +get_status_class(Data) when is_integer(Data), Data > 0 -> + hb_util:bin(prometheus_http:status_class(Data)); +get_status_class(Data) when is_binary(Data) -> + case catch binary_to_integer(Data) of + {_, _} -> + <<"unknown">>; + Status -> + get_status_class(Status) + end; +get_status_class(Data) when is_atom(Data) -> + atom_to_binary(Data); +get_status_class(_) -> + <<"unknown">>. \ No newline at end of file diff --git a/src/ar_http_sup.erl b/src/hb_http_client_sup.erl similarity index 82% rename from src/ar_http_sup.erl rename to src/hb_http_client_sup.erl index fe1d356d5..54c060610 100644 --- a/src/ar_http_sup.erl +++ b/src/hb_http_client_sup.erl @@ -1,5 +1,5 @@ %%% @doc The supervisor for the gun HTTP client wrapper. --module(ar_http_sup). +-module(hb_http_client_sup). -behaviour(supervisor). -export([start_link/1, init/1]). @@ -16,4 +16,4 @@ start_link(Opts) -> supervisor:start_link({local, ?MODULE}, ?MODULE, Opts). init(Opts) -> - {ok, {{one_for_one, 5, 10}, [?CHILD(ar_http, worker, Opts)]}}. + {ok, {{one_for_one, 5, 10}, [?CHILD(hb_http_client, worker, Opts)]}}. diff --git a/src/hb_http_multi.erl b/src/hb_http_multi.erl new file mode 100644 index 000000000..549ff9756 --- /dev/null +++ b/src/hb_http_multi.erl @@ -0,0 +1,284 @@ +%%% @doc An interface for resolving requests across multiple HTTP servers, either +%%% concurrently or sequentially, and processing the results in a configurable +%%% manner. +%%% +%%% The `Config' message for a call to `request/5' may contain the following +%%% fields: +%%% +%%% - `multirequest-nodes': A list of nodes to request from. +%%% - `multirequest-responses': The number of responses to gather. +%%% - `multirequest-stop-after': Whether to stop after the required number of +%%% responses. +%%% - `multirequest-parallel': Whether to run the requests in parallel. +%%% - `multirequest-admissible': A message to resolve against the response. +%%% - `multirequest-admissible-status': The statuses that are admissible. +%%% +%%% The `admissible' message is executed as a `base' message, with its `path' +%%% field moved to the request (or set to `is-admissible' if not present): +%%% ``` +%%% resolve(Base, Response#{ <<"path">> => Base/path OR /is-admissible }, Opts) +%%% ''' +-module(hb_http_multi). +-export([request/5]). +-include("include/hb.hrl"). + +%% @doc Dispatch the same HTTP request to many nodes. Can be configured to +%% await responses from all nodes or just one, and to halt all requests after +%% after it has received the required number of responses, or to leave all +%% requests running until they have all completed. Additionally, filters can +%% be applied to the responses to determine if they are admissible -- both on +%% `status' only, or as an AO-Core resolution on the response message. +%% +%% Default: Race for first response. +%% +%% Expects a config message of the following form: +%% /Nodes/1..n: Hostname | #{ hostname => Hostname, address => Address } +%% /Responses: Number of responses to gather +%% /Stop-After: Should we stop after the required number of responses? +%% /Parallel: Should we run the requests in parallel? +request(Config, Method, Path, Message, Opts) -> + #{ + nodes := Nodes, + responses := Responses, + stop_after := StopAfter, + admissible := Admissible, + admissible_status := Statuses, + parallel := Parallel + } = multirequest_opts(Config, Message, Opts), + MultirequestMsg = + hb_message:without_unless_signed( + lists:filter( + fun(<<"multirequest-", _/binary>>) -> true; (_) -> false end, + hb_maps:keys(Message) + ), + Message, + Opts + ), + ?event(debug_multi, + {multirequest_opts_parsed, + {config, Config}, + {method, Method}, + {path, Path}, + {raw_message, Message}, + {message_to_send, MultirequestMsg} + }), + AllResults = + if Parallel -> + parallel_multirequest( + Nodes, + Responses, + StopAfter, + Method, + Path, + MultirequestMsg, + Admissible, + Statuses, + Opts + ); + true -> + serial_multirequest( + Nodes, + Responses, + Method, + Path, + MultirequestMsg, + Admissible, + Statuses, + Opts + ) + end, + ?event(http, {multirequest_results, {results, AllResults}}), + case AllResults of + [] -> {error, no_viable_responses}; + Results -> if Responses == 1 -> hd(Results); true -> Results end + end. + +%% @doc Get the multirequest options from the config or message. The options in +%% the message take precidence over the options in the config. +multirequest_opts(Config, Message, Opts) -> + Opts#{ + nodes => + multirequest_opt(<<"nodes">>, Config, Message, #{}, Opts), + responses => + multirequest_opt(<<"responses">>, Config, Message, 1, Opts), + stop_after => + multirequest_opt(<<"stop-after">>, Config, Message, true, Opts), + admissible => + multirequest_opt(<<"admissible">>, Config, Message, undefined, Opts), + admissible_status => + multirequest_opt(<<"admissible-status">>, Config, Message, <<"All">>, Opts), + parallel => + multirequest_opt(<<"parallel">>, Config, Message, false, Opts) + }. + +%% @doc Get a value for a multirequest option from the config or message. +multirequest_opt(Key, Config, Message, Default, Opts) -> + hb_ao:get_first( + [ + {Message, <<"multirequest-", Key/binary>>}, + {Config, Key} + ], + Default, + Opts#{ hashpath => ignore } + ). + +%% @doc Check if a response is admissible, according to the configuration. First, +%% we check the Erlang response status to check for `ok'. If the response is +%% not `ok', it is not admissible. +%% +%% If the response is `ok', we check the status and the response message against +%% the configuration. +is_admissible(ok, Res, Admissible, Statuses, Opts) -> + ?event(debug_multi, + {is_admissible, + {response, Res}, + {admissible, Admissible}, + {statuses, Statuses} + } + ), + AdmissibleStatus = admissible_status(Res, Statuses), + ?event(debug_multi, {admissible_status, {result, AdmissibleStatus}}), + AdmissibleResponse = admissible_response(Res, Admissible, Opts), + ?event(debug_multi, {admissible_response, {result, AdmissibleResponse}}), + AdmissibleStatus andalso AdmissibleResponse; +is_admissible(_, _, _, _, _) -> false. + +%% @doc Serially request a message, collecting responses until the required +%% number of responses have been gathered. Ensure that the statuses are +%% allowed, according to the configuration. +serial_multirequest(_Nodes, 0, _Method, _Path, _Message, _Admissible, _Statuses, _Opts) -> []; +serial_multirequest([], _, _Method, _Path, _Message, _Admissible, _Statuses, _Opts) -> []; +serial_multirequest([Node|Nodes], Remaining, Method, Path, Message, Admissible, Statuses, Opts) -> + {ErlStatus, Res} = hb_http:request(Method, Node, Path, Message, Opts), + case is_admissible(ErlStatus, Res, Admissible, Statuses, Opts) of + true -> + ?event(http, {admissible_status, {response, Res}}), + [ + {ErlStatus, Res} + | + serial_multirequest( + Nodes, + Remaining - 1, + Method, + Path, + Message, + Admissible, + Statuses, + Opts + ) + ]; + false -> + ?event(http, {inadmissible_status, {response, Res}}), + serial_multirequest( + Nodes, + Remaining, + Method, + Path, + Message, + Admissible, + Statuses, + Opts + ) + end. + +%% @doc Dispatch the same HTTP request to many nodes in parallel. +parallel_multirequest(Nodes, Responses, StopAfter, Method, Path, Message, Admissible, Statuses, Opts) -> + Ref = make_ref(), + Parent = self(), + Procs = + lists:map( + fun(Node) -> + spawn( + fun() -> + Res = hb_http:request(Method, Node, Path, Message, Opts), + receive no_reply -> stopping + after 0 -> Parent ! {Ref, self(), Res} + end + end + ) + end, + Nodes + ), + parallel_responses([], Procs, Ref, Responses, StopAfter, Admissible, Statuses, Opts). + +%% @doc Check if a status is allowed, according to the configuration. Statuses +%% can be a single integer, a comma-separated list of integers, or the string +%% `All'. +admissible_status(_, <<"All">>) -> true; +admissible_status(_ResponseMsg = #{ <<"status">> := Status }, Statuses) -> + admissible_status(Status, Statuses); +admissible_status(Status, Statuses) when is_integer(Statuses) -> + admissible_status(Status, [Statuses]); +admissible_status(Status, Statuses) when is_binary(Status) -> + admissible_status(binary_to_integer(Status), Statuses); +admissible_status(Status, Statuses) when is_binary(Statuses) -> + % Convert the statuses to a list of integers. + admissible_status( + Status, + lists:map(fun binary_to_integer/1, binary:split(Statuses, <<",">>)) + ); +admissible_status(Status, Statuses) when is_list(Statuses) -> + lists:member(Status, Statuses). + +%% @doc If an `admissable` message is set for the request, check if the response +%% adheres to it. Else, return `true'. +admissible_response(_Response, undefined, _Opts) -> true; +admissible_response(Response, Msg, Opts) -> + Path = hb_maps:get(<<"path">>, Msg, <<"is-admissible">>, Opts), + Req = Response#{ <<"path">> => Path }, + Base = hb_message:without_unless_signed([<<"path">>], Msg, Opts), + ?event(debug_multi, + {executing_admissible_message, {message, Base}, {req, Req}} + ), + case hb_ao:resolve(Base, Req, Opts) of + {ok, Res} when is_atom(Res) or is_binary(Res) -> + ?event(debug_multi, {admissible_result, {result, Res}}), + hb_util:atom(Res) == true; + {error, Reason} -> + ?event(debug_multi, {admissible_error, {reason, Reason}}), + false + end. + +%% @doc Collect the necessary number of responses, and stop workers if +%% configured to do so. +parallel_responses(Res, Procs, Ref, 0, false, _Admissible, _Statuses, _Opts) -> + lists:foreach(fun(P) -> P ! no_reply end, Procs), + empty_inbox(Ref), + {ok, Res}; +parallel_responses(Res, Procs, Ref, 0, true, _Admissible, _Statuses, _Opts) -> + lists:foreach(fun(P) -> exit(P, kill) end, Procs), + empty_inbox(Ref), + Res; +parallel_responses(Res, Procs, Ref, Awaiting, StopAfter, Admissible, Statuses, Opts) -> + receive + {Ref, Pid, {Status, NewRes}} -> + case is_admissible(Status, NewRes, Admissible, Statuses, Opts) of + true -> + parallel_responses( + [NewRes | Res], + lists:delete(Pid, Procs), + Ref, + Awaiting - 1, + StopAfter, + Admissible, + Statuses, + Opts + ); + false -> + parallel_responses( + Res, + lists:delete(Pid, Procs), + Ref, + Awaiting, + StopAfter, + Admissible, + Statuses, + Opts + ) + end +end. + +%% @doc Empty the inbox of the current process for all messages with the given +%% reference. +empty_inbox(Ref) -> + receive {Ref, _} -> empty_inbox(Ref) after 0 -> ok end. \ No newline at end of file diff --git a/src/hb_http_server.erl b/src/hb_http_server.erl index d3ac8f6c3..599e7db70 100644 --- a/src/hb_http_server.erl +++ b/src/hb_http_server.erl @@ -1,35 +1,168 @@ -%%% @doc A router that attaches a HTTP server to the Converge resolver. -%%% Because Converge is built to speak in HTTP semantics, this module +%%% @doc A router that attaches a HTTP server to the AO-Core resolver. +%%% Because AO-Core is built to speak in HTTP semantics, this module %%% only has to marshal the HTTP request into a message, and then -%%% pass it to the Converge resolver. +%%% pass it to the AO-Core resolver. %%% -%%% `hb_http:reply/3' is used to respond to the client, handling the +%%% `hb_http:reply/4' is used to respond to the client, handling the %%% process of converting a message back into an HTTP response. %%% -%%% The router uses an `Opts` message as its Cowboy initial state, +%%% The router uses an `Opts' message as its Cowboy initial state, %%% such that changing it on start of the router server allows for %%% the execution parameters of all downstream requests to be controlled. -module(hb_http_server). --export([start/0, start/1, allowed_methods/2, init/2, set_opts/1]). --export([start_test_node/0, start_test_node/1]). +-export([start/0, start/1, allowed_methods/2, init/2]). +-export([set_opts/1, set_opts/2, get_opts/0, get_opts/1]). +-export([set_default_opts/1, set_proc_server_id/1]). +-export([start_node/0, start_node/1]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). -%% @doc Starts the HTTP server. Optionally accepts an `Opts` message, which +%% @doc Starts the HTTP server. Optionally accepts an `Opts' message, which %% is used as the source for server configuration settings, as well as the -%% `Opts` argument to use for all Converge resolution requests downstream. +%% `Opts' argument to use for all AO-Core resolution requests downstream. start() -> - start(#{ priv_wallet => hb:wallet(hb_opts:get(key_location)) }). + ?event(http, {start_store, <<"cache-mainnet">>}), + Loaded = + case hb_opts:load(Loc = hb_opts:get(hb_config_location, <<"config.flat">>)) of + {ok, Conf} -> + ?event(boot, {loaded_config, Loc, Conf}), + Conf; + {error, Reason} -> + ?event(boot, {failed_to_load_config, Loc, Reason}), + #{} + end, + MergedConfig = + hb_maps:merge( + hb_opts:default_message_with_env(), + Loaded + ), + %% Apply store defaults before starting store + StoreOpts = hb_opts:get(store, no_store, MergedConfig), + StoreDefaults = hb_opts:get(store_defaults, #{}, MergedConfig), + UpdatedStoreOpts = + case StoreOpts of + no_store -> no_store; + _ when is_list(StoreOpts) -> hb_store_opts:apply(StoreOpts, StoreDefaults); + _ -> StoreOpts + end, + hb_store:start(UpdatedStoreOpts), + PrivWallet = + hb:wallet( + hb_opts:get( + priv_key_location, + <<"hyperbeam-key.json">>, + Loaded + ) + ), + maybe_greeter(MergedConfig, PrivWallet), + start( + Loaded#{ + priv_wallet => PrivWallet, + store => UpdatedStoreOpts, + port => hb_opts:get(port, 8734, Loaded), + cache_writers => [hb_util:human_id(ar_wallet:to_address(PrivWallet))] + } + ). start(Opts) -> - {ok, Listener, _Port} = new_server(Opts), + application:ensure_all_started([ + kernel, + stdlib, + inets, + ssl, + ranch, + cowboy, + gun, + os_mon + ]), + hb:init(), + BaseOpts = set_default_opts(Opts), + {ok, Listener, _Port} = new_server(BaseOpts), {ok, Listener}. +%% @doc Print the greeter message to the console if we are not running tests. +maybe_greeter(MergedConfig, PrivWallet) -> + case hb_features:test() of + false -> + print_greeter(MergedConfig, PrivWallet); + true -> + ok + end. + +%% @doc Print the greeter message to the console. Includes the version, operator +%% address, URL to access the node, and the wider configuration (including the +%% keys inherited from the default configuration). +print_greeter(Config, PrivWallet) -> + FormattedConfig = hb_format:term(Config, Config, 2), + io:format("~n" + "===========================================================~n" + "== ██╗ ██╗██╗ ██╗██████╗ ███████╗██████╗ ==~n" + "== ██║ ██║╚██╗ ██╔╝██╔══██╗██╔════╝██╔══██╗ ==~n" + "== ███████║ ╚████╔╝ ██████╔╝█████╗ ██████╔╝ ==~n" + "== ██╔══██║ ╚██╔╝ ██╔═══╝ ██╔══╝ ██╔══██╗ ==~n" + "== ██║ ██║ ██║ ██║ ███████╗██║ ██║ ==~n" + "== ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝╚═╝ ╚═╝ ==~n" + "== ==~n" + "== ██████╗ ███████╗ █████╗ ███╗ ███╗ VERSION: ==~n" + "== ██╔══██╗██╔════╝██╔══██╗████╗ ████║ v~p. ==~n" + "== ██████╔╝█████╗ ███████║██╔████╔██║ ==~n" + "== ██╔══██╗██╔══╝ ██╔══██║██║╚██╔╝██║ EAT GLASS, ==~n" + "== ██████╔╝███████╗██║ ██║██║ ╚═╝ ██║ BUILD THE ==~n" + "== ╚═════╝ ╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝ FUTURE. ==~n" + "===========================================================~n" + "== Node activate at: ~s ==~n" + "== Operator: ~s ==~n" + "===========================================================~n" + "== Config: ==~n" + "===========================================================~n" + " ~s~n" + "===========================================================~n", + [ + ?HYPERBEAM_VERSION, + string:pad( + lists:flatten( + io_lib:format( + "http://~s:~p", + [ + hb_opts:get(host, <<"localhost">>, Config), + hb_opts:get(port, 8734, Config) + ] + ) + ), + 35, leading, $ + ), + hb_util:human_id(ar_wallet:to_address(PrivWallet)), + FormattedConfig + ] + ). + +%% @doc Trigger the creation of a new HTTP server node. Accepts a `NodeMsg' +%% message, which is used to configure the server. This function executed the +%% `start' hook on the node, giving it the opportunity to modify the `NodeMsg' +%% before it is used to configure the server. The `start' hook expects gives and +%% expects the node message to be in the `body' key. new_server(RawNodeMsg) -> - NodeMsg = - maps:merge( - hb_opts:default_message(), + RawNodeMsgWithDefaults = + hb_maps:merge( + hb_opts:default_message_with_env(), RawNodeMsg#{ only => local } ), + HookMsg = #{ <<"body">> => RawNodeMsgWithDefaults }, + NodeMsg = + case dev_hook:on(<<"start">>, HookMsg, RawNodeMsgWithDefaults) of + {ok, #{ <<"body">> := NodeMsgAfterHook }} -> NodeMsgAfterHook; + Unexpected -> + ?event(http, + {failed_to_start_server, + {unexpected_hook_result, Unexpected} + } + ), + throw( + {failed_to_start_server, + {unexpected_hook_result, Unexpected} + } + ) + end, + % Put server ID into node message so it's possible to update current server hb_http:start(), ServerID = hb_util:human_id( @@ -41,50 +174,77 @@ new_server(RawNodeMsg) -> ) ) ), - Dispatcher = - cowboy_router:compile( - [ - % {HostMatch, list({PathMatch, Handler, InitialState})} - {'_', [ - { - "/metrics/[:registry]", - prometheus_cowboy2_handler, - #{} - }, - {'_', ?MODULE, ServerID} - ]} - ] - ), + % Put server ID into node message so it's possible to update current server + % params + NodeMsgWithID = hb_maps:put(http_server, ServerID, NodeMsg), + Dispatcher = cowboy_router:compile([{'_', [{'_', ?MODULE, ServerID}]}]), ProtoOpts = #{ - env => #{dispatch => Dispatcher, node_msg => NodeMsg}, - metrics_callback => - fun prometheus_cowboy2_instrumenter:observe/1, - stream_handlers => [cowboy_metrics_h, cowboy_stream_h] + env => #{dispatch => Dispatcher, node_msg => NodeMsgWithID}, + stream_handlers => [cowboy_stream_h], + max_connections => infinity, + idle_timeout => hb_opts:get(idle_timeout, 300000, NodeMsg) }, + PrometheusOpts = + case hb_opts:get(prometheus, not hb_features:test(), NodeMsg) of + true -> + ?event(prometheus, + {starting_prometheus, {test_mode, hb_features:test()}} + ), + % Attempt to start the prometheus application, if possible. + try + application:ensure_all_started([prometheus, prometheus_cowboy]), + ProtoOpts#{ + metrics_callback => + fun prometheus_cowboy2_instrumenter:observe/1, + stream_handlers => [cowboy_metrics_h, cowboy_stream_h] + } + catch + Type:Reason -> + % If the prometheus application is not started, we can + % still start the HTTP server, but we won't have any + % metrics. + ?event(prometheus, + {prometheus_not_started, {type, Type}, {reason, Reason}} + ), + ProtoOpts + end; + false -> + ?event(prometheus, + {prometheus_not_started, {test_mode, hb_features:test()}} + ), + ProtoOpts + end, + DefaultProto = + case hb_features:http3() of + true -> http3; + false -> http2 + end, {ok, Port, Listener} = - case Protocol = hb_opts:get(protocol, no_proto, NodeMsg) of + case Protocol = hb_opts:get(protocol, DefaultProto, NodeMsg) of http3 -> - start_http3(ServerID, ProtoOpts, NodeMsg); + start_http3(ServerID, PrometheusOpts, NodeMsg); Pro when Pro =:= http2; Pro =:= http1 -> - % The HTTP/2 server has fallback mode to 1.1 as necessary. - start_http2(ServerID, ProtoOpts, NodeMsg); + % The HTTP/2 server has fallback mode to 1.1 as necessary. + start_http2(ServerID, PrometheusOpts, NodeMsg); _ -> {error, {unknown_protocol, Protocol}} end, - ?event(debug, + ?event(http, {http_server_started, {listener, Listener}, {server_id, ServerID}, {port, Port}, - {protocol, Protocol} + {protocol, Protocol}, + {store, hb_opts:get(store, no_store, NodeMsg)} } ), {ok, Listener, Port}. start_http3(ServerID, ProtoOpts, _NodeMsg) -> - ?event(debug, {start_http3, ServerID}), + ?event(http, {start_http3, ServerID}), Parent = self(), ServerPID = spawn(fun() -> + application:ensure_all_started(quicer), {ok, Listener} = cowboy:start_quic( ServerID, TransOpts = #{ @@ -100,12 +260,18 @@ start_http3(ServerID, ProtoOpts, _NodeMsg) -> ServerID, 1024, ranch:normalize_opts( - maps:to_list(TransOpts#{ port => GivenPort }) + hb_maps:to_list(TransOpts#{ port => GivenPort }) ), ProtoOpts, [] ), ranch_server:set_addr(ServerID, {<<"localhost">>, GivenPort}), + % Bypass ranch's requirement to have a connection supervisor define + % to support updating protocol opts. + % Quicer doesn't use a connection supervisor, so we just spawn one + % that does nothing. + ConnSup = spawn(fun() -> http3_conn_sup_loop() end), + ranch_server:set_connections_sup(ServerID, ConnSup), Parent ! {ok, GivenPort}, receive stop -> stopped end end), @@ -114,106 +280,380 @@ start_http3(ServerID, ProtoOpts, _NodeMsg) -> {error, {timeout, staring_http3_server, ServerID}} end. +http3_conn_sup_loop() -> + receive + _ -> + % Ignore any other messages + http3_conn_sup_loop() + end. + start_http2(ServerID, ProtoOpts, NodeMsg) -> - ?event(debug, {start_http2, ServerID}), - {ok, Listener} = cowboy:start_clear( + ?event(http, {start_http2, ServerID}), + StartRes = cowboy:start_clear( ServerID, [ {port, Port = hb_opts:get(port, 8734, NodeMsg)} ], ProtoOpts ), - {ok, Port, Listener}. + case StartRes of + {ok, Listener} -> + ?event(debug_router_info, {http2_started, {listener, Listener}, {port, Port}}), + {ok, Port, Listener}; + {error, {already_started, Listener}} -> + ?event(http, {http2_already_started, {listener, Listener}}), + ?event(debug_router_info, + {restarting, + {id, ServerID}, + {node_msg, NodeMsg} + } + ), + cowboy:set_env(ServerID, node_msg, #{}), + % {ok, Port, Listener} + cowboy:stop_listener(ServerID), + start_http2(ServerID, ProtoOpts, NodeMsg) + end. +%% @doc Entrypoint for all HTTP requests. Receives the Cowboy request option and +%% the server ID, which can be used to lookup the node message. init(Req, ServerID) -> - NodeMsg = cowboy:get_env(ServerID, node_msg, no_node_msg), - % Parse the HTTP request into HyerBEAM's message format. - ReqSingleton = hb_http:req_to_tabm_singleton(Req, NodeMsg), - ?event(http, {http_inbound, ReqSingleton}), - {ok, Res} = dev_meta:handle(NodeMsg, ReqSingleton), - hb_http:reply(Req, Res). - -%% @doc Return the complete Ranch ETS table for the node for debugging. -ranch_ets() -> - case ets:info(ranch_server) of - undefined -> []; - _ -> ets:tab2list(ranch_server) + case cowboy_req:method(Req) of + <<"OPTIONS">> -> cors_reply(Req, ServerID); + _ -> + {ok, Body} = read_body(Req), + handle_request(Req, Body, ServerID) + end. + +%% @doc Helper to grab the full body of a HTTP request, even if it's chunked. +read_body(Req) -> read_body(Req, <<>>). +read_body(Req0, Acc) -> + case cowboy_req:read_body(Req0) of + {ok, Data, _Req} -> {ok, << Acc/binary, Data/binary >>}; + {more, Data, Req} -> read_body(Req, << Acc/binary, Data/binary >>) + end. + +%% @doc Reply to CORS preflight requests. +cors_reply(Req, _ServerID) -> + Req2 = cowboy_req:reply(204, #{ + <<"access-control-allow-origin">> => <<"*">>, + <<"access-control-allow-headers">> => <<"*">>, + <<"access-control-allow-methods">> => + <<"GET, POST, PUT, DELETE, OPTIONS, PATCH">> + }, Req), + ?event(http_debug, {cors_reply, {req, Req}, {req2, Req2}}), + {ok, Req2, no_state}. + +%% @doc Handle all non-CORS preflight requests as AO-Core requests. Execution +%% starts by parsing the HTTP request into HyerBEAM's message format, then +%% passing the message directly to `meta@1.0' which handles calling AO-Core in +%% the appropriate way. +handle_request(RawReq, Body, ServerID) -> + % Insert the start time into the request so that it can be used by the + % `hb_http' module to calculate the duration of the request. + StartTime = os:system_time(millisecond), + Req = RawReq#{ start_time => StartTime }, + NodeMsg = get_opts(#{ http_server => ServerID }), + put(server_id, ServerID), + case {cowboy_req:path(RawReq), cowboy_req:qs(RawReq)} of + {<<"/">>, <<>>} -> + % If the request is for the root path, serve a redirect to the default + % request of the node. + Req2 = cowboy_req:reply( + 302, + #{ + <<"location">> => + hb_opts:get( + default_request, + <<"/~hyperbuddy@1.0/dashboard">>, + NodeMsg + ) + }, + RawReq + ), + {ok, Req2, no_state}; + _ -> + % The request is of normal AO-Core form, so we parse it and invoke + % the meta@1.0 device to handle it. + ?event(http, + { + http_inbound, + {cowboy_req, {explicit, Req}, {body, {string, Body}}} + } + ), + TracePID = hb_tracer:start_trace(), + % Parse the HTTP request into HyerBEAM's message format. + ReqSingleton = + try hb_http:req_to_tabm_singleton(Req, Body, NodeMsg) + catch ParseError:ParseDetails:ParseStacktrace -> + {parse_error, ParseError, ParseDetails, ParseStacktrace} + end, + try + case ReqSingleton of + {parse_error, PType, PDetails, PStacktrace} -> + erlang:raise(PType, PDetails, PStacktrace); + _ -> + ok + end, + CommitmentCodec = hb_http:accept_to_codec(ReqSingleton, NodeMsg), + ?event(http, + {parsed_singleton, + {req_singleton, ReqSingleton}, + {accept_codec, CommitmentCodec}}, + #{trace => TracePID} + ), + % hb_tracer:record_step(TracePID, request_parsing), + % Invoke the meta@1.0 device to handle the request. + {ok, Res} = + dev_meta:handle( + NodeMsg#{ + commitment_device => CommitmentCodec, + trace => TracePID + }, + ReqSingleton + ), + hb_http:reply(Req, ReqSingleton, Res, NodeMsg) + catch + Type:Details:Stacktrace -> + handle_error( + Req, + ReqSingleton, + Type, + Details, + Stacktrace, + NodeMsg + ) + end end. +%% @doc Return a 500 error response to the client. +handle_error(Req, Singleton, Type, Details, Stacktrace, NodeMsg) -> + DetailsStr = hb_util:bin(hb_format:message(Details, NodeMsg, 1)), + StacktraceStr = hb_util:bin(hb_format:trace(Stacktrace)), + ErrorMsg = + #{ + <<"status">> => 500, + <<"type">> => hb_util:bin(hb_format:message(Type)), + <<"details">> => DetailsStr, + <<"stacktrace">> => StacktraceStr + }, + ErrorBin = hb_format:error(ErrorMsg, NodeMsg), + ?event( + http_error, + {returning_500_error, + {string, + hb_format:indent_lines( + <<"\n", ErrorBin/binary, "\n">>, + 1 + ) + } + } + ), + % Remove leading and trailing noise from the stacktrace and details. + FormattedErrorMsg = + ErrorMsg#{ + <<"stacktrace">> => hb_util:bin(hb_format:remove_noise(StacktraceStr)), + <<"details">> => hb_util:bin(hb_format:remove_noise(DetailsStr)) + }, + hb_http:reply(Req, Singleton, FormattedErrorMsg, NodeMsg). + +%% @doc Return the list of allowed methods for the HTTP server. allowed_methods(Req, State) -> - {[<<"GET">>, <<"POST">>, <<"PUT">>, <<"DELETE">>], Req, State}. + { + [<<"GET">>, <<"POST">>, <<"PUT">>, <<"DELETE">>, <<"OPTIONS">>, <<"PATCH">>], + Req, + State + }. -%% @doc Update the `Opts` map that the HTTP server uses for all future -%% requests. +%% @doc Merges the provided `Opts' with uncommitted values from `Request', +%% preserves the http_server value, and updates node_history by prepending +%% the `Request'. If a server reference exists, updates the Cowboy environment +%% variable 'node_msg' with the resulting options map. set_opts(Opts) -> - ServerRef = hb_opts:get(http_server, no_server_ref, Opts), - cowboy:set_env(ServerRef, opts, Opts). + case hb_opts:get(http_server, no_server_ref, Opts) of + no_server_ref -> + ok; + ServerRef -> + ok = cowboy:set_env(ServerRef, node_msg, Opts) + end. +set_opts(Request, Opts) -> + PreparedOpts = + hb_opts:mimic_default_types( + Opts, + false, + Opts + ), + PreparedRequest = + hb_opts:mimic_default_types( + hb_message:uncommitted(Request), + false, + Opts + ), + MergedOpts = + maps:merge( + PreparedOpts, + PreparedRequest + ), + ?event(set_opts, {merged_opts, {explicit, MergedOpts}}), + History = + hb_opts:get(node_history, [], Opts) + ++ [ hb_private:reset(maps:without([node_history], PreparedRequest)) ], + FinalOpts = MergedOpts#{ + http_server => hb_opts:get(http_server, no_server, Opts), + node_history => History + }, + {set_opts(FinalOpts), FinalOpts}. -%%% Tests +%% @doc Get the node message for the current process. +get_opts() -> + get_opts(#{ http_server => get(server_id) }). +get_opts(NodeMsg) -> + ServerRef = hb_opts:get(http_server, no_server_ref, NodeMsg), + cowboy:get_env(ServerRef, node_msg, no_node_msg). -test_opts(Opts) -> - rand:seed(default), - % Generate a random port number between 42000 and 62000 to use +%% @doc Initialize the server ID for the current process. +set_proc_server_id(ServerID) -> + put(server_id, ServerID). + +%% @doc Apply the default node message to the given opts map. +set_default_opts(Opts) -> + % Create a temporary opts map that does not include the defaults. + TempOpts = Opts#{ only => local }, + % Generate a random port number between 10000 and 30000 to use % for the server. - Port = 10000 + rand:uniform(20000), - Wallet = ar_wallet:new(), + Port = + case hb_opts:get(port, no_port, TempOpts) of + no_port -> + rand:seed(exsplus, erlang:system_time(microsecond)), + 10000 + rand:uniform(50000); + PassedPort -> PassedPort + end, + Wallet = + case hb_opts:get(priv_wallet, no_viable_wallet, TempOpts) of + no_viable_wallet -> ar_wallet:new(); + PassedWallet -> PassedWallet + end, + Store = + case hb_opts:get(store, no_store, TempOpts) of + no_store -> + hb_store:start(Stores = [hb_test_utils:test_store()]), + Stores; + PassedStore -> PassedStore + end, + ?event({set_default_opts, + {given, TempOpts}, + {port, Port}, + {store, Store}, + {wallet, Wallet} + }), Opts#{ - % Generate a random port number between 8000 and 9000. port => Port, - store => - {hb_store_fs, - #{ - prefix => - <<"TEST-cache-", (integer_to_binary(Port))/binary>> - } - }, - priv_wallet => Wallet + store => Store, + priv_wallet => Wallet, + address => hb_util:human_id(ar_wallet:to_address(Wallet)), + force_signed => true }. %% @doc Test that we can start the server, send a message, and get a response. -start_test_node() -> - start_test_node(#{}). -start_test_node(Opts) -> +start_node() -> + start_node(#{}). +start_node(Opts) -> application:ensure_all_started([ kernel, stdlib, inets, ssl, - debugger, ranch, cowboy, gun, - prometheus, - prometheus_cowboy, - os_mon, - rocksdb + os_mon ]), hb:init(), hb_sup:start_link(Opts), - ServerOpts = test_opts(Opts), + ServerOpts = set_default_opts(Opts), {ok, _Listener, Port} = new_server(ServerOpts), <<"http://localhost:", (integer_to_binary(Port))/binary, "/">>. -raw_http_access_test() -> - URL = start_test_node(#{ protocol => http1 }), - TX = - ar_bundles:serialize( - hb_message:convert( - #{ - path => <<"Key1">>, - <<"Key1">> => #{ <<"Key2">> => <<"Value1">> } - }, - tx, - converge, - #{} - ) - ), - {ok, {{_, 200, _}, _, Body}} = - httpc:request( - post, - {iolist_to_binary(URL), [], "application/octet-stream", TX}, - [], - [{body_format, binary}] - ), - Msg = hb_message:convert(ar_bundles:deserialize(Body), converge, tx, #{}), - ?assertEqual(<<"Value1">>, hb_converge:get(<<"Key2">>, Msg, #{})). \ No newline at end of file +%%% Tests +%%% The following only covering the HTTP server initialization process. For tests +%%% of HTTP server requests/responses, see `hb_http.erl'. + +%% @doc Ensure that the `start' hook can be used to modify the node options. We +%% do this by creating a message with a device that has a `start' key. This +%% key takes the message's body (the anticipated node options) and returns a +%% modified version of that body, which will be used to configure the node. We +%% then check that the node options were modified as we expected. +set_node_opts_test() -> + Node = + start_node(#{ + on => #{ + <<"start">> => #{ + <<"device">> => + #{ + <<"start">> => + fun(_, #{ <<"body">> := NodeMsg }, _) -> + {ok, #{ + <<"body">> => + NodeMsg#{ <<"test-success">> => true } + }} + end + } + } + } + }), + {ok, LiveOpts} = hb_http:get(Node, <<"/~meta@1.0/info">>, #{}), + ?assert(hb_ao:get(<<"test-success">>, LiveOpts, false, #{})). + +%% @doc Test the set_opts/2 function that merges request with options, +%% manages node history, and updates server state. +set_opts_test() -> + DefaultOpts = hb_opts:default_message_with_env(), + start_node(DefaultOpts#{ + priv_wallet => Wallet = ar_wallet:new(), + port => rand:uniform(10000) + 10000 + }), + Opts = get_opts(#{ + http_server => hb_util:human_id(ar_wallet:to_address(Wallet)) + }), + NodeHistory = hb_opts:get(node_history, [], Opts), + ?event(debug_node_history, {node_history_length, length(NodeHistory)}), + ?assert(length(NodeHistory) == 0), + % Test case 1: Empty node_history case + Request1 = #{ + <<"hello">> => <<"world">> + }, + {ok, UpdatedOpts1} = set_opts(Request1, Opts), + NodeHistory1 = hb_opts:get(node_history, not_found, UpdatedOpts1), + Key1 = hb_opts:get(<<"hello">>, not_found, UpdatedOpts1), + ?event(debug_node_history, {node_history_length, length(NodeHistory1)}), + ?assert(length(NodeHistory1) == 1), + ?assert(Key1 == <<"world">>), + % Test case 2: Non-empty node_history case + Request2 = #{ + <<"hello2">> => <<"world2">> + }, + {ok, UpdatedOpts2} = set_opts(Request2, UpdatedOpts1), + NodeHistory2 = hb_opts:get(node_history, not_found, UpdatedOpts2), + Key2 = hb_opts:get(<<"hello2">>, not_found, UpdatedOpts2), + ?event(debug_node_history, {node_history_length, length(NodeHistory2)}), + ?assert(length(NodeHistory2) == 2), + ?assert(Key2 == <<"world2">>), + % Test case 3: Non-empty node_history case + {ok, UpdatedOpts3} = set_opts(#{}, UpdatedOpts2#{ <<"hello3">> => <<"world3">> }), + NodeHistory3 = hb_opts:get(node_history, not_found, UpdatedOpts3), + Key3 = hb_opts:get(<<"hello3">>, not_found, UpdatedOpts3), + ?event(debug_node_history, {node_history_length, length(NodeHistory3)}), + ?assert(length(NodeHistory3) == 3), + ?assert(Key3 == <<"world3">>). + +restart_server_test() -> + Wallet = ar_wallet:new(), + BaseOpts = #{ + <<"test-key">> => <<"server-1">>, + priv_wallet => Wallet + }, + _ = start_node(BaseOpts), + N2 = start_node(BaseOpts#{ <<"test-key">> => <<"server-2">> }), + ?assertEqual( + {ok, <<"server-2">>}, + hb_http:get(N2, <<"/~meta@1.0/info/test-key">>, #{}) + ). \ No newline at end of file diff --git a/src/hb_http_signature.erl b/src/hb_http_signature.erl deleted file mode 100644 index c80151e6d..000000000 --- a/src/hb_http_signature.erl +++ /dev/null @@ -1,1121 +0,0 @@ -%%% @doc This module implements HTTP Message Signatures -%%% as described in RFC-9421 https://datatracker.ietf.org/doc/html/rfc9421 - --module(hb_http_signature). - -% Signing/Verifying --export([authority/3, sign/2, sign/3, verify/2, verify/3]). -% Mapping --export([sf_signature/1, sf_signature_params/2, sf_signature_param/1]). - -% https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.7-14 --define(EMPTY_QUERY_PARAMS, $?). -% https://datatracker.ietf.org/doc/html/rfc9421#name-signature-parameters --define(SIGNATURE_PARAMS, [created, expired, nonce, alg, keyid, tag]). - --include("include/hb.hrl"). - --include_lib("eunit/include/eunit.hrl"). - --type fields() :: #{ - binary() | atom() | string() => binary() | atom() | string() -}. --type request_message() :: #{ - url => binary(), - method => binary(), - headers => fields(), - trailers => fields(), - is_absolute_form => boolean() -}. --type response_message() :: #{ - status => integer(), - headers => fields(), - trailers => fields() -}. --type component_identifier() :: { - item, - {string, binary()}, - {binary(), integer() | boolean() | {string | token | binary, binary()}} -}. - -%%% A map that contains signature parameters metadata as described -%%% in https://datatracker.ietf.org/doc/html/rfc9421#name-signature-parameters -%%% -%%% All values are optional, but in our use-case "alg" and "keyid" will -%%% almost always be provided. -%%% -%%% #{ -%%% created => 1733165109, % a unix timestamp -%%% expires => 1733165209, % a unix timestamp -%%% nonce => <<"foobar">, -%%% alg => <<"rsa-pss-sha512">>, -%%% keyid => <<"6eVuWgpNgv3bxfNgFrIiTkzE8Yb0V2omShxS4uKyzpw">> -%%% tag => <<"HyperBEAM">> -%%% } --type signature_params() :: #{atom() | binary() | string() => binary() | integer()}. - -%%% The state encapsulated as the "Authority". -%%% It includes an ordered list of parsed component identifiers, used for extracting values -%%% from the Request/Response Message Context, as well as the signature parameters -%%% used when creating the signature and encode in the signature base. -%%% -%%% This is effectively the State of an Authority, used to sign a Request/Response Message -%%% Context. -%%% -%%% #{ -%%% component_identifiers => [{item, {string, <<"@method">>}, []}] -%%% sig_params => #{ -%%% created => 1733165109, % a unix timestamp -%%% expires => 1733165209, % a unix timestamp -%%% nonce => <<"foobar">, -%%% alg => <<"rsa-pss-sha512">>, -%%% keyid => <<"6eVuWgpNgv3bxfNgFrIiTkzE8Yb0V2omShxS4uKyzpw">> -%%% tag => <<"HyperBEAM">> -%%% } -%%% } --type authority_state() :: #{ - component_identifiers => [component_identifier()], - % TODO: maybe refine this to be more explicit w.r.t valid signature params - sig_params => signature_params(), - key => binary() -}. - -%%% @doc A helper to validate and produce an "Authority" State --spec authority( - [binary() | component_identifier()], - #{binary() => binary() | integer()}, - {} %TODO: type out a key_pair -) -> authority_state(). -authority(ComponentIdentifiers, SigParams, PubKey = {KeyType = {ALG, _}, _Pub}) when is_atom(ALG) -> - % Only the public key is provided, so use an stub binary for private - % which will trigger errors downstream if it's needed, which is what we want - authority(ComponentIdentifiers, SigParams, {{KeyType, <<>>, PubKey}, PubKey}); -authority(ComponentIdentifiers, SigParams, PrivKey = {KeyType = {ALG, _}, _Priv, Pub}) when is_atom(ALG) -> - % Only the private key was provided, so derive the public from private - authority(ComponentIdentifiers, SigParams, {PrivKey, {KeyType, Pub}}); -authority(ComponentIdentifiers, SigParams, KeyPair = {{_, _, _}, {_, _}}) -> - #{ - % parse each component identifier into a Structured Field Item: - % - % <<"\"Example-Dict\";key=\"foo\"">> -> {item, {string, <<"Example-Dict">>}, [{<<"key">>, {string, <<"foo">>}}]} - % See hb_http_structuted_fields for parsed Structured Fields full data structures - % - % sf_item/1 handles when the argument is already parsed. - % This provides a feedback loop, in case any encoded component identifier is - % not properly encoded - component_identifiers => lists:map(fun sf_item/1, ComponentIdentifiers), - % TODO: add checks to allow only valid signature parameters - % https://datatracker.ietf.org/doc/html/rfc9421#name-signature-parameters - sig_params => SigParams, - % TODO: validate the key is supported? - key_pair => KeyPair - }. - -%%% @doc using the provided Authority and Request Message Context, and create a Signature and SignatureInput -%%% that can be used to additional signatures to a corresponding HTTP Message --spec sign(authority_state(), request_message()) -> {ok, {binary(), binary(), binary()}}. -sign(Authority, Req) -> - sign(Authority, Req, #{}). -%%% @doc using the provided Authority and Request/Response Messages Context, create a Name, Signature and SignatureInput -%%% that can be used to additional signatures to a corresponding HTTP Message --spec sign(authority_state(), request_message(), response_message()) -> {ok, {binary(), binary(), binary()}}. -sign(Authority, Req, Res) -> - {Priv, {KeyType, PubKey}} = maps:get(key_pair, Authority), - % Create the signature base and signature-input values - SigParamsWithKeyAndAlg = maps:merge( - maps:get(sig_params, Authority), - % TODO: determine alg based on KeyType from authority - % TODO: is there a more turn-key way to get the wallet address - #{ alg => <<"rsa-pss-sha512">>, keyid => hb_util:encode(bin(ar_wallet:to_address(PubKey, KeyType))) } - ), - ?no_prod(<<"Is the wallet address as keyid kosher here?">>), - AuthorityWithSigParams = maps:put(sig_params, SigParamsWithKeyAndAlg, Authority), - {SignatureInput, SignatureBase} = signature_base(AuthorityWithSigParams, Req, Res), - % Now perform the actual signing - Signature = ar_wallet:sign(Priv, SignatureBase, sha512), - {ok, {SignatureInput, Signature}}. - -%%% @doc same verify/3, but with an empty Request Message Context -verify(Verifier, Msg) -> - % Assume that the Msg is a response message, and use an empty Request message context - % - % A corollary is that a signature containing any components from the request will produce - % an error. It is the caller's responsibility to provide the required Message Context - % in order to verify the signature - verify(Verifier, #{}, Msg). - -%%% @doc Given the signature name, and the Request/Response Message Context -%%% verify the named signature by constructing the signature base and comparing -verify(#{ sig_name := SigName, key := Key }, Req, Res) -> - % Signature and Signature-Input fields are each themself a dictionary structured field. - % Ergo, we can use our same utilities to extract the value at the desired key, in this case, - % the signature name. Because our utilities already implement the relevant portions - % of RFC-9421, we get the error handling here as well. - % - % See https://datatracker.ietf.org/doc/html/rfc9421#section-3.2-3.2 - SigNameParams = [{<<"key">>, {string, bin(SigName)}}], - SignatureIdentifier = {item, {string, <<"signature">>}, SigNameParams}, - SignatureInputIdentifier = {item, {string, <<"signature-input">>}, SigNameParams}, - % extract signature and signature params - case {extract_field(SignatureIdentifier, Req, Res), extract_field(SignatureInputIdentifier, Req, Res)} of - {{ok, {_, EncodedSignature}}, {ok, {_, SignatureInput}}} -> - % The signature may be encoded ie. as binary, so we need to parse it further - % as a structured field - {item, {_, Signature}, _} = hb_http_structured_fields:parse_item(EncodedSignature), - % The value encoded within signature input is also a structured field, - % specifically an inner list that encodes the ComponentIdentifiers - % and the Signature Params. - % - % So we must parse this value, and then use it to construct the signature base - [{list, ComponentIdentifiers, SigParams}] = hb_http_structured_fields:parse_list(SignatureInput), - % TODO: HACK convert parsed sig params into a map that authority() can handle - % maybe authority() should handle both parsed and unparsed SigParams, similar to ComponentIdentifiers - SigParamsMap = lists:foldl( - % TODO: does not support SF decimal params - fun - ({Name, {_Kind, Value}}, Map) -> maps:put(Name, Value, Map); - ({Name, Value}, Map) -> maps:put(Name, Value, Map) - end, - #{}, - SigParams - ), - % Construct the signature base using the parsed parameters - Authority = authority(ComponentIdentifiers, SigParamsMap, Key), - {_, SignatureBase} = signature_base(Authority, Req, Res), - {_Priv, Pub} = maps:get(key_pair, Authority), - % Now verify the signature base signed with the provided key matches the signature - ar_wallet:verify(Pub, SignatureBase, Signature, sha512); - % An issue with parsing the signature - {SignatureErr, {ok, _}} -> SignatureErr; - % An issue with parsing the signature input - {{ok, _}, SignatureInputErr} -> SignatureInputErr; - % An issue with parsing both, so just return the first one from the signature parsing - % TODO: maybe could merge the errors? - {SignatureErr, _} -> SignatureErr - end. - -%%% @doc create the signature base that will be signed in order to create the Signature and SignatureInput. -%%% -%%% This implements a portion of RFC-9421 -%%% See https://datatracker.ietf.org/doc/html/rfc9421#name-creating-the-signature-base -signature_base(Authority, Req, Res) when is_map(Authority) -> - ComponentIdentifiers = maps:get(component_identifiers, Authority), - ComponentsLine = signature_components_line(ComponentIdentifiers, Req, Res), - ParamsLine = signature_params_line(ComponentIdentifiers, maps:get(sig_params, Authority)), - SignatureBase = join_signature_base(ComponentsLine, ParamsLine), - {ParamsLine, SignatureBase}. - -join_signature_base(ComponentsLine, ParamsLine) -> - SignatureBase = <>/binary, <<"\"@signature-params\": ">>/binary, ParamsLine/binary>>, - SignatureBase. - -%%% @doc Given a list of Component Identifiers and a Request/Response Message context, create the -%%% "signature-base-line" portion of the signature base -%%% TODO: catch duplicate identifier: https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.1 -%%% -%%% See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.1 -signature_components_line(ComponentIdentifiers, Req, Res) -> - ComponentsLines = lists:map( - fun(ComponentIdentifier) -> - % TODO: handle errors? - {ok, {I, V}} = identifier_to_component(ComponentIdentifier, Req, Res), - <>/binary, V/binary>> - end, - ComponentIdentifiers - ), - ComponentsLine = lists:join(<<"\n">>, ComponentsLines), - bin(ComponentsLine). - -%%% @doc construct the "signature-params-line" part of the signature base. -%%% -%%% See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.3.2.4 -signature_params_line(ComponentIdentifiers, SigParams) -> - SfList = sf_signature_params(ComponentIdentifiers, SigParams), - Res = hb_http_structured_fields:list(SfList), - bin(Res). - -%%% @doc Given a Component Identifier and a Request/Response Messages Context -%%% extract the value represented by the Component Identifier, from the Messages Context, -%%% and return the normalized form of the identifier, along with the extracted encoded value. -%%% -%%% Generally speaking, a Component Identifier may reference a "Derived" Component, a Message Field, -%%% or a sub-component of a Message Field. -%%% -%%% Since a Component Identifier is itself a Structured Field, it may also specify parameters, which are -%%% used to describe behavior such as which Message to derive a field or sub-component of the field, -%%% and how to encode the value as part of the signature base. -identifier_to_component(Identifier, Req, Res) when is_list(Identifier) -> - identifier_to_component(list_to_binary(Identifier), Req, Res); -identifier_to_component(Identifier, Req, Res) when is_atom(Identifier) -> - identifier_to_component(atom_to_binary(Identifier), Req, Res); -identifier_to_component(Identifier, Req, Res) when is_binary(Identifier) -> - identifier_to_component(hb_http_structured_fields:parse_item(Identifier), Req, Res); -identifier_to_component(ParsedIdentifier = {item, {_Kind, Value}, _Params}, Req, Res) -> - case Value of - <<$@, _R/bits>> -> derive_component(ParsedIdentifier, Req, Res); - _ -> extract_field(ParsedIdentifier, Req, Res) - end. - -%%% @doc Given a Component Identifier and a Request/Response Messages Context -%%% extract the value represented by the Component Identifier, from the Messages Context, -%%% specifically a field on a Message within the Messages Context, -%%% and return the normalized form of the identifier, along with the extracted encoded value. -%%% -%%% This implements a portion of RFC-9421 -%%% See https://datatracker.ietf.org/doc/html/rfc9421#name-http-fields -extract_field(Identifier, Req, Res) when map_size(Res) == 0 -> - extract_field(Identifier, Req, Res); -extract_field({item, {_Kind, IParsed}, IParams}, Req, Res) -> - [IsStrictFormat, IsByteSequenceEncoded, DictKey] = [ - find_sf_strict_format_param(IParams), - find_sf_byte_sequence_param(IParams), - find_sf_key_param(IParams) - ], - case (IsStrictFormat orelse DictKey =/= false) andalso IsByteSequenceEncoded of - true -> - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.2 - {conflicting_params_error, <<"Component Identifier parameter 'bs' MUST not be used with 'sf' or 'key'">>}; - _ -> - Lowered = lower_bin(IParsed), - NormalizedItem = hb_http_structured_fields:item({item, {string, Lowered}, IParams}), - [IsRequestIdentifier, IsTrailerField] = [find_sf_request_param(IParams), find_sf_trailer_param(IParams)], - % There may be multiple fields that match the identifier on the Msg, - % so we filter, instead of find - MaybeRawFields = lists:filter( - fun({Key, _Value}) -> Key =:= Lowered end, - % Field names are normalized to lowercase in the signature base and also are case insensitive. - % So by converting all the names to lowercase here, we simoultaneously normalize them, and prepare - % them for comparison in one pass. - [ - {lower_bin(Key), Value} - % TODO: how can we maintain the order msg fields, especially in the case where there are - % multiple fields with the same name, and order must be preserved - || {Key, Value} <- maps:to_list( - maps:get( - % The field will almost certainly be a header, but could also be a trailer - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-18.10.1 - case IsTrailerField of true -> trailers; false -> headers end, - % The header may exist on any message in the context of the signature - % which could be the Request or Response Message - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-18.8.1 - case IsRequestIdentifier of true -> Req; false -> Res end - ) - ) - ] - ), - case MaybeRawFields of - [] -> - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.6 - {field_not_found_error, <<"Component Identifier for a field MUST be present on the message">>}; - FieldPairs -> - % The Field was found, but we still need to potentially parse it - % (it could be a Structured Field) and potentially extract - % subsequent values ie. specific dictionary key and its parameters, or further encode it - case - extract_field_value( - [bin(Value) || {_Key, Value} <- FieldPairs], - [DictKey, IsStrictFormat, IsByteSequenceEncoded] - ) - of - {ok, Extracted} -> {ok, {bin(NormalizedItem), bin(Extracted)}}; - E -> E - end - end - end. - -%%% @doc Extract values from the field and return the normalized field, -%%% along with encoded value -extract_field_value(RawFields, [Key, IsStrictFormat, IsByteSequenceEncoded]) -> - % TODO: (maybe this already works?) empty string for empty header - HasKey = case Key of false -> false; _ -> true end, - case not (HasKey orelse IsStrictFormat orelse IsByteSequenceEncoded) of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-5 - true -> - Normalized = [trim_and_normalize(Field) || Field <- RawFields], - {ok, bin(lists:join(<<", ">>, Normalized))}; - _ -> - case IsByteSequenceEncoded of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.3-2 - true -> - SfList = [ - {item, {binary, trim_and_normalize(Field)}, []} - || Field <- RawFields - ], - sf_encode(SfList); - _ -> - Combined = bin(lists:join(<<", ">>, RawFields)), - case sf_parse(Combined) of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.1-3 - {error, _} -> - {sf_parsing_error, <<"Component Identifier value could not be parsed as a structured field">>}; - {ok, SF} -> - case Key of - % Not accessing a key, so just re-serialize, which should - % properly format the data in Strict-Formatting style - false -> sf_encode(SF); - _ -> extract_dictionary_field_value(SF, Key) - end - end - end - end. - -%%% @doc Extract a value from a Structured Field, and return the normalized field, -%%% along with the encoded value -extract_dictionary_field_value(StructuredField = [Elem | _Rest], Key) -> - case Elem of - {Name, _} when is_binary(Name) -> - case lists:keyfind(Key, 1, StructuredField) of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.2-5 - false -> - {sf_dicionary_key_not_found_error, - <<"Component Identifier references key not found in dictionary structured field">>}; - {_, Value} -> - sf_encode(Value) - end; - _ -> - {sf_not_dictionary_error, <<"Component Identifier cannot reference key on a non-dictionary structured field">>} - end. - -%%% @doc Given a Component Identifier and a Request/Response Messages Context -%%% extract the value represented by the Component Identifier, from the Messages Context, -%%% specifically a "Derived" Component within the Messages Context, -%%% and return the normalized form of the identifier, along with the extracted encoded value. -%%% -%%% This implements a portion of RFC-9421 -%%% See https://datatracker.ietf.org/doc/html/rfc9421#name-derived-components -derive_component(Identifier, Req, Res) when map_size(Res) == 0 -> - derive_component(Identifier, Req, Res, req); -derive_component(Identifier, Req, Res) -> - derive_component(Identifier, Req, Res, res). -derive_component({item, {_Kind, IParsed}, IParams}, Req, Res, Subject) -> - case find_sf_request_param(IParams) andalso Subject =:= req of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.2.2.5.2.3 - true -> - {req_identifier_error, - <<"A Component Identifier may not contain a req parameter if the target is a request message">>}; - _ -> - Lowered = lower_bin(IParsed), - NormalizedItem = hb_http_structured_fields:item({item, {string, Lowered}, IParams}), - Result = - case Lowered of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.2.1 - <<"@method">> -> - {ok, upper_bin(maps:get(method, Req))}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.4.1 - <<"@target-uri">> -> - {ok, bin(maps:get(url, Req))}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.6.1 - <<"@authority">> -> - URI = uri_string:parse(maps:get(url, Req)), - Authority = maps:get(host, URI), - {ok, lower_bin(Authority)}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.8.1 - <<"@scheme">> -> - URI = uri_string:parse(maps:get(url, Req)), - Scheme = maps:get(scheme, URI), - {ok, lower_bin(Scheme)}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.10.1 - <<"@request-target">> -> - URI = uri_string:parse(maps:get(url, Req)), - % If message contains the absolute form value, then - % the value must be the absolut url - % - % TODO: maybe better way to distinguish besides a flag - % on the request? - % - % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.5-10 - RequestTarget = - case maps:get(is_absolute_form, Req, false) of - true -> maps:get(url, Req); - _ -> lists:join($?, [maps:get(path, URI), maps:get(query, URI, ?EMPTY_QUERY_PARAMS)]) - end, - {ok, bin(RequestTarget)}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.12.1 - <<"@path">> -> - URI = uri_string:parse(maps:get(url, Req)), - Path = maps:get(path, URI), - {ok, bin(Path)}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.14.1 - <<"@query">> -> - URI = uri_string:parse(maps:get(url, Req)), - % No query params results in a "?" value - % See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.7-14 - Query = - case maps:get(query, URI, <<>>) of - <<>> -> ?EMPTY_QUERY_PARAMS; - Q -> Q - end, - {ok, bin(Query)}; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.16.1 - <<"@query-param">> -> - case find_sf_name_param(IParams) of - % The name parameter MUST be provided when specifiying a @query-param - % Derived Component. See https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.8-1 - false -> - {req_identifier_error, <<"@query_param Derived Component Identifier must specify a name parameter">>}; - Name -> - URI = uri_string:parse(maps:get(url, Req)), - QueryParams = uri_string:dissect_query(maps:get(query, URI, "")), - QueryParam = - case lists:keyfind(Name, 1, QueryParams) of - {_, QP} -> QP; - % An missing or empty query param value results in - % an empty string value in the signature base - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.8-4 - _ -> "" - end, - {ok, bin(QueryParam)} - end; - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2-4.18.1 - <<"@status">> -> - case Subject =:= req of - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.2.9-8 - true -> - {res_identifier_error, <<"@status Derived Component must not be used if target is a request message">>}; - _ -> - Status = maps:get(status, Res, <<"200">>), - {ok, Status} - end - end, - case Result of - {ok, V} -> {ok, {bin(NormalizedItem), V}}; - E -> E - end - end. - -%%% -%%% Strucutured Field Utilities -%%% - -%%% @doc construct the structured field Parameter for the signature parameter, -%%% checking whether the parameter name is valid according RFC-9421 -%%% -%%% See https://datatracker.ietf.org/doc/html/rfc9421#section-2.3-3 -sf_signature_param({Name, Param}) -> - NormalizedName = bin(Name), - NormalizedNames = lists:map(fun bin/1, ?SIGNATURE_PARAMS), - case lists:member(NormalizedName, NormalizedNames) of - false -> {unknown_signature_param, NormalizedName}; - % all signature params are either integer or string values - true -> case Param of - I when is_integer(I) -> {ok, {NormalizedName, Param}}; - P when is_atom(P) orelse is_list(P) orelse is_binary(P) -> {ok, {NormalizedName, {string, bin(P)}}}; - P -> {invalid_signature_param_value, P} - end - end. - -%%% @doc construct the structured field List for the -%%% "signature-params-line" part of the signature base. -%%% -%%% Can be parsed into a binary by simply passing to hb_structured_fields:list/1 -%%% -%%% See https://datatracker.ietf.org/doc/html/rfc9421#section-2.5-7.3.2.4 -sf_signature_params(ComponentIdentifiers, SigParams) when is_map(SigParams) -> - AsList = maps:to_list(SigParams), - Sorted = lists:sort(fun({Key1, _}, {Key2, _}) -> Key1 < Key2 end, AsList), - sf_signature_params(ComponentIdentifiers, Sorted); -sf_signature_params(ComponentIdentifiers, SigParams) when is_list(SigParams) -> - [ - { - list, - lists:map( - fun(ComponentIdentifier) -> - {item, {_Kind, Value}, Params} = sf_item(ComponentIdentifier), - {item, {string, lower_bin(Value)}, Params} - end, - ComponentIdentifiers - ), - lists:foldl( - fun (RawParam, Params) -> - case sf_signature_param(RawParam) of - {ok, Param} -> Params ++ [Param]; - % Ignore unknown signature parameters - {unknown_signature_param, _} -> Params - % TODO: what to do about invalid_signature_param_value? - % For now will cause badmatch - end - end, - [], - SigParams - ) - } - ]. - -% TODO: should this also handle the Signature already being encoded? -sf_signature(Signature) -> - {item, {binary, Signature}, []}. - -%%% @doc Attempt to parse the binary into a data structure that represents -%%% an HTTP Structured Field. -%%% -%%% Lacking some sort of "hint", there isn't a way to know which "kind" of Structured Field -%%% the binary is, apriori. So we simply try each parser, and return the first invocation that -%%% doesn't result in an error. -%%% -%%% If no parser is successful, then we return an error tuple -sf_parse(Raw) when is_list(Raw) -> sf_parse(list_to_binary(Raw)); -sf_parse(Raw) when is_binary(Raw) -> - Parsers = [ - fun hb_http_structured_fields:parse_list/1, - fun hb_http_structured_fields:parse_dictionary/1, - fun hb_http_structured_fields:parse_item/1 - ], - sf_parse(Parsers, Raw). - -sf_parse([], _Raw) -> - {error, undefined}; -sf_parse([Parser | Rest], Raw) -> - case catch Parser(Raw) of - % skip parsers that fail - {'EXIT', _} -> sf_parse(Rest, Raw); - Parsed -> {ok, Parsed} - end. - -%%% @doc Attempt to encode the data structure into an HTTP Structured Field. -%%% This is the inverse of sf_parse. -sf_encode(StructuredField = {list, _, _}) -> - % The value is an inner_list, and so needs to be wrapped with an outer list - % before being serialized - sf_encode(fun hb_http_structured_fields:list/1, [StructuredField]); -sf_encode(StructuredField = {item, _, _}) -> - sf_encode(fun hb_http_structured_fields:item/1, StructuredField); -sf_encode(StructuredField = [Elem | _Rest]) -> - sf_encode( - % Both an sf list and dictionary is represented in Erlang as a List of pairs - % but a dictionary's members will always be a pair whose first value - % is a binary, so we can match on that to determine which serializer to use - case Elem of - {Name, _} when is_binary(Name) -> fun hb_http_structured_fields:dictionary/1; - _ -> fun hb_http_structured_fields:list/1 - end, - StructuredField - ). -sf_encode(Serializer, StructuredField) -> - case catch Serializer(StructuredField) of - {'EXIT', _} -> {error, <<"Could not serialize into structured field">>}; - Parsed -> {ok, Parsed} - end. - -%%% @doc Attempt to parse the provided value into an HTTP Structured Field Item -sf_item(SfItem = {item, {_Kind, _Parsed}, _Params}) -> - SfItem; -sf_item(ComponentIdentifier) when is_list(ComponentIdentifier) -> - sf_item(list_to_binary(ComponentIdentifier)); -sf_item(ComponentIdentifier) when is_binary(ComponentIdentifier) -> - sf_item(hb_http_structured_fields:parse_item(ComponentIdentifier)). - -%%% @doc Given a parameter Name, extract the Parameter value from the HTTP Structured Field -%%% data structure. -%%% -%%% If no value is found, then false is returned -find_sf_param(Name, Params, Default) when is_list(Name) -> - find_sf_param(list_to_binary(Name), Params, Default); -find_sf_param(Name, Params, Default) -> - % [{<<"name">>,{string,<<"baz">>}}] - case lists:keyfind(Name, 1, Params) of - false -> Default; - {_, {string, Value}} -> Value; - {_, {token, Value}} -> Value; - {_, {binary, Value}} -> Value; - {_, Value} -> Value - end. - -%%% -%%% https://datatracker.ietf.org/doc/html/rfc9421#section-6.5.2-1 -%%% using functions allows encapsulating default values -%%% -find_sf_strict_format_param(Params) -> find_sf_param(<<"sf">>, Params, false). -find_sf_key_param(Params) -> find_sf_param(<<"key">>, Params, false). -find_sf_byte_sequence_param(Params) -> find_sf_param(<<"bs">>, Params, false). -find_sf_trailer_param(Params) -> find_sf_param(<<"tr">>, Params, false). -find_sf_request_param(Params) -> find_sf_param(<<"req">>, Params, false). -find_sf_name_param(Params) -> find_sf_param(<<"name">>, Params, false). - -%%% -%%% Data Utilities -%%% - -% https://datatracker.ietf.org/doc/html/rfc9421#section-2.1-5 -trim_and_normalize(Bin) -> - binary:replace(trim_ws(Bin), <<$\n>>, <<" ">>, [global]). - -upper_bin(Item) when is_atom(Item) -> upper_bin(atom_to_list(Item)); -upper_bin(Item) when is_binary(Item) -> upper_bin(binary_to_list(Item)); -upper_bin(Item) when is_list(Item) -> bin(string:uppercase(Item)). - -lower_bin(Item) when is_atom(Item) -> lower_bin(atom_to_list(Item)); -lower_bin(Item) when is_binary(Item) -> lower_bin(binary_to_list(Item)); -lower_bin(Item) when is_list(Item) -> bin(string:lowercase(Item)). - -bin(Item) when is_atom(Item) -> atom_to_binary(Item, utf8); -bin(Item) when is_integer(Item) -> - case Item of - % Treat integer as an ASCII code - N when N > 0 andalso N < 256 -> <>; - N -> integer_to_binary(N) - end; -bin(Item) -> - iolist_to_binary(Item). - -%%% @doc Recursively trim space characters from the beginning of the binary -trim_ws(<<$\s, Bin/bits>>) -> trim_ws(Bin); -%%% @doc No space characters at the beginning so now trim them from the end -%%% recrusively -trim_ws(Bin) -> trim_ws_end(Bin, byte_size(Bin) - 1). - -trim_ws_end(_, -1) -> - <<>>; -trim_ws_end(Value, N) -> - case binary:at(Value, N) of - $\s -> - trim_ws_end(Value, N - 1); - % No more space characters matches on the end - % So extract the bytes up to N, and this is our trimmed value - _ -> - S = N + 1, - <> = Value, - Trimmed - end. - -%%% -%%% TESTS -%%% - -trim_ws_test() -> - ?assertEqual(<<"hello world">>, trim_ws(<<" hello world ">>)), - ?assertEqual(<<>>, trim_ws(<<"">>)), - ?assertEqual(<<>>, trim_ws(<<" ">>)), - ok. - -sign_test() -> - Req = #{ - url => <<"https://foo.bar/id-123/Data?another=one&fizz=buzz">>, - method => "get", - headers => #{ - <<"foo">> => <<"req-b-bar">> - }, - trailers => #{} - }, - Res = #{ - status => 202, - headers => #{ - "fizz" => "res-l-bar", - <<"Foo">> => "a=1, b=2;x=1;y=2, c=(a b c), d" - }, - trailers => #{} - }, - % Ensure both parsed, and serialized SFs are handled - ComponentIdentifiers = [ - {item, {string, <<"@method">>}, []}, - <<"\"@path\"">>, - {item, {string, <<"foo">>}, [{<<"req">>, true}]}, - "\"foo\";key=\"a\"" - ], - SigParams = #{}, - Key = hb:wallet(), - Authority = authority(ComponentIdentifiers, SigParams, Key), - - ?assertMatch( - {ok, {_SignatureInput, _Signature}}, - sign(Authority, Req, Res) - ), - ok. - -verify_test() -> - Req = #{ - url => <<"https://foo.bar/id-123/Data?another=one&fizz=buzz">>, - method => "get", - headers => #{ - <<"foo">> => <<"req-b-bar">> - }, - trailers => #{} - }, - Res = #{ - status => 202, - headers => #{ - "fizz" => "res-l-bar", - <<"Foo">> => "a=1, b=2;x=1;y=2, c=(a b c), d" - }, - trailers => #{} - }, - ComponentIdentifiers = [ - {item, {string, <<"@method">>}, []}, - <<"\"@path\"">>, - {item, {string, <<"foo">>}, [{<<"req">>, true}]}, - "\"foo\";key=\"a\"" - ], - SigParams = #{}, - Key = {_Priv, Pub} = hb:wallet(), - Authority = authority(ComponentIdentifiers, SigParams, Key), - - % Create the signature and signature input - % TODO: maybe return the SF data structures instead, to make appending to headers easier? - % OR we could wrap behind an api ie. sf_dictionary_put(Key, SfValue, Dict) - {ok, {SignatureInput, Signature}} = sign(Authority, Req, Res), - SigName = <<"awesome">>, - [ParsedSignatureInput] = hb_http_structured_fields:parse_list(SignatureInput), - NewHeaders = maps:merge( - maps:get(headers, Res), - #{ - % https://datatracker.ietf.org/doc/html/rfc9421#section-4.2-1 - <<"signature">> => bin(hb_http_structured_fields:dictionary(#{ SigName => {item, {binary, Signature}, []} })), - <<"signature-input">> => bin(hb_http_structured_fields:dictionary(#{ SigName => ParsedSignatureInput })) - } - ), - - SignedRes = maps:put(headers, NewHeaders, Res), - Result = verify(#{ sig_name => SigName, key => Pub }, Req, SignedRes), - ?assert(Result), - ok. - -join_signature_base_test() -> - ParamsLine = - <<"(\"@method\" \"@path\" \"foo\";req \"foo\";key=\"a\");created=1733165109501;nonce=\"foobar\";keyid=\"key1\"">>, - ComponentsLine = <<"\"@method\": GET\n\"@path\": /id-123/Data\n\"foo\";req: req-b-bar\n\"foo\";key=\"a\": 1">>, - ?assertEqual( - <>/binary, <<"\"@signature-params\": ">>/binary, ParamsLine/binary>>, - join_signature_base(ComponentsLine, ParamsLine) - ). - -signature_components_line_test() -> - Req = #{ - url => <<"https://foo.bar/id-123/Data?another=one&fizz=buzz">>, - method => "get", - headers => #{ - <<"foo">> => <<"req-b-bar">> - }, - trailers => #{} - }, - Res = #{ - status => 202, - headers => #{ - "fizz" => "res-l-bar", - <<"Foo">> => "a=1, b=2;x=1;y=2, c=(a b c), d" - }, - trailers => #{} - }, - ComponentIdentifiers = [ - <<"\"@method\"">>, - <<"\"@path\"">>, - % parsed SF items are also handled - {item, {string, <<"foo">>}, [{<<"req">>, true}]}, - <<"\"foo\";key=\"a\"">> - ], - ?assertEqual( - <<"\"@method\": GET\n\"@path\": /id-123/Data\n\"foo\";req: req-b-bar\n\"foo\";key=\"a\": 1">>, - signature_components_line(ComponentIdentifiers, Req, Res) - ). - -signature_params_line_test() -> - Params = #{created => 1733165109501, nonce => "foobar", keyid => "key1"}, - ContentIdentifiers = [ - <<"\"Content-Length\"">>, <<"\"@method\"">>, "\"@Path\"", "\"content-type\";req", "\"example-dict\";sf" - ], - Result = signature_params_line(ContentIdentifiers, Params), - ?assertEqual( - <<"(\"content-length\" \"@method\" \"@path\" \"content-type\";req \"example-dict\";sf);created=1733165109501;keyid=\"key1\";nonce=\"foobar\"">>, - Result - ). - -extract_field_msg_access_test() -> - Req = #{ - url => <<"https://foo.bar/id-123/Data?another=one&fizz=buzz">>, - method => "get", - headers => #{ - <<"foo">> => <<"req-b-bar">> - }, - trailers => #{ - another => <<"req-tr-atom-one">> - } - }, - Res = #{ - status => 202, - headers => #{ - "fizz" => "res-l-bar", - "A-field" => " first\none", - "a-field" => " second " - }, - trailers => #{ - <<"Woo">> => <<"res-tr-uppercase-woo">> - } - }, - % req header + binary key + binary value - ?assertEqual( - {ok, {<<"\"foo\";req">>, <<"req-b-bar">>}}, - extract_field({item, {string, <<"foo">>}, [{<<"req">>, true}]}, Req, Res) - ), - - % req trailer + atom key + binary value - ?assertEqual( - {ok, {<<"\"another\";req;tr">>, <<"req-tr-atom-one">>}}, - extract_field({item, {string, <<"another">>}, [{<<"req">>, true}, {<<"tr">>, true}]}, Req, Res) - ), - - % res header + list key + list value - ?assertEqual( - {ok, {<<"\"fizz\"">>, <<"res-l-bar">>}}, - extract_field({item, {string, <<"fizz">>}, []}, Req, Res) - ), - - % res trailer + binary uppercase key + binary value - ?assertEqual( - {ok, {<<"\"woo\";tr">>, <<"res-tr-uppercase-woo">>}}, - extract_field({item, {string, <<"woo">>}, [{<<"tr">>, true}]}, Req, Res) - ), - - % multiple fields, with obs and newlines - ?assertEqual( - {ok, {<<"\"a-field\"">>, <<"first one, second">>}}, - extract_field({item, {string, <<"a-field">>}, []}, Req, Res) - ). - -extract_field_bs_test() -> - Req = #{}, - Res = #{ - status => 202, - headers => #{ - <<"Foo">> => "foobar", - <<"A-Field">> => "first", - <<"a-field">> => "second", - <<"b-field">> => "first, second" - }, - trailers => #{} - }, - - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.3-4 - - ?assertEqual( - {ok, {<<"\"foo\";bs">>, <<":Zm9vYmFy:">>}}, - extract_field({item, {string, <<"foo">>}, [{<<"bs">>, true}]}, Req, Res) - ), - - ?assertEqual( - {ok, {<<"\"a-field\";bs">>, <<":Zmlyc3Q=:, :c2Vjb25k:">>}}, - extract_field({item, {string, <<"a-field">>}, [{<<"bs">>, true}]}, Req, Res) - ), - - ?assertEqual( - {ok, {<<"\"b-field\";bs">>, <<":Zmlyc3QsIHNlY29uZA==:">>}}, - extract_field({item, {string, <<"b-field">>}, [{<<"bs">>, true}]}, Req, Res) - ). - -extract_field_sf_test() -> - Req = #{}, - Res = #{ - status => 202, - headers => #{ - <<"Foo">> => "a=1, b=2;x=1;y=2, c=(a b c), d" - }, - trailers => #{} - }, - % https://datatracker.ietf.org/doc/html/rfc9421#section-2.1.2-6 - ?assertEqual( - {ok, {<<"\"foo\"">>, <<"a=1, b=2;x=1;y=2, c=(a b c), d">>}}, - extract_field({item, {string, <<"foo">>}, []}, Req, Res) - ), - - ?assertEqual( - {ok, {<<"\"foo\";sf">>, <<"a=1, b=2;x=1;y=2, c=(a b c), d=?1">>}}, - extract_field({item, {string, <<"foo">>}, [{<<"sf">>, true}]}, Req, Res) - ), - - ?assertEqual( - {ok, {<<"\"foo\";key=\"a\"">>, <<"1">>}}, - extract_field({item, {string, <<"foo">>}, [{<<"key">>, {string, <<"a">>}}]}, Req, Res) - ), - % inner-list - ?assertEqual( - {ok, {<<"\"foo\";key=\"c\"">>, <<"(a b c)">>}}, - extract_field({item, {string, <<"foo">>}, [{<<"key">>, {string, <<"c">>}}]}, Req, Res) - ), - % boolean - ?assertEqual( - {ok, {<<"\"foo\";key=\"d\"">>, <<"?1">>}}, - extract_field({item, {string, <<"foo">>}, [{<<"key">>, {string, <<"d">>}}]}, Req, Res) - ), - % params - ?assertEqual( - {ok, {<<"\"foo\";key=\"b\"">>, <<"2;x=1;y=2">>}}, - extract_field({item, {string, <<"foo">>}, [{<<"key">>, {string, <<"b">>}}]}, Req, Res) - ). - -extract_field_error_conflicting_params_test() -> - Req = #{ - url => <<"https://foo.bar/id-123/Data?another=one&fizz=buzz">>, - method => "get", - headers => #{ - <<"foo">> => "a=1, b=2;x=1;y=2, c=(a b c), d" - }, - trailers => #{} - }, - Res = #{ - status => 202, - headers => #{}, - trailers => #{} - }, - Expected = conflicting_params_error, - {E, _} = extract_field({item, {string, <<"foo">>}, [{<<"bs">>, true}, {<<"sf">>, true}]}, Req, Res), - ?assertEqual(Expected, E), - - {E2, _} = extract_field({item, {string, <<"foo">>}, [{<<"bs">>, true}, {<<"key">>, {string, <<"foo">>}}]}, Req, Res), - ?assertEqual(Expected, E2). - -extract_field_error_field_not_found_test() -> - Req = #{ - url => <<"https://foo.bar/id-123/Data?another=one&fizz=buzz">>, - method => "get", - headers => #{ - <<"foo">> => "req-b-bar" - }, - trailers => #{} - }, - Res = #{ - status => 202, - headers => #{}, - trailers => #{} - }, - Expected = field_not_found_error, - % req headers - {E, _} = extract_field({item, {string, <<"not-foo">>}, [{<<"req">>, true}]}, Req, Res), - ?assertEqual(Expected, E), - % req trailers - {E2, _} = extract_field({item, {string, <<"not-foo">>}, [{<<"req">>, true}, {<<"tr">>, true}]}, Req, Res), - ?assertEqual(Expected, E2), - % res headers - {E3, _} = extract_field({item, {string, <<"not-foo">>}, []}, Req, Res), - ?assertEqual(Expected, E3), - % res trailers - {E4, _} = extract_field({item, {string, <<"not-foo">>}, [{<<"tr">>, true}]}, Req, Res), - ?assertEqual(Expected, E4). - -extract_field_error_not_sf_dictionary_test() -> - Req = #{ - url => <<"https://foo.bar/id-123/Data?another=one&fizz=buzz">>, - method => "get", - headers => #{ - <<"foo">> => "req-b-bar" - }, - trailers => #{} - }, - Res = #{ - status => 202, - headers => #{}, - trailers => #{} - }, - Expected = sf_not_dictionary_error, - {E, _M} = extract_field({item, {string, <<"foo">>}, [{<<"req">>, true}, {<<"key">>, {string, <<"smth">>}}]}, Req, Res), - ?assertEqual(Expected, E). - -extract_field_error_sf_dictionary_key_not_found_test() -> - Req = #{ - url => <<"https://foo.bar/id-123/Data?another=one&fizz=buzz">>, - method => "get", - headers => #{ - <<"foo">> => "a=1, b=2;x=1;y=2, c=(a b c), d" - }, - trailers => #{} - }, - Res = #{ - status => 202, - headers => #{}, - trailers => #{} - }, - Expected = sf_dicionary_key_not_found_error, - {E, _M} = extract_field({item, {string, <<"foo">>}, [{<<"req">>, true}, {<<"key">>, {string, <<"smth">>}}]}, Req, Res), - ?assertEqual(Expected, E). - -derive_component_test() -> - Url = <<"https://foo.bar/id-123/Data?another=one&fizz=buzz">>, - Req = #{ - url => Url, - method => "get", - headers => #{} - }, - Res = #{ - status => 202 - }, - - % normalize method (uppercase) + method - ?assertEqual( - {ok, {<<"\"@method\"">>, <<"GET">>}}, - derive_component({item, {string, <<"@method">>}, []}, Req, Res) - ), - - ?assertEqual( - {ok, {<<"\"@target-uri\"">>, Url}}, - derive_component({item, {string, <<"@target-uri">>}, []}, Req, Res) - ), - - ?assertEqual( - {ok, {<<"\"@authority\"">>, <<"foo.bar">>}}, - derive_component({item, {string, <<"@authority">>}, []}, Req, Res) - ), - - ?assertEqual( - {ok, {<<"\"@scheme\"">>, <<"https">>}}, - derive_component({item, {string, <<"@scheme">>}, []}, Req, Res) - ), - - ?assertEqual( - {ok, {<<"\"@request-target\"">>, <<"/id-123/Data?another=one&fizz=buzz">>}}, - derive_component({item, {string, <<"@request-target">>}, []}, Req, Res) - ), - - % absolute form - ?assertEqual( - {ok, {<<"\"@request-target\"">>, Url}}, - derive_component({item, {string, <<"@request-target">>}, []}, maps:merge(Req, #{is_absolute_form => true}), Res) - ), - - ?assertEqual( - {ok, {<<"\"@path\"">>, <<"/id-123/Data">>}}, - derive_component({item, {string, <<"@path">>}, []}, Req, Res) - ), - - ?assertEqual( - {ok, {<<"\"@query\"">>, <<"another=one&fizz=buzz">>}}, - derive_component({item, {string, <<"@query">>}, []}, Req, Res) - ), - - % no query params - ?assertEqual( - {ok, {<<"\"@query\"">>, <<"?">>}}, - derive_component({item, {string, <<"@query">>}, []}, maps:merge(Req, #{url => <<"https://foo.bar/id-123/Data">>}), Res) - ), - - % empty query params - ?assertEqual( - {ok, {<<"\"@query\"">>, <<"?">>}}, - derive_component({item, {string, <<"@query">>}, []}, maps:merge(Req, #{url => <<"https://foo.bar/id-123/Data?">>}), Res) - ), - - ?assertEqual( - {ok, {<<"\"@query-param\";name=\"fizz\"">>, <<"buzz">>}}, - derive_component({item, {string, <<"@query-param">>}, [{<<"name">>, {string, <<"fizz">>}}]}, Req, Res) - ), - - % normalize identifier (lowercase) + @status - ?assertEqual( - {ok, {<<"\"@status\"">>, 202}}, - derive_component({item, {string, <<"@Status">>}, []}, Req, Res) - ), - ok. - -derive_component_error_req_param_on_request_target_test() -> - Result = derive_component({item, {string, <<"@query-param">>}, [{<<"req">>, true}]}, #{}, #{}, req), - ?assertEqual( - {req_identifier_error, <<"A Component Identifier may not contain a req parameter if the target is a request message">>}, - Result - ). - -derive_component_error_query_param_no_name_test() -> - Result = derive_component({item, {string, <<"@query-param">>}, [{<<"noname">>, {string, <<"foo">>}}]}, #{}, #{}, req), - ?assertEqual( - {req_identifier_error, <<"@query_param Derived Component Identifier must specify a name parameter">>}, - Result - ). - -derive_component_error_status_req_target_test() -> - Result = derive_component({item, {string, <<"@status">>}, []}, #{}, #{}, req), - {E, _M} = Result, - ?assertEqual(res_identifier_error, E). diff --git a/src/hb_json.erl b/src/hb_json.erl new file mode 100644 index 000000000..180f2704f --- /dev/null +++ b/src/hb_json.erl @@ -0,0 +1,13 @@ +%%% @doc Wrapper for encoding and decoding JSON. Supports maps and Jiffy's old +%%% `ejson' format. This module abstracts the underlying JSON library, allowing +%%% us to switch between libraries as needed in the future. +-module(hb_json). +-export([encode/1, decode/1, decode/2]). + +%% @doc Takes a term in Erlang's native form and encodes it as a JSON string. +encode(Term) -> + iolist_to_binary(json:encode(Term)). + +%% @doc Takes a JSON string and decodes it into an Erlang term. +decode(Bin) -> json:decode(Bin). +decode(Bin, _Opts) -> decode(Bin). \ No newline at end of file diff --git a/src/hb_keccak.erl b/src/hb_keccak.erl new file mode 100644 index 000000000..8d4048c1f --- /dev/null +++ b/src/hb_keccak.erl @@ -0,0 +1,86 @@ +-module(hb_keccak). +-export([sha3_256/1]). +-export([keccak_256/1]). +-export([key_to_ethereum_address/1]). +-include_lib("eunit/include/eunit.hrl"). + +-on_load(init/0). + +-define(APPNAME, keccak). +-define(LIBNAME, keccak_nif). + +%% NIF Initialization +init() -> + SoName = filename:join([code:priv_dir(hb), "hb_keccak"]), + erlang:load_nif(SoName, 0). + +sha3_256(_Bin) -> + erlang:nif_error(not_loaded). + +keccak_256(_Bin) -> + erlang:nif_error(not_loaded). + +to_hex(Bin) when is_binary(Bin) -> + binary:encode_hex(Bin). + +key_to_ethereum_address(Key) when is_binary(Key) -> + <<_Prefix: 1/binary, NoCompressionByte/binary>> = Key, + Prefix = hb_util:to_hex(hb_keccak:keccak_256(NoCompressionByte)), + Last40 = binary:part(Prefix, byte_size(Prefix) - 40, 40), + + Hash = hb_keccak:keccak_256(Last40), + HashHex = hb_util:to_hex(Hash), + + ChecksumAddress = hash_to_checksum_address(Last40, HashHex), + ChecksumAddress. + +hash_to_checksum_address(Last40, Hash) when + is_binary(Last40), + is_binary(Hash), + byte_size(Last40) =:= 40 -> + + Checksummed = lists:zip(binary:bin_to_list(Last40), binary:bin_to_list(binary:part(Hash, 0, 40))), + Formatted = lists:map(fun({Char, H}) -> + case H >= $8 of + true -> string:to_upper([Char]); + false -> [Char] + end + end, Checksummed), + <<"0x", (list_to_binary(lists:append(Formatted)))/binary>>. + +%% Test functions +keccak_256_test() -> + Input = <<"testing">>, + Expected = <<"5F16F4C7F149AC4F9510D9CF8CF384038AD348B3BCDC01915F95DE12DF9D1B02">>, + Actual = to_hex(hb_keccak:keccak_256(Input)), + ?assertEqual(Expected, Actual). + +keccak_256_key_test() -> + Input = <<"BAoixXds4JhW42pzlLb83B3-I21lX78j3Q7cPaoFiCjMgjYwYLDj-xL132J147ifZFwRBmzmEMC8eYAXzbRNWuA">>, + BinaryInput = hb_util:decode(Input), + <<_Prefix: 1/binary, NoCompressionByte/binary>> = BinaryInput, + + Prefix = hb_keccak:keccak_256(NoCompressionByte), + PrefixHex = hb_util:to_hex(Prefix), + ?assertEqual(PrefixHex, <<"12f9afe6abd38444cab38e8cb7b4360f7f6298de2e7a11009270f35f189bd77e">>), + + Last40 = binary:part(PrefixHex, byte_size(PrefixHex) - 40, 40), + ?assertEqual(Last40, <<"b7b4360f7f6298de2e7a11009270f35f189bd77e">>), + + Hash = hb_keccak:keccak_256(Last40), + HashHex = hb_util:to_hex(Hash), + + ChecksumAddress = hash_to_checksum_address(Last40, HashHex), + ?assertEqual(ChecksumAddress, <<"0xb7B4360F7F6298dE2e7a11009270F35F189Bd77E">>). + +keccak_256_key_to_address_test() -> + Input = <<"BAoixXds4JhW42pzlLb83B3-I21lX78j3Q7cPaoFiCjMgjYwYLDj-xL132J147ifZFwRBmzmEMC8eYAXzbRNWuA">>, + ChecksumAddress = key_to_ethereum_address(hb_util:decode(Input)), + ?assertEqual(ChecksumAddress, <<"0xb7B4360F7F6298dE2e7a11009270F35F189Bd77E">>). + +sha3_256_test() -> + %% "abc" => known SHA3-256 hash from NIST + Input = <<"testing">>, + Expected = <<"7F5979FB78F082E8B1C676635DB8795C4AC6FABA03525FB708CB5FD68FD40C5E">>, + Actual = to_hex(hb_keccak:sha3_256(Input)), + ?assertEqual(Expected, Actual). diff --git a/src/hb_link.erl b/src/hb_link.erl new file mode 100644 index 000000000..24527f6c2 --- /dev/null +++ b/src/hb_link.erl @@ -0,0 +1,217 @@ +%%% @doc Utility functions for working with links. +-module(hb_link). +-export([is_link_key/1, remove_link_specifier/1]). +-export([normalize/2, normalize/3]). +-export([decode_all_links/1]). +-export([format/1, format/2, format/3]). +-export([format_unresolved/1, format_unresolved/2, format_unresolved/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Takes a message and ensures that it is normalized: +%% +%% - All literal (binary) lazily-loadable values are in-memory. +%% - All submaps are represented as links, optionally offloading their local +%% values to the cache. +%% - All other values are left unchanged (including their potential types). +%% +%% The response is a non-recursive, fully loaded message. It may still contain +%% types, but all submessages are guaranteed to be linkified. This stands in +%% contrast to `linkify', which takes a structured message and returns a message +%% with structured links. +normalize(Msg, Opts) when is_map(Opts) -> + normalize(Msg, hb_opts:get(linkify_mode, offload, Opts), Opts). + +normalize(Msg, false, _Opts) -> + Msg; +normalize(Msg, Mode, Opts) when is_map(Msg) -> + maps:merge( + maps:with([<<"commitments">>, <<"priv">>], Msg), + maps:from_list( + lists:map( + fun({Key, {link, ID, LinkOpts = #{ <<"type">> := <<"link">> }}}) -> + % The value is a link. Deconstruct it and ensure it is + % normalized (lazy links are made greedy, and both are + % returned in binary TABM form). + NormKey = hb_util:bin(Key), + UnderlyingID = + case maps:get(<<"lazy">>, LinkOpts, false) of + true -> + case hb_cache:read(ID, Opts) of + {ok, Underlying} when ?IS_ID(Underlying) -> + Underlying; + Err -> + throw( + {could_not_read_lazy_link, + {key, Key}, + {lazy_id, ID}, + {error, Err} + } + ) + end; + false -> + % The ID given is already in 'greedy' form. + % We embed it in the result unchanged. + ID + end, + ?event(debug_linkify, {link_normalized, Key, UnderlyingID}), + {<< NormKey/binary, "+link">>, UnderlyingID}; + ({Key, V}) when is_map(V) or is_list(V) -> + ?event(debug_linkify, {linkifying_submessage, Key}), + % The value is a submessage that we have in local memory. + % We must offload it such that it is cached, and + % referenced by a link. + % We start by normalizing the child message, generating + % its IDs by proxy. + NormChild = normalize(V, Mode, Opts), + NormKey = hb_util:bin(Key), + % Generate the ID of the normalized child message. + ID = hb_message:id(NormChild, all, Opts), + % If we are in `offload' mode, we write the message to the + % cache. If we are in `discard' mode, we simply drop the + % nested message. + case Mode of + discard -> do_nothing; + offload -> + % Write the child to the store to ensure its + % storage and availability. + hb_cache:write(NormChild, Opts) + end, + ?event(debug_linkify, {generated_link, {key, Key}, {id, ID}}), + {<>, ID}; + ({Key, V}) when ?IS_LINK(V) -> + % The link is not a submap. We load it such that it is + % local in-memory. This clause is used when we are + % normalizing a lazily-loaded message. + {Key, hb_cache:ensure_loaded(V, Opts)}; + ({Key, V}) -> + % The value is a primitive type. We do not need to do + % anything. + {Key, V} + end, + maps:to_list(maps:without([<<"commitments">>, <<"priv">>], Msg)) + ) + ) + ); +normalize(OtherVal, Mode, Opts) when is_list(OtherVal) -> + lists:map(fun(X) -> normalize(X, Mode, Opts) end, OtherVal); +normalize(OtherVal, _Mode, _Opts) -> + OtherVal. + +%% @doc Decode links embedded in the headers of a message. +decode_all_links(Msg) when is_map(Msg) -> + maps:from_list( + lists:map( + fun({Key, MaybeID}) -> + case is_link_key(Key) of + true -> + NewKey = binary:part(Key, 0, byte_size(Key) - 5), + {NewKey, + { + link, + MaybeID, + #{ + <<"type">> => <<"link">>, + <<"lazy">> => false + } + } + }; + _ -> {Key, MaybeID} + end + end, + maps:to_list(Msg) + ) + ); +decode_all_links(List) when is_list(List) -> + lists:map(fun(X) -> decode_all_links(X) end, List); +decode_all_links(OtherVal) -> + OtherVal. + +%% @doc Determine if a key is an encoded link. +is_link_key(Key) when byte_size(Key) >= 5 -> + binary:part(Key, byte_size(Key) - 5, 5) =:= <<"+link">>; +is_link_key(_) -> false. + +%% @doc Remove any `+link` suffixes from a key. +remove_link_specifier(Key) -> + case is_link_key(Key) of + true -> binary:part(Key, 0, byte_size(Key) - 5); + false -> Key + end. + +%% @doc Format a link as a short string suitable for printing. Checks the node +%% options (optionally) given, to see if it should resolve the link to a value +%% before printing. +format(Link) -> format(Link, #{}). +format(Link, Opts) -> + format(Link, Opts, 0). +format(Link, Opts, Indent) -> + case hb_opts:get(debug_resolve_links, false, Opts) of + true -> + try + hb_format:message( + hb_cache:ensure_all_loaded(Link, Opts), + Opts, + Indent + ) + catch + _:_ -> << "!UNRESOLVABLE! ", (format_unresolved(Link, Opts))/binary >> + end; + false -> format_unresolved(Link, Opts, Indent) + end. + +%% @doc Format a link without resolving it. +format_unresolved(Link) -> + format_unresolved(Link, #{}). +format_unresolved({link, ID, Opts}, BaseOpts) -> + format_unresolved({link, ID, Opts}, BaseOpts, 0). +format_unresolved({link, ID, Opts}, BaseOpts, Indent) -> + hb_util:bin( + hb_format:indent( + "~s~s: ~s", + [ + case maps:get(<<"lazy">>, Opts, false) of + true -> <<"Lazy link">>; + false -> <<"Link">> + end, + case maps:get(<<"type">>, Opts, no_type) of + no_type -> <<>>; + Type -> <<" (to ", (hb_util:bin(Type))/binary, ")" >> + end, + ID + ], + BaseOpts, + Indent + ) + ). + +%%% Tests + +offload_linked_message_test() -> + Opts = #{}, + Msg = #{ + <<"immediate-key">> => <<"immediate-value">>, + <<"link-key">> => #{ + <<"immediate-key-2">> => <<"link-value">>, + <<"link-key-2">> => #{ + <<"immediate-key-3">> => <<"link-value-2">> + } + } + }, + Offloaded = normalize(Msg, offload, Opts), + Structured = hb_message:convert(Offloaded, <<"structured@1.0">>, tabm, Opts), + ?event(linkify, {test_recvd_linkified, {msg, Structured}}), + Loaded = hb_cache:ensure_all_loaded(Structured, Opts), + ?event(linkify, {test_recvd_loaded, {msg, Loaded}}), + ?assertEqual(Msg, Loaded). + +offload_list_test() -> + Opts = #{}, + Msg = #{ + <<"list-key">> => [1.0, 2.0, 3.0] + }, + TABM = hb_message:convert(Msg, tabm, <<"structured@1.0">>, Opts), + Linkified = normalize(TABM, offload, Opts), + Msg2 = hb_message:convert(Linkified, <<"structured@1.0">>, tabm, Opts), + Res = hb_cache:ensure_all_loaded(Msg2, Opts), + ?assertEqual(Msg, Res). diff --git a/src/hb_maps.erl b/src/hb_maps.erl new file mode 100644 index 000000000..391e018dc --- /dev/null +++ b/src/hb_maps.erl @@ -0,0 +1,335 @@ +%%% @doc An abstraction for working with maps in HyperBEAM, matching the +%%% generic `maps' module, but additionally supporting the resolution of +%%% links as they are encountered. These functions must be used extremely +%%% carefully. In virtually all circumstances, the `hb_ao:resolve/3' or +%%% `hb_ao:get/3' functions should be used instead, as they will execute the +%%% full AO-Core protocol upon requests (normalizing keys, applying the +%%% appropriate device's functions, as well as resolving links). By using this +%%% module's functions, you are implicitly making the assumption that the message +%%% in question is of the `~message@1.0' form, ignoring any other keys that its +%%% actual device may present. This module is intended for the extremely rare +%%% circumstances in which the additional overhead of the full AO-Core +%%% execution cycle is not acceptable, and the data in question is known to +%%% conform to the `~message@1.0' form. +%%% +%%% If you do not understand any/all of the above, you are in the wrong place! +%%% Utilise the `hb_ao' module and read the documentation therein, saving +%%% yourself from the inevitable issues that will arise from using this +%%% module without understanding the full implications. You have been warned. +-module(hb_maps). +-export([get/2, get/3, get/4, put/3, put/4, find/2, find/3]). +-export([is_key/2, is_key/3, keys/1, keys/2, values/1, values/2]). +-export([map/2, map/3, filter/2, filter/3, filtermap/2, filtermap/3]). +-export([fold/3, fold/4, take/2, take/3, size/1, size/2]). +-export([merge/2, merge/3, remove/2, remove/3]). +-export([with/2, with/3, without/2, without/3, update_with/3, update_with/4]). +-export([from_list/1, to_list/1, to_list/2]). +-include_lib("eunit/include/eunit.hrl"). + +-spec get(Key :: term(), Map :: map()) -> term(). +get(Key, Map) -> + get(Key, Map, undefined). + +-spec get(Key :: term(), Map :: map(), Default :: term()) -> term(). +get(Key, Map, Default) -> + get(Key, Map, Default, #{}). + +%% @doc Get a value from a map, resolving links as they are encountered in both +%% the TABM encoded link format, as well as the structured type. +-spec get( + Key :: term(), + Map :: map(), + Default :: term(), + Opts :: map() +) -> term(). +get(Key, Map, Default, Opts) -> + hb_cache:ensure_loaded( + maps:get( + Key, + hb_cache:ensure_loaded(Map, Opts), + Default + ), + Opts + ). + +-spec find(Key :: term(), Map :: map()) -> {ok, term()} | error. +find(Key, Map) -> + find(Key, Map, #{}). + +-spec find(Key :: term(), Map :: map(), Opts :: map()) -> {ok, term()} | error. +find(Key, Map, Opts) -> + hb_cache:ensure_loaded(maps:find(Key, hb_cache:ensure_loaded(Map, Opts)), Opts). + +-spec put(Key :: term(), Value :: term(), Map :: map()) -> map(). +put(Key, Value, Map) -> + put(Key, Value, Map, #{}). + +-spec put( + Key :: term(), + Value :: term(), + Map :: map(), + Opts :: map() +) -> map(). +put(Key, Value, Map, Opts) -> + maps:put(Key, Value, hb_cache:ensure_loaded(Map, Opts)). + +-spec is_key(Key :: term(), Map :: map()) -> boolean(). +is_key(Key, Map) -> + is_key(Key, Map, #{}). + +-spec is_key(Key :: term(), Map :: map(), Opts :: map()) -> boolean(). +is_key(Key, Map, Opts) -> + maps:is_key(Key, hb_cache:ensure_loaded(Map, Opts)). + +-spec keys(Map :: map()) -> [term()]. +keys(Map) -> + keys(Map, #{}). + +-spec keys(Map :: map(), Opts :: map()) -> [term()]. +keys(Map, Opts) -> + maps:keys(hb_cache:ensure_loaded(Map, Opts)). + +-spec values(Map :: map()) -> [term()]. +values(Map) -> values(Map, #{}). + +-spec values(Map :: map(), Opts :: map()) -> [term()]. +values(Map, Opts) -> + maps:values(hb_cache:ensure_loaded(Map, Opts)). + +-spec size(Map :: map()) -> non_neg_integer(). +size(Map) -> + size(Map, #{}). + +-spec size(Map :: map(), Opts :: map()) -> non_neg_integer(). +size(Map, Opts) -> + maps:size(hb_cache:ensure_loaded(Map, Opts)). + +-spec map( + Fun :: fun((Key :: term(), Value :: term()) -> term()), + Map :: map() +) -> map(). +map(Fun, Map) -> + map(Fun, Map, #{}). + +-spec map( + Fun :: fun((Key :: term(), Value :: term()) -> term()), + Map :: map(), + Opts :: map() +) -> map(). +map(Fun, Map, Opts) -> + maps:map( + fun(K, V) -> Fun(K, hb_cache:ensure_loaded(V, Opts)) end, + hb_cache:ensure_loaded(Map, Opts) + ). + +-spec merge(Map1 :: map(), Map2 :: map()) -> map(). +merge(Map1, Map2) -> + merge(Map1, Map2, #{}). + +-spec merge(Map1 :: map(), Map2 :: map(), Opts :: map()) -> map(). +merge(Map1, Map2, Opts) -> + maps:merge(hb_cache:ensure_loaded(Map1, Opts), hb_cache:ensure_loaded(Map2, Opts)). + +-spec remove(Key :: term(), Map :: map()) -> map(). +remove(Key, Map) -> + remove(Key, Map, #{}). + +-spec remove(Key :: term(), Map :: map(), Opts :: map()) -> map(). +remove(Key, Map, Opts) -> + maps:remove(Key, hb_cache:ensure_loaded(Map, Opts)). + +-spec with(Keys :: [term()], Map :: map()) -> map(). +with(Keys, Map) -> + with(Keys, Map, #{}). + +-spec with(Keys :: [term()], Map :: map(), Opts :: map()) -> map(). +with(Keys, Map, Opts) -> + maps:with(Keys, hb_cache:ensure_loaded(Map, Opts)). + +-spec without(Keys :: [term()], Map :: map()) -> map(). +without(Keys, Map) -> + without(Keys, Map, #{}). + +-spec without(Keys :: [term()], Map :: map(), Opts :: map()) -> map(). +without(Keys, Map, Opts) -> + maps:without(Keys, hb_cache:ensure_loaded(Map, Opts)). + +-spec filter( + Fun :: fun((Key :: term(), Value :: term()) -> boolean()), + Map :: map() +) -> map(). +filter(Fun, Map) -> + filter(Fun, Map, #{}). + +-spec filter( + Fun :: fun((Key :: term(), Value :: term()) -> boolean()), + Map :: map(), + Opts :: map() +) -> map(). +filter(Fun, Map, Opts) -> + maps:filtermap( + fun(K, V) -> + case Fun(K, Loaded = hb_cache:ensure_loaded(V, Opts)) of + true -> {true, Loaded}; + false -> false + end + end, + hb_cache:ensure_loaded(Map, Opts) + ). + +-spec filtermap( + Fun :: fun((Key :: term(), Value :: term()) -> {boolean(), term()}), + Map :: map() +) -> map(). +filtermap(Fun, Map) -> + filtermap(Fun, Map, #{}). + +-spec filtermap( + Fun :: fun((Key :: term(), Value :: term()) -> {boolean(), term()}), + Map :: map(), + Opts :: map() +) -> map(). +filtermap(Fun, Map, Opts) -> + maps:filtermap( + fun(K, V) -> Fun(K, hb_cache:ensure_loaded(V, Opts)) end, + hb_cache:ensure_loaded(Map, Opts) + ). + +-spec fold( + Fun :: fun((Key :: term(), Value :: term(), Acc :: term()) -> term()), + Acc :: term(), + Map :: map() +) -> term(). +fold(Fun, Acc, Map) -> + fold(Fun, Acc, Map, #{}). + +-spec fold( + Fun :: fun((Key :: term(), Value :: term(), Acc :: term()) -> term()), + Acc :: term(), + Map :: map(), + Opts :: map() +) -> term(). +fold(Fun, Acc, Map, Opts) -> + maps:fold( + fun(K, V, CurrAcc) -> Fun(K, hb_cache:ensure_loaded(V, Opts), CurrAcc) end, + Acc, + hb_cache:ensure_loaded(Map, Opts) + ). + +-spec take(N :: non_neg_integer(), Map :: map()) -> map(). +take(N, Map) -> + take(N, Map, #{}). + +-spec take(N :: non_neg_integer(), Map :: map(), Opts :: map()) -> map(). +take(N, Map, Opts) -> + maps:take(N, hb_cache:ensure_loaded(Map, Opts)). + +-spec update_with( + Key :: term(), + Fun :: fun((Value :: term()) -> term()), + Map :: map() +) -> map(). +update_with(Key, Fun, Map) -> + update_with(Key, Fun, Map, #{}). + +-spec update_with( + Key :: term(), + Fun :: fun((Value :: term()) -> term()), + Map :: map(), + Opts :: map() +) -> map(). +update_with(Key, Fun, Map, Opts) -> + maps:update_with(Key, Fun, hb_cache:ensure_loaded(Map, Opts), Opts). + +-spec from_list(List :: [{Key :: term(), Value :: term()}]) -> map(). +from_list(List) -> + maps:from_list(List). + +-spec to_list(Map :: map()) -> [{Key :: term(), Value :: term()}]. +to_list(Map) -> + to_list(Map, #{}). + +-spec to_list(Map :: map(), Opts :: map()) -> [{Key :: term(), Value :: term()}]. +to_list(Map, Opts) -> + maps:to_list(hb_cache:ensure_loaded(Map, Opts)). + +%%% Tests + +get_with_link_test() -> + Bin = <<"TEST DATA">>, + Opts = #{}, + {ok, Location} = hb_cache:write(Bin, Opts), + Map = #{ 1 => 1, 2 => {link, Location, #{}}, 3 => 3 }, + ?assertEqual(Bin, get(2, Map)). + +map_with_link_test() -> + Bin = <<"TEST DATA">>, + Opts = #{}, + {ok, Location} = hb_cache:write(Bin, Opts), + Map = #{ 1 => 1, 2 => {link, Location, #{}}, 3 => 3 }, + ?assertEqual(#{1 => 1, 2 => Bin, 3 => 3}, map(fun(_K, V) -> V end, Map, #{})). + +get_with_typed_link_test() -> + Bin = <<"123">>, + Opts = #{}, + {ok, Location} = hb_cache:write(Bin, Opts), + Map = #{ 1 => 1, 2 => {link, Location, #{ <<"type">> => integer }}, 3 => 3 }, + ?assertEqual(123, get(2, Map, undefined)). + +resolve_on_link_test() -> + Msg = #{ <<"test-key">> => <<"test-value">> }, + Opts = #{}, + {ok, ID} = hb_cache:write(Msg, Opts), + ?assertEqual( + {ok, <<"test-value">>}, + hb_ao:resolve({link, ID, #{}}, <<"test-key">>, #{}) + ). + +filter_with_link_test() -> + Bin = <<"TEST DATA">>, + Opts = #{}, + {ok, Location} = hb_cache:write(Bin, Opts), + Map = #{ 1 => 1, 2 => {link, Location, #{}}, 3 => 3 }, + ?assertEqual(#{1 => 1, 3 => 3}, filter(fun(_, V) -> V =/= Bin end, Map)). + +filtermap_with_link_test() -> + Bin = <<"TEST DATA">>, + Opts = #{}, + {ok, Location} = hb_cache:write(Bin, Opts), + Map = #{ 1 => 1, 2 => {link, Location, #{}}, 3 => 3 }, + ?assertEqual( + #{2 => <<"FOUND">>}, + filtermap( + fun(_, <<"TEST DATA">>) -> {true, <<"FOUND">>}; + (_K, _V) -> false + end, + Map + ) + ). + +fold_with_typed_link_test() -> + Bin = <<"123">>, + Opts = #{}, + {ok, Location} = hb_cache:write(Bin, Opts), + Map = #{ 1 => 1, 2 => {link, Location, #{ <<"type">> => integer }}, 3 => 3 }, + ?assertEqual(127, fold(fun(_, V, Acc) -> V + Acc end, 0, Map)). + +filter_passively_loads_test() -> + Bin = <<"TEST DATA">>, + Opts = #{}, + {ok, Location} = hb_cache:write(Bin, Opts), + Map = #{ 1 => 1, 2 => {link, Location, #{}}, 3 => 3 }, + ?assertEqual( + #{1 => 1, 2 => <<"TEST DATA">>, 3 => 3}, + filter(fun(_, _) -> true end, Map) + ). + +filtermap_passively_loads_test() -> + Bin = <<"TEST DATA">>, + Opts = #{}, + {ok, Location} = hb_cache:write(Bin, Opts), + Map = #{ 1 => 1, 2 => {link, Location, #{}}, 3 => 3 }, + ?assertEqual( + #{ 1 => 1, 2 => <<"TEST DATA">>, 3 => 3 }, + filtermap(fun(_, V) -> {true, V} end, Map) + ). \ No newline at end of file diff --git a/src/hb_message.erl b/src/hb_message.erl index 2bb5ef627..676e99dfb 100644 --- a/src/hb_message.erl +++ b/src/hb_message.erl @@ -1,17 +1,17 @@ %%% @doc This module acts an adapter between messages, as modeled in the -%%% Converge Protocol, and their uderlying binary representations and formats. +%%% AO-Core protocol, and their uderlying binary representations and formats. %%% %%% Unless you are implementing a new message serialization codec, you should %%% not need to interact with this module directly. Instead, use the -%%% `hb_converge' interfaces to interact with all messages. The `dev_message' +%%% `hb_ao' interfaces to interact with all messages. The `dev_message' %%% module implements a device interface for abstracting over the different %%% message formats. %%% %%% `hb_message' and the HyperBEAM caches can interact with multiple different %%% types of message formats: %%% -%%% - Richly typed Converge messages. +%%% - Richly typed AO-Core structured messages. %%% - Arweave transations. %%% - ANS-104 data items. %%% - HTTP Signed Messages. @@ -28,22 +28,23 @@ %%% %%% The structure of the conversions is as follows: %%% -%%% ``` -%%% Arweave TX/ANS-104 ==> hb_codec_tx:from/1 ==> TABM -%%% HTTP Signed Message ==> hb_codec_http:from/1 ==> TABM -%%% Flat Maps ==> hb_codec_flat:from/1 ==> TABM +%%% 
+%%%     Arweave TX/ANS-104 ==> dev_codec_ans104:from/1 ==> TABM
+%%%     HTTP Signed Message ==> dev_codec_httpsig_conv:from/1 ==> TABM
+%%%     Flat Maps ==> dev_codec_flat:from/1 ==> TABM
 %%% 
-%%%     TABM ==> hb_codec_converge:to/1 ==> Converge Message
-%%%     Converge Message ==> hb_codec_converge:from/1 ==> TABM
+%%%     TABM ==> dev_codec_structured:to/1 ==> AO-Core Message
+%%%     AO-Core Message ==> dev_codec_structured:from/1 ==> TABM
 %%% 
-%%%     TABM ==> hb_codec_tx:to/1 ==> Arweave TX/ANS-104
-%%%     TABM ==> hb_codec_http:to/1 ==> HTTP Signed Message
-%%%     TABM ==> hb_codec_flat:to/1 ==> Flat Maps
-%%% '''
+%%%     TABM ==> dev_codec_ans104:to/1 ==> Arweave TX/ANS-104
+%%%     TABM ==> dev_codec_httpsig_conv:to/1 ==> HTTP Signed Message
+%%%     TABM ==> dev_codec_flat:to/1 ==> Flat Maps
+%%%     ...
+%%%
%%% %%% Additionally, this module provides a number of utility functions for %%% manipulating messages. For example, `hb_message:sign/2' to sign a message of -%%% arbitrary type, or `hb_message:format/1' to print a Converge/TABM message in +%%% arbitrary type, or `hb_formatter:format_msg/1' to print an AO-Core/TABM message in %%% a human-readable format. %%% %%% The `hb_cache' module is responsible for storing and retrieving messages in @@ -51,235 +52,380 @@ %%% backend, but each works with simple key-value pairs. Subsequently, the %%% `hb_cache' module uses TABMs as the internal format for storing and %%% retrieving messages. +%%% +%%% Test vectors to ensure the functioning of this module and the codecs that +%%% interact with it are found in `hb_message_test_vectors.erl'. -module(hb_message). --export([convert/3, convert/4, unsigned/1, attestations/1]). --export([sign/2, verify/1, type/1, minimize/1, normalize_keys/1]). --export([signers/1, serialize/1, serialize/2, deserialize/1, deserialize/2]). --export([match/2, match/3]). +-export([id/1, id/2, id/3]). +-export([convert/3, convert/4, uncommitted/1, uncommitted/2, committed/3]). +-export([with_only_committers/2, with_only_committers/3, commitment_devices/2]). +-export([verify/1, verify/2, verify/3, commit/2, commit/3, signers/2, type/1, minimize/1]). +-export([normalize_commitments/2, is_signed_key/3]). +-export([commitment/2, commitment/3, commitments/3]). +-export([with_only_committed/2, without_unless_signed/3]). +-export([with_commitments/3, without_commitments/3]). +-export([diff/3, match/2, match/3, match/4, find_target/3]). %%% Helpers: -export([default_tx_list/0, filter_default_keys/1]). %%% Debugging tools: --export([print/1, format/1, format/2]). +-export([print/1]). -include("include/hb.hrl"). --include_lib("eunit/include/eunit.hrl"). %% @doc Convert a message from one format to another. Taking a message in the %% source format, a target format, and a set of opts. If not given, the source -%% is assumed to be `converge`. Additional codecs can be added by ensuring they -%% are part of the `Opts` map -- either globally, or locally for a computation. +%% is assumed to be `structured@1.0'. Additional codecs can be added by ensuring they +%% are part of the `Opts' map -- either globally, or locally for a computation. %% %% The encoding happens in two phases: %% 1. Convert the message to a TABM. %% 2. Convert the TABM to the target format. %% -%% The conversion to a TABM is done by the `converge' codec, which is always +%% The conversion to a TABM is done by the `structured@1.0' codec, which is always %% available. The conversion from a TABM is done by the target codec. convert(Msg, TargetFormat, Opts) -> - convert(Msg, TargetFormat, converge, Opts). + convert(Msg, TargetFormat, <<"structured@1.0">>, Opts). +convert(Msg, TargetFormat, tabm, Opts) -> + OldPriv = + if is_map(Msg) -> maps:get(<<"priv">>, Msg, #{}); + true -> #{} + end, + from_tabm(Msg, TargetFormat, OldPriv, Opts); convert(Msg, TargetFormat, SourceFormat, Opts) -> - TABM = convert_to_tabm(Msg, SourceFormat, Opts), + OldPriv = + if is_map(Msg) -> maps:get(<<"priv">>, Msg, #{}); + true -> #{} + end, + TABM = + to_tabm( + case is_map(Msg) of + true -> hb_maps:without([<<"priv">>], Msg, Opts); + false -> Msg + end, + SourceFormat, + Opts + ), case TargetFormat of - tabm -> TABM; - _ -> convert_to_target(TABM, TargetFormat, Opts) + tabm -> restore_priv(TABM, OldPriv, Opts); + _ -> from_tabm(TABM, TargetFormat, OldPriv, Opts) end. -convert_to_tabm(Msg, SourceFormat, Opts) -> - SourceCodecMod = get_codec(SourceFormat, Opts), - case SourceCodecMod:from(Msg) of - TypicalMsg when is_map(TypicalMsg) -> - minimize(filter_default_keys(TypicalMsg)); - OtherTypeRes -> OtherTypeRes +to_tabm(Msg, SourceFormat, Opts) -> + {SourceCodecMod, Params} = conversion_spec_to_req(SourceFormat, Opts), + % We use _from_ here because the codecs are labelled from the perspective + % of their own format. `dev_codec_ans104:from/1' will convert _from_ + % an ANS-104 message _into_ a TABM. + case SourceCodecMod:from(Msg, Params, Opts) of + {ok, TypicalMsg} when is_map(TypicalMsg) -> + TypicalMsg; + {ok, OtherTypeRes} -> OtherTypeRes end. -convert_to_target(Msg, TargetFormat, Opts) -> - TargetCodecMod = get_codec(TargetFormat, Opts), - TargetCodecMod:to(Msg). - -%% @doc Return the unsigned version of a message in Converge format. -unsigned(Bin) when is_binary(Bin) -> Bin; -unsigned(Msg) -> - maps:remove([signed_id, signature, owner], Msg). - -%% @doc Return a sub-map of the attestation-related keys in a message. -attestations(Msg) -> - maps:with([signed_id, signature, owner], Msg). +from_tabm(Msg, TargetFormat, OldPriv, Opts) -> + {TargetCodecMod, Params} = conversion_spec_to_req(TargetFormat, Opts), + % We use the _to_ function here because each of the codecs we may call in + % this step are labelled from the perspective of the target format. For + % example, `dev_codec_httpsig:to/1' will convert _from_ a TABM to an + % HTTPSig message. + case TargetCodecMod:to(Msg, Params, Opts) of + {ok, TypicalMsg} when is_map(TypicalMsg) -> + restore_priv(TypicalMsg, OldPriv, Opts); + {ok, OtherTypeRes} -> OtherTypeRes + end. -%% @doc Get a codec from the options. -get_codec(TargetFormat, Opts) -> - case hb_opts:get(codecs, #{}, Opts) of - #{ TargetFormat := CodecMod } -> CodecMod; - _ -> throw({message_codec_not_found, TargetFormat}) +%% @doc Add the existing `priv' sub-map back to a converted message, honoring +%% any existing `priv' sub-map that may already be present. +restore_priv(Msg, EmptyPriv, _Opts) when map_size(EmptyPriv) == 0 -> Msg; +restore_priv(Msg, OldPriv, Opts) -> + MsgPriv = hb_maps:get(<<"priv">>, Msg, #{}, Opts), + ?event({restoring_priv, {msg_priv, MsgPriv}, {old_priv, OldPriv}}), + NewPriv = hb_util:deep_merge(MsgPriv, OldPriv, Opts), + ?event({new_priv, NewPriv}), + Msg#{ <<"priv">> => NewPriv }. + +%% @doc Get a codec device and request params from the given conversion request. +%% Expects conversion spec to either be a binary codec name, or a map with a +%% `device' key and other parameters. Additionally honors the `always_bundle' +%% key in the node message if present. +conversion_spec_to_req(Spec, Opts) when is_binary(Spec) or (Spec == tabm) -> + conversion_spec_to_req(#{ <<"device">> => Spec }, Opts); +conversion_spec_to_req(Spec, Opts) -> + try + Device = + hb_maps:get( + <<"device">>, + Spec, + no_codec_device_in_conversion_spec, + Opts + ), + { + case Device of + tabm -> tabm; + _ -> + hb_ao:message_to_device( + #{ + <<"device">> => Device + }, + Opts + ) + end, + hb_maps:without([<<"device">>], Spec, Opts) + } + catch _:_ -> + throw({message_codec_not_extractable, Spec}) end. -%% @doc Pretty-print a message. -print(Msg) -> print(Msg, 0). -print(Msg, Indent) -> - io:format(standard_error, "~s", [lists:flatten(format(Msg, Indent))]). - -%% @doc Format a message for printing, optionally taking an indentation level -%% to start from. -format(Item) -> format(Item, 0). -format(Bin, Indent) when is_binary(Bin) -> - hb_util:format_indented( - hb_util:format_binary(Bin), - Indent - ); -format(Map, Indent) when is_map(Map) -> - % Define helper functions for formatting elements of the map. - ValOrUndef = - fun(Key) -> - case dev_message:get(Key, Map) of - {ok, Val} -> - case hb_util:short_id(Val) of - undefined -> Val; - ShortID -> ShortID - end; - {error, _} -> undefined - end +%% @doc Return the ID of a message. +id(Msg) -> id(Msg, uncommitted). +id(Msg, Opts) when is_map(Opts) -> id(Msg, uncommitted, Opts); +id(Msg, Committers) -> id(Msg, Committers, #{}). +id(Msg, RawCommitters, Opts) -> + CommSpec = + case RawCommitters of + none -> #{ <<"committers">> => <<"none">> }; + uncommitted -> #{ <<"committers">> => <<"none">> }; + unsigned -> #{ <<"committers">> => <<"none">> }; + all -> #{ <<"committers">> => <<"all">> }; + signed -> #{ <<"committers">> => <<"all">> }; + List when is_list(List) -> #{ <<"committers">> => List } end, - FilterUndef = - fun(List) -> - lists:filter(fun({_, undefined}) -> false; (_) -> true end, List) - end, - % Prepare the metadata row for formatting. - % Note: We try to get the IDs _if_ they are *already* in the map. We do not - % force calculation of the IDs here because that may cause significant - % overhead unless the `debug_ids` option is set. - IDMetadata = - case hb_opts:get(debug_ids, false, #{}) of - false -> - [ - {<<"#P">>, ValOrUndef(hashpath)}, - {<<"*U">>, ValOrUndef(unsigned_id)}, - {<<"*S">>, ValOrUndef(id)} - ]; - true -> - {ok, UID} = dev_message:unsigned_id(Map), - {ok, ID} = dev_message:id(Map), - [ - {<<"#P">>, hb_util:short_id(ValOrUndef(hashpath))}, - {<<"*U">>, hb_util:short_id(UID)} - ] ++ - case ID of - UID -> []; - _ -> [{<<"*S">>, hb_util:short_id(ID)}] - end - end, - SignerMetadata = - case signers(Map) of - [] -> []; - [Signer] -> - [{<<"Sig">>, hb_util:short_id(Signer)}]; - Signers -> - [ - { - <<"Sigs">>, - string:join(lists:map(fun hb_util:short_id/1, Signers), ", ") - } - ] - end, - % Concatenate the present metadata rows. - Metadata = FilterUndef(lists:flatten([IDMetadata, SignerMetadata])), - % Format the metadata row. - Header = - hb_util:format_indented("Message [~s] {", - [ - string:join( - [ - io_lib:format("~s: ~s", [Lbl, Val]) - || {Lbl, Val} <- Metadata - ], - ", " - ) - ], - Indent + ?event({getting_id, {msg, Msg}, {spec, CommSpec}}), + {ok, ID} = + dev_message:id( + Msg, + CommSpec#{ <<"path">> => <<"id">> }, + Opts ), - % Put the path and device rows into the output at the _top_ of the map. - PriorityKeys = [{<<"Path">>, ValOrUndef(path)}, {<<"Device">>, ValOrUndef(device)}], - FooterKeys = - case hb_private:from_message(Map) of - PrivMap when map_size(PrivMap) == 0 -> []; - PrivMap -> [{<<"!Private!">>, PrivMap}] - end, - % Concatenate the path and device rows with the rest of the key values. - KeyVals = - FilterUndef(PriorityKeys) ++ - maps:to_list( - minimize(Map, - [owner, signature, id, unsigned_id, hashpath, path, device] - ++ [<<"Device">>, <<"Path">>] % Hack: Until key capitalization is fixed. - ) - ) ++ FooterKeys, - % Format the remaining 'normal' keys and values. - Res = lists:map( - fun({Key, Val}) -> - NormKey = hb_converge:to_key(Key, #{ error_strategy => ignore }), - KeyStr = - case NormKey of - undefined -> - io_lib:format("~p [!!! INVALID KEY !!!]", [Key]); - _ -> - hb_converge:key_to_binary(Key) + hb_util:human_id(ID). + +%% @doc Normalize the IDs in a message, ensuring that there is at least one +%% unsigned ID present. By forcing this work to occur in strategically positioned +%% places, we avoid the need to recalculate the IDs for every `hb_message:id` +%% call. +normalize_commitments(Msg, Opts) when is_map(Msg) -> + NormMsg = + maps:map( + fun(Key, Val) when Key == <<"commitments">> orelse Key == <<"priv">> -> + Val; + (_Key, Val) -> normalize_commitments(Val, Opts) + end, + Msg + ), + case hb_maps:get(<<"commitments">>, NormMsg, not_found, Opts) of + not_found -> + {ok, #{ <<"commitments">> := Commitments }} = + dev_message:commit( + NormMsg, + #{ <<"type">> => <<"unsigned">> }, + Opts + ), + NormMsg#{ <<"commitments">> => Commitments }; + _ -> NormMsg + end; +normalize_commitments(Msg, Opts) when is_list(Msg) -> + lists:map(fun(X) -> normalize_commitments(X, Opts) end, Msg); +normalize_commitments(Msg, _Opts) -> + Msg. + +%% @doc Return a message with only the committed keys. If no commitments are +%% present, the message is returned unchanged. This means that you need to +%% check if the message is: +%% - Committed +%% - Verifies +%% ...before using the output of this function as the 'canonical' message. This +%% is such that expensive operations like signature verification are not +%% performed unless necessary. +with_only_committed(Msg, Opts) when is_map(Msg) -> + ?event({with_only_committed, {msg, Msg}, {opts, Opts}}), + Comms = hb_maps:get(<<"commitments">>, Msg, not_found, Opts), + case is_map(Msg) andalso Comms /= not_found of + true -> + try + CommittedKeys = + hb_message:committed( + Msg, + #{ <<"commitments">> => <<"all">> }, + Opts + ), + % Add the ao-body-key to the committed list if it is not + % already present. + ?event(debug_bundle, {committed_keys, CommittedKeys, {msg, Msg}}), + {ok, + with_links( + [<<"commitments">> | CommittedKeys], + Msg, + Opts + ) + } + catch Class:Reason:St -> + {error, + {could_not_normalize, + {class, Class}, + {reason, Reason}, + {msg, Msg}, + {stacktrace, St} + } + } + end; + false -> {ok, Msg} + end; +with_only_committed(Msg, _) -> + % If the message is not a map, it cannot be signed. + {ok, Msg}. + +%% @doc Filter keys from a map that do not match either the list of keys or +%% their relative `+link` variants. +with_links(Keys, Map, Opts) -> + hb_maps:with( + Keys ++ + lists:map( + fun(Key) -> + <<(hb_link:remove_link_specifier(Key))/binary, "+link">> end, - hb_util:format_indented( - "~s => ~s~n", - [ - lists:flatten([KeyStr]), - case Val of - NextMap when is_map(NextMap) -> - hb_util:format_map(NextMap, Indent + 2); - _ when (byte_size(Val) == 32) or (byte_size(Val) == 43) -> - Short = hb_util:short_id(Val), - io_lib:format("~s [*]", [Short]); - _ when byte_size(Val) == 87 -> - io_lib:format("~s [#p]", [hb_util:short_id(Val)]); - Bin when is_binary(Bin) -> - hb_util:format_binary(Bin); - Other -> - io_lib:format("~p", [Other]) + Keys + ), + Map, + Opts + ). + +%% @doc Return the message with only the specified committers attached. +with_only_committers(Msg, Committers) -> + with_only_committers(Msg, Committers, #{}). +with_only_committers(Msg, Committers, Opts) when is_map(Msg) -> + NewCommitments = + hb_maps:filter( + fun(_, #{ <<"committer">> := Committer }) -> + lists:member(Committer, Committers); + (_, _) -> false + end, + hb_maps:get(<<"commitments">>, Msg, #{}, Opts), + Opts + ), + Msg#{ <<"commitments">> => NewCommitments }; +with_only_committers(Msg, _Committers, _Opts) -> + throw({unsupported_message_type, Msg}). + +%% @doc Determine whether a specific key is part of a message's commitments. +is_signed_key(Key, Msg, Opts) -> + lists:member(Key, hb_message:committed(Msg, all, Opts)). + +%% @doc Remove the any of the given keys that are not signed from a message. +without_unless_signed(Key, Msg, Opts) when not is_list(Key) -> + without_unless_signed([Key], Msg, Opts); +without_unless_signed(Keys, Msg, Opts) -> + SignedKeys = hb_message:committed(Msg, all, Opts), + maps:without( + lists:filter(fun(K) -> not lists:member(K, SignedKeys) end, Keys), + Msg + ). + +%% @doc Sign a message with the given wallet. +commit(Msg, WalletOrOpts) -> + commit( + Msg, + WalletOrOpts, + hb_opts:get( + commitment_device, + no_viable_commitment_device, + case is_map(WalletOrOpts) of + true -> WalletOrOpts; + false -> #{ priv_wallet => WalletOrOpts } + end + ) + ). +commit(Msg, Wallet, Format) when not is_map(Wallet) -> + commit(Msg, #{ priv_wallet => Wallet }, Format); +commit(Msg, Opts, CodecName) when is_binary(CodecName) -> + commit(Msg, Opts, #{ <<"commitment-device">> => CodecName }); +commit(Msg, Opts, Spec) -> + {ok, Signed} = + dev_message:commit( + Msg, + Spec#{ + <<"commitment-device">> => + case hb_maps:get(<<"commitment-device">>, Spec, none, Opts) of + none -> + case hb_maps:get(<<"device">>, Spec, none, Opts) of + none -> + throw( + { + no_commitment_device_in_codec_spec, + Spec + } + ); + Device -> Device + end; + CommitmentDevice -> CommitmentDevice end - ], - Indent + 1 - ) - end, - KeyVals + }, + Opts + ), + Signed. + +%% @doc Return the list of committed keys from a message. +committed(Msg, all, Opts) -> + committed(Msg, #{ <<"committers">> => <<"all">> }, Opts); +committed(Msg, none, Opts) -> + committed(Msg, #{ <<"committers">> => <<"none">> }, Opts); +committed(Msg, List, Opts) when is_list(List) -> + committed(Msg, #{ <<"commitments">> => List }, Opts); +committed(Msg, CommittersMsg, Opts) -> + ?event( + {committed, + {msg, {explicit, Msg}}, + {committers_msg, {explicit, CommittersMsg}}, + {opts, Opts} + } ), - case Res of - [] -> lists:flatten(Header ++ " [Empty] }"); - _ -> - lists:flatten( - Header ++ ["\n"] ++ Res ++ hb_util:format_indented("}", Indent) - ) - end; -format(Item, Indent) -> - % Whatever we have is not a message map. - hb_util:format_indented("[UNEXPECTED VALUE] ~p", [Item], Indent). - -%% @doc Return the signers of a message. For now, this is just the signer -%% of the message itself. In the future, we will support multiple signers. -signers(Msg) when is_map(Msg) -> - case {maps:find(owner, Msg), maps:find(signature, Msg)} of - {_, error} -> []; - {error, _} -> []; - {{ok, Owner}, {ok, _}} -> [ar_wallet:to_address(Owner)] - end; -signers(TX) when is_record(TX, tx) -> - ar_bundles:signer(TX); -signers(_) -> []. - -%% @doc Sign a message with the given wallet. Only supports the `tx' format -%% at the moment. -sign(Msg, Wallet) -> - TX = convert(Msg, tx, #{}), - SignedTX = ar_bundles:sign_item(TX, Wallet), - convert(SignedTX, converge, tx, #{}). - -%% @doc Verify a message. -verify(Msg) -> - TX = convert(Msg, tx, converge, #{}), - ar_bundles:verify_item(TX). - -%% @doc Return the type of a message. + {ok, CommittedKeys} = dev_message:committed(Msg, CommittersMsg, Opts), + CommittedKeys. + +%% @doc wrapper function to verify a message. +verify(Msg) -> verify(Msg, all). +verify(Msg, Committers) -> + verify(Msg, Committers, #{}). +verify(Msg, all, Opts) -> + verify(Msg, <<"all">>, Opts); +verify(Msg, signers, Opts) -> + verify(Msg, hb_message:signers(Msg, Opts), Opts); +verify(Msg, Committers, Opts) when not is_map(Committers) -> + verify( + Msg, + #{ + <<"committers">> => + case ?IS_ID(Committers) of + true -> [Committers]; + false -> Committers + end + }, + Opts + ); +verify(Msg, Spec, Opts) -> + ?event(verify, {verify, {spec, Spec}}), + {ok, Res} = + dev_message:verify( + Msg, + Spec, + Opts + ), + Res. + +%% @doc Return the unsigned version of a message in AO-Core format. +uncommitted(Msg) -> uncommitted(Msg, #{}). +uncommitted(Bin, _Opts) when is_binary(Bin) -> Bin; +uncommitted(Msg, Opts) -> + hb_maps:remove(<<"commitments">>, Msg, Opts). + +%% @doc Return all of the committers on a message that have 'normal', 256 bit, +%% addresses. +signers(Msg, Opts) -> + hb_util:ok(dev_message:committers(Msg, #{}, Opts)). + +%% @doc Pretty-print a message. +print(Msg) -> print(Msg, 0). +print(Msg, Indent) -> + io:format(standard_error, "~s", [lists:flatten(hb_format:message(Msg, #{}, Indent))]). + +%% @doc Return the type of an encoded message. type(TX) when is_record(TX, tx) -> tx; type(Binary) when is_binary(Binary) -> binary; type(Msg) when is_map(Msg) -> @@ -287,7 +433,7 @@ type(Msg) when is_map(Msg) -> fun({_, Value}) -> is_map(Value) end, lists:filter( fun({Key, _}) -> not hb_private:is_private(Key) end, - maps:to_list(Msg) + hb_maps:to_list(Msg) ) ), case IsDeep of @@ -300,16 +446,32 @@ type(Msg) when is_map(Msg) -> %% `strict': All keys in both maps be present and match. %% `only_present': Only present keys in both maps must match. %% `primary': Only the primary map's keys must be present. +%% Returns `true` or `{ErrType, Err}`. match(Map1, Map2) -> match(Map1, Map2, strict). match(Map1, Map2, Mode) -> + match(Map1, Map2, Mode, #{}). +match(Map1, Map2, Mode, Opts) -> + try unsafe_match(Map1, Map2, Mode, [], Opts) + catch _:Details -> Details + end. + +%% @doc Match two maps, returning `true' if they match, or throwing an error +%% if they do not. +unsafe_match(Map1, Map2, Mode, Path, Opts) -> Keys1 = - maps:keys( - NormMap1 = minimize(normalize(hb_converge:ensure_message(Map1))) + hb_maps:keys( + NormMap1 = hb_util:lower_case_key_map(minimize( + normalize(hb_ao:normalize_keys(Map1, Opts), Opts), + [<<"content-type">>, <<"ao-body-key">>] + ), Opts) ), Keys2 = - maps:keys( - NormMap2 = minimize(normalize(hb_converge:ensure_message(Map2))) + hb_maps:keys( + NormMap2 = hb_util:lower_case_key_map(minimize( + normalize(hb_ao:normalize_keys(Map2, Opts), Opts), + [<<"content-type">>, <<"ao-body-key">>] + ), Opts) ), PrimaryKeysPresent = (Mode == primary) andalso @@ -317,28 +479,56 @@ match(Map1, Map2, Mode) -> fun(Key) -> lists:member(Key, Keys1) end, Keys1 ), + ?event(match, + {match, + {keys1, Keys1}, + {keys2, Keys2}, + {mode, Mode}, + {primary_keys_present, PrimaryKeysPresent}, + {msg1, Map1}, + {msg2, Map2} + } + ), case (Keys1 == Keys2) or (Mode == only_present) or PrimaryKeysPresent of true -> lists:all( fun(Key) -> - Val1 = maps:get(Key, NormMap1, not_found), - Val2 = maps:get(Key, NormMap2, not_found), + ?event(match, {matching_key, Key}), + Val1 = + hb_ao:normalize_keys( + hb_maps:get(Key, NormMap1, not_found, Opts), + Opts + ), + Val2 = + hb_ao:normalize_keys( + hb_maps:get(Key, NormMap2, not_found, Opts), + Opts + ), BothPresent = (Val1 =/= not_found) and (Val2 =/= not_found), case (not BothPresent) and (Mode == only_present) of true -> true; false -> case is_map(Val1) andalso is_map(Val2) of - true -> match(Val1, Val2); + true -> + unsafe_match(Val1, Val2, Mode, Path ++ [Key], Opts); false -> - case Val1 == Val2 of - true -> true; - false -> - ?event( + case {Val1, Val2} of + {V, V} -> true; + {V, '_'} when V =/= not_found -> true; + {'_', V} when V =/= not_found -> true; + {'_', '_'} -> true; + _ -> + throw( {value_mismatch, - {key, Val1, Val2} + hb_format:short_id( + hb_path:to_binary( + Path ++ [Key] + ) + ), + {val1, Val1}, + {val2, Val2} } - ), - false + ) end end end @@ -346,36 +536,191 @@ match(Map1, Map2, Mode) -> Keys1 ); false -> - ?event({keys_mismatch, {keys1, Keys1}, {keys2, Keys2}}), - false + throw( + {keys_mismatch, + {path, hb_format:short_id(hb_path:to_binary(Path))}, + {keys1, Keys1}, + {keys2, Keys2} + } + ) end. matchable_keys(Map) -> - lists:sort(lists:map(fun hb_converge:key_to_binary/1, maps:keys(Map))). - -%% @doc Normalize the keys in a map. Also takes a list of keys and returns a -%% sorted list of normalized keys if the input is a list. -normalize_keys(Keys) when is_list(Keys) -> - lists:sort(lists:map(fun hb_converge:key_to_binary/1, Keys)); -normalize_keys(Map) -> - maps:from_list( - lists:map( - fun({Key, Value}) -> - {hb_converge:key_to_binary(Key), Value} + lists:sort(lists:map(fun hb_ao:normalize_key/1, hb_maps:keys(Map))). + +%% @doc Return the numeric differences between two messages, matching deeply +%% across nested messages. If the values are non-numeric, the new value is +%% returned if the values are different. Keys found only in the first message +%% are dropped, as they have 'changed' to absence. +diff(Msg1, Msg2, Opts) when is_map(Msg1) andalso is_map(Msg2) -> + maps:filtermap( + fun(Key, Val2) -> + case hb_maps:get(Key, Msg1, not_found, Opts) of + Val2 -> + % The key is present in both maps, and the values match. + false; + not_found -> + % The key is net-new in Map2. + {true, Val2}; + Val1 when is_number(Val1) andalso is_number(Val2) -> + % The key is present in both maps, and the values are numbers; + % return the difference. + {true, Val2 - Val1}; + Val1 when is_map(Val1) andalso is_map(Val2) -> + % The key is present in both maps, and the values are maps; + % return the difference. + {true, diff(Val1, Val2, Opts)}; + _ -> + % The key is present in both maps, and the values do not + % match. Return the new value. + {true, Val2} + end + end, + Msg2 + ); +diff(_Val1, _Val2, _Opts) -> + not_found. + +%% @doc Filter messages that do not match the 'spec' given. The underlying match +%% is performed in the `only_present' mode, such that match specifications only +%% need to specify the keys that must be present. +with_commitments(ID, Msg, Opts) when ?IS_ID(ID) -> + with_commitments([ID], Msg, Opts); +with_commitments(Spec, Msg = #{ <<"commitments">> := Commitments }, Opts) -> + ?event({with_commitments, {spec, Spec}, {commitments, Commitments}}), + FilteredCommitments = + hb_maps:filter( + fun(ID, CommMsg) -> + if is_list(Spec) -> + lists:member(ID, Spec); + is_map(Spec) -> + match(Spec, CommMsg, primary, Opts) == true + end end, - maps:to_list(Map) - ) - ). + Commitments, + Opts + ), + ?event({with_commitments, {filtered_commitments, FilteredCommitments}}), + Msg#{ <<"commitments">> => FilteredCommitments }; +with_commitments(_Spec, Msg, _Opts) -> + Msg. + +%% @doc Filter messages that match the 'spec' given. Inverts the `with_commitments/2' +%% function, such that only messages that do _not_ match the spec are returned. +without_commitments(Spec, Msg = #{ <<"commitments">> := Commitments }, Opts) -> + ?event({without_commitments, {spec, Spec}, {msg, Msg}, {commitments, Commitments}}), + FilteredCommitments = + hb_maps:without( + hb_maps:keys( + hb_maps:get( + <<"commitments">>, + with_commitments(Spec, Msg, Opts), + #{}, + Opts + ) + ), + Commitments + ), + ?event({without_commitments, {filtered_commitments, FilteredCommitments}}), + Msg#{ <<"commitments">> => FilteredCommitments }; +without_commitments(_Spec, Msg, _Opts) -> + Msg. + +%% @doc Extract a commitment from a message given a `committer' or `commitment' +%% ID, or a spec message to match against. Returns only the first matching +%% commitment, or `not_found'. +commitment(ID, Msg) -> + commitment(ID, Msg, #{}). +commitment(ID, Link, Opts) when ?IS_LINK(Link) -> + commitment(ID, hb_cache:ensure_loaded(Link, Opts), Opts); +commitment(ID, #{ <<"commitments">> := Commitments }, Opts) + when is_binary(ID), is_map_key(ID, Commitments) -> + hb_maps:get( + ID, + Commitments, + not_found, + Opts + ); +commitment(Spec, Msg, Opts) -> + Matches = commitments(Spec, Msg, Opts), + ?event(debug_commitment, {commitment, {spec, Spec}, {matches, Matches}}), + if + map_size(Matches) == 0 -> not_found; + map_size(Matches) == 1 -> + CommID = hd(hb_maps:keys(Matches)), + {ok, CommID, hb_util:ok(hb_maps:find(CommID, Matches, Opts))}; + true -> + ?event(commitment, {multiple_matches, {matches, Matches}}), + multiple_matches + end; +commitment(_Spec, _Msg, _Opts) -> + % The message has no commitments, so the spec can never match. + not_found. + +%% @doc Return a list of all commitments that match the spec. +commitments(ID, Link, Opts) when ?IS_LINK(Link) -> + commitments(ID, hb_cache:ensure_loaded(Link, Opts), Opts); +commitments(CommitterID, Msg, Opts) when is_binary(CommitterID) -> + commitments(#{ <<"committer">> => CommitterID }, Msg, Opts); +commitments(Spec, #{ <<"commitments">> := Commitments }, Opts) -> + hb_maps:filtermap( + fun(_ID, CommMsg) -> + case match(Spec, CommMsg, primary, Opts) of + true -> {true, CommMsg}; + _ -> false + end + end, + Commitments, + Opts + ); +commitments(_Spec, _Msg, _Opts) -> + #{}. + +%% @doc Return the devices for which there are commitments on a message. +commitment_devices(#{ <<"commitments">> := Commitments }, Opts) -> + lists:map( + fun(CommMsg) -> + hb_ao:get(<<"commitment-device">>, CommMsg, Opts) + end, + maps:values(Commitments) + ); +commitment_devices(_Msg, _Opts) -> + []. + +%% @doc Implements a standard pattern in which the target for an operation is +%% found by looking for a `target' key in the request. If the target is `self', +%% or not present, the operation is performed on the original message. Otherwise, +%% the target is expected to be a key in the message, and the operation is +%% performed on the value of that key. +find_target(Self, Req, Opts) -> + GetOpts = Opts#{ + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>] + }, + {ok, + case hb_maps:get(<<"target">>, Req, <<"self">>, GetOpts) of + <<"self">> -> Self; + Key -> + hb_maps:get( + Key, + Req, + hb_maps:get(<<"body">>, Req, GetOpts), + GetOpts + ) + end + }. %% @doc Remove keys from the map that can be regenerated. Optionally takes an %% additional list of keys to include in the minimization. minimize(Msg) -> minimize(Msg, []). minimize(RawVal, _) when not is_map(RawVal) -> RawVal; minimize(Map, ExtraKeys) -> - NormKeys = normalize_keys(?REGEN_KEYS) ++ normalize_keys(ExtraKeys), + NormKeys = + lists:map(fun hb_ao:normalize_key/1, ?REGEN_KEYS) + ++ lists:map(fun hb_ao:normalize_key/1, ExtraKeys), maps:filter( fun(Key, _) -> - (not lists:member(hb_converge:key_to_binary(Key), NormKeys)) + (not lists:member(hb_ao:normalize_key(Key), NormKeys)) andalso (not hb_private:is_private(Key)) end, maps:map(fun(_K, V) -> minimize(V) end, Map) @@ -383,10 +728,12 @@ minimize(Map, ExtraKeys) -> %% @doc Return a map with only the keys that necessary, without those that can %% be regenerated. -normalize(Map) -> - NormalizedMap = normalize_keys(Map), +normalize(Map, Opts) when is_map(Map) orelse is_list(Map) -> + NormalizedMap = hb_ao:normalize_keys(Map, Opts), FilteredMap = filter_default_keys(NormalizedMap), - maps:with(matchable_keys(FilteredMap), FilteredMap). + hb_maps:with(matchable_keys(FilteredMap), FilteredMap); +normalize(Other, _Opts) -> + Other. %% @doc Remove keys from a map that have the default values found in the tx %% record. @@ -394,7 +741,7 @@ filter_default_keys(Map) -> DefaultsMap = default_tx_message(), maps:filter( fun(Key, Value) -> - case maps:find(hb_converge:key_to_binary(Key), DefaultsMap) of + case hb_maps:find(hb_ao:normalize_key(Key), DefaultsMap) of {ok, Value} -> false; _ -> true end @@ -404,376 +751,10 @@ filter_default_keys(Map) -> %% @doc Get the normalized fields and default values of the tx record. default_tx_message() -> - normalize_keys(maps:from_list(default_tx_list())). + hb_maps:from_list(default_tx_list()). -%% @doc Get the ordered list of fields and default values of the tx record. +%% @doc Get the ordered list of fields as AO-Core keys and default values of +%% the tx record. default_tx_list() -> - lists:zip(record_info(fields, tx), tl(tuple_to_list(#tx{}))). - -%% @doc Serialize a message to a binary representation, either as JSON or the -%% binary format native to the message/bundles spec in use. -serialize(M) -> serialize(M, binary). -serialize(M, json) -> - jiffy:encode(ar_bundles:item_to_json_struct(M)); -serialize(M, binary) -> - ar_bundles:serialize(convert(M, tx, #{})). - -%% @doc Deserialize a message from a binary representation. -deserialize(B) -> deserialize(B, binary). -deserialize(J, json) -> - {JSONStruct} = jiffy:decode(J), - ar_bundles:json_struct_to_item(JSONStruct); -deserialize(B, binary) -> - convert(ar_bundles:deserialize(B), converge, tx, #{}). - -%%% Tests - -%% @doc Test that the filter_default_keys/1 function removes TX fields -%% that have the default values found in the tx record, but not those that -%% have been set by the user. -default_keys_removed_test() -> - TX = #tx { unsigned_id = << 1:256 >>, last_tx = << 2:256 >> }, - TXMap = #{ - unsigned_id => TX#tx.unsigned_id, - last_tx => TX#tx.last_tx, - <<"owner">> => TX#tx.owner, - <<"target">> => TX#tx.target, - data => TX#tx.data - }, - FilteredMap = filter_default_keys(TXMap), - ?assertEqual(<< 1:256 >>, maps:get(unsigned_id, FilteredMap)), - ?assertEqual(<< 2:256 >>, maps:get(last_tx, FilteredMap, not_found)), - ?assertEqual(not_found, maps:get(<<"owner">>, FilteredMap, not_found)), - ?assertEqual(not_found, maps:get(<<"target">>, FilteredMap, not_found)). - -minimization_test() -> - Msg = #{ - unsigned_id => << 1:256 >>, - <<"id">> => << 2:256 >> - }, - MinimizedMsg = minimize(Msg), - ?event({minimized, MinimizedMsg}), - ?assertEqual(0, maps:size(MinimizedMsg)). - - -basic_map_codec_test(Codec) -> - Msg = #{ normal_key => <<"NORMAL_VALUE">> }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(hb_message:match(Msg, Decoded)). - -%% @doc Test that we can convert a message into a tx record and back. -single_layer_message_to_encoding_test(Codec) -> - Msg = #{ - last_tx => << 2:256 >>, - owner => << 3:4096 >>, - target => << 4:256 >>, - data => <<"DATA">>, - <<"Special-Key">> => <<"SPECIAL_VALUE">> - }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(hb_message:match(Msg, Decoded)). - -% %% @doc Test that different key encodings are converted to their corresponding -% %% TX fields. -% key_encodings_to_tx_test() -> -% Msg = #{ -% <<"last_tx">> => << 2:256 >>, -% <<"Owner">> => << 3:4096 >>, -% <<"Target">> => << 4:256 >> -% }, -% TX = message_to_tx(Msg), -% ?event({key_encodings_to_tx, {msg, Msg}, {tx, TX}}), -% ?assertEqual(maps:get(<<"last_tx">>, Msg), TX#tx.last_tx), -% ?assertEqual(maps:get(<<"Owner">>, Msg), TX#tx.owner), -% ?assertEqual(maps:get(<<"Target">>, Msg), TX#tx.target). - -%% @doc Test that the message matching function works. -match_test(Codec) -> - Msg = #{ a => 1, b => 2 }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(Msg, Decoded)). - -match_modes_test() -> - Msg1 = #{ a => 1, b => 2 }, - Msg2 = #{ a => 1 }, - Msg3 = #{ a => 1, b => 2, c => 3 }, - ?assert(match(Msg1, Msg2, only_present)), - ?assert(not match(Msg2, Msg1, strict)), - ?assert(match(Msg1, Msg3, primary)), - ?assert(not match(Msg3, Msg1, primary)). - -%% @doc Structured field parsing tests. -structured_field_atom_parsing_test(Codec) -> - Msg = #{ highly_unusual_http_header => highly_unusual_value }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(Msg, Decoded)). - -structured_field_decimal_parsing_test(Codec) -> - Msg = #{ integer_field => 1234567890 }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(Msg, Decoded)). - -binary_to_binary_test(Codec) -> - % Serialization must be able to turn a raw binary into a TX, then turn - % that TX back into a binary and have the result match the original. - Bin = <<"THIS IS A BINARY, NOT A NORMAL MESSAGE">>, - Encoded = convert(Bin, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assertEqual(Bin, Decoded). - -%% @doc Test that the data field is correctly managed when we have multiple -%% uses for it (the 'data' key itself, as well as keys that cannot fit in -%% tags). -message_with_large_keys_test(Codec) -> - Msg = #{ - <<"normal_key">> => <<"normal_value">>, - <<"large_key">> => << 0:((1 + 1024) * 8) >>, - <<"another_large_key">> => << 0:((1 + 1024) * 8) >>, - <<"another_normal_key">> => <<"another_normal_value">> - }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(Msg, Decoded)). - -%% @doc Check that large keys and data fields are correctly handled together. -nested_message_with_large_keys_and_data_test(Codec) -> - Msg = #{ - <<"normal_key">> => <<"normal_value">>, - <<"large_key">> => << 0:(1024 * 16) >>, - <<"another_large_key">> => << 0:(1024 * 16) >>, - <<"another_normal_key">> => <<"another_normal_value">>, - data => <<"Hey from the data field!">> - }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?event({matching, {input, Msg}, {output, Decoded}}), - ?assert(match(Msg, Decoded)). - -simple_nested_message_test(Codec) -> - Msg = #{ - a => <<"1">>, - nested => #{ <<"b">> => <<"1">> }, - c => <<"3">> - }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?event({matching, {input, Msg}, {output, Decoded}}), - ?assert( - match( - Msg, - Decoded - ) - ). - -%% @doc Test that the data field is correctly managed when we have multiple -%% uses for it (the 'data' key itself, as well as keys that cannot fit in -%% tags). -nested_message_with_large_data_test(Codec) -> - Msg = #{ - <<"tx_depth">> => <<"outer">>, - data => #{ - <<"tx_map_item">> => - #{ - <<"tx_depth">> => <<"inner">>, - <<"large_data_inner">> => << 0:((1 + 1024) * 8) >> - }, - <<"large_data_outer">> => << 0:((1 + 1024) * 8) >> - } - }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(Msg, Decoded)). - -%% @doc Test that we can convert a 3 layer nested message into a tx record and back. -deeply_nested_message_with_data_test(Codec) -> - Msg = #{ - <<"tx_depth">> => <<"outer">>, - data => #{ - <<"tx_map_item">> => - #{ - <<"tx_depth">> => <<"inner">>, - data => #{ - <<"tx_depth">> => <<"innermost">>, - data => <<"DATA">> - } - } - } - }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(Msg, Decoded)). - -nested_structured_fields_test(Codec) -> - NestedMsg = #{ a => #{ b => 1 } }, - Encoded = convert(NestedMsg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(NestedMsg, Decoded)). - -nested_message_with_large_keys_test(Codec) -> - Msg = #{ - a => <<"1">>, - long_data => << 0:((1 + 1024) * 8) >>, - nested => #{ <<"b">> => <<"1">> }, - c => <<"3">> - }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(Msg, Decoded)). - -%% @doc Test that we can convert a signed tx into a message and back. -signed_tx_encode_decode_verify_test(Codec) -> - TX = #tx { - data = <<"TEST_DATA">>, - tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}] - }, - SignedTX = ar_bundles:sign_item(TX, hb:wallet()), - Encoded = convert(SignedTX, Codec, tx, #{}), - ?assert(ar_bundles:verify_item(SignedTX)), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(verify(Decoded)). - -signed_message_encode_decode_verify_test(Codec) -> - Msg = #{ - data => <<"TEST_DATA">>, - tags => [{<<"TEST_KEY">>, <<"TEST_VALUE">>}] - }, - SignedMsg = hb_message:sign(Msg, hb:wallet()), - Encoded = convert(SignedMsg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(SignedMsg, Decoded)). - -tabm_converge_ids_equal_test() -> - Msg = #{ - data => <<"TEST_DATA">>, - deep_data => #{ - data => <<"DEEP_DATA">>, - complex_key => 1337, - list => [1,2,3] - } - }, - ?assertEqual( - dev_message:unsigned_id(Msg), - dev_message:unsigned_id(convert(Msg, tabm, converge, #{})) - ). - -signed_deep_tx_serialize_and_deserialize_test(Codec) -> - TX = #tx { - tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}], - data = #{ - <<"NESTED_TX">> => - #tx { - data = <<"NESTED_DATA">>, - tags = [{<<"NESTED_KEY">>, <<"NESTED_VALUE">>}] - } - } - }, - SignedTX = ar_bundles:deserialize( - ar_bundles:sign_item(TX, hb:wallet()) - ), - ?assert(ar_bundles:verify_item(SignedTX)), - Encoded = convert(SignedTX, Codec, tx, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert( - match( - convert(SignedTX, converge, tx, #{}), - Decoded - ) - ). - -unsigned_id_test(Codec) -> - Msg = #{ data => <<"TEST_DATA">> }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assertEqual( - dev_message:unsigned_id(Decoded), - dev_message:unsigned_id(Msg) - ). - -% signed_id_test_disabled() -> -% TX = #tx { -% data = <<"TEST_DATA">>, -% tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}] -% }, -% SignedTX = ar_bundles:sign_item(TX, hb:wallet()), -% ?assert(ar_bundles:verify_item(SignedTX)), -% SignedMsg = hb_codec_tx:from(SignedTX), -% ?assertEqual( -% hb_util:encode(ar_bundles:id(SignedTX, signed)), -% hb_util:id(SignedMsg, signed) -% ). - -message_with_simple_list_test(Codec) -> - Msg = #{ a => [<<"1">>, <<"2">>, <<"3">>] }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(Msg, Decoded)). - -empty_string_in_tag_test(Codec) -> - Msg = - #{ - dev => - #{ - <<"stderr">> => <<"">>, - <<"stdin">> => <<"b">>, - <<"stdout">> => <<"c">> - } - }, - Encoded = convert(Msg, Codec, converge, #{}), - Decoded = convert(Encoded, converge, Codec, #{}), - ?assert(match(Msg, Decoded)). - - -%%% Test helpers - -test_codecs() -> - [converge, tx, flat, http]. - -generate_test_suite(Suite) -> - lists:map( - fun(CodecName) -> - {foreach, - fun() -> ok end, - fun(_) -> ok end, - [ - { - atom_to_list(CodecName) ++ ": " ++ Desc, - fun() -> Test(CodecName) end - } - || - {Desc, Test} <- Suite - ] - } - end, - test_codecs() - ). - -message_suite_test_() -> - generate_test_suite([ - {"basic map codec test", fun basic_map_codec_test/1}, - {"match test", fun match_test/1}, - {"single layer message to encoding test", fun single_layer_message_to_encoding_test/1}, - {"message with large keys test", fun message_with_large_keys_test/1}, - {"nested message with large keys and data test", fun nested_message_with_large_keys_and_data_test/1}, - {"simple nested message test", fun simple_nested_message_test/1}, - {"nested message with large data test", fun nested_message_with_large_data_test/1}, - {"deeply nested message with data test", fun deeply_nested_message_with_data_test/1}, - {"structured field atom parsing test", fun structured_field_atom_parsing_test/1}, - {"structured field decimal parsing test", fun structured_field_decimal_parsing_test/1}, - {"binary to binary test", fun binary_to_binary_test/1}, - {"nested structured fields test", fun nested_structured_fields_test/1}, - {"nested message with large keys test", fun nested_message_with_large_keys_test/1}, - {"message with simple list test", fun message_with_simple_list_test/1}, - {"empty string in tag test", fun empty_string_in_tag_test/1}, - {"signed item to message and back test", fun signed_message_encode_decode_verify_test/1}, - {"signed item to tx and back test", fun signed_tx_encode_decode_verify_test/1}, - {"signed deep serialize and deserialize test", fun signed_deep_tx_serialize_and_deserialize_test/1}, - {"unsigned id test", fun unsigned_id_test/1} - ]). - -simple_test() -> - basic_map_codec_test(http). + Keys = lists:map(fun hb_ao:normalize_key/1, record_info(fields, tx)), + lists:zip(Keys, tl(tuple_to_list(#tx{}))). \ No newline at end of file diff --git a/src/hb_message_test_vectors.erl b/src/hb_message_test_vectors.erl new file mode 100644 index 000000000..4cab747d9 --- /dev/null +++ b/src/hb_message_test_vectors.erl @@ -0,0 +1,1604 @@ +%%% @doc A battery of test vectors for message codecs, implementing the +%%% `message@1.0' encoding and commitment APIs. Additionally, this module +%%% houses tests that ensure the general functioning of the `hb_message' API. +-module(hb_message_test_vectors). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% @doc Test invocation function, making it easier to run a specific test. +%% Disable/enable as needed. +run_test() -> + hb:init(), + nested_structured_fields_test( + #{ <<"device">> => <<"json@1.0">>, <<"bundle">> => true }, + test_opts(normal) + ). + +%% @doc Return a list of codecs to test. Disable these as necessary if you need +%% to test the functionality of a single codec, etc. +test_codecs() -> + [ + <<"structured@1.0">>, + <<"httpsig@1.0">>, + #{ <<"device">> => <<"httpsig@1.0">>, <<"bundle">> => true }, + <<"flat@1.0">>, + <<"ans104@1.0">>, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + <<"json@1.0">>, + #{ <<"device">> => <<"json@1.0">>, <<"bundle">> => true } + ]. + +%% @doc Return a set of options for testing, taking the codec name as an +%% argument. We do not presently use the codec name in the test, but we may +%% wish to do so in the future. +suite_test_opts() -> + [ + #{ + name => normal, + desc => <<"Default opts">>, + opts => test_opts(normal) + } + ]. +suite_test_opts(OptsName) -> + [ O || O = #{ name := OName } <- suite_test_opts(), OName == OptsName ]. + +test_opts(normal) -> + #{ + store => hb_test_utils:test_store(), + priv_wallet => hb:wallet() + }. + +test_suite() -> + [ + % Basic operations + {<<"Binary to binary">>, + fun binary_to_binary_test/2}, + {<<"Match">>, + fun match_test/2}, + {<<"Basic message encoding and decoding">>, + fun basic_message_codec_test/2}, + {<<"Priv survives conversion">>, + fun priv_survives_conversion_test/2}, + {<<"Message with body">>, + fun set_body_codec_test/2}, + {<<"Message with large keys">>, + fun message_with_large_keys_test/2}, + {<<"Structured field atom parsing">>, + fun structured_field_atom_parsing_test/2}, + {<<"Structured field decimal parsing">>, + fun structured_field_decimal_parsing_test/2}, + {<<"Unsigned id">>, + fun unsigned_id_test/2}, + % Nested structures + {<<"Simple nested message">>, + fun simple_nested_message_test/2}, + {<<"Message with simple embedded list">>, + fun message_with_simple_embedded_list_test/2}, + {<<"Nested empty map">>, + fun nested_empty_map_test/2}, + {<<"Empty body">>, + fun empty_body_test/2}, + {<<"Nested structured fields">>, + fun nested_structured_fields_test/2}, + {<<"Single layer message to encoding">>, + fun single_layer_message_to_encoding_test/2}, + {<<"Nested body list">>, + fun nested_body_list_test/2}, + {<<"Empty string in nested tag">>, + fun empty_string_in_nested_tag_test/2}, + {<<"Deep typed message ID">>, + fun deep_typed_message_id_test/2}, + {<<"Encode small balance table">>, + fun encode_small_balance_table_test/2}, + {<<"Encode large balance table">>, + fun encode_large_balance_table_test/2}, + {<<"Normalize commitments">>, + fun normalize_commitments_test/2}, + % Signed messages + {<<"Signed message to message and back">>, + fun signed_message_encode_decode_verify_test/2}, + {<<"Specific order signed message">>, + fun specific_order_signed_message_test/2}, + {<<"Specific order deeply nested signed message">>, + fun specific_order_deeply_nested_signed_message_test/2}, + {<<"Signed only committed data field">>, + fun signed_only_committed_data_field_test/2}, + {<<"Signed simple nested message">>, + fun simple_signed_nested_message_test/2}, + {<<"Signed nested message">>, + fun signed_nested_message_with_child_test/2}, + {<<"Committed keys">>, + fun committed_keys_test/2}, + {<<"Committed empty keys">>, + fun committed_empty_keys_test/2}, + {<<"Signed list HTTP response">>, + fun signed_list_test/2}, + {<<"Sign node message">>, + fun sign_node_message_test/2}, + {<<"Complex signed message">>, + fun complex_signed_message_test/2}, + {<<"Nested message with large keys">>, + fun nested_message_with_large_keys_test/2}, + {<<"Signed nested complex signed message">>, + fun verify_nested_complex_signed_test/2}, + % Complex structures + {<<"Nested message with large keys and content">>, + fun nested_message_with_large_keys_and_content_test/2}, + {<<"Nested message with large content">>, + fun nested_message_with_large_content_test/2}, + {<<"Deeply nested message with content">>, + fun deeply_nested_message_with_content_test/2}, + {<<"Deeply nested message with only content">>, + fun deeply_nested_message_with_only_content/2}, + {<<"Signed deep serialize and deserialize">>, + fun signed_deep_message_test/2}, + {<<"Signed nested data key">>, + fun signed_nested_data_key_test/2}, + {<<"Signed message with hashpath">>, + fun hashpath_sign_verify_test/2}, + {<<"Message with derived components">>, + fun signed_message_with_derived_components_test/2}, + {<<"Large body committed keys">>, + fun large_body_committed_keys_test/2}, + {<<"Signed with inner signed">>, + fun signed_with_inner_signed_message_test/2}, + {<<"Recursive nested list">>, + fun recursive_nested_list_test/2}, + {<<"Sign links">>, + fun sign_links_test/2}, + {<<"ID of linked message">>, + fun id_of_linked_message_test/2}, + {<<"Sign deep message from lazy cache read">>, + fun sign_deep_message_from_lazy_cache_read_test/2}, + {<<"ID of deep message and link message match">>, + fun id_of_deep_message_and_link_message_match_test/2}, + {<<"Signed non-bundle is bundlable">>, + fun signed_non_bundle_is_bundlable_test/2}, + {<<"Bundled ordering">>, + fun bundled_ordering_test/2}, + {<<"Codec round-trip conversion is idempotent">>, + fun codec_roundtrip_conversion_is_idempotent_test/2}, + {<<"Bundled and unbundled IDs differ">>, + fun bundled_and_unbundled_ids_differ_test/2}, + {<<"Tabm conversion is idempotent">>, + fun tabm_conversion_is_idempotent_test/2} + ]. + +%% @doc Organizes a test battery for the `hb_message' module and its codecs. +suite_test_() -> + hb_test_utils:suite_with_opts( + codec_test_suite( + test_codecs(), + normal + ), + suite_test_opts(normal) + ). + +%% @doc Run the test suite for a set of codecs, using the given options type. +%% Unlike normal `hb_test_utils:suite_with_opts/2' users, this suite generator +%% creates a new options message for each individual test, such that stores +%% are completely isolated from each other. +codec_test_suite(Codecs, OptsType) -> + lists:flatmap( + fun(CodecName) -> + lists:map(fun({Desc, Test}) -> + TestName = + binary_to_list( + << (suite_name(CodecName))/binary, ": ", Desc/binary >> + ), + TestSpecificOpts = test_opts(OptsType), + { + Desc, + TestName, + fun(_SuiteOpts) -> Test(CodecName, TestSpecificOpts) end + } + end, test_suite()) + end, + Codecs + ). + +%% @doc Create a name for a suite from a codec spec. +suite_name(CodecSpec) when is_binary(CodecSpec) -> CodecSpec; +suite_name(CodecSpec) when is_map(CodecSpec) -> + CodecName = maps:get(<<"device">>, CodecSpec, <<"[! NO CODEC !]">>), + case maps:get(<<"bundle">>, CodecSpec, false) of + false -> CodecName; + true -> << CodecName/binary, " (bundle)">> + end. + +%%% Codec-specific/misc. tests + +%% @doc Tests a message transforming function to ensure that it is idempotent. +%% Runs the conversion a total of 3 times, ensuring that the result remains +%% unchanged. This function takes transformation functions that result in +%% `{ok, Res}`-form messages, as well as bare message results. +is_idempotent(Func, Msg, Opts) -> + Run = fun(M) -> case Func(M) of {ok, Res} -> Res; Res -> Res end end, + After1 = Run(Msg), + After2 = Run(After1), + After3 = Run(After2), + MatchRes1 = hb_message:match(After1, After2, strict, Opts), + MatchRes2 = hb_message:match(After2, After3, strict, Opts), + ?event({is_idempotent, {match_res1, MatchRes1}, {match_res2, MatchRes2}}), + MatchRes1 andalso MatchRes2. + +%% @doc Ensure that converting a message to/from TABM multiple times repeatedly +%% does not alter the message's contents. +tabm_conversion_is_idempotent_test(_Codec, Opts) -> + From = fun(M) -> hb_message:convert(M, <<"structured@1.0">>, tabm, Opts) end, + To = fun(M) -> hb_message:convert(M, tabm, <<"structured@1.0">>, Opts) end, + SimpleMsg = #{ <<"a">> => <<"x">>, <<"b">> => <<"y">>, <<"c">> => <<"z">> }, + ComplexMsg = + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + Signed = hb_message:commit( + #{ + <<"type">> => <<"Message">>, + <<"function">> => <<"fac">>, + <<"parameters">> => #{ + <<"a">> => 1 + }, + <<"content-type">> => <<"application/html">>, + <<"body">> => + << + """ + +

Hello, multiline message

+ + """ + >> + }, + Opts, + <<"structured@1.0">> + ) + }, + ?assert(is_idempotent(From, SimpleMsg, Opts)), + ?assert(is_idempotent(From, Signed, Opts)), + ?assert(is_idempotent(From, ComplexMsg, Opts)), + ?assert(is_idempotent(To, SimpleMsg, Opts)), + ?assert(is_idempotent(To, Signed, Opts)), + ?assert(is_idempotent(To, ComplexMsg, Opts)). + +%% @doc Ensure that converting a message to a codec, then back to TABM multiple +%% times results in the same message being returned. This test differs from its +%% TABM form, as it shuttles (`to-from-to-...`), while the TABM test repeatedly +%% encodes in a single direction (`to->to->...`). +codec_roundtrip_conversion_is_idempotent_test(Codec, Opts) -> + Roundtrip = + fun(M) -> + hb_message:convert( + hb_message:convert(M, Codec, <<"structured@1.0">>, Opts), + <<"structured@1.0">>, + Codec, + Opts + ) + end, + SimpleMsg = #{ <<"a">> => <<"x">>, <<"b">> => <<"y">>, <<"c">> => <<"z">> }, + ComplexMsg = + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + Signed = hb_message:commit( + #{ + <<"type">> => <<"Message">>, + <<"function">> => <<"fac">>, + <<"parameters">> => #{ + <<"a">> => 1 + }, + <<"content-type">> => <<"application/html">>, + <<"body">> => + << + """ + +

Hello, multiline message

+ + """ + >> + }, + Opts, + Codec + ) + }, + ?assert(is_idempotent(Roundtrip, SimpleMsg, Opts)), + ?assert(is_idempotent(Roundtrip, Signed, Opts)), + ?assert(is_idempotent(Roundtrip, ComplexMsg, Opts)). + +%% @doc Test that the filter_default_keys/1 function removes TX fields +%% that have the default values found in the tx record, but not those that +%% have been set by the user. +default_keys_removed_test() -> + TX = #tx { unsigned_id = << 1:256 >>, anchor = << 2:256 >> }, + TXMap = #{ + <<"unsigned_id">> => TX#tx.unsigned_id, + <<"anchor">> => TX#tx.anchor, + <<"owner">> => TX#tx.owner, + <<"target">> => TX#tx.target, + <<"data">> => TX#tx.data + }, + FilteredMap = hb_message:filter_default_keys(TXMap), + ?assertEqual(<< 1:256 >>, hb_maps:get(<<"unsigned_id">>, FilteredMap)), + ?assertEqual(<< 2:256 >>, hb_maps:get(<<"anchor">>, FilteredMap, not_found)), + ?assertEqual(not_found, hb_maps:get(<<"owner">>, FilteredMap, not_found)), + ?assertEqual(not_found, hb_maps:get(<<"target">>, FilteredMap, not_found)). + +minimization_test() -> + Msg = #{ + <<"unsigned_id">> => << 1:256 >>, + <<"id">> => << 2:256 >> + }, + MinimizedMsg = hb_message:minimize(Msg), + ?event({minimized, MinimizedMsg}), + ?assertEqual(1, hb_maps:size(MinimizedMsg)). + +match_modes_test() -> + Msg1 = #{ <<"a">> => 1, <<"b">> => 2 }, + Msg2 = #{ <<"a">> => 1 }, + Msg3 = #{ <<"a">> => 1, <<"b">> => 2, <<"c">> => 3 }, + ?assert(hb_message:match(Msg1, Msg2, only_present)), + ?assert(hb_message:match(Msg2, Msg1, strict) =/= true), + ?assert(hb_message:match(Msg1, Msg3, primary)), + ?assert(hb_message:match(Msg3, Msg1, primary) =/= true). + +basic_message_codec_test(Codec, Opts) -> + Msg = #{ <<"normal_key">> => <<"NORMAL_VALUE">> }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +set_body_codec_test(Codec, Opts) -> + Msg = #{ <<"body">> => <<"NORMAL_VALUE">>, <<"test-key">> => <<"Test-Value">> }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +%% @doc Test that we can convert a message into a tx record and back. +single_layer_message_to_encoding_test(Codec, Opts) -> + Msg = #{ + <<"anchor">> => << 2:256 >>, + <<"target">> => << 4:256 >>, + <<"data">> => <<"DATA">>, + <<"special-key">> => <<"SPECIAL_VALUE">> + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?event({matching, {input, Msg}, {output, Decoded}}), + MatchRes = hb_message:match(Msg, Decoded, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes). + +signed_only_committed_data_field_test(Codec, Opts) -> + Msg = hb_message:commit(#{ <<"data">> => <<"DATA">> }, Opts, Codec), + ?event({signed_msg, Msg}), + {ok, OnlyCommitted} = hb_message:with_only_committed(Msg, Opts), + ?event({only_committed, OnlyCommitted}), + Encoded = hb_message:convert(OnlyCommitted, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + MatchRes = hb_message:match(Msg, OnlyCommitted, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes), + ?assert(hb_message:verify(OnlyCommitted, all, Opts)). + +signed_nested_data_key_test(Codec, Opts) -> + Msg = + #{ + <<"outer-data">> => <<"outer">>, + <<"body">> => + #{ + <<"inner-data">> => <<"inner">>, + <<"data">> => <<"DATA">> + } + }, + Signed = hb_message:commit(Msg, Opts, Codec), + ?event({signed, Signed}), + Encoded = hb_message:convert(Signed, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + LoadedMsg = hb_cache:ensure_all_loaded(Decoded, Opts), + ?event({matching, {input, Msg}, {output, LoadedMsg}}), + ?assert(hb_message:match(Msg, LoadedMsg, primary, Opts)). + +% %% @doc Test that different key encodings are converted to their corresponding +% %% TX fields. +% key_encodings_to_tx_test() -> +% Msg = #{ +% <<"last_tx">> => << 2:256 >>, +% <<"owner">> => << 3:4096 >>, +% <<"target">> => << 4:256 >> +% }, +% TX = message_to_tx(Msg), +% ?event({key_encodings_to_tx, {msg, Msg}, {tx, TX}}), +% ?assertEqual(hb_maps:get(<<"last_tx">>, Msg), TX#tx.last_tx), +% ?assertEqual(hb_maps:get(<<"owner">>, Msg), TX#tx.owner), +% ?assertEqual(hb_maps:get(<<"target">>, Msg), TX#tx.target). + +%% @doc Test that the message matching function works. +match_test(Codec, Opts) -> + Msg = #{ <<"a">> => 1, <<"b">> => 2 }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +binary_to_binary_test(Codec, Opts) -> + % Serialization must be able to turn a raw binary into a TX, then turn + % that TX back into a binary and have the result match the original. + Bin = <<"THIS IS A BINARY, NOT A NORMAL MESSAGE">>, + Encoded = hb_message:convert(Bin, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?assertEqual(Bin, Decoded). + +%% @doc Structured field parsing tests. +structured_field_atom_parsing_test(Codec, Opts) -> + Msg = #{ highly_unusual_http_header => highly_unusual_value }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +structured_field_decimal_parsing_test(Codec, Opts) -> + Msg = #{ integer_field => 1234567890 }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +%% @doc Test that the data field is correctly managed when we have multiple +%% uses for it (the 'data' key itself, as well as keys that cannot fit in +%% tags). +message_with_large_keys_test(Codec, Opts) -> + Msg = #{ + <<"normal_key">> => <<"normal_value">>, + <<"large_key">> => << 0:((1 + 1024) * 8) >>, + <<"another_large_key">> => << 0:((1 + 1024) * 8) >>, + <<"another_normal_key">> => <<"another_normal_value">> + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +%% @doc Check that a nested signed message with an embedded typed list can +%% be further nested and signed. We then encode and decode the message. This +%% tests a large portion of the complex type encodings that HyperBEAM uses +%% together. +verify_nested_complex_signed_test(Codec, Opts) -> + Msg = + hb_message:commit(#{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + Inner = hb_message:commit( + #{ + <<"type">> => <<"Message">>, + <<"function">> => <<"fac">>, + <<"parameters">> => #{ + <<"a">> => 1 + }, + <<"content-type">> => <<"application/html">>, + <<"body">> => + << + """ + +

Hello, multiline message

+ + """ + >> + }, + Opts, + Codec + ) + }, + Opts, + Codec + ), + ?event({signed, Msg}), + ?event({inner, Inner}), + % Ensure that the messages verify prior to conversion. + LoadedInitialInner = hb_cache:ensure_all_loaded(Inner, Opts), + ?assert(hb_message:verify(Inner, all, Opts)), + ?assert(hb_message:verify(LoadedInitialInner, all, Opts)), + % % Test encoding and decoding. + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + LoadedMsg = hb_cache:ensure_all_loaded(Decoded, Opts), + ?event({loaded, LoadedMsg}), + % % Ensure that the decoded message matches. + MatchRes = hb_message:match(Msg, Decoded, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes), + ?assert(hb_message:verify(Decoded, all, Opts)), + % % Ensure that both of the messages can be verified (and retreived). + FoundInner = hb_maps:get(<<"body">>, Msg, not_found, Opts), + LoadedFoundInner = hb_cache:ensure_all_loaded(FoundInner, Opts), + % Verify that the fully loaded version of the inner message, and the one + % gained by applying `hb_maps:get` match and verify. + ?assert(hb_message:match(Inner, FoundInner, primary, Opts)), + ?assert(hb_message:match(FoundInner, LoadedFoundInner, primary, Opts)), + ?assert(hb_message:verify(Inner, all, Opts)), + ?assert(hb_message:verify(LoadedFoundInner, all, Opts)), + ?assert(hb_message:verify(FoundInner, all, Opts)). + +%% @doc Check that large keys and data fields are correctly handled together. +nested_message_with_large_keys_and_content_test(Codec, Opts) -> + MainBodyKey = + case Codec of + <<"ans104@1.0">> -> <<"data">>; + _ -> <<"body">> + end, + Msg = #{ + <<"normal_key">> => <<"normal_value">>, + <<"large_key">> => << 0:(1024 * 16) >>, + <<"another_large_key">> => << 0:(1024 * 16) >>, + <<"another_normal_key">> => <<"another_normal_value">>, + MainBodyKey => <<"Hey from the data field!">> + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +simple_nested_message_test(Codec, Opts) -> + Msg = #{ + <<"a">> => <<"1">>, + <<"nested">> => #{ <<"b">> => <<"1">> }, + <<"c">> => <<"3">> + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +simple_signed_nested_message_test(Codec, Opts) -> + Msg = + hb_message:commit( + #{ + <<"a">> => <<"1">>, + <<"nested">> => #{ <<"b">> => <<"1">> }, + <<"c">> => <<"3">> + }, + Opts, + Codec + ), + ?assert(hb_message:verify(Msg, all, Opts)), + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + MatchRes = hb_message:match(Msg, Decoded, primary, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes), + ?assert(hb_message:verify(Decoded, all, Opts)). + +signed_nested_message_with_child_test(Codec, Opts) -> + Msg = #{ + <<"outer-a">> => <<"1">>, + <<"nested">> => + hb_message:commit( + #{ <<"inner-b">> => <<"1">>, <<"inner-list">> => [1, 2, 3] }, + Opts, + Codec + ), + <<"outer-c">> => <<"3">> + }, + hb_cache:write(Msg, Opts), + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + MatchRes = hb_message:match(Msg, Decoded, primary, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes), + ?assert(hb_message:verify(Decoded, all, Opts)). + +nested_empty_map_test(Codec, Opts) -> + Msg = #{ <<"body">> => #{ <<"empty-map-test">> => #{}}}, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + MatchRes = hb_message:match(Msg, Decoded, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes). + +empty_body_test(Codec, Opts) -> + Msg = #{ <<"body">> => <<>> }, + Signed = hb_message:commit(Msg, Opts, Codec), + Encoded = hb_message:convert(Signed, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + MatchRes = hb_message:match(Signed, Decoded, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes). + +%% @doc Test that the data field is correctly managed when we have multiple +%% uses for it (the 'data' key itself, as well as keys that cannot fit in +%% tags). +nested_message_with_large_content_test(Codec, Opts) -> + MainBodyKey = + case Codec of + <<"ans104@1.0">> -> <<"data">>; + _ -> <<"body">> + end, + Msg = #{ + <<"depth">> => <<"outer">>, + MainBodyKey => #{ + <<"map_item">> => + #{ + <<"depth">> => <<"inner">>, + <<"large_data_inner">> => << 0:((1 + 1024) * 8) >> + }, + <<"large_data_outer">> => << 0:((1 + 1024) * 8) >> + } + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +%% @doc Test that we can convert a 3 layer nested message into a tx record and back. +deeply_nested_message_with_content_test(Codec, Opts) -> + MainBodyKey = + case Codec of + <<"ans104@1.0">> -> <<"data">>; + _ -> <<"body">> + end, + Msg = #{ + <<"depth">> => <<"outer">>, + MainBodyKey => #{ + <<"map_item">> => + #{ + <<"depth">> => <<"inner">>, + MainBodyKey => #{ + <<"depth">> => <<"innermost">>, + MainBodyKey => <<"DATA">> + } + } + } + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +deeply_nested_message_with_only_content(Codec, Opts) -> + MainBodyKey = + case Codec of + <<"ans104@1.0">> -> <<"data">>; + _ -> <<"body">> + end, + Msg = #{ + <<"depth1">> => <<"outer">>, + MainBodyKey => #{ + MainBodyKey => #{ + MainBodyKey => <<"depth2-body">> + } + } + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +nested_structured_fields_test(Codec, Opts) -> + NestedMsg = #{ <<"a">> => #{ <<"b">> => 1 } }, + Encoded = hb_message:convert(NestedMsg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, NestedMsg}, {output, Decoded}}), + ?assert(hb_message:match(NestedMsg, Decoded, strict, Opts)). + +nested_message_with_large_keys_test(Codec, Opts) -> + Msg = #{ + <<"a">> => <<"1">>, + <<"long_data">> => << 0:((1 + 1024) * 8) >>, + <<"nested">> => #{ <<"b">> => <<"1">> }, + <<"c">> => <<"3">> + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +signed_message_encode_decode_verify_test(Codec, Opts) -> + Msg = #{ + <<"test-1">> => <<"TEST VALUE 1">>, + <<"test-2">> => <<"TEST VALUE 2">>, + <<"test-3">> => <<"TEST VALUE 3">>, + <<"test-4">> => <<"TEST VALUE 4">>, + <<"test-5">> => <<"TEST VALUE 5">> + }, + SignedMsg = + hb_message:commit( + Msg, + Opts, + Codec + ), + ?event({signed_msg, SignedMsg}), + ?assertEqual(true, hb_message:verify(SignedMsg, all, Opts)), + Encoded = hb_message:convert(SignedMsg, Codec, <<"structured@1.0">>, Opts), + ?event({msg_encoded_as_codec, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assertEqual(true, hb_message:verify(Decoded, all, Opts)), + ?event({matching, {input, SignedMsg}, {encoded, Encoded}, {decoded, Decoded}}), + ?event({http, {string, dev_codec_httpsig_conv:encode_http_msg(SignedMsg, Opts)}}), + MatchRes = hb_message:match(SignedMsg, Decoded, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes). + +specific_order_signed_message_test(RawCodec, Opts) -> + Msg = #{ + <<"key-1">> => <<"DATA-1">>, + <<"key-2">> => <<"DATA-2">>, + <<"key-3">> => <<"DATA-3">> + }, + Codec = + if is_map(RawCodec) -> RawCodec; + true -> #{ <<"device">> => RawCodec } + end, + SignedMsg = + hb_message:commit( + Msg, + Opts, + Codec#{ <<"committed">> => [<<"key-3">>, <<"key-1">>, <<"key-2">>] } + ), + ?event({signed_msg, SignedMsg}), + ?event({http, {string, dev_codec_httpsig_conv:encode_http_msg(SignedMsg, Opts)}}), + ?assert(hb_message:verify(SignedMsg, all, Opts)). + +specific_order_deeply_nested_signed_message_test(RawCodec, Opts) -> + Msg = #{ + <<"key-1">> => <<"DATA-1">>, + <<"key-2">> => #{ <<"body">> => [1,2] }, + <<"key-3">> => <<"DATA-3">>, + <<"key-4">> => #{ <<"body">> => [1,2,3,4] }, + <<"key-5">> => <<"DATA-5">> + }, + Codec = + if is_map(RawCodec) -> RawCodec; + true -> #{ <<"device">> => RawCodec } + end, + SignedMsg = + hb_message:commit( + Msg, + Opts, + Codec#{ + <<"committed">> => + [ + <<"key-3">>, + <<"key-5">>, + <<"key-1">>, + <<"key-2">>, + <<"key-4">> + ] + } + ), + ?event({signed_msg, SignedMsg}), + ?assert(hb_message:verify(SignedMsg, all, Opts)). + +complex_signed_message_test(Codec, Opts) -> + Msg = #{ + <<"data">> => <<"TEST DATA">>, + <<"deep-data">> => #{ + <<"data">> => <<"DEEP DATA">>, + <<"complex-key">> => 1337, + <<"list">> => [1,2,3] + } + }, + SignedMsg = + hb_message:commit( + Msg, + Opts, + Codec + ), + Encoded = hb_message:convert(SignedMsg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assertEqual(true, hb_message:verify(Decoded, all, Opts)), + ?event({matching, {input, SignedMsg}, {output, Decoded}}), + MatchRes = hb_message:match(SignedMsg, Decoded, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes). + +% multisignature_test(Codec) -> +% Wallet1 = ar_wallet:new(), +% Wallet2 = ar_wallet:new(), +% Msg = #{ +% <<"data">> => <<"TEST_DATA">>, +% <<"test_key">> => <<"TEST_VALUE">> +% }, +% {ok, SignedMsg} = +% dev_message:commit( +% Msg, +% #{ <<"commitment-device">> => Codec }, +% #{ priv_wallet => Wallet1 } +% ), +% ?event({signed_msg, SignedMsg}), +% {ok, MsgSignedTwice} = +% dev_message:commit( +% SignedMsg, +% #{ <<"commitment-device">> => Codec }, +% #{ priv_wallet => Wallet2 } +% ), +% ?event({signed_msg_twice, MsgSignedTwice}), +% ?assert(verify(MsgSignedTwice)), +% {ok, Committers} = dev_message:committers(MsgSignedTwice), +% ?event({committers, Committers}), +% ?assert(lists:member(hb_util:human_id(ar_wallet:to_address(Wallet1)), Committers)), +% ?assert(lists:member(hb_util:human_id(ar_wallet:to_address(Wallet2)), Committers)). + +deep_multisignature_test() -> + % Only the `httpsig@1.0' codec supports multisignatures. + Opts = test_opts(normal), + Codec = <<"httpsig@1.0">>, + Wallet1 = ar_wallet:new(), + Wallet2 = ar_wallet:new(), + Msg = #{ + <<"data">> => <<"TEST_DATA">>, + <<"test-key">> => <<"TEST_VALUE">>, + <<"body">> => #{ + <<"nested-key">> => <<"NESTED_VALUE">> + } + }, + SignedMsg = + hb_message:commit( + Msg, + Opts#{ priv_wallet => Wallet1 }, + Codec + ), + ?event({signed_msg, SignedMsg}), + MsgSignedTwice = + hb_message:commit( + SignedMsg, + Opts#{ priv_wallet => Wallet2 }, + Codec + ), + ?event({signed_msg_twice, MsgSignedTwice}), + ?assert(hb_message:verify(MsgSignedTwice, all, Opts)), + Committers = hb_message:signers(MsgSignedTwice, Opts), + ?event({committers, Committers}), + ?assert(lists:member(hb_util:human_id(ar_wallet:to_address(Wallet1)), Committers)), + ?assert(lists:member(hb_util:human_id(ar_wallet:to_address(Wallet2)), Committers)). + +deep_typed_message_id_test(Codec, Opts) -> + Msg = #{ + <<"data">> => <<"TEST DATA">>, + <<"deep-data">> => #{ + <<"data">> => <<"DEEP DATA">>, + <<"complex-key">> => 1337, + <<"list">> => [1,2,3] + } + }, + InitID = hb_message:id(Msg, none, Opts), + ?event({init_id, InitID}), + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + DecodedID = hb_message:id(Decoded, none, Opts), + ?event({decoded_id, DecodedID}), + ?event({stages, {init, Msg}, {encoded, Encoded}, {decoded, Decoded}}), + ?assertEqual( + InitID, + DecodedID + ). + +signed_deep_message_test(Codec, Opts) -> + Msg = #{ + <<"test-key">> => <<"TEST_VALUE">>, + <<"body">> => #{ + <<"nested-1">> => + #{ + <<"body">> => <<"NESTED BODY">>, + <<"nested-2">> => <<"NESTED-2">> + }, + <<"nested-3">> => <<"NESTED-3">> + } + }, + EncDec = + hb_message:convert( + hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + <<"structured@1.0">>, + Codec, + Opts + ), + ?event({enc_dec, EncDec}), + SignedMsg = + hb_message:commit( + EncDec, + Opts, + Codec + ), + ?event({signed_msg, SignedMsg}), + {ok, Res} = dev_message:verify(SignedMsg, #{ <<"committers">> => <<"all">>}, Opts), + ?event({verify_res, Res}), + ?assertEqual(true, hb_message:verify(SignedMsg, all, Opts)), + ?event({verified, SignedMsg}), + Encoded = hb_message:convert(SignedMsg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + {ok, DecodedRes} = + dev_message:verify( + Decoded, + #{ <<"committers">> => <<"all">>}, + Opts + ), + ?event({verify_decoded_res, DecodedRes}), + MatchRes = hb_message:match(SignedMsg, Decoded, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes). + +signed_list_test(Codec, Opts) -> + Msg = #{ <<"key-with-list">> => [1.0, 2.0, 3.0] }, + Signed = hb_message:commit(Msg, Opts, Codec), + ?assert(hb_message:verify(Signed, all, Opts)), + Encoded = hb_message:convert(Signed, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assert(hb_message:verify(Decoded, all, Opts)), + ?assert(hb_message:match(Signed, Decoded, strict, Opts)). + +unsigned_id_test(Codec, Opts) -> + Msg = #{ <<"data">> => <<"TEST_DATA">> }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?assertEqual( + dev_message:id(Decoded, #{ <<"committers">> => <<"none">>}, Opts), + dev_message:id(Msg, #{ <<"committers">> => <<"none">>}, Opts) + ). + +% signed_id_test_disabled() -> +% TX = #tx { +% data = <<"TEST_DATA">>, +% tags = [{<<"TEST_KEY">>, <<"TEST_VALUE">>}] +% }, +% SignedTX = ar_bundles:sign_item(TX, hb:wallet()), +% ?assert(ar_bundles:verify_item(SignedTX)), +% SignedMsg = hb_codec_tx:from(SignedTX), +% ?assertEqual( +% hb_util:encode(ar_bundles:id(SignedTX, signed)), +% hb_util:id(SignedMsg, signed) +% ). + +message_with_simple_embedded_list_test(Codec, Opts) -> + Msg = #{ <<"list">> => [<<"value-1">>, <<"value-2">>, <<"value-3">>] }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +empty_string_in_nested_tag_test(Codec, Opts) -> + Msg = + #{ + <<"dev">> => + #{ + <<"stderr">> => <<"aa">>, + <<"stdin">> => <<"b">>, + <<"stdout">> => <<"c">> + } + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +hashpath_sign_verify_test(Codec, Opts) -> + Msg = + #{ + <<"test_key">> => <<"TEST_VALUE">>, + <<"body">> => #{ + <<"nested_key">> => + #{ + <<"body">> => <<"NESTED_DATA">>, + <<"nested_key">> => <<"NESTED_VALUE">> + }, + <<"nested_key2">> => <<"NESTED_VALUE2">> + }, + <<"priv">> => #{ + <<"hashpath">> => + hb_path:hashpath( + hb_util:human_id(crypto:strong_rand_bytes(32)), + hb_util:human_id(crypto:strong_rand_bytes(32)), + fun hb_crypto:sha256_chain/2, + #{} + ) + } + }, + ?event({msg, {explicit, Msg}}), + SignedMsg = hb_message:commit(Msg, Opts, Codec), + ?event({signed_msg, {explicit, SignedMsg}}), + {ok, Res} = dev_message:verify(SignedMsg, #{ <<"committers">> => <<"all">>}, Opts), + ?event({verify_res, {explicit, Res}}), + ?assert(hb_message:verify(SignedMsg, all, Opts)), + ?event({verified, {explicit, SignedMsg}}), + Encoded = hb_message:convert(SignedMsg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assert(hb_message:verify(Decoded, all, Opts)), + ?assert( + hb_message:match( + SignedMsg, + Decoded, + strict, + Opts + ) + ). + +normalize_commitments_test(Codec, Opts) -> + Msg = #{ + <<"a">> => #{ + <<"b">> => #{ + <<"c">> => 1, + <<"d">> => #{ + <<"e">> => 2 + }, + <<"f">> => 3 + }, + <<"g">> => 4 + }, + <<"h">> => 5 + }, + NormMsg = hb_message:normalize_commitments(Msg, Opts), + ?event({norm_msg, NormMsg}), + ?assert(hb_message:verify(NormMsg, all, Opts)), + ?assert(maps:is_key(<<"commitments">>, NormMsg)), + ?assert(maps:is_key(<<"commitments">>, maps:get(<<"a">>, NormMsg))), + ?assert( + maps:is_key( + <<"commitments">>, + maps:get(<<"b">>, maps:get(<<"a">>, NormMsg)) + ) + ). + +signed_message_with_derived_components_test(Codec, Opts) -> + Msg = #{ + <<"path">> => <<"/test">>, + <<"authority">> => <<"example.com">>, + <<"scheme">> => <<"https">>, + <<"method">> => <<"GET">>, + <<"target-uri">> => <<"/test">>, + <<"request-target">> => <<"/test">>, + <<"status">> => <<"200">>, + <<"reason-phrase">> => <<"OK">>, + <<"body">> => <<"TEST_DATA">>, + <<"content-digest">> => <<"TEST_DIGEST">>, + <<"normal">> => <<"hello">> + }, + SignedMsg = + hb_message:commit( + Msg, + Opts, + Codec + ), + ?event({signed_msg, SignedMsg}), + ?assert(hb_message:verify(SignedMsg, all, Opts)), + Encoded = hb_message:convert(SignedMsg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assert(hb_message:verify(Decoded, all, Opts)), + ?assert( + hb_message:match( + SignedMsg, + Decoded, + strict, + Opts + ) + ). + +committed_keys_test(Codec, Opts) -> + Msg = #{ <<"a">> => 1, <<"b">> => 2, <<"c">> => 3 }, + Signed = hb_message:commit(Msg, Opts, Codec), + CommittedKeys = hb_message:committed(Signed, all, Opts), + ?event({committed_keys, CommittedKeys}), + ?assert(hb_message:verify(Signed, all, Opts)), + ?assert(lists:member(<<"a">>, CommittedKeys)), + ?assert(lists:member(<<"b">>, CommittedKeys)), + ?assert(lists:member(<<"c">>, CommittedKeys)), + MsgToFilter = Signed#{ <<"bad-key">> => <<"BAD VALUE">> }, + ?assert( + not lists:member( + <<"bad-key">>, + hb_message:committed(MsgToFilter, all, Opts) + ) + ). + +committed_empty_keys_test(Codec, Opts) -> + Msg = #{ + <<"very">> => <<>>, + <<"exciting">> => #{}, + <<"values">> => [], + <<"non-empty">> => <<"TEST">> + }, + Signed = hb_message:commit(Msg, Opts, Codec), + ?assert(hb_message:verify(Signed, all, Opts)), + CommittedKeys = hb_message:committed(Signed, all, Opts), + ?event({committed_keys, CommittedKeys}), + ?event({signed, Signed}), + ?assert(lists:member(<<"very">>, CommittedKeys)), + ?assert(lists:member(<<"exciting">>, CommittedKeys)), + ?assert(lists:member(<<"values">>, CommittedKeys)), + ?assert(lists:member(<<"non-empty">>, CommittedKeys)). + +deeply_nested_committed_keys_test() -> + Opts = (test_opts(normal))#{ + store => [ + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + } + ] + }, + Msg = #{ + <<"a">> => 1, + <<"b">> => #{ <<"c">> => #{ <<"d">> => <<0:((1 + 1024) * 1024)>> } }, + <<"e">> => <<0:((1 + 1024) * 1024)>> + }, + Signed = hb_message:commit(Msg, Opts, <<"httpsig@1.0">>), + {ok, WithOnlyCommitted} = hb_message:with_only_committed(Signed, Opts), + Committed = hb_message:committed(Signed, all, Opts), + ToCompare = hb_maps:without([<<"commitments">>], WithOnlyCommitted), + ?event( + {msgs, + {base, Msg}, + {signed, Signed}, + {committed, Committed}, + {with_only_committed, WithOnlyCommitted}, + {to_compare, ToCompare} + } + ), + ?assert( + hb_message:match( + Msg, + ToCompare, + strict, + Opts + ) + ). + +signed_with_inner_signed_message_test(Codec, Opts) -> + Msg = + hb_message:commit( + #{ + <<"a">> => 1, + <<"inner">> => + hb_maps:merge( + InnerSigned = + hb_message:commit( + #{ + <<"c">> => <<"abc">>, + <<"e">> => 5 + %<<"body">> => <<"inner-body">> + % <<"inner-2">> => #{ + % <<"body">> => <<"inner-2-body">> + % } + }, + Opts, + Codec + ), + % Uncommitted keys that should be ripped out of the inner + % message by `with_only_committed'. These should still be + % present in the `with_only_committed' outer message. + % For now, only `httpsig@1.0' supports stripping + % non-committed keys. + case Codec of + <<"httpsig@1.0">> -> + #{ <<"f">> => 6, <<"g">> => 7}; + #{ <<"device">> := <<"httpsig@1.0">> } -> + #{ <<"f">> => 6, <<"g">> => 7}; + _ -> #{} + end + ) + }, + Opts, + Codec + ), + ?event({initial_msg, Msg}), + % 1. Verify the outer message without changes. + ?assert(hb_message:verify(Msg, all, Opts)), + % 2. Convert the message to the format and back. + Encoded = hb_message:convert(Msg, Codec, Opts), + ?event({encoded, Encoded}), + %?event({encoded_body, {string, hb_maps:get(<<"body">>, Encoded)}}, #{}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + {ok, InnerFromDecoded} = + hb_message:with_only_committed( + hb_maps:get(<<"inner">>, Decoded, not_found, Opts), + Opts + ), + ?event({verify_inner, {original, InnerSigned}, {from_decoded, InnerFromDecoded}}), + % 3. Verify the outer message after decode. + MatchRes = + hb_message:match( + InnerSigned, + InnerFromDecoded, + primary, + Opts + ), + ?event({match_result, MatchRes}), + ?assert(MatchRes), + ?assert(hb_message:verify(InnerFromDecoded, all, Opts)), + ?assert(hb_message:verify(Decoded, all, Opts)), + % 4. If the message is not a bundle, verify the inner message from the + % converted message, applying `with_only_committed` first. + Inner = hb_maps:get(<<"inner">>, Msg, not_found, Opts), + {ok, CommittedInner} = + hb_message:with_only_committed( + Inner, + Opts + ), + ?event({committed_inner, CommittedInner}), + ?event({inner_committers, hb_message:signers(CommittedInner, Opts)}), + ?assert(hb_message:verify(CommittedInner, signers, Opts)), + InnerDecoded = hb_maps:get(<<"inner">>, Decoded, not_found, Opts), + ?event({inner_decoded, InnerDecoded}), + % Applying `with_only_committed' should verify the inner message. + {ok, CommittedInnerOnly} = + hb_message:with_only_committed( + InnerDecoded, + Opts + ), + ?assert(hb_message:verify(CommittedInnerOnly, signers, Opts)). + +large_body_committed_keys_test(Codec, Opts) -> + case Codec of + <<"httpsig@1.0">> -> + Msg = #{ + <<"a">> => 1, + <<"b">> => 2, + <<"c">> => #{ <<"d">> => << 1:((1 + 1024) * 1024) >> } + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + Signed = hb_message:commit(Decoded, Opts, Codec), + ?event({signed, Signed}), + CommittedKeys = hb_message:committed(Signed, all, Opts), + ?assert(lists:member(<<"a">>, CommittedKeys)), + ?assert(lists:member(<<"b">>, CommittedKeys)), + ?assert(lists:member(<<"c">>, CommittedKeys)), + MsgToFilter = Signed#{ <<"bad-key">> => <<"BAD VALUE">> }, + ?assert( + not lists:member( + <<"bad-key">>, + hb_message:committed(MsgToFilter, all, Opts) + ) + ); + _ -> + skip + end. + +sign_node_message_test(Codec, Opts) -> + Msg = hb_message:commit(hb_opts:default_message_with_env(), Opts, Codec), + ?event({committed, Msg}), + ?assert(hb_message:verify(Msg, all, Opts)), + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({final, Decoded}), + MatchRes = hb_message:match(Msg, Decoded, strict, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes), + ?assert(hb_message:verify(Decoded, all, Opts)). + +nested_body_list_test(Codec, Opts) -> + Msg = #{ + <<"body">> => + [ + #{ + <<"test-key">> => + <<"TEST VALUE #", (integer_to_binary(X))/binary>> + } + || + X <- lists:seq(1, 3) + ] + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event(encoded, {encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +recursive_nested_list_test(Codec, Opts) -> + % This test is to ensure that the codec can handle arbitrarily deep nested + % lists. + Msg = #{ + <<"body">> => + [ + [ + [ + << + "TEST VALUE #", + (integer_to_binary(X))/binary, + "-", + (integer_to_binary(Y))/binary, + "-", + (integer_to_binary(Z))/binary + >> + || + Z <- lists:seq(1, 3) + ] + || + Y <- lists:seq(1, 3) + ] + || + X <- lists:seq(1, 3) + ] + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event(encoded, {encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)). + +priv_survives_conversion_test(<<"ans104@1.0">>, _Opts) -> skip; +priv_survives_conversion_test(<<"json@1.0">>, _Opts) -> skip; +priv_survives_conversion_test(#{ <<"device">> := <<"ans104@1.0">> }, _Opts) -> + skip; +priv_survives_conversion_test(#{ <<"device">> := <<"json@1.0">> }, _Opts) -> + skip; +priv_survives_conversion_test(Codec, Opts) -> + Msg = #{ + <<"data">> => <<"TEST_DATA">>, + <<"priv">> => #{ <<"test_key">> => <<"TEST_VALUE">> } + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({decoded, Decoded}), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)), + ?assertMatch( + #{ <<"test_key">> := <<"TEST_VALUE">> }, + maps:get(<<"priv">>, Decoded) + ). + +encode_balance_table(Size, Codec, Opts) -> + Msg = + #{ + hb_util:encode(crypto:strong_rand_bytes(32)) => + rand:uniform(1_000_000_000_000_000) + || + _ <- lists:seq(1, Size) + }, + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, {explicit, Encoded}}), + Decoded = + hb_message:uncommitted( + hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + Opts + ), + ?event({decoded, {explicit, Decoded}}), + ?assert(hb_message:match(Msg, Decoded, if_present, Opts)). + +encode_small_balance_table_test(Codec, Opts) -> + encode_balance_table(5, Codec, Opts). + +encode_large_balance_table_test(<<"ans104@1.0">>, _Opts) -> + skip; +encode_large_balance_table_test(#{ <<"device">> := <<"ans104@1.0">> }, _Opts) -> + skip; +encode_large_balance_table_test(Codec, Opts) -> + encode_balance_table(1000, Codec, Opts). + +sign_links_test(#{ <<"bundle">> := true }, _Opts) -> + skip; +sign_links_test(Codec, Opts) -> + % Make a message with definitively non-accessible lazy-loadable links. Sign + % it, ensuring that we can produce signatures and IDs without having the + % data directly in memory. + Msg = #{ + <<"immediate-key">> => <<"immediate-value">>, + <<"submap+link">> => hb_util:human_id(crypto:strong_rand_bytes(32)) + }, + Signed = hb_message:commit(Msg, Opts, Codec), + ?event({signed, Signed}), + ?assert(hb_message:verify(Signed, all, Opts)). + +bundled_and_unbundled_ids_differ_test(Codec = #{ <<"bundle">> := true }, Opts) -> + SignatureType = + case maps:get(<<"device">>, Codec, undefined) of + <<"ans104@1.0">> -> <<"rsa-pss-sha256">>; + _ -> <<"hmac-sha256">> + end, + Msg = #{ + <<"immediate-key">> => <<"immediate-value">>, + <<"nested">> => #{ + <<"immediate-key-2">> => <<"immediate-value-2">> + } + }, + SignedNoBundle = + hb_message:commit( + Msg, + Opts, + maps:without([<<"bundle">>], Codec) + ), + SignedBundled = hb_message:commit(Msg, Opts, Codec), + ?event({signed_no_bundle, SignedNoBundle}), + ?event({signed_bundled, SignedBundled}), + {ok, UnbundledID, _} = + hb_message:commitment( + #{ <<"type">> => SignatureType }, + SignedNoBundle, + Opts + ), + {ok, BundledID, _} = + hb_message:commitment( + #{ <<"type">> => SignatureType }, + SignedBundled, + Opts + ), + ?event({unbundled_id, UnbundledID}), + ?event({bundled_id, BundledID}), + ?assertNotEqual(UnbundledID, BundledID); +bundled_and_unbundled_ids_differ_test(_Codec, _Opts) -> + skip. + +id_of_linked_message_test(#{ <<"bundle">> := true }, _Opts) -> + skip; +id_of_linked_message_test(Codec, Opts) -> + Msg = #{ + <<"immediate-key">> => <<"immediate-value">>, + <<"link-key">> => + {link, hb_util:human_id(crypto:strong_rand_bytes(32)), #{ + <<"type">> => <<"link">>, + <<"lazy">> => false + }} + }, + UnsignedID = hb_message:id(Msg, Opts), + ?event({id, UnsignedID}), + EncMsg = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + DecMsg = hb_message:convert(EncMsg, <<"structured@1.0">>, Codec, Opts), + UnsignedID2 = hb_message:id(DecMsg, Opts), + ?assertEqual(UnsignedID, UnsignedID2). + +sign_deep_message_from_lazy_cache_read_test(#{ <<"bundle">> := true }, _Opts) -> + skip; +sign_deep_message_from_lazy_cache_read_test(Codec, Opts) -> + Msg = #{ + <<"immediate-key">> => <<"immediate-value">>, + <<"link-key">> => #{ + <<"immediate-key-2">> => <<"link-value">>, + <<"link-key-2">> => #{ + <<"immediate-key-3">> => <<"link-value-2">> + } + } + }, + % Write the message to the store to ensure that we get lazy-loadable links. + {ok, Path} = hb_cache:write(Msg, Opts), + {ok, ReadMsg} = hb_cache:read(Path, Opts), + ?event({read, ReadMsg}), + Signed = hb_message:commit(ReadMsg, Opts, Codec), + ?event({signed, Signed}), + ?assert( + lists:all( + fun({_K, Value}) -> not is_map(Value) end, + maps:to_list(maps:without([<<"commitments">>, <<"priv">>], Signed)) + ) + ), + ?assert(hb_message:verify(Signed, all, Opts)). + +id_of_deep_message_and_link_message_match_test(_Codec, Opts) -> + Msg = #{ + <<"immediate-key">> => <<"immediate-value">>, + <<"link-key">> => #{ + <<"immediate-key-2">> => <<"immediate-value-2">>, + <<"link-key-2">> => #{ + <<"immediate-key-3">> => <<"immediate-value-3">> + } + } + }, + Linkified = hb_link:normalize(Msg, offload, Opts), + ?event(linkify, {test_recvd_linkified, {msg, Linkified}}), + BaseID = hb_message:id(Msg, Opts), + ?event(linkify, {test_recvd_nonlink_id, {id, BaseID}}), + LinkID = hb_message:id(Linkified, Opts), + ?event(linkify, {test_recvd_link_id, {id, LinkID}}), + ?assertEqual(BaseID, LinkID). + +signed_non_bundle_is_bundlable_test( + Codec = #{ <<"device">> := <<"httpsig@1.0">>, <<"bundle">> := true }, + Opts) -> + Msg = + hb_message:commit( + #{ + <<"target">> => hb_util:human_id(crypto:strong_rand_bytes(32)), + <<"type">> => <<"Message">>, + <<"function">> => <<"fac">>, + <<"parameters">> => [5.0] + }, + Opts, + maps:get(<<"device">>, Codec) + ), + Encoded = + hb_message:convert( + Msg, + Codec, + <<"structured@1.0">>, + Opts + ), + Decoded = + hb_message:convert( + Encoded, + <<"structured@1.0">>, + maps:get(<<"device">>, Codec), + Opts + ), + ?assert(hb_message:match(Msg, Decoded, strict, Opts)), + ?assert(hb_message:verify(Decoded, all, Opts)); +signed_non_bundle_is_bundlable_test(_Codec, _Opts) -> + skip. + +%% Ensure that we can write a message with multiple commitments to the store, +%% then read back all of the written commitments by loading the message's +%% unsigned ID. +find_multiple_commitments_test_disabled() -> + Opts = test_opts(normal), + Store = hb_opts:get(store, no_store, Opts), + hb_store:reset(Store), + Msg = #{ + <<"a">> => 1, + <<"b">> => 2, + <<"c">> => 3 + }, + Sig1 = hb_message:commit(Msg, Opts#{ priv_wallet => ar_wallet:new() }), + {ok, _} = hb_cache:write(Sig1, Opts), + Sig2 = hb_message:commit(Msg, Opts#{ priv_wallet => ar_wallet:new() }), + {ok, _} = hb_cache:write(Sig2, Opts), + {ok, ReadMsg} = hb_cache:read(hb_message:id(Msg, none, Opts), Opts), + LoadedCommitments = hb_cache:ensure_all_loaded(ReadMsg, Opts), + ?event(debug_commitments, {read, LoadedCommitments}), + ok. + +%% @doc Ensure that a httpsig@1.0 message which is bundled and requests an +%% invalid ordering of keys is normalized to a valid ordering. +bundled_ordering_test(Codec = #{ <<"bundle">> := true }, Opts) -> + % Opts = (test_opts(normal))#{ + % store => [ + % #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-TEST">> } + % ] + % }, + Msg = + hb_message:commit( + #{ + <<"a">> => <<"1">>, + <<"b">> => <<"2">>, + <<"b-2">> => #{ <<"nested">> => #{ <<"n">> => <<"2">> } }, + <<"c">> => <<"3">>, + <<"c-2">> => #{ <<"nested">> => #{ <<"n">> => <<"3">> } }, + <<"d">> => <<"4">> + }, + Opts, + Codec#{ + <<"committed">> => [ + <<"a">>, + <<"b">>, + <<"b-2">>, + <<"c">>, + <<"c-2">>, + <<"d">> + ] + } + ), + ?event({committed, Msg}), + Encoded = hb_message:convert(Msg, Codec, <<"structured@1.0">>, Opts), + ?event({encoded, Encoded}), + ?event({http, {string, dev_codec_httpsig_conv:encode_http_msg(Msg, Opts)}}), + Decoded = hb_message:convert(Encoded, <<"structured@1.0">>, Codec, Opts), + ?event({matching, {input, Msg}, {output, Decoded}}), + MatchRes = hb_message:match(Msg, Decoded, primary, Opts), + ?event({match_result, MatchRes}), + ?assert(MatchRes), + ?assert(hb_message:verify(Decoded, all, Opts)); +bundled_ordering_test(_Codec, _Opts) -> + skip. \ No newline at end of file diff --git a/src/hb_name.erl b/src/hb_name.erl new file mode 100644 index 000000000..e85c05162 --- /dev/null +++ b/src/hb_name.erl @@ -0,0 +1,205 @@ +%%% @doc An abstraction for name registration/deregistration in HyperBEAM. +%%% Its motivation is to provide a way to register names that are not necessarily +%%% atoms, but can be any term (for example: hashpaths or `process@1.0' IDs). +%%% An important characteristic of these functions is that they are atomic: +%%% There can only ever be one registrant for a given name at a time. +-module(hb_name). +-export([start/0, register/1, register/2, unregister/1, lookup/1, all/0]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-define(NAME_TABLE, hb_name_registry). + +%% Initialize ETS table for non-atom registrations +start() -> + try ets:info(?NAME_TABLE) of + undefined -> start_ets(); + _ -> ok + catch + error:badarg -> start_ets() + end. + +%% Start the ETS table. +start_ets() -> + ets:new(?NAME_TABLE, [ + named_table, + public, + {keypos, 1}, + {write_concurrency, true}, % Safe as key-writes are atomic. + {read_concurrency, true} + ]), + ok. + +%%% @doc Register a name. If the name is already registered, the registration +%%% will fail. The name can be any Erlang term. +register(Name) -> + start(), + ?MODULE:register(Name, self()). + +register(Name, Pid) when is_atom(Name) -> + try erlang:register(Name, Pid) of + true -> ok + catch + error:badarg -> error % Name already registered + end; +register(Name, Pid) -> + start(), + case ets:insert_new(?NAME_TABLE, {Name, Pid}) of + true -> ok; + false -> error + end. + +%%% @doc Unregister a name. +unregister(Name) when is_atom(Name) -> + catch erlang:unregister(Name), + ets:delete(?NAME_TABLE, Name), % Cleanup if atom was in ETS + ok; +unregister(Name) -> + start(), + ets:delete(?NAME_TABLE, Name), + ok. + +%%% @doc Lookup a name -> PID. +lookup(Name) when is_atom(Name) -> + case whereis(Name) of + undefined -> + % Check ETS for atom-based names + start(), + ets_lookup(Name); + Pid -> Pid + end; +lookup(Name) -> + start(), + ets_lookup(Name). + +ets_lookup(Name) -> + case ets:lookup(?NAME_TABLE, Name) of + [{Name, Pid}] -> + case is_process_alive(Pid) of + true -> Pid; + false -> + ets:delete(?NAME_TABLE, Name), + undefined + end; + [] -> undefined + end. + +%% @doc List the names in the registry. +all() -> + Registered = + ets:tab2list(?NAME_TABLE) ++ + lists:filtermap( + fun(Name) -> + case whereis(Name) of + undefined -> false; + Pid -> {true, {Name, Pid}} + end + end, + erlang:registered() + ), + lists:filter( + fun({_, Pid}) -> is_process_alive(Pid) end, + Registered + ). + +%%% Tests + +-define(CONCURRENT_REGISTRATIONS, 10). + +basic_test(Term) -> + ?assertEqual(ok, hb_name:register(Term)), + ?assertEqual(self(), hb_name:lookup(Term)), + ?assertEqual(error, hb_name:register(Term)), + hb_name:unregister(Term), + ?assertEqual(undefined, hb_name:lookup(Term)). + +atom_test() -> + basic_test(atom). + +term_test() -> + basic_test({term, os:timestamp()}). + +concurrency_test() -> + Name = {concurrent_test, os:timestamp()}, + SuccessCount = length([R || R <- spawn_test_workers(Name), R =:= ok]), + ?assertEqual(1, SuccessCount), + ?assert(is_pid(hb_name:lookup(Name))), + hb_name:unregister(Name). + + +spawn_test_workers(Name) -> + Self = self(), + Names = + [ + case Name of + random -> {random_name, rand:uniform(1000000)}; + _ -> Name + end + || + _ <- lists:seq(1, ?CONCURRENT_REGISTRATIONS) + ], + Pids = + [ + spawn( + fun() -> + Result = hb_name:register(ProcName), + Self ! {result, self(), Result}, + % Stay alive to prevent cleanup for a period. + timer:sleep(500) + end + ) + || + ProcName <- Names + ], + [ + receive {result, Pid, Res} -> Res after 100 -> timeout end + || + Pid <- Pids + ]. + +dead_process_test() -> + Name = {dead_process_test, os:timestamp()}, + {Pid, Ref} = spawn_monitor(fun() -> hb_name:register(Name), ok end), + receive {'DOWN', Ref, process, Pid, _} -> ok end, + ?assertEqual(undefined, hb_name:lookup(Name)). + +cleanup_test() -> + {setup, + fun() -> + Name = {cleanup_test, os:timestamp()}, + {Pid, Ref} = spawn_monitor(fun() -> timer:sleep(1000) end), + ?assertEqual(ok, hb_name:register(Name, Pid)), + {Name, Pid, Ref} + end, + fun({Name, _, _}) -> + hb_name:unregister(Name) + end, + fun({Name, Pid, Ref}) -> + {"Auto-cleanup on process death", + fun() -> + exit(Pid, kill), + receive {'DOWN', Ref, process, Pid, _} -> ok end, + ?assertEqual(undefined, wait_for_cleanup(Name, 10)) + end} + end + }. + +wait_for_cleanup(Name, Retries) -> + case Retries > 0 of + true -> + case hb_name:lookup(Name) of + undefined -> undefined; + _ -> + timer:sleep(100), + wait_for_cleanup(Name, Retries - 1) + end; + false -> undefined + end. + +all_test() -> + hb_name:register(test_name, self()), + ?assert(lists:member({test_name, self()}, hb_name:all())), + BaseRegistered = length(hb_name:all()), + spawn_test_workers(random), + ?assertEqual(BaseRegistered + ?CONCURRENT_REGISTRATIONS, length(hb_name:all())), + timer:sleep(1000), + ?assertEqual(BaseRegistered, length(hb_name:all())). diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 315d28614..6d262593b 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -13,65 +13,180 @@ %%% deterministic behavior impossible, the caller should fail the execution %%% with a refusal to execute. -module(hb_opts). --export([get/1, get/2, get/3, default_message/0]). --include_lib("eunit/include/eunit.hrl"). +-export([get/1, get/2, get/3, as/2, identities/1, load/1, load/2, load_bin/2]). +-export([default_message/0, default_message_with_env/0, mimic_default_types/3]). +-export([ensure_node_history/2]). +-export([check_required_opts/2]). +-include("include/hb.hrl"). + +%%% Environment variables that can be used to override the default message. +-ifdef(TEST). +-define(DEFAULT_PRINT_OPTS, [error, http_error]). +-else. +-define(DEFAULT_PRINT_OPTS, + [error, http_error, http_short, compute_short, push_short, copycat_short] +). +-endif. + +-ifdef(AO_PROFILING). +-define(DEFAULT_TRACE_TYPE, ao). +-else. +-define(DEFAULT_TRACE_TYPE, erlang). +-endif. + +-define(DEFAULT_PRIMARY_STORE, #{ + <<"name">> => <<"cache-mainnet/lmdb">>, + <<"store-module">> => hb_store_lmdb +}). +-define(ENV_KEYS, + #{ + priv_key_location => {"HB_KEY", "hyperbeam-key.json"}, + hb_config_location => {"HB_CONFIG", "config.flat"}, + port => {"HB_PORT", fun erlang:list_to_integer/1, "8734"}, + mode => {"HB_MODE", fun list_to_existing_atom/1}, + debug_print => + {"HB_PRINT", + fun + ({preparsed, Parsed}) -> Parsed; + (Str) when Str == "1" -> true; + (Str) when Str == "true" -> true; + (Str) -> + lists:map( + fun(Topic) -> list_to_atom(Topic) end, + string:tokens(Str, ",") + ) + end, + {preparsed, ?DEFAULT_PRINT_OPTS} + }, + lua_scripts => {"LUA_SCRIPTS", "scripts"}, + lua_tests => {"LUA_TESTS", fun dev_lua_test:parse_spec/1, tests}, + default_index => + { + "HB_INDEX", + fun("ui") -> + #{ + <<"device">> => <<"hyperbuddy@1.0">> + }; + ("text") -> + #{ + <<"device">> => <<"hyperbuddy@1.0">>, + <<"path">> => <<"format">> + }; + (Str) -> + case string:tokens(Str, "/") of + [Device, Path] -> + #{ <<"device">> => Device, <<"path">> => Path }; + [Device] -> + #{ <<"device">> => Device } + end + end, + "ui" + } + } +). + +%% @doc Return the default message with all environment variables set. +default_message_with_env() -> + maps:fold( + fun(Key, _Spec, NodeMsg) -> + case global_get(Key, undefined, #{}) of + undefined -> NodeMsg; + Value -> NodeMsg#{ Key => Value } + end + end, + default_message(), + ?ENV_KEYS + ). %% @doc The default configuration options of the hyperbeam node. default_message() -> #{ %%%%%%%% Functional options %%%%%%%% - %% What protocol should the node use for HTTP requests? - %% Options: http1, http2, http3 - protocol => http2, + hb_config_location => <<"config.flat">>, + initialized => true, + %% What HTTP client should the node use? + %% Options: gun, httpc + http_client => gun, %% Scheduling mode: Determines when the SU should inform the recipient %% that an assignment has been scheduled for a message. - %% Options: aggressive(!), local_confirmation, remote_confirmation + %% Options: aggressive(!), local_confirmation, remote_confirmation, + %% disabled scheduling_mode => local_confirmation, - %% Compute mode: Determines whether the CU should attempt to execute - %% more messages on a process after it has returned a result. + %% Compute mode: Determines whether the process device should attempt to + %% execute more messages on a process after it has returned a result. %% Options: aggressive, lazy compute_mode => lazy, %% Choice of remote nodes for tasks that are not local to hyperbeam. - http_host => <<"localhost">>, gateway => <<"https://arweave.net">>, - bundler => <<"https://up.arweave.net">>, + bundler_ans104 => <<"https://up.arweave.net:443">>, %% Location of the wallet keyfile on disk that this node will use. - key_location => <<"hyperbeam-key.json">>, - %% Default page limit for pagination of results from the APIs. - %% Currently used in the SU devices. - default_page_limit => 5, + priv_key_location => <<"hyperbeam-key.json">>, %% The time-to-live that should be specified when we register %% ourselves as a scheduler on the network. - scheduler_location_ttl => 60 * 60 * 24 * 30, + %% Default: 7 days. + scheduler_location_ttl => (60 * 60 * 24 * 7) * 1000, %% Preloaded devices for the node to use. These names override %% resolution of devices via ID to the default implementations. - preloaded_devices => - #{ - <<"Test-Device/1.0">> => dev_test, - <<"Message/1.0">> => dev_message, - <<"Stack/1.0">> => dev_stack, - <<"Multipass/1.0">> => dev_multipass, - <<"Scheduler/1.0">> => dev_scheduler, - <<"Process/1.0">> => dev_process, - <<"WASM-64/1.0">> => dev_wasm, - <<"WASI/1.0">> => dev_wasi, - <<"JSON-Iface/1.0">> => dev_json_iface, - <<"Dedup/1.0">> => dev_dedup, - <<"Router/1.0">> => dev_router, - <<"Cron">> => dev_cron, - <<"PODA">> => dev_poda, - <<"Monitor">> => dev_monitor, - <<"Push">> => dev_mu, - <<"Compute">> => dev_cu, - <<"P4">> => dev_p4 - }, - codecs => - #{ - converge => hb_codec_converge, - tx => hb_codec_tx, - flat => hb_codec_flat, - http => hb_codec_http - }, + preloaded_devices => [ + #{<<"name">> => <<"arweave@2.9-pre">>, <<"module">> => dev_arweave}, + #{<<"name">> => <<"apply@1.0">>, <<"module">> => dev_apply}, + #{<<"name">> => <<"auth-hook@1.0">>, <<"module">> => dev_auth_hook}, + #{<<"name">> => <<"ans104@1.0">>, <<"module">> => dev_codec_ans104}, + #{<<"name">> => <<"compute@1.0">>, <<"module">> => dev_cu}, + #{<<"name">> => <<"cache@1.0">>, <<"module">> => dev_cache}, + #{<<"name">> => <<"cacheviz@1.0">>, <<"module">> => dev_cacheviz}, + #{<<"name">> => <<"cookie@1.0">>, <<"module">> => dev_codec_cookie}, + #{<<"name">> => <<"cron@1.0">>, <<"module">> => dev_cron}, + #{<<"name">> => <<"dedup@1.0">>, <<"module">> => dev_dedup}, + #{<<"name">> => <<"delegated-compute@1.0">>, <<"module">> => dev_delegated_compute}, + #{<<"name">> => <<"faff@1.0">>, <<"module">> => dev_faff}, + #{<<"name">> => <<"flat@1.0">>, <<"module">> => dev_codec_flat}, + #{<<"name">> => <<"genesis-wasm@1.0">>, <<"module">> => dev_genesis_wasm}, + #{<<"name">> => <<"greenzone@1.0">>, <<"module">> => dev_green_zone}, + #{<<"name">> => <<"httpsig@1.0">>, <<"module">> => dev_codec_httpsig}, + #{<<"name">> => <<"http-auth@1.0">>, <<"module">> => dev_codec_http_auth}, + #{<<"name">> => <<"hook@1.0">>, <<"module">> => dev_hook}, + #{<<"name">> => <<"hyperbuddy@1.0">>, <<"module">> => dev_hyperbuddy}, + #{<<"name">> => <<"copycat@1.0">>, <<"module">> => dev_copycat}, + #{<<"name">> => <<"json@1.0">>, <<"module">> => dev_codec_json}, + #{<<"name">> => <<"json-iface@1.0">>, <<"module">> => dev_json_iface}, + #{<<"name">> => <<"local-name@1.0">>, <<"module">> => dev_local_name}, + #{<<"name">> => <<"lookup@1.0">>, <<"module">> => dev_lookup}, + #{<<"name">> => <<"lua@5.3a">>, <<"module">> => dev_lua}, + #{<<"name">> => <<"manifest@1.0">>, <<"module">> => dev_manifest}, + #{<<"name">> => <<"message@1.0">>, <<"module">> => dev_message}, + #{<<"name">> => <<"meta@1.0">>, <<"module">> => dev_meta}, + #{<<"name">> => <<"monitor@1.0">>, <<"module">> => dev_monitor}, + #{<<"name">> => <<"multipass@1.0">>, <<"module">> => dev_multipass}, + #{<<"name">> => <<"name@1.0">>, <<"module">> => dev_name}, + #{<<"name">> => <<"node-process@1.0">>, <<"module">> => dev_node_process}, + #{<<"name">> => <<"p4@1.0">>, <<"module">> => dev_p4}, + #{<<"name">> => <<"patch@1.0">>, <<"module">> => dev_patch}, + #{<<"name">> => <<"poda@1.0">>, <<"module">> => dev_poda}, + #{<<"name">> => <<"process@1.0">>, <<"module">> => dev_process}, + #{<<"name">> => <<"profile@1.0">>, <<"module">> => dev_profile}, + #{<<"name">> => <<"push@1.0">>, <<"module">> => dev_push}, + #{<<"name">> => <<"query@1.0">>, <<"module">> => dev_query}, + #{<<"name">> => <<"relay@1.0">>, <<"module">> => dev_relay}, + #{<<"name">> => <<"router@1.0">>, <<"module">> => dev_router}, + #{<<"name">> => <<"scheduler@1.0">>, <<"module">> => dev_scheduler}, + #{<<"name">> => <<"simple-pay@1.0">>, <<"module">> => dev_simple_pay}, + #{<<"name">> => <<"snp@1.0">>, <<"module">> => dev_snp}, + #{<<"name">> => <<"stack@1.0">>, <<"module">> => dev_stack}, + #{<<"name">> => <<"structured@1.0">>, <<"module">> => dev_codec_structured}, + #{<<"name">> => <<"test-device@1.0">>, <<"module">> => dev_test}, + #{<<"name">> => <<"volume@1.0">>, <<"module">> => dev_volume}, + #{<<"name">> => <<"secret@1.0">>, <<"module">> => dev_secret}, + #{<<"name">> => <<"wasi@1.0">>, <<"module">> => dev_wasi}, + #{<<"name">> => <<"wasm-64@1.0">>, <<"module">> => dev_wasm}, + #{<<"name">> => <<"whois@1.0">>, <<"module">> => dev_whois} + ], + %% Default execution cache control options + cache_control => [<<"no-cache">>, <<"no-store">>], + cache_lookup_hueristics => false, + % Should we await in-progress executions, rather than re-running? + % Has three settings: false, only `named' executions, or all executions. + await_inprogress => named, %% Should the node attempt to access data from remote caches for %% client requests? access_remote_cache_for_client => false, @@ -81,27 +196,168 @@ default_message() -> trusted_device_signers => [], %% What should the node do if a client error occurs? client_error_strategy => throw, - %% Default execution cache control options - cache_control => [<<"no-cache">>, <<"no-store">>], %% HTTP request options http_connect_timeout => 5000, - http_response_timeout => 30000, http_keepalive => 120000, http_request_send_timeout => 60000, - http_default_remote_port => 8734, - http_port => 8734, + port => 8734, + wasm_allow_aot => false, + %% Options for the relay device + relay_http_client => httpc, + %% The default codec to use for commitment signatures. + commitment_device => <<"httpsig@1.0">>, %% Dev options mode => debug, + profiling => true, + % Every modification to `Opts' called directly by the node operator + % should be recorded here. + node_history => [], debug_stack_depth => 40, + debug_print => false, debug_print_map_line_threshold => 30, debug_print_binary_max => 60, debug_print_indent => 2, - debug_print => false, - cache_results => false, - stack_print_prefixes => ["hb", "dev", "ar"], + stack_print_prefixes => ["hb", "dev", "ar", "maps"], debug_print_trace => short, % `short` | `false`. Has performance impact. - short_trace_len => 5, - debug_ids => true + debug_trace_type => ?DEFAULT_TRACE_TYPE, + short_trace_len => 20, + debug_metadata => true, + debug_ids => true, + debug_committers => true, + debug_show_priv => if_present, + debug_resolve_links => true, + debug_print_fail_mode => long, + trusted => #{}, + snp_enforced_keys => [ + firmware, kernel, + initrd, append, + vmm_type, guest_features + ], + routes => [ + #{ + % Routes for the genesis-wasm device to use a local CU, if requested. + <<"template">> => <<"/result/.*">>, + <<"node">> => #{ <<"prefix">> => <<"http://localhost:6363">> } + }, + #{ + % Routes for the genesis-wasm device to use a local CU, if requested. + <<"template">> => <<"/snapshot/.*">>, + <<"node">> => #{ <<"prefix">> => <<"http://localhost:6363">> } + }, + #{ + % Routes for the genesis-wasm device to use a local CU, if requested. + <<"template">> => <<"/dry-run.*">>, + <<"node">> => #{ <<"prefix">> => <<"http://localhost:6363">> } + }, + #{ + % Routes for GraphQL requests to use a remote GraphQL API. + <<"template">> => <<"/graphql">>, + <<"nodes">> => + [ + #{ + <<"prefix">> => <<"https://ao-search-gateway.goldsky.com">>, + <<"opts">> => #{ http_client => httpc, protocol => http2 } + }, + #{ + <<"prefix">> => <<"https://arweave-search.goldsky.com">>, + <<"opts">> => #{ http_client => httpc, protocol => http2 } + }, + #{ + <<"prefix">> => <<"https://arweave.net">>, + <<"opts">> => #{ http_client => gun, protocol => http2 } + } + ] + }, + #{ + % Routes for Arweave transaction requests to use a remote gateway. + <<"template">> => <<"/arweave">>, + <<"node">> => + #{ + <<"match">> => <<"^/arweave">>, + <<"with">> => <<"https://arweave.net">>, + <<"opts">> => #{ http_client => httpc, protocol => http2 } + } + }, + #{ + % Routes for raw data requests to use a remote gateway. + <<"template">> => <<"/raw">>, + <<"node">> => + #{ + <<"prefix">> => <<"https://arweave.net">>, + <<"opts">> => #{ http_client => gun, protocol => http2 } + } + } + ], + store => + [ + ?DEFAULT_PRIMARY_STORE, + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-mainnet">> + }, + #{ + <<"store-module">> => hb_store_gateway, + <<"subindex">> => [ + #{ + <<"name">> => <<"Data-Protocol">>, + <<"value">> => <<"ao">> + } + ], + <<"local-store">> => [?DEFAULT_PRIMARY_STORE] + }, + #{ + <<"store-module">> => hb_store_gateway, + <<"local-store">> => [?DEFAULT_PRIMARY_STORE] + } + ], + priv_store => + [ + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-priv">> + } + ], + %default_index => #{ <<"device">> => <<"hyperbuddy@1.0">> }, + % Should we use the latest cached state of a process when computing? + process_now_from_cache => false, + % Should we trust the GraphQL API when converting to ANS-104? Some GQL + % services do not provide the `anchor' or `last_tx' fields, so their + % responses are not verifiable. + ans104_trust_gql => true, + http_extra_opts => + #{ + force_message => true, + cache_control => [<<"always">>] + }, + % Should the node store all signed messages? + store_all_signed => true, + % Should the node use persistent processes? + process_workers => false, + % Options for the router device + router_opts => #{ + routes => [] + }, + on => #{ + <<"request">> => #{ + <<"device">> => <<"auth-hook@1.0">>, + <<"path">> => <<"request">>, + <<"when">> => #{ + <<"keys">> => [<<"authorization">>, <<"!">>] + }, + <<"secret-provider">> => + #{ + <<"device">> => <<"http-auth@1.0">>, + <<"access-control">> => + #{ <<"device">> => <<"http-auth@1.0">> } + } + } + } + % Should the node track and expose prometheus metrics? + % We do not set this explicitly, so that the hb_features:test() value + % can be used to determine if we should expose metrics instead, + % dynamically changing the configuration based on whether we are running + % tests or not. To override this, set the `prometheus' option explicitly. + % prometheus => false }. %% @doc Get an option from the global options, optionally overriding with a @@ -113,81 +369,365 @@ default_message() -> %% `prefer' defaults to `local'. get(Key) -> ?MODULE:get(Key, undefined). get(Key, Default) -> ?MODULE:get(Key, Default, #{}). -get(Key, Default, Opts = #{ only := local }) -> +get(Key, Default, Opts) when is_binary(Key) -> + try binary_to_existing_atom(Key, utf8) of + AtomKey -> do_get(AtomKey, Default, Opts) + catch + error:badarg -> do_get(Key, Default, Opts) + end; +get(Key, Default, Opts) -> + do_get(Key, Default, Opts). +do_get(Key, Default, Opts = #{ <<"only">> := Only }) -> + do_get(Key, Default, maps:remove(<<"only">>, Opts#{ only => Only })); +do_get(Key, Default, Opts = #{ <<"prefer">> := Prefer }) -> + do_get(Key, Default, maps:remove(<<"prefer">>, Opts#{ prefer => Prefer })); +do_get(Key, Default, Opts = #{ only := local }) -> case maps:find(Key, Opts) of {ok, Value} -> Value; error -> Default end; -get(Key, Default, #{ only := global }) -> - case global_get(Key, hb_opts_not_found) of +do_get(Key, Default, Opts = #{ only := global }) -> + case global_get(Key, hb_opts_not_found, Opts) of hb_opts_not_found -> Default; Value -> Value end; -get(Key, Default, Opts = #{ prefer := global }) -> - case ?MODULE:get(Key, hb_opts_not_found, #{ only => global }) of - hb_opts_not_found -> ?MODULE:get(Key, Default, Opts#{ only => local }); +do_get(Key, Default, Opts = #{ prefer := global }) -> + case do_get(Key, hb_opts_not_found, #{ only => global }) of + hb_opts_not_found -> do_get(Key, Default, Opts#{ only => local }); Value -> Value end; -get(Key, Default, Opts = #{ prefer := local }) -> - case ?MODULE:get(Key, hb_opts_not_found, Opts#{ only => local }) of +do_get(Key, Default, Opts = #{ prefer := local }) -> + case do_get(Key, hb_opts_not_found, Opts#{ only => local }) of hb_opts_not_found -> - ?MODULE:get(Key, Default, Opts#{ only => global }); + do_get(Key, Default, Opts#{ only => global }); Value -> Value end; -get(Key, Default, Opts) -> +do_get(Key, Default, Opts) -> % No preference was set in Opts, so we default to local. - ?MODULE:get(Key, Default, Opts#{ prefer => local }). + do_get(Key, Default, Opts#{ prefer => local }). --define(ENV_KEYS, - #{ - key_location => {"HB_KEY", "hyperbeam-key.json"}, - http_port => {"HB_PORT", fun erlang:list_to_integer/1, "8734"}, - store => - {"HB_STORE", - fun(Dir) -> - { - hb_store_fs, - #{ prefix => Dir } - } +%% @doc Get an environment variable or configuration key. Depending on whether +%% the value is derived from an environment variable, we may be able to cache +%% the result in the process dictionary. +global_get(Key, Default, Opts) -> + case erlang:get({processed_env, Key}) of + {cached, Value} -> Value; + undefined -> + % Thee value is not cached, so we need to process it. + {IsCachable, Value} = + case maps:get(Key, ?ENV_KEYS, Default) of + Default -> {false, config_lookup(Key, Default, Opts)}; + {EnvKey, ValParser, DefaultValue} when is_function(ValParser) -> + {true, ValParser( + cached_os_env( + EnvKey, + normalize_default(DefaultValue) + ) + )}; + {EnvKey, ValParser} when is_function(ValParser) -> + case cached_os_env(EnvKey, not_found) of + not_found -> {false, config_lookup(Key, Default, Opts)}; + V -> {true, ValParser(V)} + end; + {EnvKey, DefaultValue} -> + {true, cached_os_env(EnvKey, DefaultValue)} end, - "TEST-cache" - }, - mode => - {"HB_MODE", fun list_to_existing_atom/1}, - debug_print => - {"HB_PRINT", - fun - (Str) when Str == "1" -> true; - (Str) when Str == "true" -> true; - (Str) -> string:tokens(Str, ",") - end - } - } -). + % Cache the result if it is immutable and return. + if IsCachable -> erlang:put({processed_env, Key}, {cached, Value}); + true -> ok + end, + Value + end. -%% @doc Get an environment variable or configuration key. -global_get(Key, Default) -> - case maps:get(Key, ?ENV_KEYS, Default) of - Default -> config_lookup(Key, Default); - {EnvKey, ValParser, DefaultValue} when is_function(ValParser) -> - ValParser(os:getenv(EnvKey, DefaultValue)); - {EnvKey, ValParser} when is_function(ValParser) -> - case os:getenv(EnvKey, not_found) of - not_found -> config_lookup(Key, Default); - Value -> ValParser(Value) - end; - {EnvKey, DefaultValue} -> - os:getenv(EnvKey, DefaultValue) +%% @doc Cache the result of os:getenv/1 in the process dictionary, as it never +%% changes during the lifetime of a node. +cached_os_env(Key, DefaultValue) -> + case erlang:get({os_env, Key}) of + {cached, false} -> DefaultValue; + {cached, Value} -> Value; + undefined -> + % The process dictionary returns `undefined' for a key that is not + % set, so we need to check the environment and store the result. + erlang:put({os_env, Key}, {cached, os:getenv(Key)}), + % We recurse to follow the normal path. + cached_os_env(Key, DefaultValue) end. +%% @doc Get an option from environment variables, optionally consulting the +%% `hb_features' of the node if a conditional default tuple is provided. +normalize_default({conditional, Feature, IfTest, Else}) -> + case hb_features:enabled(Feature) of + true -> IfTest; + false -> Else + end; +normalize_default(Default) -> Default. + %% @doc An abstraction for looking up configuration variables. In the future, %% this is the function that we will want to change to support a more dynamic %% configuration system. -config_lookup(Key, Default) -> maps:get(Key, default_message(), Default). +config_lookup(Key, Default, _Opts) -> maps:get(Key, default_message(), Default). + +%% @doc Parse a `flat@1.0' encoded file into a map, matching the types of the +%% keys to those in the default message. +load(Path) -> load(Path, #{}). +load(Path, Opts) -> + {ok, Device} = path_to_device(Path), + case file:read_file(Path) of + {ok, Bin} -> + load_bin(Device, Bin, Opts); + _ -> {error, not_found} + end. + +%% @doc Convert a path to a device from its file extension. If no extension is +%% provided, we default to `flat@1.0'. +path_to_device(Path) -> + case binary:split(hb_util:bin(Path), <<".">>, []) of + [_, Extension] -> + ?event(debug_node_msg, + {path_to_device, + {path, Path}, + {extension, Extension} + } + ), + extension_to_device(Extension); + _ -> {ok, <<"flat@1.0">>} + end. + +%% @doc Convert a file extension to a device name. +extension_to_device(Ext) -> + extension_to_device(Ext, maps:get(preloaded_devices, default_message())). +extension_to_device(_, []) -> {error, not_found}; +extension_to_device(Ext, [#{ <<"name">> := Name }|Rest]) -> + case binary:match(Name, Ext) of + nomatch -> extension_to_device(Ext, Rest); + {0, _} -> {ok, Name} + end. + +%% @doc Parse a given binary with a device (defaulting to `flat@1.0') into a +%% node message. Types are converted to match those in the default message, if +%% applicable. +load_bin(Bin, Opts) -> + load_bin(<<"flat@1.0">>, Bin, Opts). +load_bin(<<"flat@1.0">>, Bin, Opts) -> + % Trim trailing whitespace from each line in the file. + Ls = + lists:map( + fun(Line) -> string:trim(Line, trailing) end, + binary:split(Bin, <<"\n">>, [global]) + ), + try dev_codec_flat:deserialize(iolist_to_binary(lists:join(<<"\n">>, Ls))) of + {ok, Map} -> + {ok, mimic_default_types(Map, new_atoms, Opts)} + catch + error:B -> {error, B} + end; +load_bin(Device, Bin, Opts) -> + try + { + ok, + mimic_default_types( + hb_cache:ensure_all_loaded( + hb_message:convert(Bin, <<"structured@1.0">>, Device, Opts), + Opts + ), + new_atoms, + Opts + ) + } + catch error:B -> {error, B} + end. + +%% @doc Mimic the types of the default message for a given map. +mimic_default_types(Map, Mode, Opts) -> + Default = default_message_with_env(), + hb_maps:from_list(lists:map( + fun({Key, Value}) -> + NewKey = try hb_util:key_to_atom(Key, Mode) catch _:_ -> Key end, + NewValue = + case hb_maps:get(NewKey, Default, not_found, Opts) of + not_found -> Value; + DefaultValue when is_atom(DefaultValue) -> + hb_util:atom(Value); + DefaultValue when is_integer(DefaultValue) -> + hb_util:int(Value); + DefaultValue when is_float(DefaultValue) -> + hb_util:float(Value); + DefaultValue when is_binary(DefaultValue) -> + Value; + _ -> Value + end, + {NewKey, NewValue} + end, + hb_maps:to_list(Map, Opts) + )). + +%% @doc Find a given identity from the `identities' map, and return the options +%% merged with the sub-options for that identity. +as(Identity, Opts) -> + case identities(Opts) of + #{ Identity := SubOpts } -> + ?event({found_identity_sub_opts_are, SubOpts}), + {ok, maps:merge(Opts, mimic_default_types(SubOpts, new_atoms, Opts))}; + _ -> + {error, not_found} + end. + +%% @doc Find all known IDs and their sub-options from the `priv_ids' map. Allows +%% the identities to be named, or based on addresses. The results are normalized +%% such that the map returned by this function contains both mechanisms for +%% finding an identity and its sub-options. Additionally, sub-options are also +%% normalized such that the `address' property is present and accurate for all +%% given identities. +identities(Opts) -> + identities(hb:wallet(), Opts). +identities(Default, Opts) -> + Named = ?MODULE:get(identities, #{}, Opts), + % Generate an address-based map of identities. + Addresses = + maps:from_list(lists:filtermap( + fun({_Name, SubOpts}) -> + case maps:find(priv_wallet, SubOpts) of + {ok, Wallet} -> + Addr = hb_util:human_id(ar_wallet:to_address(Wallet)), + {true, {Addr, SubOpts}}; + error -> false + end + end, + maps:to_list(Named) + )), + % Merge the named and address-based maps. Normalize each result to ensure + % that the `address' property is present and accurate. + Identities = + maps:map( + fun(_NameOrID, SubOpts) -> + case maps:find(priv_wallet, SubOpts) of + {ok, Wallet} -> + SubOpts#{ <<"address">> => hb_util:human_id(Wallet) }; + error -> SubOpts + end + end, + maps:merge(Named, Addresses) + ), + ?event({identities_without_default, Identities}), + % Add a default identity if one is not already present. + DefaultWallet = ?MODULE:get(priv_wallet, Default, Opts), + case maps:find(DefaultID = hb_util:human_id(DefaultWallet), Identities) of + {ok, _} -> Identities; + error -> + Identities#{ + DefaultID => #{ + priv_wallet => DefaultWallet + }, + <<"default">> => #{ + priv_wallet => DefaultWallet + } + } + end. + +%% @doc Utility function to check for required options in a list. +%% Takes a list of {Name, Value} pairs and returns: +%% - {ok, Opts} when all required options are present (Value =/= not_found) +%% - {error, ErrorMsg} with a message listing all missing options when any are not_found +%% @param KeyValuePairs A list of {Name, Value} pairs to check. +%% @param Opts The original options map to return if validation succeeds. +%% @returns `{ok, Opts}' if all required options are present, or +%% `{error, <<"Missing required parameters: ", MissingOptsStr/binary>>}' +%% where `MissingOptsStr' is a comma-separated list of missing option names. +-spec check_required_opts(list({binary(), term()}), map()) -> + {ok, map()} | {error, binary()}. +check_required_opts(KeyValuePairs, Opts) -> + MissingOpts = lists:filtermap( + fun({Name, Value}) -> + case Value of + not_found -> {true, Name}; + _ -> false + end + end, + KeyValuePairs + ), + case MissingOpts of + [] -> + {ok, Opts}; + _ -> + MissingOptsStr = binary:list_to_bin( + lists:join(<<", ">>, MissingOpts) + ), + ErrorMsg = <<"Missing required opts: ", MissingOptsStr/binary>>, + {error, ErrorMsg} + end. + +%% @doc Ensures all items in a node history meet required configuration options. +%% +%% This function verifies that the first item (complete opts) contains all required +%% configuration options and that their values match the expected format. Then it +%% validates that subsequent history items (which represent differences) never +%% modify any of the required keys from the first item. +%% +%% Validation is performed in two steps: +%% 1. Checks that the first item has all required keys and valid values +%% 2. Verifies that subsequent items don't modify any required keys from the first item +%% +%% @param Opts The complete options map (will become first item in history) +%% @param RequiredOpts A map of options that must be present and unchanging +%% @returns {ok, <<"valid">>} when validation passes +%% @returns {error, <<"missing_keys">>} when required keys are missing from first item +%% @returns {error, <<"invalid_values">>} when first item values don't match requirements +%% @returns {error, <<"modified_required_key">>} when history items modify required keys +%% @returns {error, <<"validation_failed">>} when other validation errors occur +-spec ensure_node_history(NodeHistory :: list() | term(), RequiredOpts :: map()) -> + {ok, binary()} | {error, binary()}. +ensure_node_history(Opts, RequiredOpts) -> + ?event(validate_history_items, {required_opts, RequiredOpts}), + maybe + % Get the node history from the options + NodeHistory = hb_opts:get(node_history, [], Opts), + % Add the Opts to the node history to validate all items + NodeHistoryWithOpts = [ Opts | NodeHistory ], + % Normalize required options + NormalizedRequiredOpts ?= hb_ao:normalize_keys(RequiredOpts), + % Normalize all node history items once + NormalizedNodeHistory ?= lists:map( + fun(Item) -> + hb_ao:normalize_keys(Item) + end, + NodeHistoryWithOpts + ), + % Get the first item (complete opts) and remaining items (differences) + [FirstItem | RemainingItems] = NormalizedNodeHistory, + % Step 2: Validate first item values match requirements + FirstItemValuesMatch = hb_message:match(NormalizedRequiredOpts, FirstItem, primary), + true ?= (FirstItemValuesMatch == true) orelse {error, values_invalid}, + % Step 3: Check that remaining items don't modify required keys + NoRequiredKeysModified = lists:all( + fun(HistoryItem) -> + % For each required key, if it exists in this history item, + % it must match the value from the first item + hb_message:match(RequiredOpts, HistoryItem, only_present) + end, + RemainingItems + ), + true ?= NoRequiredKeysModified orelse {error, required_key_modified}, + % If we've made it this far, everything is valid + ?event({validate_node_history_items, all_items_valid}), + {ok, valid} + else + {error, values_invalid} -> + ?event({validate_node_history_items, validation_failed, invalid_values}), + {error, invalid_values}; + {error, required_key_modified} -> + ?event({validate_node_history_items, validation_failed, required_key_modified}), + {error, modified_required_key}; + _ -> + ?event({validate_node_history_items, validation_failed, unknown}), + {error, validation_failed} + end. %%% Tests +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + global_get_test() -> ?assertEqual(debug, ?MODULE:get(mode)), ?assertEqual(debug, ?MODULE:get(mode, production)), @@ -215,4 +755,169 @@ global_preference_test() -> ?assertEqual(undefined, ?MODULE:get(test_key, undefined, Global)), ?assertNotEqual(incorrect, ?MODULE:get(mode, undefined, Global#{ mode => incorrect })), - ?assertNotEqual(undefined, ?MODULE:get(mode, undefined, Global)). \ No newline at end of file + ?assertNotEqual(undefined, ?MODULE:get(mode, undefined, Global)). + +load_flat_test() -> + % File contents: + % port: 1234 + % host: https://ao.computer + % await-inprogress: false + {ok, Conf} = load("test/config.flat", #{}), + ?event({loaded, {explicit, Conf}}), + % Ensure we convert types as expected. + ?assertEqual(1234, hb_maps:get(port, Conf)), + % A binary + ?assertEqual(<<"https://ao.computer">>, hb_maps:get(host, Conf)), + % An atom, where the key contained a header-key `-' rather than a `_'. + ?assertEqual(false, hb_maps:get(await_inprogress, Conf)). + +load_json_test() -> + {ok, Conf} = load("test/config.json", #{}), + ?event(debug_node_msg, {loaded, Conf}), + ?assertEqual(1234, hb_maps:get(port, Conf)), + ?assertEqual(9001, hb_maps:get(example, Conf)), + % A binary + ?assertEqual(<<"https://ao.computer">>, hb_maps:get(host, Conf)), + % An atom, where the key contained a header-key `-' rather than a `_'. + ?assertEqual(false, hb_maps:get(await_inprogress, Conf)), + % Ensure that a store with `ao-types' is loaded correctly. + ?assertMatch( + [#{ <<"store-module">> := hb_store_fs }|_], + hb_maps:get(store, Conf) + ). + +as_identity_test() -> + DefaultWallet = ar_wallet:new(), + TestWallet1 = ar_wallet:new(), + TestWallet2 = ar_wallet:new(), + TestID2 = hb_util:human_id(TestWallet2), + Opts = #{ + test_key => 0, + priv_wallet => DefaultWallet, + identities => #{ + <<"testname-1">> => #{ + priv_wallet => TestWallet1, + test_key => 1 + }, + TestID2 => #{ + priv_wallet => TestWallet2, + test_key => 2 + } + } + }, + ?event({base_opts, Opts}), + Identities = identities(Opts), + ?event({identities, Identities}), + % The number of identities should be 5: `default`, its ID, `testname-1`, + % and its ID, and just the ID of `TestWallet2`. + ?assertEqual(5, maps:size(Identities)), + % The wallets for each of the names should be the same as the wallets we + % provided. We also check that the settings are applied correctly. + ?assertMatch( + {ok, #{ priv_wallet := DefaultWallet, test_key := 0 }}, + as(<<"default">>, Opts) + ), + ?assertMatch( + {ok, #{ priv_wallet := DefaultWallet, test_key := 0 }}, + as(hb_util:human_id(DefaultWallet), Opts) + ), + ?assertMatch( + {ok, #{ priv_wallet := TestWallet1, test_key := 1 }}, + as(<<"testname-1">>, Opts) + ), + ?assertMatch( + {ok, #{ priv_wallet := TestWallet1, test_key := 1 }}, + as(hb_util:human_id(TestWallet1), Opts) + ), + ?assertMatch( + {ok, #{ priv_wallet := TestWallet2, test_key := 2 }}, + as(TestID2, Opts) + ). + +ensure_node_history_test() -> + % Define some test data + RequiredOpts = #{ + key1 => + #{ + <<"type">> => <<"string">>, + <<"value">> => <<"value1">> + }, + key2 => <<"value2">> + }, + % Test case: All items have required options + ValidOpts = + #{ + <<"key1">> => + #{ + <<"type">> => <<"string">>, + <<"value">> => <<"value1">> + }, + <<"key2">> => <<"value2">>, + <<"extra">> => <<"value">>, + node_history => [ + #{ + <<"key1">> => + #{ + <<"type">> => <<"string">>, + <<"value">> => <<"value1">> + }, + <<"key2">> => <<"value2">>, + <<"extra">> => <<"value">> + }, + #{ + <<"key1">> => + #{ + <<"type">> => <<"string">>, + <<"value">> => <<"value1">> + }, + <<"key2">> => <<"value2">> + } + ] + }, + ?assertEqual({ok, valid}, ensure_node_history(ValidOpts, RequiredOpts)), + ?event({valid_items, ValidOpts}), + % Test Missing items + MissingItems = + #{ + <<"key1">> => + #{ + <<"type">> => <<"string">>, + <<"value">> => <<"value1">> + }, + node_history => [ + #{ + <<"key1">> => + #{ + <<"type">> => <<"string">>, + <<"value">> => <<"value1">> + } + % missing key2 + + } + ] + }, + ?assertEqual({error, invalid_values}, ensure_node_history(MissingItems, RequiredOpts)), + ?event({missing_items, MissingItems}), + % Test Invalid items + InvalidItems = + #{ + <<"key1">> => + #{ + <<"type">> => <<"string">>, + <<"value">> => <<"value">> + }, + <<"key2">> => <<"value2">>, + node_history => + [ + #{ + <<"key1">> => + #{ + <<"type">> => <<"string">>, + <<"value">> => <<"value2">> + }, + <<"key2">> => <<"value3">> + } + ] + }, + ?assertEqual({error, invalid_values}, ensure_node_history(InvalidItems, RequiredOpts)). +-endif. \ No newline at end of file diff --git a/src/hb_path.erl b/src/hb_path.erl index 297449730..f2788842f 100644 --- a/src/hb_path.erl +++ b/src/hb_path.erl @@ -4,7 +4,7 @@ %%% %%% A HashPath is a rolling Merkle list of the messages that have been applied %%% in order to generate a given message. Because applied messages can -%%% themselves be the result of message applications with the Converge Protocol, +%%% themselves be the result of message applications with the AO-Core protocol, %%% the HashPath can be thought of as the tree of messages that represent the %%% history of a given message. The initial message on a HashPath is referred to %%% by its ID and serves as its user-generated 'root'. @@ -12,11 +12,12 @@ %%% Specifically, the HashPath can be generated by hashing the previous HashPath %%% and the current message. This means that each message in the HashPath is %%% dependent on all previous messages. -%%% ``` +%%% 
 %%%     Msg1.HashPath = Msg1.ID
 %%%     Msg3.HashPath = Msg1.Hash(Msg1.HashPath, Msg2.ID)
-%%%     Msg3.{...} = Converge.apply(Msg1, Msg2)
-%%%     ...'''
+%%%     Msg3.{...} = AO-Core.apply(Msg1, Msg2)
+%%%     ...
+%%%
%%% %%% A message's ID itself includes its HashPath, leading to the mixing of %%% a Msg2's merkle list into the resulting Msg3's HashPath. This allows a single @@ -28,12 +29,14 @@ %%% message. When Msg2's are applied to a Msg1, the resulting Msg3's HashPath %%% will be generated according to Msg1's algorithm choice. -module(hb_path). --export([hashpath/2, hashpath/3, hashpath/4, hashpath_alg/1]). --export([hd/2, tl/2, push_request/2, queue_request/2, pop_request/2]). --export([priv_remaining/2, priv_original/2, priv_store_original/2]). --export([priv_store_original/3, priv_store_remaining/2]). +-export([hashpath/2, hashpath/3, hashpath/4, hashpath_alg/2]). +-export([hd/2, tl/2]). +-export([push_request/2, push_request/3]). +-export([queue_request/2, queue_request/3]). +-export([pop_request/2]). +-export([priv_remaining/2, priv_store_remaining/2, priv_store_remaining/3]). -export([verify_hashpath/2]). --export([term_to_path_parts/1, term_to_path_parts/2, from_message/2]). +-export([term_to_path_parts/1, term_to_path_parts/2, from_message/3]). -export([matches/2, to_binary/1, regex_matches/2, normalize/1]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -46,13 +49,13 @@ hd(Msg2, Opts) -> case pop_request(Msg2, Opts) of undefined -> undefined; {Head, _} -> - % `term_to_path` returns the full path, so we need to take the - % `hd` of our `Head`. + % `term_to_path' returns the full path, so we need to take the + % `hd' of our `Head'. erlang:hd(term_to_path_parts(Head, Opts)) end. %% @doc Return the message without its first path element. Note that this -%% is the only transformation in Converge that does _not_ make a log of its +%% is the only transformation in AO-Core that does _not_ make a log of its %% transformation. Subsequently, the message's IDs will not be verifiable %% after executing this transformation. %% This may or may not be the mainnet behavior we want. @@ -62,112 +65,96 @@ tl(Msg2, Opts) when is_map(Msg2) -> {_, Rest} -> Rest end; tl(Path, Opts) when is_list(Path) -> - case tl(#{ path => Path }, Opts) of + case tl(#{ <<"path">> => Path }, Opts) of [] -> undefined; undefined -> undefined; - #{ path := Rest } -> Rest + #{ <<"path">> := Rest } -> Rest end. -%% @doc Return the `Remaining-Path' of a message, from its hidden `Converge' -%% key. Does not use the `get` or set `hb_private` functions, such that it -%% can be safely used inside the main Converge resolve function. +%% @doc Return the `Remaining-Path' of a message, from its hidden `AO-Core' +%% key. Does not use the `get' or set `hb_private' functions, such that it +%% can be safely used inside the main AO-Core resolve function. priv_remaining(Msg, Opts) -> Priv = hb_private:from_message(Msg), - Converge = maps:get(<<"Converge">>, Priv, #{}), - maps:get(<<"Remaining-Path">>, Converge, undefined). + AOCore = hb_maps:get(<<"ao-core">>, Priv, #{}, Opts), + hb_maps:get(<<"remaining">>, AOCore, undefined, Opts). -%% @doc Return the `Original-Path' of a message, from its hidden `Converge' -%% key. -priv_original(Msg, Opts) -> - Priv = hb_private:from_message(Msg), - Converge = maps:get(<<"Converge">>, Priv, #{}), - maps:get(<<"Original-Path">>, Converge, undefined). - -%% @doc Store the `Original-Path' (and optionally `Remaining-Path') of a message -%% in its hidden `Converge' key. -priv_store_original(Msg, OriginalPath) -> - priv_store_original(Msg, OriginalPath, undefined). -priv_store_original(Msg, OriginalPath, RemainingPath) -> - Priv = hb_private:from_message(Msg), - Converge = maps:get(<<"Converge">>, Priv, #{}), - Msg#{ - priv => - Priv#{ - <<"Converge">> => - Converge#{ - <<"Original-Path">> => OriginalPath, - <<"Remaining-Path">> => RemainingPath - } - } - }. - -%% @doc Store the remaining path of a message in its hidden `Converge' key. +%% @doc Store the remaining path of a message in its hidden `AO-Core' key. priv_store_remaining(Msg, RemainingPath) -> + priv_store_remaining(Msg, RemainingPath, #{}). +priv_store_remaining(Msg, RemainingPath, Opts) -> Priv = hb_private:from_message(Msg), - Converge = maps:get(<<"Converge">>, Priv, #{}), + AOCore = hb_maps:get(<<"ao-core">>, Priv, #{}, Opts), Msg#{ - priv => + <<"priv">> => Priv#{ - <<"Converge">> => - Converge#{ - <<"Remaining-Path">> => RemainingPath + <<"ao-core">> => + AOCore#{ + <<"remaining">> => RemainingPath } } }. -%% @doc Return the internal ID of a binary as it will be written to our -%% stores. -data_id(Bin, _Opts) when is_binary(Bin) -> - % Default hashpath for a binary message is its SHA2-256 hash. - hb_util:human_id(hb_crypto:sha256(Bin)). - %%% @doc Add an ID of a Msg2 to the HashPath of another message. hashpath(Bin, _Opts) when is_binary(Bin) -> % Default hashpath for a binary message is its SHA2-256 hash. hb_util:human_id(hb_crypto:sha256(Bin)); hashpath(RawMsg1, Opts) -> - Msg1 = hb_converge:ensure_message(RawMsg1), - case dev_message:get(hashpath, Msg1) of - {ok, ignore} -> - throw({hashpath_set_to_ignore, {msg1, Msg1}, {opts, Opts}}); - {ok, Hashpath} -> Hashpath; + Msg1 = hb_ao:normalize_keys(RawMsg1, Opts), + case hb_private:from_message(Msg1) of + #{ <<"hashpath">> := HP } -> HP; _ -> - try hb_util:ok(dev_message:id(Msg1)) + % Note: We do not use `hb_message:id' here because it will call + % hb_ao:resolve, which will call `hashpath' recursively. + try + hb_util:human_id( + hb_util:ok( + dev_message:id( + Msg1, + #{ <<"commitments">> => <<"all">> }, + Opts + ) + ) + ) catch - _A:_B:_ST -> throw({badarg, {unsupported_type, Msg1}}) + A:B:ST -> + throw( + {badarg, + {unsupported_type, Msg1}, + {error, A}, + {details, B}, + {stacktrace, ST} + } + ) end end. -% hashpath(Msg1, Msgs, Opts) when is_list(Msgs) -> -% HashpathAlg = hashpath_alg(Msg1), -% lists:foldl( -% fun(Msg, Acc) -> -% ?event({generating_hashpath, {current, Acc}, {msg, Msg}}), -% hashpath(Acc, hashpath(Msg, Opts), HashpathAlg, Opts) -% end, -% hashpath(Msg1, Opts), -% Msgs -% ); -hashpath(Msg1, Msg2ID, Opts) when is_map(Msg1) -> +hashpath(Msg1, Msg2, Opts) when is_map(Msg1) -> Msg1Hashpath = hashpath(Msg1, Opts), - HashpathAlg = hashpath_alg(Msg1), - hashpath(Msg1Hashpath, Msg2ID, HashpathAlg, Opts); + HashpathAlg = hashpath_alg(Msg1, Opts), + hashpath(Msg1Hashpath, Msg2, HashpathAlg, Opts); hashpath(Msg1, Msg2, Opts) -> throw({hashpath_not_viable, Msg1, Msg2, Opts}). hashpath(Msg1, Msg2, HashpathAlg, Opts) when is_map(Msg2) -> - {ok, Msg2WithoutMeta} = dev_message:remove(Msg2, #{ items => ?CONVERGE_KEYS }), - ?event({generating_msg2_hashpath_with_keys, maps:keys(Msg2WithoutMeta)}), - case {map_size(Msg2WithoutMeta), hd(Msg2, Opts)} of - {0, Key} when Key =/= undefined -> - hashpath(Msg1, to_binary(Key), HashpathAlg, Opts); + Msg2WithoutMeta = hb_maps:without(?AO_CORE_KEYS, Msg2, Opts), + ReqPath = from_message(request, Msg2, Opts), + case {map_size(Msg2WithoutMeta), ReqPath} of + {0, _} when ReqPath =/= undefined -> + hashpath(Msg1, to_binary(hd(ReqPath)), HashpathAlg, Opts); _ -> - {ok, Msg2ID} = dev_message:id(Msg2), - hashpath(Msg1, Msg2ID, HashpathAlg, Opts) + {ok, Msg2ID} = + dev_message:id( + Msg2, + #{ <<"commitments">> => <<"all">> }, + Opts + ), + hashpath(Msg1, hb_util:human_id(Msg2ID), HashpathAlg, Opts) end; -hashpath(Msg1Hashpath, Msg2ID, HashpathAlg, _Opts) -> +hashpath(Msg1Hashpath, HumanMsg2ID, HashpathAlg, Opts) -> + ?event({hashpath, {msg1hp, {explicit, Msg1Hashpath}}, {msg2id, {explicit, HumanMsg2ID}}}), HP = - case term_to_path_parts(Msg1Hashpath) of - [_] -> - << Msg1Hashpath/binary, "/", Msg2ID/binary >>; + case term_to_path_parts(Msg1Hashpath, Opts) of + [_] -> + << Msg1Hashpath/binary, "/", HumanMsg2ID/binary >>; [Prev1, Prev2] -> % Calculate the new base of the hashpath. We check whether the key is % a human-readable binary ID, or a path part, and convert or pass @@ -181,16 +168,16 @@ hashpath(Msg1Hashpath, Msg2ID, HashpathAlg, _Opts) -> end ), HumanNewBase = hb_util:human_id(NativeNewBase), - << HumanNewBase/binary, "/", Msg2ID/binary >> + << HumanNewBase/binary, "/", HumanMsg2ID/binary >> end, - ?event({generated_hashpath, HP, {msg1hp, Msg1Hashpath}, {msg2id, Msg2ID}}), + ?event({generated_hashpath, HP, {msg1hp, Msg1Hashpath}, {msg2id, HumanMsg2ID}}), HP. %%% @doc Get the hashpath function for a message from its HashPath-Alg. %%% If no hashpath algorithm is specified, the protocol defaults to %%% `sha-256-chain'. -hashpath_alg(Msg) -> - case dev_message:get(<<"Hashpath-Alg">>, Msg) of +hashpath_alg(Msg, Opts) -> + case dev_message:get(<<"hashpath-alg">>, Msg, Opts) of {ok, <<"sha-256-chain">>} -> fun hb_crypto:sha256_chain/2; {ok, <<"accumulate-256">>} -> @@ -201,19 +188,21 @@ hashpath_alg(Msg) -> %%% @doc Add a message to the head (next to execute) of a request path. push_request(Msg, Path) -> - maps:put(path, term_to_path_parts(Path) ++ from_message(request, Msg), Msg). + push_request(Msg, Path, #{}). +push_request(Msg, Path, Opts) -> + hb_maps:put(<<"path">>, term_to_path_parts(Path, Opts) ++ from_message(request, Msg, Opts), Msg, Opts). %%% @doc Pop the next element from a request path or path list. pop_request(undefined, _Opts) -> undefined; pop_request(Msg, Opts) when is_map(Msg) -> %?event({popping_request, {msg, Msg}, {opts, Opts}}), - case pop_request(from_message(request, Msg), Opts) of + case pop_request(from_message(request, Msg, Opts), Opts) of undefined -> undefined; {undefined, _} -> undefined; {Head, []} -> {Head, undefined}; {Head, Rest} -> ?event({popped_request, Head, Rest}), - {Head, maps:put(path, Rest, Msg)} + {Head, hb_maps:put(<<"path">>, Rest, Msg, Opts)} end; pop_request([], _Opts) -> undefined; pop_request([Head|Rest], _Opts) -> @@ -221,15 +210,17 @@ pop_request([Head|Rest], _Opts) -> %%% @doc Queue a message at the back of a request path. `path' is the only %%% key that we cannot use dev_message's `set/3' function for (as it expects -%%% the compute path to be there), so we use `maps:put/3' instead. +%%% the compute path to be there), so we use `hb_maps:put/3' instead. queue_request(Msg, Path) -> - maps:put(path, from_message(request, Msg) ++ term_to_path_parts(Path), Msg). + queue_request(Msg, Path, #{}). +queue_request(Msg, Path, Opts) -> + hb_maps:put(<<"path">>, from_message(request, Msg, Opts) ++ term_to_path_parts(Path), Msg, Opts). %%% @doc Verify the HashPath of a message, given a list of messages that %%% represent its history. verify_hashpath([Msg1, Msg2, Msg3|Rest], Opts) -> CorrectHashpath = hashpath(Msg1, Msg2, Opts), - FromMsg3 = from_message(hashpath, Msg3), + FromMsg3 = from_message(hashpath, Msg3, Opts), CorrectHashpath == FromMsg3 andalso case Rest of [] -> true; @@ -237,42 +228,39 @@ verify_hashpath([Msg1, Msg2, Msg3|Rest], Opts) -> end. %% @doc Extract the request path or hashpath from a message. We do not use -%% Converge for this resolution because this function is called from inside Converge +%% AO-Core for this resolution because this function is called from inside AO-Core %% itself. This imparts a requirement: the message's device must store a %% viable hashpath and path in its Erlang map at all times, unless the message %% is directly from a user (in which case paths and hashpaths will not have %% been assigned yet). -from_message(hashpath, Msg) -> hashpath(Msg, #{}); -from_message(request, #{ path := [] }) -> undefined; -from_message(request, #{ path := Path }) when is_list(Path) -> - term_to_path_parts(Path); -from_message(request, #{ path := Other }) -> - term_to_path_parts(Other); -from_message(request, #{ <<"path">> := Path }) -> term_to_path_parts(Path); -from_message(request, #{ <<"Path">> := Path }) -> term_to_path_parts(Path); -from_message(request, _) -> - undefined. +from_message(Type, Link, Opts) when ?IS_LINK(Link) -> + from_message(Type, hb_cache:ensure_loaded(Link, Opts), Opts); +from_message(hashpath, Msg, Opts) -> hashpath(Msg, Opts); +from_message(request, #{ path := Path }, Opts) -> term_to_path_parts(Path, Opts); +from_message(request, #{ <<"path">> := Path }, Opts) -> term_to_path_parts(Path, Opts); +from_message(request, #{ <<"Path">> := Path }, Opts) -> term_to_path_parts(Path, Opts); +from_message(request, _, _Opts) -> undefined. %% @doc Convert a term into an executable path. Supports binaries, lists, and %% atoms. Notably, it does not support strings as lists of characters. term_to_path_parts(Path) -> term_to_path_parts(Path, #{ error_strategy => throw }). +term_to_path_parts(Link, Opts) when ?IS_LINK(Link) -> + term_to_path_parts(hb_cache:ensure_loaded(Link, Opts), Opts); +term_to_path_parts([], _Opts) -> undefined; +term_to_path_parts(<<>>, _Opts) -> undefined; term_to_path_parts(<<"/">>, _Opts) -> []; term_to_path_parts(Binary, Opts) when is_binary(Binary) -> case binary:match(Binary, <<"/">>) of nomatch -> [Binary]; _ -> term_to_path_parts( - lists:filter( - fun(Part) -> byte_size(Part) > 0 end, - binary:split(Binary, <<"/">>, [global]) - ), + binary:split(Binary, <<"/">>, [global, trim_all]), Opts ) end; -term_to_path_parts([], _Opts) -> undefined; term_to_path_parts(Path = [ASCII | _], _Opts) when is_integer(ASCII) -> - [list_to_binary(Path)]; + [hb_ao:normalize_key(Path)]; term_to_path_parts(List, Opts) when is_list(List) -> lists:flatten(lists:map( fun(Part) -> @@ -282,134 +270,146 @@ term_to_path_parts(List, Opts) when is_list(List) -> )); term_to_path_parts(Atom, _Opts) when is_atom(Atom) -> [Atom]; term_to_path_parts(Integer, _Opts) when is_integer(Integer) -> - [integer_to_binary(Integer)]. + [hb_ao:normalize_key(Integer)]; +term_to_path_parts({as, DevName, Msgs}, _Opts) -> + [{as, hb_ao:normalize_key(DevName), Msgs}]. %% @doc Convert a path of any form to a binary. to_binary(Path) -> Parts = binary:split(do_to_binary(Path), <<"/">>, [global, trim_all]), iolist_to_binary(lists:join(<<"/">>, Parts)). -do_to_binary(String = [ASCII|_]) when is_integer(ASCII) -> - to_binary(list_to_binary(String)); do_to_binary(Path) when is_list(Path) -> - iolist_to_binary( - lists:join( - "/", - lists:filtermap( - fun(Part) -> - case do_to_binary(Part) of - <<>> -> false; - BinPart -> {true, BinPart} - end - end, - Path - ) - ) - ); + case hb_util:is_string_list(Path) of + false -> + iolist_to_binary( + lists:join( + "/", + lists:filtermap( + fun(Part) -> + case do_to_binary(Part) of + <<>> -> false; + BinPart -> {true, BinPart} + end + end, + Path + ) + ) + ); + true -> + to_binary(list_to_binary(Path)) + end; do_to_binary(Path) when is_binary(Path) -> Path; do_to_binary(Other) -> - hb_converge:key_to_binary(Other). + hb_ao:normalize_key(Other). %% @doc Check if two keys match. matches(Key1, Key2) -> - hb_util:to_lower(hb_converge:key_to_binary(Key1)) == - hb_util:to_lower(hb_converge:key_to_binary(Key2)). + hb_util:to_lower(hb_ao:normalize_key(Key1)) == + hb_util:to_lower(hb_ao:normalize_key(Key2)). %% @doc Check if two keys match using regex. regex_matches(Path1, Path2) -> - NormP1 = normalize(Path1), - NormP2 = normalize(Path2), + NormP1 = normalize(hb_ao:normalize_key(Path1)), + NormP2 = + case hb_ao:normalize_key(Path2) of + Normalized = <<"^", _/binary>> -> Normalized; + Normalized -> normalize(Normalized) + end, try re:run(NormP1, NormP2) =/= nomatch catch _A:_B:_C -> false end. %% @doc Normalize a path to a binary, removing the leading slash if present. normalize(Path) -> - case hb_converge:key_to_binary(Path) of + case iolist_to_binary([Path]) of BinPath = <<"/", _/binary>> -> BinPath; Binary -> <<"/", Binary/binary>> end. %%% TESTS - hashpath_test() -> - Msg1 = #{ <<"empty">> => <<"message">> }, - Msg2 = #{ <<"exciting">> => <<"message2">> }, + Msg1 = #{ priv => #{<<"empty">> => <<"message">>} }, + Msg2 = #{ priv => #{<<"exciting">> => <<"message2">>} }, Hashpath = hashpath(Msg1, Msg2, #{}), ?assert(is_binary(Hashpath) andalso byte_size(Hashpath) == 87). hashpath_direct_msg2_test() -> - Msg1 = #{ <<"Base">> => <<"Message">> }, - Msg2 = #{ path => <<"Base">> }, + Msg1 = #{ <<"base">> => <<"message">> }, + Msg2 = #{ <<"path">> => <<"base">> }, Hashpath = hashpath(Msg1, Msg2, #{}), [_, KeyName] = term_to_path_parts(Hashpath), - ?assert(matches(KeyName, <<"Base">>)). + ?assert(matches(KeyName, <<"base">>)). multiple_hashpaths_test() -> Msg1 = #{ <<"empty">> => <<"message">> }, Msg2 = #{ <<"exciting">> => <<"message2">> }, - Msg3 = #{ hashpath => hashpath(Msg1, Msg2, #{}) }, + Msg3 = #{ priv => #{<<"hashpath">> => hashpath(Msg1, Msg2, #{}) } }, Msg4 = #{ <<"exciting">> => <<"message4">> }, Msg5 = hashpath(Msg3, Msg4, #{}), ?assert(is_binary(Msg5)). verify_hashpath_test() -> - Msg1 = #{ <<"TEST">> => <<"INITIAL">> }, - Msg2 = #{ <<"FirstApplied">> => <<"Msg2">> }, - Msg3 = #{ hashpath => hashpath(Msg1, Msg2, #{}) }, - Msg4 = #{ hashpath => hashpath(Msg2, Msg3, #{}) }, - Msg3Fake = #{ hashpath => hashpath(Msg4, Msg2, #{}) }, + Msg1 = #{ <<"test">> => <<"initial">> }, + Msg2 = #{ <<"firstapplied">> => <<"msg2">> }, + Msg3 = #{ priv => #{<<"hashpath">> => hashpath(Msg1, Msg2, #{})} }, + Msg4 = #{ priv => #{<<"hashpath">> => hashpath(Msg2, Msg3, #{})} }, + Msg3Fake = #{ priv => #{<<"hashpath">> => hashpath(Msg4, Msg2, #{})} }, ?assert(verify_hashpath([Msg1, Msg2, Msg3, Msg4], #{})), ?assertNot(verify_hashpath([Msg1, Msg2, Msg3Fake, Msg4], #{})). validate_path_transitions(X, Opts) -> {Head, X2} = pop_request(X, Opts), - ?assertEqual(a, Head), + ?assertEqual(<<"a">>, Head), {H2, X3} = pop_request(X2, Opts), - ?assertEqual(b, H2), + ?assertEqual(<<"b">>, H2), {H3, X4} = pop_request(X3, Opts), - ?assertEqual(c, H3), + ?assertEqual(<<"c">>, H3), ?assertEqual(undefined, pop_request(X4, Opts)). pop_from_message_test() -> - validate_path_transitions(#{ path => [a, b, c] }, #{}). + validate_path_transitions(#{ <<"path">> => [<<"a">>, <<"b">>, <<"c">>] }, #{}). pop_from_path_list_test() -> - validate_path_transitions([a, b, c], #{}). + validate_path_transitions([<<"a">>, <<"b">>, <<"c">>], #{}). hd_test() -> - ?assertEqual(a, hd(#{ path => [a, b, c] }, #{})), - ?assertEqual(undefined, hd(#{ path => undefined }, #{})). + ?assertEqual(<<"a">>, hd(#{ <<"path">> => [<<"a">>, <<"b">>, <<"c">>] }, #{})), + ?assertEqual(undefined, hd(#{ <<"path">> => undefined }, #{})). tl_test() -> - ?assertMatch([b, c], maps:get(path, tl(#{ path => [a, b, c] }, #{}))), - ?assertEqual(undefined, tl(#{ path => [] }, #{})), - ?assertEqual(undefined, tl(#{ path => a }, #{})), - ?assertEqual(undefined, tl(#{ path => undefined }, #{})), + ?assertMatch([<<"b">>, <<"c">>], hb_maps:get(<<"path">>, tl(#{ <<"path">> => [<<"a">>, <<"b">>, <<"c">>] }, #{}))), + ?assertEqual(undefined, tl(#{ <<"path">> => [] }, #{})), + ?assertEqual(undefined, tl(#{ <<"path">> => <<"a">> }, #{})), + ?assertEqual(undefined, tl(#{ <<"path">> => undefined }, #{})), - ?assertEqual([b, c], tl([a, b, c], #{ })), - ?assertEqual(undefined, tl([c], #{ })). + ?assertEqual([<<"b">>, <<"c">>], tl([<<"a">>, <<"b">>, <<"c">>], #{ })), + ?assertEqual(undefined, tl([<<"c">>], #{ })). to_binary_test() -> - ?assertEqual(<<"a/b/c">>, to_binary([a, b, c])), + ?assertEqual(<<"a/b/c">>, to_binary([<<"a">>, <<"b">>, <<"c">>])), ?assertEqual(<<"a/b/c">>, to_binary(<<"a/b/c">>)), - ?assertEqual(<<"a/b/c">>, to_binary([<<"a">>, b, [<<"c">>]])), - ?assertEqual(<<"a/b/c">>, to_binary(["a", b, <<"c">>])), + ?assertEqual(<<"a/b/c">>, to_binary([<<"a">>, <<"b">>, <<"c">>])), + ?assertEqual(<<"a/b/c">>, to_binary(["a", <<"b">>, <<"c">>])), ?assertEqual(<<"a/b/b/c">>, to_binary([<<"a">>, [<<"b">>, <<"//b">>], <<"c">>])). term_to_path_parts_test() -> - ?assert(matches([a, b, c], term_to_path_parts(<<"a/b/c">>))), - ?assert(matches([a, b, c], term_to_path_parts([<<"a">>, <<"b">>, <<"c">>]))), - ?assert(matches([a, b, c], term_to_path_parts(["a", b, <<"c">>]))), - ?assert(matches([a, b, b, c], term_to_path_parts([[<<"/a">>, [<<"b">>, <<"//b">>], <<"c">>]]))), + ?assert(matches([<<"a">>, <<"b">>, <<"c">>], + term_to_path_parts(<<"a/b/c">>))), + ?assert(matches([<<"a">>, <<"b">>, <<"c">>], + term_to_path_parts([<<"a">>, <<"b">>, <<"c">>]))), + ?assert(matches([<<"a">>, <<"b">>, <<"c">>], + term_to_path_parts(["a", <<"b">>, <<"c">>]))), + ?assert(matches([<<"a">>, <<"b">>, <<"b">>, <<"c">>], + term_to_path_parts([[<<"/a">>, [<<"b">>, <<"//b">>], <<"c">>]]))), ?assertEqual([], term_to_path_parts(<<"/">>)). % calculate_multistage_hashpath_test() -> -% Msg1 = #{ <<"Base">> => <<"Message">> }, -% Msg2 = #{ path => <<"2">> }, -% Msg3 = #{ path => <<"3">> }, -% Msg4 = #{ path => <<"4">> }, +% Msg1 = #{ <<"base">> => <<"message">> }, +% Msg2 = #{ <<"path">> => <<"2">> }, +% Msg3 = #{ <<"path">> => <<"3">> }, +% Msg4 = #{ <<"path">> => <<"4">> }, % Msg5 = hashpath(Msg1, [Msg2, Msg3, Msg4], #{}), % ?assert(is_binary(Msg5)), % Msg3Path = <<"3">>, diff --git a/src/hb_persistent.erl b/src/hb_persistent.erl index 48f3a448f..c73299210 100644 --- a/src/hb_persistent.erl +++ b/src/hb_persistent.erl @@ -1,130 +1,202 @@ -%%% @doc Creates and manages long-lived Converge resolution processes. +%%% @doc Creates and manages long-lived AO-Core resolution processes. %%% These can be useful for situations where a message is large and expensive %%% to serialize and deserialize, or when executions should be deliberately %%% serialized to avoid parallel executions of the same computation. This -%%% module is called during the core `hb_converge' execution process, so care +%%% module is called during the core `hb_ao' execution process, so care %%% must be taken to avoid recursive spawns/loops. %%% %%% Built using the `pg' module, which is a distributed Erlang process group %%% manager. -module(hb_persistent). +-export([start_monitor/0, start_monitor/1, stop_monitor/1]). -export([find_or_register/3, unregister_notify/4, await/4, notify/4]). -export([group/3, start_worker/3, start_worker/2, forward_work/2]). --export([default_grouper/3, default_worker/3]). +-export([default_grouper/3, default_worker/3, default_await/5]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). %% @doc Ensure that the `pg' module is started. -start() -> pg:start(pg). +start() -> hb_name:start(). + +%% @doc Start a monitor that prints the current members of the group every +%% n seconds. +start_monitor() -> + start_monitor(global). +start_monitor(Group) -> + start_monitor(Group, #{}). +start_monitor(Group, Opts) -> + start(), + ?event({worker_monitor, {start_monitor, Group, hb_name:all()}}), + spawn(fun() -> do_monitor(Group, #{}, Opts) end). + +stop_monitor(PID) -> + PID ! stop. + +do_monitor(Group, Last, Opts) -> + Groups = lists:map(fun({Name, _}) -> Name end, hb_name:all()), + New = + hb_maps:from_list( + lists:map( + fun(G) -> + Pid = hb_name:lookup(G), + { + G, + #{ + pid => Pid, + messages => + case Pid of + undefined -> 0; + _ -> + length( + element(2, + erlang:process_info(Pid, messages) + ) + ) + end + } + + } + end, + case Group of + global -> Groups; + TargetGroup -> + case lists:member(TargetGroup, Groups) of + true -> [TargetGroup]; + false -> [] + end + end + ) + ), + Delta = + hb_maps:filter( + fun(G, NewState) -> + case hb_maps:get(G, Last, []) of + NewState -> false; + _ -> true + end + end, + New, + Opts + ), + case hb_maps:size(Delta, Opts) of + 0 -> ok; + Deltas -> + io:format(standard_error, "== Sitrep ==> ~p named processes. ~p changes. ~n", + [hb_maps:size(New, Opts), Deltas]), + hb_maps:map( + fun(G, #{pid := P, messages := Msgs}) -> + io:format(standard_error, "[~p: ~p] #M: ~p~n", [G, P, Msgs]) + end, + Delta, + Opts + ), + io:format(standard_error, "~n", []) + end, + timer:sleep(1000), + receive stop -> stopped + after 0 -> do_monitor(Group, New, Opts) + end. %% @doc Register the process to lead an execution if none is found, otherwise %% signal that we should await resolution. find_or_register(Msg1, Msg2, Opts) -> GroupName = group(Msg1, Msg2, Opts), find_or_register(GroupName, Msg1, Msg2, Opts). +find_or_register(ungrouped_exec, _Msg1, _Msg2, _Opts) -> + {leader, ungrouped_exec}; find_or_register(GroupName, _Msg1, _Msg2, Opts) -> - Self = self(), - case find_execution(GroupName, Opts) of - {ok, [Leader|_]} when Leader =/= Self -> - ?event({found_leader, GroupName, {leader, Leader}}), - {wait, Leader}; - {ok, [Leader|_]} when Leader =:= Self -> - {infinite_recursion, GroupName}; + case hb_opts:get(await_inprogress, false, Opts) of + false -> {leader, GroupName}; _ -> - ?event( - worker, - { - register_resolver, - {group, GroupName} - }, - Opts - ), - register_groupname(GroupName, Opts), - {leader, GroupName} + Self = self(), + case find_execution(GroupName, Opts) of + {ok, Leader} when Leader =/= Self -> + ?event({found_leader, GroupName, {leader, Leader}}), + {wait, Leader}; + {ok, Leader} when Leader =:= Self -> + {infinite_recursion, GroupName}; + _ -> + ?event({register_resolver, {group, GroupName}}), + register_groupname(GroupName, Opts), + {leader, GroupName} + end end. %% @doc Unregister as the leader for an execution and notify waiting processes. +unregister_notify(ungrouped_exec, _Msg2, _Msg3, _Opts) -> ok; unregister_notify(GroupName, Msg2, Msg3, Opts) -> - % ?event( - % {unregister_notify, - % {group, GroupName}, - % {msg3, Msg3}, - % {opts, Opts} - % } - % ), unregister_groupname(GroupName, Opts), notify(GroupName, Msg2, Msg3, Opts). %% @doc Find a group with the given name. find_execution(Groupname, _Opts) -> start(), - case pg:get_local_members(Groupname) of - [] -> not_found; - Procs -> {ok, Procs} + case hb_name:lookup(Groupname) of + undefined -> not_found; + Pid -> {ok, Pid} end. %% @doc Calculate the group name for a Msg1 and Msg2 pair. Uses the Msg1's %% `group' function if it is found in the `info', otherwise uses the default. group(Msg1, Msg2, Opts) -> Grouper = - maps:get(grouper, hb_converge:info(Msg1, Opts), fun default_grouper/3), + hb_maps:get(grouper, hb_ao:info(Msg1, Opts), fun default_grouper/3, Opts), apply( Grouper, - hb_converge:truncate_args(Grouper, [Msg1, Msg2, Opts]) + hb_ao:truncate_args(Grouper, [Msg1, Msg2, Opts]) ). -%% @doc Register for performing a Converge resolution. +%% @doc Register for performing an AO-Core resolution. register_groupname(Groupname, _Opts) -> ?event({registering_as, Groupname}), - pg:join(Groupname, self()). + hb_name:register(Groupname). -%% @doc Unregister for being the leader on a Converge resolution. +%% @doc Unregister for being the leader on an AO-Core resolution. unregister(Msg1, Msg2, Opts) -> start(), unregister_groupname(group(Msg1, Msg2, Opts), Opts). unregister_groupname(Groupname, _Opts) -> ?event({unregister_resolver, {explicit, Groupname}}), - pg:leave(Groupname, self()). + hb_name:unregister(Groupname). %% @doc If there was already an Erlang process handling this execution, %% we should register with them and wait for them to notify us of %% completion. await(Worker, Msg1, Msg2, Opts) -> + % Get the device's await function, if it exists. + AwaitFun = + hb_maps:get( + await, + hb_ao:info(Msg1, Opts), + fun default_await/5, + Opts + ), % Calculate the compute path that we will wait upon resolution of. % Register with the process. GroupName = group(Msg1, Msg2, Opts), % set monitor to a worker, so we know if it exits _Ref = erlang:monitor(process, Worker), Worker ! {resolve, self(), GroupName, Msg2, Opts}, - ?event(worker, - {await_resolution, - {group, GroupName}, - {worker, Worker}, - {msg1, Msg1}, - {msg2, Msg2}, - {opts, Opts} - } - ), + AwaitFun(Worker, GroupName, Msg1, Msg2, Opts). + +%% @doc Default await function that waits for a resolution from a worker. +default_await(Worker, GroupName, Msg1, Msg2, Opts) -> % Wait for the result. receive - {'DOWN', _R, process, Worker, _Reason} -> - ?event(worker, + {resolved, _, GroupName, Msg2, Res} -> + worker_event(GroupName, {resolved_await, Res}, Msg1, Msg2, Opts), + Res; + {'DOWN', _R, process, Worker, Reason} -> + ?event( {leader_died, {group, GroupName}, - {leader, Worker} - } - ), - {error, leader_died}; - {resolved, _, GroupName, Msg2, Msg3} -> - ?event(worker, - {resolved_await, - {group, GroupName}, - {msg2, Msg2}, - {msg3, Msg3} + {leader, Worker}, + {reason, Reason}, + {request, Msg2} } ), - Msg3 + {error, leader_died} end. %% @doc Check our inbox for processes that are waiting for the resolution @@ -132,8 +204,15 @@ await(Worker, Msg1, Msg2, Opts) -> %% 1. Notify on group name alone. %% 2. Notify on group name and Msg2. notify(GroupName, Msg2, Msg3, Opts) -> + case is_binary(GroupName) of + true -> + ?event({notifying_all, {group, GroupName}}); + false -> + ok + end, receive {resolve, Listener, GroupName, Msg2, _ListenerOpts} -> + ?event({notifying_listener, {listener, Listener}, {group, GroupName}}), send_response(Listener, GroupName, Msg2, Msg3), notify(GroupName, Msg2, Msg3, Opts) after 0 -> @@ -142,7 +221,7 @@ notify(GroupName, Msg2, Msg3, Opts) -> end. %% @doc Forward requests to a newly delegated execution process. -forward_work(NewPID, _Opts) -> +forward_work(NewPID, Opts) -> Gather = fun Gather() -> receive @@ -159,10 +238,7 @@ forward_work(NewPID, _Opts) -> ), case length(ToForward) > 0 of true -> - ?event( - worker, - {fwded, {requests, length(ToForward)}, {pid, NewPID}} - ); + ?event({fwded, {reqs, length(ToForward)}, {pid, NewPID}}, Opts); false -> ok end, ok. @@ -190,13 +266,14 @@ start_worker(GroupName, Msg, Opts) -> ), WorkerPID = spawn( fun() -> - % If the device's info contains a `worker` function we + % If the device's info contains a `worker' function we % use that instead of the default implementation. WorkerFun = - maps:get( + hb_maps:get( worker, - hb_converge:info(Msg, Opts), - Def = fun default_worker/3 + hb_ao:info(Msg, Opts), + Def = fun default_worker/3, + Opts ), ?event(worker, {new_worker, @@ -212,16 +289,17 @@ start_worker(GroupName, Msg, Opts) -> register_groupname(GroupName, Opts), apply( WorkerFun, - hb_converge:truncate_args( + hb_ao:truncate_args( WorkerFun, [ GroupName, Msg, - maps:merge(Opts, #{ + hb_maps:merge(Opts, #{ is_worker => true, spawn_worker => false, allow_infinite => true - }) + }, + Opts) ] ) ) @@ -232,14 +310,7 @@ start_worker(GroupName, Msg, Opts) -> %% @doc A server function for handling persistent executions. default_worker(GroupName, Msg1, Opts) -> Timeout = hb_opts:get(worker_timeout, 10000, Opts), - ?event(worker, - { - default_worker_waiting_for_req, - {group, GroupName}, - {msg1, Msg1}, - {opts, Opts} - } - ), + worker_event(GroupName, default_worker_waiting_for_req, Msg1, undefined, Opts), receive {resolve, Listener, GroupName, Msg2, ListenerOpts} -> ?event(worker, @@ -249,10 +320,10 @@ default_worker(GroupName, Msg1, Opts) -> } ), Res = - hb_converge:resolve( + hb_ao:resolve( Msg1, Msg2, - maps:merge(ListenerOpts, Opts) + hb_maps:merge(ListenerOpts, Opts, Opts) ), send_response(Listener, GroupName, Msg2, Res), notify(GroupName, Msg2, Res, Opts), @@ -288,14 +359,32 @@ default_worker(GroupName, Msg1, Opts) -> end. %% @doc Create a group name from a Msg1 and Msg2 pair as a tuple. -default_grouper(Msg1, Msg2, _Opts) -> +default_grouper(Msg1, Msg2, Opts) -> %?event({calculating_default_group_name, {msg1, Msg1}, {msg2, Msg2}}), - % Use Erlang's `phash2` to hash the result of the Grouper function. - % `phash2` is relatively fast and ensures that the group name is short for - % storage in `pg`. In production we should only use a hash with a larger + % Use Erlang's `phash2' to hash the result of the Grouper function. + % `phash2' is relatively fast and ensures that the group name is short for + % storage in `pg'. In production we should only use a hash with a larger % output range to avoid collisions. ?no_prod("Using a hash for group names is not secure."), - erlang:phash2({Msg1, Msg2}). + case hb_opts:get(await_inprogress, true, Opts) of + true -> + erlang:phash2( + { + hb_maps:without([<<"priv">>], Msg1, Opts), + hb_maps:without([<<"priv">>], Msg2, Opts) + } + ); + _ -> ungrouped_exec + end. + +%% @doc Log an event with the worker process. If we used the default grouper +%% function, we should also include the Msg1 and Msg2 in the event. If we did not, +%% we assume that the group name expresses enough information to identify the +%% request. +worker_event(Group, Data, Msg1, Msg2, Opts) when is_integer(Group) -> + ?event(worker, {worker_event, Group, Data, {msg1, Msg1}, {msg2, Msg2}}, Opts); +worker_event(Group, Data, _, _, Opts) -> + ?event(worker, {worker_event, Group, Data}, Opts). %%% Tests @@ -304,7 +393,7 @@ test_device(Base) -> #{ info => fun() -> - maps:merge( + hb_maps:merge( #{ grouper => fun(M1, _M2, _Opts) -> @@ -315,7 +404,7 @@ test_device(Base) -> ) end, slow_key => - fun(_, #{ wait := Wait }) -> + fun(_, #{ <<"wait">> := Wait }) -> ?event({slow_key_wait_started, Wait}), receive after Wait -> {ok, @@ -329,7 +418,7 @@ test_device(Base) -> end end, self => - fun(M1, #{ wait := Wait }) -> + fun(M1, #{ <<"wait">> := Wait }) -> ?event({self_waiting, {wait, Wait}}), receive after Wait -> ?event({self_returning, M1, {wait, Wait}}), @@ -345,7 +434,7 @@ spawn_test_client(Msg1, Msg2, Opts) -> TestParent = self(), spawn_link(fun() -> ?event({new_concurrent_test_resolver, Ref, {executing, Msg2}}), - Res = hb_converge:resolve(Msg1, Msg2, Opts), + Res = hb_ao:resolve(Msg1, Msg2, Opts), ?event({test_worker_got_result, Ref, {result, Res}}), TestParent ! {result, Ref, Res} end), @@ -357,8 +446,8 @@ wait_for_test_result(Ref) -> %% @doc Test merging and returning a value with a persistent worker. deduplicated_execution_test() -> TestTime = 200, - Msg1 = #{ device => test_device() }, - Msg2 = #{ path => [slow_key], wait => TestTime }, + Msg1 = #{ <<"device">> => test_device() }, + Msg2 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => TestTime }, T0 = hb:now(), Ref1 = spawn_test_client(Msg1, Msg2), receive after 100 -> ok end, @@ -374,12 +463,12 @@ deduplicated_execution_test() -> %% @doc Test spawning a default persistent worker. persistent_worker_test() -> TestTime = 200, - Msg1 = #{ device => test_device() }, + Msg1 = #{ <<"device">> => test_device() }, link(start_worker(Msg1, #{ static_worker => true })), receive after 10 -> ok end, - Msg2 = #{ path => [slow_key], wait => TestTime }, - Msg3 = #{ path => [slow_key], wait => trunc(TestTime*1.1) }, - Msg4 = #{ path => [slow_key], wait => trunc(TestTime*1.2) }, + Msg2 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => TestTime }, + Msg3 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.1) }, + Msg4 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.2) }, T0 = hb:now(), Ref1 = spawn_test_client(Msg1, Msg2), Ref2 = spawn_test_client(Msg1, Msg3), @@ -395,10 +484,10 @@ persistent_worker_test() -> spawn_after_execution_test() -> ?event(<<"">>), TestTime = 500, - Msg1 = #{ device => test_device() }, - Msg2 = #{ path => [self], wait => TestTime }, - Msg3 = #{ path => [slow_key], wait => trunc(TestTime*1.1) }, - Msg4 = #{ path => [slow_key], wait => trunc(TestTime*1.2) }, + Msg1 = #{ <<"device">> => test_device() }, + Msg2 = #{ <<"path">> => <<"self">>, <<"wait">> => TestTime }, + Msg3 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.1) }, + Msg4 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.2) }, T0 = hb:now(), Ref1 = spawn_test_client( diff --git a/src/hb_private.erl b/src/hb_private.erl index ca5e8c857..636611507 100644 --- a/src/hb_private.erl +++ b/src/hb_private.erl @@ -7,97 +7,211 @@ %%% should _not_ be used for encoding state that makes the execution of a %%% device non-deterministic (unless you are sure you know what you are doing). %%% -%%% The `set` and `get` functions of this module allow you to run those keys -%%% as converge paths if you would like to have private `devices` in the +%%% The `set' and `get' functions of this module allow you to run those keys +%%% as AO-Core paths if you would like to have private `devices' in the %%% messages non-public zone. %%% -%%% See `hb_converge' for more information about the Converge Protocol +%%% See `hb_ao' for more information about the AO-Core protocol %%% and private elements of messages. - -module(hb_private). +-export([opts/1]). -export([from_message/1, reset/1, is_private/1]). --export([get/3, get/4, set/4, set/3, set_priv/2]). +-export([get/3, get/4, set/4, set/3, set_priv/2, merge/3]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). %% @doc Return the `private' key from a message. If the key does not exist, an %% empty map is returned. -from_message(Msg) when is_map(Msg) -> maps:get(priv, Msg, #{}); +from_message(Msg) when is_map(Msg) -> + case maps:is_key(<<"priv">>, Msg) of + true -> maps:get(<<"priv">>, Msg, #{}); + false -> maps:get(priv, Msg, #{}) + end; from_message(_NonMapMessage) -> #{}. %% @doc Helper for getting a value from the private element of a message. Uses -%% Converge resolve under-the-hood, removing the private specifier from the +%% AO-Core resolve under-the-hood, removing the private specifier from the %% path if it exists. get(Key, Msg, Opts) -> get(Key, Msg, not_found, Opts). get(InputPath, Msg, Default, Opts) -> - Path = hb_path:term_to_path_parts(remove_private_specifier(InputPath)), - ?event({get_private, {in, InputPath}, {out, Path}}), % Resolve the path against the private element of the message. - Resolve = - hb_converge:resolve( + Resolved = + hb_util:deep_get( + remove_private_specifier(InputPath, Opts), from_message(Msg), - #{ path => Path }, - priv_converge_opts(Opts) + opts(Opts) ), - case Resolve of - {error, _} -> Default; - {ok, Value} -> Value + case Resolved of + not_found -> Default; + Value -> Value end. %% @doc Helper function for setting a key in the private element of a message. set(Msg, InputPath, Value, Opts) -> - Path = remove_private_specifier(InputPath), + Path = remove_private_specifier(InputPath, Opts), Priv = from_message(Msg), - NewPriv = hb_converge:set(Priv, Path, Value, priv_converge_opts(Opts)), - set(Msg, NewPriv, Opts). + ?event({set_private, {in, Path}, {out, Path}, {value, Value}, {opts, Opts}}), + NewPriv = hb_util:deep_set(Path, Value, Priv, opts(Opts)), + ?event({set_private_res, {out, NewPriv}}), + set_priv(Msg, NewPriv). set(Msg, PrivMap, Opts) -> CurrentPriv = from_message(Msg), - NewPriv = hb_converge:set(CurrentPriv, PrivMap, priv_converge_opts(Opts)), + ?event({set_private, {in, PrivMap}, {opts, Opts}}), + NewPriv = hb_util:deep_merge(CurrentPriv, PrivMap, opts(Opts)), + ?event({set_private_res, {out, NewPriv}}), set_priv(Msg, NewPriv). +%% @doc Merge the private elements of two messages into one. The keys in the +%% second message will override the keys in the first message. The base keys +%% from the first message will be preserved, but the keys in the second message +%% will be lost. +merge(Msg1, Msg2, Opts) -> + % Merge the private elements of the two messages. + Merged = + hb_util:deep_merge( + from_message(Msg1), + from_message(Msg2), + opts(Opts) + ), + % Set the merged private element on the first message. + set_priv(Msg1, Merged). + %% @doc Helper function for setting the complete private element of a message. +set_priv(Msg, PrivMap) + when map_size(PrivMap) =:= 0 andalso not is_map_key(<<"priv">>, Msg) -> + Msg; set_priv(Msg, PrivMap) -> - Msg#{ priv => PrivMap }. + Msg#{ <<"priv">> => PrivMap }. %% @doc Check if a key is private. is_private(Key) -> - case hb_converge:key_to_binary(Key) of + try hb_util:bin(Key) of <<"priv", _/binary>> -> true; _ -> false + catch _:_ -> false end. %% @doc Remove the first key from the path if it is a private specifier. -remove_private_specifier(InputPath) -> - case is_private(hd(Path = hb_path:term_to_path_parts(InputPath))) of +remove_private_specifier(InputPath, Opts) -> + case is_private(hd(Path = hb_path:term_to_path_parts(InputPath, Opts))) of true -> tl(Path); false -> Path end. %% @doc The opts map that should be used when resolving paths against the -%% private element of a message. -priv_converge_opts(Opts) -> - Opts#{ hashpath => ignore, cache_control => [<<"no-cache">>, <<"no-store">>] }. +%% private element of a message. We add the `priv_store' option if set, such that +%% evaluations are not inadvertently persisted in public storage but this module +%% can still access data from the normal stores. This mechanism requires that +%% the priv_store is writable. We also ensure that no cache entries are +%% generated from downstream AO-Core resolutions. +opts(Opts) -> + PrivStore = + case hb_opts:get(priv_store, undefined, Opts) of + undefined -> []; + PrivateStores when is_list(PrivateStores) -> PrivateStores; + PrivateStore -> [PrivateStore] + end, + BaseStore = + case hb_opts:get(store, [], Opts) of + SingleStore when is_map(SingleStore) -> [SingleStore]; + Stores when is_list(Stores) -> Stores + end, + NormStore = PrivStore ++ BaseStore, + Opts#{ + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>], + store => NormStore + }. -%% @doc Unset all of the private keys in a message. -reset(Msg) -> - maps:without( - lists:filter(fun is_private/1, maps:keys(Msg)), - Msg - ). +%% @doc Unset all of the private keys in a message or deep Erlang term. +%% This function operates on all types of data, such that it can be used on +%% non-message terms to ensure that `priv` elements can _never_ pass through. +reset(Msg) when is_map(Msg) -> + maps:map( + fun(_Key, Val) -> reset(Val) end, + maps:without( + lists:filter(fun is_private/1, maps:keys(Msg)), + Msg + ) + ); +reset(List) when is_list(List) -> + % Check if any of the terms in the list are private specifiers, return an + % empty list if so. + case lists:any(fun is_private/1, List) of + true -> []; + false -> + % The list itself is safe. Check each of the children. + lists:map(fun reset/1, List) + end; +reset(Tuple) when is_tuple(Tuple) -> + list_to_tuple(reset(tuple_to_list(Tuple))); +reset(NonMapMessage) -> + NonMapMessage. %%% Tests set_private_test() -> - ?assertEqual(#{a => 1, priv => #{b => 2}}, set(#{a => 1}, b, 2, #{})), - Res = set(#{a => 1}, a, 1, #{}), - ?assertEqual(#{a => 1, priv => #{a => 1}}, Res), - ?assertEqual(#{a => 1, priv => #{a => 1}}, set(Res, a, 1, #{})). + ?assertEqual( + #{<<"a">> => 1, <<"priv">> => #{<<"b">> => 2}}, + set(#{<<"a">> => 1}, <<"b">>, 2, #{}) + ), + Res = set(#{<<"a">> => 1}, <<"a">>, 1, #{}), + ?assertEqual(#{<<"a">> => 1, <<"priv">> => #{<<"a">> => 1}}, Res), + ?assertEqual( + #{<<"a">> => 1, <<"priv">> => #{<<"a">> => 1}}, + set(Res, <<"a">>, 1, #{}) + ). get_private_key_test() -> - M1 = #{a => 1, priv => #{b => 2}}, - ?assertEqual(not_found, get(a, M1, #{})), - {ok, [a]} = hb_converge:resolve(M1, <<"Keys">>, #{}), - ?assertEqual(2, get(b, M1, #{})), - {error, _} = hb_converge:resolve(M1, <<"priv/a">>, #{}), - {error, _} = hb_converge:resolve(M1, priv, #{}). \ No newline at end of file + M1 = #{<<"a">> => 1, <<"priv">> => #{<<"b">> => 2}}, + ?assertEqual(not_found, get(<<"a">>, M1, #{})), + {ok, [<<"a">>]} = hb_ao:resolve(M1, <<"keys">>, #{}), + ?assertEqual(2, get(<<"b">>, M1, #{})), + {error, _} = hb_ao:resolve(M1, <<"priv/a">>, #{}), + {error, _} = hb_ao:resolve(M1, <<"priv">>, #{}). + +get_deep_key_test() -> + M1 = #{<<"a">> => 1, <<"priv">> => #{<<"b">> => #{<<"c">> => 3}}}, + ?assertEqual(3, get(<<"b/c">>, M1, #{})). + +priv_opts_store_read_link_test() -> + % Write a message to the public store. + PublicStore = [hb_test_utils:test_store(hb_store_lmdb)], + timer:sleep(1), + OnlyPrivStore = [hb_test_utils:test_store(hb_store_fs)], + ok = hb_store:write(PublicStore, <<"key">>, <<"test-message">>), + {ok, <<"test-message">>} = hb_store:read(PublicStore, <<"key">>), + % Make a link to the key in the public store. + ok = hb_store:make_link(PublicStore, <<"key">>, <<"link">>), + {ok, <<"test-message">>} = hb_store:read(PublicStore, <<"link">>), + % Read the link from the private store. First as a simple store read, then + % as a link. + Opts = #{ store => PublicStore, priv_store => OnlyPrivStore }, + PrivOpts = #{ store := PrivStore } = opts(Opts), + {ok, <<"test-message">>} = hb_store:read(PrivStore, <<"link">>), + Loaded = + hb_cache:ensure_loaded( + {link, <<"link">>, #{ <<"type">> => <<"link">>, <<"lazy">> => false }}, + PrivOpts + ), + ?assertEqual(<<"test-message">>, Loaded). + +priv_opts_cache_read_message_test() -> + hb:init(), + PublicStore = [hb_test_utils:test_store(hb_store_lmdb)], + OnlyPrivStore = [hb_test_utils:test_store(hb_store_fs)], + Opts = #{ store => PublicStore, priv_store => OnlyPrivStore }, + PrivOpts = opts(Opts), + % Use the `~scheduler@1.0' and `~process@1.0' infrastructure to write a + % complex message into the public store. + Msg = hb_cache:ensure_all_loaded(dev_process:test_aos_process(Opts), Opts), + {ok, ID} = hb_cache:write(Msg, Opts), + % Ensure we can read the message using the public store. + {ok, PubMsg} = hb_cache:read(ID, Opts), + PubMsgLoaded = hb_cache:ensure_all_loaded(PubMsg, Opts), + ?assertEqual(Msg, PubMsgLoaded), + % Read the message using the private store. + {ok, PrivMsg} = hb_cache:read(ID, PrivOpts), + PrivMsgLoaded = hb_cache:ensure_all_loaded(PrivMsg, PrivOpts), + ?assertEqual(Msg, PrivMsgLoaded). \ No newline at end of file diff --git a/src/hb_router.erl b/src/hb_router.erl index c48558cdd..022ded9a3 100644 --- a/src/hb_router.erl +++ b/src/hb_router.erl @@ -8,9 +8,10 @@ find(Type, ID) -> find(Type, ID, '_'). - -find(Type, _ID, Address) -> - case maps:get(Type, hb_opts:get(nodes), undefined) of +find(Type, ID, Address) -> + find(Type, ID, Address, #{}). +find(Type, _ID, Address, Opts) -> + case hb_maps:get(Type, hb_opts:get(nodes), undefined, Opts) of #{ Address := Node } -> {ok, Node}; undefined -> {error, service_type_not_found} end. \ No newline at end of file diff --git a/src/hb_singleton.erl b/src/hb_singleton.erl index e31687b41..36aaf7dd3 100644 --- a/src/hb_singleton.erl +++ b/src/hb_singleton.erl @@ -1,80 +1,200 @@ -%%% @doc A parser that translates Converge HTTP API requests in TABM format +%%% @doc A parser that translates AO-Core HTTP API requests in TABM format %%% into an ordered list of messages to evaluate. The details of this format -%%% are described in `docs/converge-http-api.md`. -%%% +%%% are described in `docs/ao-core-http-api.md'. +%%% %%% Syntax overview: -%%% ``` -%%% Singleton: Message containing keys and a `relative-reference` field, +%%% 
+%%%     Singleton: Message containing keys and a `path' field,
 %%%                which may also contain a query string of key-value pairs.
-%%% 
+%%%
 %%%     Path:
 %%%         - /Part1/Part2/.../PartN/ => [Part1, Part2, ..., PartN]
 %%%         - /ID/Part2/.../PartN => [ID, Part2, ..., PartN]
-%%% 
-%%%     Part: (Key | Resolution), Device?, #{ K => V}?
+%%%
+%%%     Part: (Key + Resolution), Device?, #{ K => V}?
 %%%         - Part => #{ path => Part }
-%%%         - Part+Key=Value => #{ path => Part, Key => Value }
-%%%         - Part+Key => #{ path => Part, Key => true }
-%%%         - Part+K1=V1&K2=V2 => #{ path => Part, K1 => <<"V1">>, K2 => <<"V2">> }
-%%%         - Part!Device => {as, Device, #{ path => Part }}
-%%%         - Part!D+K1=V1 => {as, D, #{ path => Part, K1 => <<"V1">> }}
-%%%         - Part+K1|Int=1 => #{ path => Part, K1 => 1 }
-%%%         - Part!D+K1|Int=1 => {as, D, #{ path => Part, K1 => 1 }}
-%%%         - (/nested/path) => Resolution of the path /nested/path
-%%%         - (/nested/path+K1=V1) => (resolve /nested/path)#{`K1 => V1}
-%%%         - (/nested/path!D+K1=V1) => (resolve /nested/path)#{K1 => V1}
-%%%         - Pt+K1|Res=(/a/b/c) => #{ path => Pt, K1 => (resolve /a/b/c) }
+%%%         - `Part&Key=Value => #{ path => Part, Key => Value }'
+%%%         - `Part=Value&... => #{ path => Part, Part => Value, ... }'
+%%%         - `Part&Key => #{ path => Part, Key => true }'
+%%%         - `Part&k1=v1&k2=v2 => #{ path => Part, k1 => `<<"v1">>', k2 => `<<"v2">>' }'
+%%%         - `Part~Device => {as, Device, #{ path => Part }}'
+%%%         - `Part~D&K1=V1 => {as, D, #{ path => Part, K1 => `<<"v1">>' }}'
+%%%         - `pt&k1+int=1 => #{ path => pt, k1 => 1 }'
+%%%         - `pt~d&k1+int=1 => {as, d, #{ path => pt, k1 => 1 }}'
+%%%         - `(/nested/path) => Resolution of the path /nested/path'
+%%%         - `(/nested/path&k1=v1) => (resolve /nested/path)#{k1 => v1}'
+%%%         - `(/nested/path~D&K1=V1) => (resolve /nested/path)#{K1 => V1}'
+%%%         - `pt&k1+res=(/a/b/c) => #{ path => pt, k1 => (resolve /a/b/c) }'
 %%%     Key:
-%%%         - Key: <<"Value">> => #{ Key => <<"Value">>, ... } for all messages
-%%%         - N.Key: <<"Value">> => #{ Key => <<"Value">>, ... } for Nth message
-%%%         - Key|Int: 1 => #{ Key => 1, ... }
-%%%         - Key|Res: /nested/path => #{ Key => (resolve /nested/path), ... }
-%%%         - N.Key|Res=(/a/b/c) => #{ Key => (resolve /a/b/c), ... }
-%%% '''
+%%%         - key: `<<"value">>' => #{ key => `<<"value">>', ... } for all messages
+%%%         - n.key: `<<"value">>' => #{ key => `<<"value">>', ... } for Nth message
+%%%         - key+int: 1 => #{ key => 1, ... }
+%%%         - key+res: /nested/path => #{ key => (resolve /nested/path), ... }
+%%%         - N.Key+res=(/a/b/c) => #{ Key => (resolve /a/b/c), ... }
+%%%
-module(hb_singleton). --export([from/1]). +-export([from/2, from_path/1, to/1]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(MAX_SEGMENT_LENGTH, 512). -%% @doc Normalize a singleton TABM message into a list of executable Converge +-type ao_message() :: map() | binary(). +-type tabm_message() :: map(). + +%% @doc Convert a list of AO-Core message into TABM message. +-spec to(list(ao_message())) -> tabm_message(). +to(Messages) -> + % Iterate through all AO-Core messages folding them into the TABM message + % Scopes contains the following map: #{Key => [StageIndex, StageIndex2...]} + % that allows to scope keys to the given stage. + {TABMMessage, _FinalIndex, Scopes} = + lists:foldl( + fun + % Special case when AO-Core message is ID + (Message, {Acc, Index, ScopedModifications}) when ?IS_ID(Message) -> + {append_path(Message, Acc), Index + 1, ScopedModifications}; + % Special case when AO-Core message contains resolve command + ({resolve, SubMessages0}, {Acc, Index, ScopedModifications}) -> + SubMessages1 = hb_maps:get(<<"path">>, to(SubMessages0)), + <<"/", SubMessages2/binary>> = SubMessages1, + SubMessages = <<"(", SubMessages2/binary, ")">>, + {append_path(SubMessages, Acc), Index + 1, ScopedModifications}; + % Regular case when message is a map + (Message, {Acc, Index, ScopedModifications}) -> + {NewMessage, NewScopedModifications} = + hb_maps:fold( + fun + (<<"path">>, PathPart, {AccIn, Scoped}) -> + {append_path(PathPart, AccIn), Scoped}; + % Specifically ignore method field from scope modifications + (<<"method">>, Value, {AccIn, Scoped}) -> + {hb_maps:put(<<"method">>, Value, AccIn), Scoped}; + (Key, {resolve, SubMessages}, {AccIn, Scoped}) -> + NewKey = <>, + NewSubMessages = hb_maps:get(<<"path">>, to(SubMessages)), + { + hb_maps:put(NewKey, NewSubMessages, AccIn), + hb_maps:update_with( + NewKey, + fun(Indexes) -> [Index | Indexes] end, + [Index], + Scoped + ) + }; + (Key, Value, {AccIn, Scoped}) -> + { + hb_maps:put(Key, Value, AccIn), + hb_maps:update_with( + Key, + fun(Indexes) -> [Index | Indexes] end, + [Index], + Scoped + ) + } + end, + {Acc, ScopedModifications}, + Message), + {NewMessage, Index + 1, NewScopedModifications} + + end, + {#{}, 0, #{}}, + Messages), + MessageWithTypeAndScopes = + hb_maps:fold( + fun + % For the case when a given Key appeared only once in scopes + (Key, [SingleIndexScope], AccIn) -> + Index = integer_to_binary(SingleIndexScope), + {Value, NewAccIn} = hb_maps:take(Key, AccIn), + {NewKey, NewValue} = + case type(Value) of + integer -> + K = <>, + V = integer_to_binary(Value), + {K, V}; + _ -> {<>, Value} + end, + hb_maps:put(NewKey, NewValue, NewAccIn); + (_Key, _Value, AccIn) -> AccIn + end, + TABMMessage, + Scopes), + MessageWithTypeAndScopes. + +append_path(PathPart, #{<<"path">> := Path} = Message) -> + hb_maps:put(<<"path">>, <>, Message); +append_path(PathPart, Message) -> + hb_maps:put(<<"path">>, <<"/", PathPart/binary>>, Message). + +type(Value) when is_binary(Value) -> binary; +type(Value) when is_integer(Value) -> integer; +type(_Value) -> unknown. + +%% @doc Normalize a singleton TABM message into a list of executable AO-Core %% messages. -from(RawMsg) -> - {ok, Path, Query} = - parse_rel_ref( - maps:get(<<"relative-reference">>, RawMsg, <<"/">>) +from(RawMsg, Opts) when is_binary(RawMsg) -> + from(#{ <<"path">> => RawMsg }, Opts); +from(RawMsg, Opts) -> + RawPath = hb_maps:get(<<"path">>, RawMsg, <<>>), + ?event(parsing, {raw_path, RawPath}), + {ok, Path, Query} = from_path(RawPath), + ?event(parsing, {parsed_path, Path, Query}), + MsgWithoutBasePath = + hb_maps:merge( + hb_maps:remove(<<"path">>, RawMsg), + Query ), - MsgWithoutRef = maps:merge( - maps:remove(<<"relative-reference">>, RawMsg), - Query - ), % 2. Decode, split, and sanitize path segments. Each yields one step message. - Msgs = lists:flatten(lists:map(fun path_messages/1, Path)), + RawMsgs = + lists:flatten( + lists:map( + fun(Msg) -> path_messages(Msg, Opts) end, + Path + ) + ), + ?event(parsing, {raw_messages, RawMsgs}), + Msgs = normalize_base(RawMsgs), + ?event(parsing, {normalized_messages, Msgs}), % 3. Type keys and values - Typed = apply_types(MsgWithoutRef), - % 4. Group keys by N-scope and global scope + Typed = apply_types(MsgWithoutBasePath, Opts), + ?event(parsing, {typed_messages, Typed}), + % 4. Group keys by N-scope and global scope ScopedModifications = group_scoped(Typed, Msgs), + ?event(parsing, {scoped_modifications, ScopedModifications}), % 5. Generate the list of messages (plus-notation, device, typed keys). - build_messages(Msgs, ScopedModifications). + Result = build_messages(Msgs, ScopedModifications, Opts), + ?event(parsing, {result, Result}), + Result. %% @doc Parse the relative reference into path, query, and fragment. -parse_rel_ref(RelativeRef) -> - {Path, QMap} = - case binary:split(RelativeRef, <<"?">>) of - [P, QStr] -> {P, cowboy_req:parse_qs(#{ qs => QStr })}; - [P] -> {P, #{}} +from_path(RelativeRef) -> + %?event(parsing, {raw_relative_ref, RawRelativeRef}), + %RelativeRef = hb_escape:decode(RawRelativeRef), + Decoded = decode_string(RelativeRef), + ?event(parsing, {parsed_relative_ref, Decoded}), + {Path, QKVList} = + case hb_util:split_depth_string_aware_single("?", Decoded) of + {_Sep, P, QStr} -> {P, cowboy_req:parse_qs(#{ qs => QStr })}; + {no_match, P, <<>>} -> {P, []} end, { ok, - lists:map(fun(Part) -> decode_string(Part) end, path_parts($/, Path)), - QMap + path_parts($/, Path), + hb_maps:from_list(QKVList) }. -%% @doc Step 2: Decode + Split + Sanitize the path. Split by `/` but avoid -%% subpath components, such that their own path parts are not dissociated from +%% @doc Step 2: Decode, split and sanitize the path. Split by `/' but avoid +%% subpath components, such that their own path parts are not dissociated from %% their parent path. -path_messages(RawBin) when is_binary(RawBin) -> - lists:map(fun parse_part/1, path_parts([$/], decode_string(RawBin))). +path_messages(Bin, Opts) when is_binary(Bin) -> + lists:map(fun(Part) -> parse_part(Part, Opts) end, path_parts([$/], Bin)). + +%% @doc Normalize the base path. +normalize_base([]) -> []; +normalize_base([First|Rest]) when ?IS_ID(First) -> [First|Rest]; +normalize_base([{as, DevID, First}|Rest]) -> [{as, DevID, First}|Rest]; +normalize_base([Subres = {resolve, _}|Rest]) -> [Subres|Rest]; +normalize_base(Rest) -> [#{}|Rest]. %% @doc Split the path into segments, filtering out empty segments and %% segments that are too long. @@ -90,13 +210,13 @@ path_parts(Sep, PathBin) when is_binary(PathBin) -> end, all_path_parts(Sep, PathBin) ), + ?event({path_parts, Res}), Res. %% @doc Extract all of the parts from the binary, given (a list of) separators. all_path_parts(_Sep, <<>>) -> []; all_path_parts(Sep, Bin) -> - {_MatchedSep, Part, Rest} = part(Sep, Bin), - [Part | all_path_parts(Sep, Rest)]. + hb_util:split_depth_string_aware(Sep, Bin). %% @doc Extract the characters from the binary until a separator is found. %% The first argument of the function is an explicit separator character, or @@ -105,76 +225,106 @@ all_path_parts(Sep, Bin) -> part(Sep, Bin) when not is_list(Sep) -> part([Sep], Bin); part(Seps, Bin) -> - part(Seps, Bin, 0, <<>>). -part(_Seps, <<>>, _Depth, CurrAcc) -> {no_match, CurrAcc, <<>>}; -part(Seps, << $\(, Rest/binary>>, Depth, CurrAcc) -> - %% Increase depth - part(Seps, Rest, Depth + 1, << CurrAcc/binary, "(" >>); -part(Seps, << $\), Rest/binary>>, Depth, CurrAcc) when Depth > 0 -> - %% Decrease depth - part(Seps, Rest, Depth - 1, << CurrAcc/binary, ")">>); -part(Seps, <>, Depth, CurrAcc) -> - case Depth == 0 andalso lists:member(C, Seps) of - true -> {C, CurrAcc, Rest}; - false -> - part(Seps, Rest, Depth, << CurrAcc/binary, C:8/integer >>) - end. + hb_util:split_depth_string_aware_single(Seps, Bin). %% @doc Step 3: Apply types to values and remove specifiers. -apply_types(Msg) -> - maps:fold( +apply_types(Msg, Opts) -> + hb_maps:fold( fun(Key, Val, Acc) -> - {_, K, V} = maybe_typed(Key, Val), - maps:put(K, V, Acc) + {_, K, V} = maybe_typed(Key, Val, Opts), + hb_maps:put(K, V, Acc, Opts) end, #{}, - Msg + Msg, + Opts ). -%% @doc Step 4: Group headers/query by N-scope. -%% `N.Key` => applies to Nth step. Otherwise => global +%% @doc Step 4: Group headers/query by N-scope. +%% `N.Key' => applies to Nth step. Otherwise => `global' group_scoped(Map, Msgs) -> {NScope, Global} = - maps:fold( + hb_maps:fold( fun(KeyBin, Val, {Ns, Gs}) -> case parse_scope(KeyBin) of {OkN, RealKey} when OkN > 0 -> - Curr = maps:get(OkN, Ns, #{}), - Ns2 = maps:put(OkN, maps:put(RealKey, Val, Curr), Ns), + Curr = hb_maps:get(OkN, Ns, #{}), + Ns2 = hb_maps:put(OkN, hb_maps:put(RealKey, Val, Curr), Ns), {Ns2, Gs}; - global -> {Ns, maps:put(KeyBin, Val, Gs)} + global -> {Ns, hb_maps:put(KeyBin, Val, Gs)} end end, {#{}, #{}}, Map ), [ - maps:merge(Global, maps:get(N, NScope, #{})) + hb_maps:merge(Global, hb_maps:get(N, NScope, #{})) || N <- lists:seq(1, length(Msgs)) ]. -%% @doc Get the scope of a key. +%% @doc Get the scope of a key. Adds 1 to account for the base message. parse_scope(KeyBin) -> case binary:split(KeyBin, <<".">>, [global]) of [Front, Remainder] -> case catch erlang:binary_to_integer(Front) of - NInt when is_integer(NInt) -> {NInt, Remainder}; + NInt when is_integer(NInt) -> {NInt + 1, Remainder}; _ -> throw({error, invalid_scope, KeyBin}) end; _ -> global end. %% @doc Step 5: Merge the base message with the scoped messages. -build_messages(Msgs, ScopedModifications) -> - do_build(1, Msgs, ScopedModifications). - -do_build(I, [], _ScopedKeys) -> []; -do_build(I, [Msg|Rest], ScopedKeys) when not is_map(Msg) -> - [Msg | do_build(I+1, Rest, ScopedKeys)]; -do_build(I, [Msg | Rest], ScopedKeys) -> - StepMsg = maps:merge(Msg, lists:nth(I, ScopedKeys)), - [StepMsg | do_build(I+1, Rest, ScopedKeys)]. +build_messages(Msgs, ScopedModifications, Opts) -> + do_build(1, Msgs, ScopedModifications, Opts). + +do_build(_, [], _, _) -> []; +do_build(I, [{as, DevID, RawMsg} | Rest], ScopedKeys, Opts) when is_map(RawMsg) -> + % We are processing an `as' message. If the path is empty, we need to + % remove it from the message and the additional message, such that AO-Core + % returns only the message with the device specifier changed. If the message + % does have a path, AO-Core will subresolve it. + RawAdditional = lists:nth(I, ScopedKeys), + {Msg, Additional} = + case hb_maps:get(<<"path">>, RawMsg, <<"">>, Opts) of + ID when ?IS_ID(ID) -> + % When we have an ID, we do not merge the globally scoped elements. + { + RawMsg, + #{} + }; + <<"">> -> + % When we have an empty path, we remove the path from both + % messages. AO-Core will then simply set the device specifier + % and not execute a subresolve. + { + hb_ao:set(RawMsg, <<"path">>, unset, Opts), + hb_ao:set(RawAdditional, <<"path">>, unset, Opts) + }; + _BasePath -> + % When we have a non-empty path, we merge the messages in + % totality. The path-part's path will be subresolved. + {RawMsg, RawAdditional} + end, + Merged = hb_maps:merge(Additional, Msg, Opts), + StepMsg = hb_message:convert( + Merged, + <<"structured@1.0">>, + Opts#{ topic => ao_internal } + ), + ?event(parsing, {build_messages, {base, Msg}, {additional, Additional}}), + [{as, DevID, StepMsg} | do_build(I + 1, Rest, ScopedKeys, Opts)]; +do_build(I, [Msg | Rest], ScopedKeys, Opts) when not is_map(Msg) -> + [Msg | do_build(I + 1, Rest, ScopedKeys, Opts)]; +do_build(I, [Msg | Rest], ScopedKeys, Opts) -> + Additional = lists:nth(I, ScopedKeys), + Merged = hb_maps:merge(Additional, Msg, Opts), + StepMsg = hb_message:convert( + Merged, + <<"structured@1.0">>, + Opts#{ topic => ao_internal } + ), + ?event(parsing, {build_messages, {base, Msg}, {additional, Additional}}), + [StepMsg | do_build(I + 1, Rest, ScopedKeys, Opts)]. %% @doc Parse a path part into a message or an ID. %% Applies the syntax rules outlined in the module doc, in the following order: @@ -182,254 +332,527 @@ do_build(I, [Msg | Rest], ScopedKeys) -> %% 2. Part subpath resolutions %% 3. Inlined key-value pairs %% 4. Device specifier -parse_part(ID) when ?IS_ID(ID) -> ID; -parse_part(Part) -> - case maybe_subpath(Part) of +parse_part(ID, _Opts) when ?IS_ID(ID) -> ID; +parse_part(Part, Opts) -> + case maybe_subpath(Part, Opts) of {resolve, Subpath} -> {resolve, Subpath}; Part -> - case part([$+, $&, $!, $|], Part) of + case part([$&, $~, $+, $ , $=], Part) of {no_match, PartKey, <<>>} -> - #{ path => PartKey }; + #{ <<"path">> => PartKey }; {Sep, PartKey, PartModBin} -> parse_part_mods( << Sep:8/integer, PartModBin/binary >>, - #{ path => PartKey } + #{ <<"path">> => PartKey }, + Opts ) end end. -%% @doc Parse part modifiers: -%% 1. `!Device` => {as, Device, Msg} -%% 2. `+K=V` => Msg#{ K => V } -parse_part_mods([], Msg) -> Msg; -parse_part_mods(<<>>, Msg) -> Msg; -parse_part_mods(<<"!", PartMods/binary>>, Msg) -> +%% @doc Parse part modifiers: +%% 1. `~Device' => `{as, Device, Msg}' +%% 2. `&K=V' => `Msg#{ K => V }' +parse_part_mods([], Msg, _Opts) -> Msg; +parse_part_mods(<<>>, Msg, _Opts) -> Msg; +parse_part_mods(<<"~", PartMods/binary>>, Msg, Opts) -> % Get the string until the end of the device specifier or end of string. - [DeviceBin, Rest] = path_parts($+, PartMods), - InlinedMsgBin = iolist_to_binary(lists:join(Rest, <<"+">>)), + {_, DeviceBin, InlinedMsgBin} = part([$&], PartMods), % Calculate the inlined keys - MsgWithInlines = parse_part_mods(InlinedMsgBin, Msg), + MsgWithInlines = parse_part_mods(<<"&", InlinedMsgBin/binary >>, Msg, Opts), % Apply the device specifier - {as, maybe_subpath(DeviceBin), MsgWithInlines}; -parse_part_mods(<< "+", InlinedMsgBin/binary >>, Msg) -> + {as, maybe_subpath(DeviceBin, Opts), MsgWithInlines}; +parse_part_mods(<< "&", InlinedMsgBin/binary >>, Msg, Opts) -> InlinedKeys = path_parts($&, InlinedMsgBin), - MsgWithInlined = + MsgWithInlined = lists:foldl( fun(InlinedKey, Acc) -> - {Key, Val} = parse_inlined_key_val(InlinedKey), - maps:put(Key, Val, Acc) + {Key, Val} = parse_inlined_key_val(InlinedKey, Opts), + ?event({inlined_key, {explicit, Key}, {explicit, Val}}), + hb_maps:put(Key, Val, Acc) end, Msg, InlinedKeys ), - MsgWithInlined. + MsgWithInlined; +parse_part_mods(<<$=, InlinedMsgBin/binary>>, M = #{ <<"path">> := Path }, Opts) + when map_size(M) =:= 1, is_binary(Path) -> + parse_part_mods(<< "&", Path/binary, "=", InlinedMsgBin/binary >>, M, Opts); +parse_part_mods(<<$+, InlinedMsgBin/binary>>, M = #{ <<"path">> := Path }, Opts) + when map_size(M) =:= 1, is_binary(InlinedMsgBin) -> + parse_part_mods(<< "&", Path/binary, "+", InlinedMsgBin/binary >>, M, Opts). %% @doc Extrapolate the inlined key-value pair from a path segment. If the %% key has a value, it may provide a type (as with typical keys), but if a -%% value is not provided, it is assumed to be a boolean `true`. -parse_inlined_key_val(Bin) -> +%% value is not provided, it is assumed to be a boolean `true'. +parse_inlined_key_val(Bin, Opts) -> case part([$=, $&], Bin) of {no_match, K, <<>>} -> {K, true}; - {$=, K, V} -> - {_, Key, Val} = maybe_typed(K, maybe_subpath(V)), + {$=, K, RawV} -> + V = unquote(RawV), + {_, Key, Val} = maybe_typed(K, maybe_subpath(V, Opts), Opts), {Key, Val} end. +%% @doc Unquote a string. +unquote(<<"\"", Inner/binary>>) -> + case binary:last(Inner) of + $" -> binary:part(Inner, 0, byte_size(Inner) - 1); + _ -> Inner + end; +unquote(Bin) -> Bin. + %% @doc Attempt Cowboy URL decode, then sanitize the result. decode_string(B) -> - case catch http_uri:decode(B) of + case catch uri_string:unquote(B) of DecodedBin when is_binary(DecodedBin) -> DecodedBin; _ -> throw({error, cannot_decode, B}) end. %% @doc Check if the string is a subpath, returning it in parsed form, %% or the original string with a specifier. -maybe_subpath(Str) when byte_size(Str) >= 2 -> +maybe_subpath(Str, Opts) when byte_size(Str) >= 2 -> case {binary:first(Str), binary:last(Str)} of {$(, $)} -> Inside = binary:part(Str, 1, byte_size(Str) - 2), - {resolve, from(#{ <<"relative-reference">> => Inside })}; + {resolve, from(#{ <<"path">> => Inside }, Opts)}; _ -> Str end; -maybe_subpath(Other) -> Other. +maybe_subpath(Other, _Opts) -> Other. %% @doc Parse a key's type (applying it to the value) and device name if present. -maybe_typed(Key, Value) -> - case part($|, Key) of +%% We allow ` ` characters as type indicators because some URL-string encoders +%% (e.g. Chrome) will encode `+` characters in a form that query-string parsers +%% interpret as ` ' characters. +maybe_typed(Key, Value, Opts) -> + case part([$+, $ ], Key) of {no_match, OnlyKey, <<>>} -> {untyped, OnlyKey, Value}; - {$|, OnlyKey, Type} -> + {_, OnlyKey, Type} -> case {Type, Value} of - {<<"Resolve">>, Subpath} -> + {<<"resolve">>, Subpath} -> % If the value needs to be resolved before it is converted, - % use the `Codec/1.0` device to resolve it. + % use the `Codec/1.0' device to resolve it. % For example: - % /a/b+k|Int=(/x/y/z)` => /a/b+k=(/x/y/z/body+Type=Int|Codec) + % `/a/b&k+Int=(/x/y/z) => /a/b&k=(/x/y/z/body&Type=Int+Codec)' {typed, OnlyKey, - {resolve, from(#{ <<"relative-reference">> => Subpath })} + {resolve, from(#{ <<"path">> => Subpath }, Opts)} }; - {_T, Bin} when is_binary(Bin) -> - {typed, OnlyKey, hb_codec_converge:decode_value(Type, Bin)} + {_T, RawValue} when is_binary(RawValue) -> + Decoded = hb_escape:decode_quotes(RawValue), + {typed, OnlyKey, dev_codec_structured:decode_value(Type, Decoded)} end end. +%% @doc Join a list of items with a separator, or return the first item if there +%% is only one item. If there are no items, return an empty binary. +maybe_join(Items, Sep) -> + case length(Items) of + 0 -> <<>>; + 1 -> hd(Items); + _ -> iolist_to_binary(lists:join(Sep, Items)) + end. + %%% Tests -%%% Simple tests +parse_explicit_message_test() -> + Singleton1 = #{ + <<"path">> => <<"/a">>, + <<"a">> => <<"b">> + }, + ?assertEqual( + [ + #{ <<"a">> => <<"b">>}, + #{ <<"path">> => <<"a">>, <<"a">> => <<"b">> } + ], + from(Singleton1, #{}) + ), + DummyID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Singleton2 = #{ + <<"path">> => <<"/", DummyID/binary, "/a">> + }, + ?assertEqual([DummyID, #{ <<"path">> => <<"a">> }], from(Singleton2, #{})), + Singleton3 = #{ + <<"path">> => <<"/", DummyID/binary, "/a">>, + <<"a">> => <<"b">> + }, + ?assertEqual( + [DummyID, #{ <<"path">> => <<"a">>, <<"a">> => <<"b">> }], + from(Singleton3, #{}) + ). + +%%% `to/1' function tests +to_suite_test_() -> + [ + fun simple_to_test/0, + fun multiple_messages_to_test/0, + fun basic_hashpath_to_test/0, + fun scoped_key_to_test/0, + fun typed_key_to_test/0, + fun subpath_in_key_to_test/0, + fun subpath_in_path_to_test/0, + fun inlined_keys_to_test/0, + fun multiple_inlined_keys_to_test/0, + fun subpath_in_inlined_to_test/0 + ]. +simple_to_test() -> + Messages = [ + #{<<"test-key">> => <<"test-value">>}, + #{<<"path">> => <<"a">>, <<"test-key">> => <<"test-value">>} + ], + Expected = #{<<"path">> => <<"/a">>, <<"test-key">> => <<"test-value">>}, + ?assertEqual(Expected, to(Messages)), + ?assertEqual(Messages, from(to(Messages), #{})). + +multiple_messages_to_test() -> + Messages = + [ + #{<<"test-key">> => <<"test-value">>}, + #{<<"path">> => <<"a">>, <<"test-key">> => <<"test-value">>}, + #{<<"path">> => <<"b">>, <<"test-key">> => <<"test-value">>}, + #{<<"path">> => <<"c">>, <<"test-key">> => <<"test-value">>} + ], + Expected = #{ + <<"path">> => <<"/a/b/c">>, + <<"test-key">> => <<"test-value">> + }, + ?assertEqual(Expected, to(Messages)), + ?assertEqual(Messages, from(to(Messages), #{})). + +basic_hashpath_to_test() -> + Messages = [ + <<"e5ohB7TgMYRoc0BLllkmAqkqLy1SrliEkOPJlNPXBQ8">>, + #{<<"method">> => <<"GET">>, <<"path">> => <<"some-other">>} + ], + Expected = #{ + <<"path">> => <<"/e5ohB7TgMYRoc0BLllkmAqkqLy1SrliEkOPJlNPXBQ8/some-other">>, + <<"method">> => <<"GET">> + }, + ?assertEqual(Expected, to(Messages)), + ?assertEqual(Messages, from(to(Messages), #{})). + +scoped_key_to_test() -> + Messages = [ + #{}, + #{<<"path">> => <<"a">>}, + #{<<"path">> => <<"b">>, <<"test-key">> => <<"test-value">>}, + #{<<"path">> => <<"c">>} + ], + Expected = #{<<"2.test-key">> => <<"test-value">>, <<"path">> => <<"/a/b/c">>}, + ?assertEqual(Expected, to(Messages)), + ?assertEqual(Messages, from(to(Messages), #{})). + +typed_key_to_test() -> + Messages = + [ + #{}, + #{<<"path">> => <<"a">>}, + #{<<"path">> => <<"b">>, <<"test-key">> => 123}, + #{<<"path">> => <<"c">>} + ], + Expected = #{<<"2.test-key+integer">> => <<"123">>, <<"path">> => <<"/a/b/c">>}, + ?assertEqual(Expected, to(Messages)), + ?assertEqual(Messages, from(to(Messages), #{})). + +subpath_in_key_to_test() -> + Messages = [ + #{}, + #{<<"path">> => <<"a">>}, + #{ + <<"path">> => <<"b">>, + <<"test-key">> => + {resolve, + [ + #{}, + #{<<"path">> => <<"x">>}, + #{<<"path">> => <<"y">>}, + #{<<"path">> => <<"z">>} + ] + } + }, + #{<<"path">> => <<"c">>} + ], + Expected = #{<<"2.test-key+resolve">> => <<"/x/y/z">>, <<"path">> => <<"/a/b/c">>}, + ?assertEqual(Expected, to(Messages)), + ?assertEqual(Messages, from(to(Messages), #{})). + +subpath_in_path_to_test() -> + Messages = [ + #{}, + #{<<"path">> => <<"a">>}, + {resolve, + [ + #{}, + #{<<"path">> => <<"x">>}, + #{<<"path">> => <<"y">>}, + #{<<"path">> => <<"z">>} + ] + }, + #{<<"path">> => <<"z">>} + ], + Expected = #{ + <<"path">> => <<"/a/(x/y/z)/z">> + }, + ?assertEqual(Expected, to(Messages)), + ?assertEqual(Messages, from(to(Messages), #{})). + +inlined_keys_to_test() -> + Messages = + [ + #{<<"method">> => <<"POST">>}, + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"a">> + }, + #{ + <<"k1">> => <<"v1">>, + <<"method">> => <<"POST">>, + <<"path">> => <<"b">> + }, + #{ + <<"k2">> => <<"v2">>, + <<"method">> => <<"POST">>, + <<"path">> => <<"c">> + } + ], + % NOTE: The implementation above does not convert the given list of messages + % into the original format, however it assures that the `to/1' and `from/1' + % operations are idempotent. + ?assertEqual(Messages, from(to(Messages), #{})). + +multiple_inlined_keys_to_test() -> + Messages = [ + #{<<"method">> => <<"POST">>}, + #{<<"method">> => <<"POST">>, <<"path">> => <<"a">>}, + #{ + <<"k1">> => <<"v1">>, + <<"k2">> => <<"v2">>, + <<"method">> => <<"POST">>, + <<"path">> => <<"b">> + } + ], + % NOTE: The implementation above does not convert the given list of messages + % into the original format, however it assures that the `to/1' and `from/1' + % operations are idempotent. + ?assertEqual(Messages, from(to(Messages), #{})). + +subpath_in_inlined_to_test() -> + Messages = [ + #{}, + #{<<"path">> => <<"part1">>}, + #{<<"b">> => + {resolve, + [#{}, + #{<<"path">> => <<"x">>}, + #{<<"path">> => <<"y">>}]}, + <<"path">> => <<"part2">>, + <<"test">> => <<"1">>}, + #{<<"path">> => <<"part3">>}], + % NOTE: The implementation above does not convert the given list of messages + % into the original format, however it assures that the `to/1' and `from/1' + % operations are idempotent. + ?assertEqual(Messages, from(to(Messages), #{})). + +%%% `from/1' function tests single_message_test() -> + % This is a singleton TABM message Req = #{ - <<"relative-reference">> => <<"/a">>, + <<"path">> => <<"/a">>, <<"test-key">> => <<"test-value">> }, - Msgs = from(Req), - ?assertEqual(1, length(Msgs)), + Msgs = from(Req, #{}), + ?assertEqual(2, length(Msgs)), ?assert(is_map(hd(Msgs))), - ?assertEqual(<<"test-value">>, maps:get(<<"test-key">>, hd(Msgs))). + ?assertEqual(<<"test-value">>, hb_maps:get(<<"test-key">>, hd(Msgs))). basic_hashpath_test() -> Hashpath = hb_util:human_id(crypto:strong_rand_bytes(32)), - Path = <<"/", Hashpath/binary, "/someOther">>, + Path = <<"/", Hashpath/binary, "/some-other">>, Req = #{ - <<"relative-reference">> => Path, + <<"path">> => Path, <<"method">> => <<"GET">> }, - Msgs = from(Req), + Msgs = from(Req, #{}), ?assertEqual(2, length(Msgs)), [Base, Msg2] = Msgs, ?assertEqual(Base, Hashpath), - ?assertEqual(<<"GET">>, maps:get(<<"method">>, Msg2)), - ?assertEqual(<<"someOther">>, maps:get(path, Msg2)). + ?assertEqual(<<"GET">>, hb_maps:get(<<"method">>, Msg2)), + ?assertEqual(<<"some-other">>, hb_maps:get(<<"path">>, Msg2)). multiple_messages_test() -> Req = #{ - <<"relative-reference">> => <<"/a/b/c">>, + <<"path">> => <<"/a/b/c">>, <<"test-key">> => <<"test-value">> }, - Msgs = from(Req), - ?assertEqual(3, length(Msgs)), - [Base, Msg2, Msg3] = Msgs, + Msgs = from(Req, #{}), + ?assertEqual(4, length(Msgs)), + [_Base, Msg1, Msg2, Msg3] = Msgs, ?assert(lists:all(fun is_map/1, Msgs)), - ?assertEqual(<<"test-value">>, maps:get(<<"test-key">>, Base)), - ?assertEqual(<<"test-value">>, maps:get(<<"test-key">>, Msg2)), - ?assertEqual(<<"test-value">>, maps:get(<<"test-key">>, Msg3)). + ?assertEqual(<<"test-value">>, hb_maps:get(<<"test-key">>, Msg1)), + ?assertEqual(<<"test-value">>, hb_maps:get(<<"test-key">>, Msg2)), + ?assertEqual(<<"test-value">>, hb_maps:get(<<"test-key">>, Msg3)). %%% Advanced key syntax tests scoped_key_test() -> Req = #{ - <<"relative-reference">> => <<"/a/b/c">>, + <<"path">> => <<"/a/b/c">>, <<"2.test-key">> => <<"test-value">> }, - Msgs = from(Req), - ?assertEqual(3, length(Msgs)), - [Msg1, Msg2, Msg3] = Msgs, - ?assertEqual(not_found, maps:get(<<"test-key">>, Msg1, not_found)), - ?assertEqual(<<"test-value">>, maps:get(<<"test-key">>, Msg2, not_found)), - ?assertEqual(not_found, maps:get(<<"test-key">>, Msg3, not_found)). + Msgs = from(Req, #{}), + ?assertEqual(4, length(Msgs)), + [_, Msg1, Msg2, Msg3] = Msgs, + ?assertEqual(not_found, hb_maps:get(<<"test-key">>, Msg1, not_found)), + ?assertEqual(<<"test-value">>, hb_maps:get(<<"test-key">>, Msg2, not_found)), + ?assertEqual(not_found, hb_maps:get(<<"test-key">>, Msg3, not_found)). typed_key_test() -> Req = #{ - <<"relative-reference">> => <<"/a/b/c">>, - <<"2.test-key|Integer">> => <<"123">> + <<"path">> => <<"/a/b/c">>, + <<"2.test-key+integer">> => <<"123">> }, - Msgs = from(Req), - ?assertEqual(3, length(Msgs)), - [Msg1, Msg2, Msg3] = Msgs, - ?assertEqual(not_found, maps:get(<<"test-key">>, Msg1, not_found)), - ?assertEqual(123, maps:get(<<"test-key">>, Msg2, not_found)), - ?assertEqual(not_found, maps:get(<<"test-key">>, Msg3, not_found)). + Msgs = from(Req, #{}), + ?assertEqual(4, length(Msgs)), + [_, Msg1, Msg2, Msg3] = Msgs, + ?assertEqual(not_found, hb_maps:get(<<"test-key">>, Msg1, not_found)), + ?assertEqual(123, hb_maps:get(<<"test-key">>, Msg2, not_found)), + ?assertEqual(not_found, hb_maps:get(<<"test-key">>, Msg3, not_found)). subpath_in_key_test() -> Req = #{ - <<"relative-reference">> => <<"/a/b/c">>, - <<"2.test-key|Resolve">> => <<"/x/y/z">> + <<"path">> => <<"/a/b/c">>, + <<"2.test-key+resolve">> => <<"/x/y/z">> }, - Msgs = from(Req), - ?assertEqual(3, length(Msgs)), - [Msg1, Msg2, Msg3] = Msgs, - ?assertEqual(not_found, maps:get(<<"test-key">>, Msg1, not_found)), + Msgs = from(Req, #{}), + ?assertEqual(4, length(Msgs)), + [_, Msg1, Msg2, Msg3] = Msgs, + ?assertEqual(not_found, hb_maps:get(<<"test-key">>, Msg1, not_found)), ?assertEqual( {resolve, [ - #{ path => <<"x">> }, - #{ path => <<"y">> }, - #{ path => <<"z">> } + #{}, + #{ <<"path">> => <<"x">> }, + #{ <<"path">> => <<"y">> }, + #{ <<"path">> => <<"z">> } ] }, - maps:get(<<"test-key">>, Msg2, not_found) + hb_maps:get(<<"test-key">>, Msg2, not_found) ), - ?assertEqual(not_found, maps:get(<<"test-key">>, Msg3, not_found)). + ?assertEqual(not_found, hb_maps:get(<<"test-key">>, Msg3, not_found)). %%% Advanced path syntax tests subpath_in_path_test() -> Req = #{ - <<"relative-reference">> => <<"/a/(x/y/z)/z">> + <<"path">> => <<"/a/(x/y/z)/z">> }, - Msgs = from(Req), - ?assertEqual(3, length(Msgs)), - [Msg1, Msg2, Msg3] = Msgs, - ?assertEqual(<<"a">>, maps:get(path, Msg1)), + Msgs = from(Req, #{}), + ?assertEqual(4, length(Msgs)), + [_, Msg1, Msg2, Msg3] = Msgs, + ?assertEqual(<<"a">>, hb_maps:get(<<"path">>, Msg1)), ?assertEqual( {resolve, [ - #{ path => <<"x">> }, - #{ path => <<"y">> }, - #{ path => <<"z">> } + #{}, + #{ <<"path">> => <<"x">> }, + #{ <<"path">> => <<"y">> }, + #{ <<"path">> => <<"z">> } ] }, Msg2 ), - ?assertEqual(<<"z">>, maps:get(path, Msg3)). + ?assertEqual(<<"z">>, hb_maps:get(<<"path">>, Msg3)). inlined_keys_test() -> Req = #{ <<"method">> => <<"POST">>, - <<"relative-reference">> => <<"/a/b+K1=V1/c+K2=V2">> + <<"path">> => <<"/a/b&k1=v1/c&k2=v2">> }, - Msgs = from(Req), - ?assertEqual(3, length(Msgs)), - [Msg1, Msg2, Msg3] = Msgs, - ?assertEqual(<<"V1">>, maps:get(<<"K1">>, Msg2)), - ?assertEqual(<<"V2">>, maps:get(<<"K2">>, Msg3)), - ?assertEqual(not_found, maps:get(<<"K1">>, Msg1, not_found)), - ?assertEqual(not_found, maps:get(<<"K2">>, Msg2, not_found)). + Msgs = from(Req, #{}), + ?assertEqual(4, length(Msgs)), + [_, Msg1, Msg2, Msg3] = Msgs, + ?assertEqual(<<"v1">>, hb_maps:get(<<"k1">>, Msg2)), + ?assertEqual(<<"v2">>, hb_maps:get(<<"k2">>, Msg3)), + ?assertEqual(not_found, hb_maps:get(<<"k1">>, Msg1, not_found)), + ?assertEqual(not_found, hb_maps:get(<<"k2">>, Msg2, not_found)). + +inlined_quoted_key_test() -> + Req = #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"/a/b&k1=\"v/1\"/c&k2=v2">> + }, + Msgs = from(Req, #{}), + ?assertEqual(4, length(Msgs)), + [_, Msg1, Msg2, Msg3] = Msgs, + ?assertEqual(<<"v/1">>, hb_maps:get(<<"k1">>, Msg2)), + ?assertEqual(<<"v2">>, hb_maps:get(<<"k2">>, Msg3)), + ?assertEqual(not_found, hb_maps:get(<<"k1">>, Msg1, not_found)), + ?assertEqual(not_found, hb_maps:get(<<"k2">>, Msg2, not_found)), + ReqB = #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"/~profile@1.0/eval=%22~meta@1.0/info%22">> + }, + MsgsB = from(ReqB, #{}), + [_, Msg2b] = MsgsB, + ?assertEqual(<<"~meta@1.0/info">>, hb_maps:get(<<"eval">>, Msg2b)). + +inlined_assumed_key_test() -> + Req = #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"/a/b=4/c&k2=v2">> + }, + Msgs = from(Req, #{}), + ?assertEqual(4, length(Msgs)), + [_, Msg1, Msg2, Msg3] = Msgs, + ?event({parsed, Msgs}), + ?assertEqual(<<"4">>, hb_maps:get(<<"b">>, Msg2)), + ?assertEqual(not_found, hb_maps:get(<<"b">>, Msg1, not_found)), + ?assertEqual(not_found, hb_maps:get(<<"b">>, Msg3, not_found)), + ReqB = #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"/a/b+integer=4/c&k2=v2">> + }, + MsgsB = from(ReqB, #{}), + [_, Msg1b, Msg2b, Msg3b] = MsgsB, + ?event({parsed, MsgsB}), + ?assertEqual(4, hb_maps:get(<<"b">>, Msg2b)), + ?assertEqual(not_found, hb_maps:get(<<"b">>, Msg1b, not_found)), + ?assertEqual(not_found, hb_maps:get(<<"b">>, Msg3b, not_found)). multiple_inlined_keys_test() -> - Path = <<"/a/b+K1=V1&K2=V2">>, + Path = <<"/a/b&k1=v1&k2=v2">>, Req = #{ <<"method">> => <<"POST">>, - <<"relative-reference">> => Path + <<"path">> => Path }, - Msgs = from(Req), - ?assertEqual(2, length(Msgs)), - [Msg1, Msg2] = Msgs, - ?assertEqual(not_found, maps:get(<<"K1">>, Msg1, not_found)), - ?assertEqual(not_found, maps:get(<<"K2">>, Msg1, not_found)), - ?assertEqual(<<"V1">>, maps:get(<<"K1">>, Msg2, not_found)), - ?assertEqual(<<"V2">>, maps:get(<<"K2">>, Msg2, not_found)). + Msgs = from(Req, #{}), + ?assertEqual(3, length(Msgs)), + [_, Msg1, Msg2] = Msgs, + ?assertEqual(not_found, hb_maps:get(<<"k1">>, Msg1, not_found)), + ?assertEqual(not_found, hb_maps:get(<<"k2">>, Msg1, not_found)), + ?assertEqual(<<"v1">>, hb_maps:get(<<"k1">>, Msg2, not_found)), + ?assertEqual(<<"v2">>, hb_maps:get(<<"k2">>, Msg2, not_found)). subpath_in_inlined_test() -> - Path = <<"/Part1/Part2+Test=1&B=(/x/y)/Part3">>, + Path = <<"/part1/part2&test=1&b=(/x/y)/part3">>, Req = #{ - <<"relative-reference">> => Path + <<"path">> => Path }, - Msgs = from(Req), - ?assertEqual(3, length(Msgs)), - [First, Second, Third] = Msgs, - ?assertEqual(<<"Part1">>, maps:get(path, First)), - ?assertEqual(<<"Part3">>, maps:get(path, Third)), + Msgs = from(Req, #{}), + ?assertEqual(4, length(Msgs)), + [_, First, Second, Third] = Msgs, + ?assertEqual(<<"part1">>, hb_maps:get(<<"path">>, First)), + ?assertEqual(<<"part3">>, hb_maps:get(<<"path">>, Third)), ?assertEqual( - {resolve, [#{ path => <<"x">> }, #{ path => <<"y">> }] }, - maps:get(<<"B">>, Second) + {resolve, [#{}, #{ <<"path">> => <<"x">> }, #{ <<"path">> => <<"y">> }] }, + hb_maps:get(<<"b">>, Second) ). path_parts_test() -> ?assertEqual( - [<<"a">>, <<"b+c=(/d/e)">>, <<"f">>], - path_parts($/, <<"/a/b+c=(/d/e)/f">>) + [<<"a">>, <<"b&c=(/d/e)">>, <<"f">>], + path_parts($/, <<"/a/b&c=(/d/e)/f">>) ), ?assertEqual([<<"a">>], path_parts($/, <<"/a">>)), ?assertEqual([<<"a">>, <<"b">>, <<"c">>], path_parts($/, <<"/a/b/c">>)), @@ -437,12 +860,12 @@ path_parts_test() -> [ <<"IYkkrqlZNW_J-4T-5eFApZOMRl5P4VjvrcOXWvIqB1Q">>, <<"msg2">> - ], + ], path_parts($/, <<"/IYkkrqlZNW_J-4T-5eFApZOMRl5P4VjvrcOXWvIqB1Q/msg2">>) ), ?assertEqual( - [<<"a">>, <<"b+K1=V1">>, <<"c+K2=V2">>], - path_parts($/, <<"/a/b+K1=V1/c+K2=V2">>) + [<<"a">>, <<"b&K1=V1">>, <<"c&K2=V2">>], + path_parts($/, <<"/a/b&K1=V1/c&K2=V2">>) ), ?assertEqual( [<<"a">>, <<"(x/y/z)">>, <<"c">>], diff --git a/src/hb_store.erl b/src/hb_store.erl index 319bd1517..0d8eb0178 100644 --- a/src/hb_store.erl +++ b/src/hb_store.erl @@ -1,36 +1,171 @@ +%%% @doc A simple abstraction layer for AO key value store operations. +%%% +%%% This interface allows us to swap out the underlying store implementation(s) +%%% as desired, without changing the API that `hb_cache` employs. Additionally, +%%% it enables node operators to customize their configuration to maximize +%%% performance, data availability, and other factors. +%%% +%%% Stores can be represented in a node's configuration as either a single +%%% message, or a (`structured@1.0') list of store messages. If a list of stores +%%% is provided, the node will cycle through each until a viable store is found +%%% to execute the given function. +%%% +%%% A valid store must implement a _subset_ of the following functions: +%%% ``` +%%% start/1: Initialize the store. +%%% stop/1: Stop any processes (etc.) that manage the store. +%%% reset/1: Restore the store to its original, empty state. +%%% scope/0: A tag describing the 'scope' of a stores search: `in_memory', +%%% `local', `remote', `arweave', etc. Used in order to allow +%%% node operators to prioritize their stores for search. +%%% make_group/2: Create a new group of keys in the store with the given ID. +%%% make_link/3: Create a link (implying one key should redirect to another) +%%% from `existing` to `new` (in that order). +%%% type/2: Return whether the value found at the given key is a +%%% `composite' (group) type, or a `simple' direct binary. +%%% read/2: Read the data at the given location, returning a binary +%%% if it is a `simple' value, or a message if it is a complex +%%% term. +%%% write/3: Write the given `key` with the associated `value` (in that +%%% order) to the store. +%%% list/2: For `composite' type keys, return a list of its child keys. +%%% path/2: Optionally transform a list of path parts into the store's +%%% canonical form. +%%% ''' +%%% Each function takes a `store' message first, containing an arbitrary set +%%% of its necessary configuration keys, as well as the `store-module' key which +%%% refers to the Erlang module that implements the store. +%%% +%%% All functions must return `ok` or `{ok, Result}`, as appropriate. Other +%%% results will lead to the store manager (this module) iterating to the next +%%% store message given by the user. If none of the given store messages are +%%% able to execute a requested service, the store manager will return +%%% `not_found`. + -module(hb_store). -export([behavior_info/1]). -export([start/1, stop/1, reset/1]). -export([filter/2, scope/2, sort/2]). --export([type/2, read/2, write/3, list/2]). +-export([type/2, read/2, write/3, list/2, match/2]). -export([path/1, path/2, add_path/2, add_path/3, join/1]). -export([make_group/2, make_link/3, resolve/2]). +-export([find/1]). -export([generate_test_suite/1, generate_test_suite/2, test_stores/0]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%%% A simple abstraction layer for AO key value store operations. -%%% This interface allows us to swap out the underlying store -%%% implementation(s) as desired. -%%% -%%% It takes a list of modules and their options, and calls the appropriate -%%% function on the first module that succeeds. If all modules fail, it returns -%%% {error, no_viable_store}. +%% @doc The number of write and read operations to perform in the benchmark. +-define(STORE_BENCH_WRITE_OPS, 100_000). +-define(STORE_BENCH_READ_OPS, 100_000). +-define(STORE_BENCH_LIST_KEYS, 100_000). +-define(STORE_BENCH_LIST_GROUP_SIZE, 10). +-define(STORE_BENCH_LIST_OPS, 20_000). +-define(BENCH_MSG_WRITE_OPS, 250). +-define(BENCH_MSG_READ_OPS, 250). +-define(BENCH_MSG_DATA_SIZE, 1024). behavior_info(callbacks) -> [ {start, 1}, {stop, 1}, {reset, 1}, {make_group, 2}, {make_link, 3}, {type, 2}, {read, 2}, {write, 3}, - {list, 2}, {path, 2}, {add_path, 3} + {list, 2}, {match, 2}, {path, 2}, {add_path, 3} ]. -define(DEFAULT_SCOPE, local). +-define(DEFAULT_RETRIES, 1). + +%% @doc Store access policies to function names. +-define(STORE_ACCESS_POLICIES, #{ + <<"read">> => [read, resolve, list, type, path, add_path, join], + <<"write">> => [write, make_link, make_group, reset, path, add_path, join], + <<"admin">> => [start, stop, reset] +}). + +%%% Store named terms registry functions. + +%% @doc Set the instance options for a given store module and name combination. +set(StoreOpts, InstanceTerm) -> + Mod = maps:get(<<"store-module">>, StoreOpts), + set( + Mod, + maps:get(<<"name">>, StoreOpts, Mod), + InstanceTerm + ). +set(StoreMod, Name, undefined) -> + StoreRef = {store, StoreMod, Name}, + erlang:erase(StoreRef), + persistent_term:erase(StoreRef); +set(StoreMod, Name, InstanceTerm) -> + StoreRef = {store, StoreMod, Name}, + put(StoreRef, InstanceTerm), + persistent_term:put(StoreRef, InstanceTerm), + ok. + +%% @doc Find or spawn a store instance by its store opts. +-ifdef(STORE_EVENTS). +find(StoreOpts) -> + {Time, Result} = timer:tc(fun() -> do_find(StoreOpts) end), + hb_event:increment(<<"store_duration">>, <<"find">>, #{}, Time), + hb_event:increment(<<"store">>, <<"find">>, #{}, 1), + Result. +-else. +find(StoreOpts) -> + do_find(StoreOpts). +-endif. + +do_find(StoreOpts = #{ <<"store-module">> := Mod }) -> + Name = maps:get(<<"name">>, StoreOpts, Mod), + LookupName = {store, Mod, Name}, + case get(LookupName) of + undefined -> + try persistent_term:get(LookupName) of + Instance1 -> + EnsuredInstance = ensure_instance_alive(StoreOpts, Instance1), + put(LookupName, EnsuredInstance), + EnsuredInstance + catch + error:badarg -> spawn_instance(StoreOpts) + end; + InstanceMessage -> + ensure_instance_alive(StoreOpts, InstanceMessage) + end. + +%% @doc Create a new instance of a store and return its term. +spawn_instance(StoreOpts = #{ <<"store-module">> := Mod }) -> + Name = maps:get(<<"name">>, StoreOpts, Mod), + try Mod:start(StoreOpts) of + ok -> ok; + {ok, InstanceMessage} -> + set(Mod, Name, InstanceMessage), + InstanceMessage; + {error, Reason} -> + ?event(error, {store_start_failed, {Mod, Name, Reason}}), + throw({store_start_failed, {Mod, Name, Reason}}) + catch error:undef -> + ok + end. + +%% @doc Handle a found instance message. If it contains a PID, we check if it +%% is alive. If it does not, we return it as is. +ensure_instance_alive(StoreOpts, InstanceMessage = #{ <<"pid">> := Pid }) -> + case is_process_alive(Pid) of + true -> InstanceMessage; + false -> spawn_instance(StoreOpts) + end; +ensure_instance_alive(_, InstanceMessage) -> + InstanceMessage. %%% Library wrapper implementations. -start(Modules) -> call_all(Modules, start, []). +%% @doc Ensure that a store, or list of stores, have all been started. +start(StoreOpts) when not is_list(StoreOpts) -> start([StoreOpts]); +start([]) -> ok; +start([StoreOpts | Rest]) -> + find(StoreOpts), + start(Rest). -stop(Modules) -> call_function(Modules, stop, []). +stop(Modules) -> + call_function(Modules, stop, []). %% @doc Takes a store object and a filter function or match spec, returning a %% new store object with only the modules that match the filter. The filter @@ -49,7 +184,26 @@ filter(Modules, Filter) -> ). %% @doc Limit the store scope to only a specific (set of) option(s). -%% Takes either a single scope or a list of scopes. +%% Takes either an Opts message or store, and either a single scope or a list +%% of scopes. +scope(Opts, Scope) when is_map(Opts) -> + case hb_opts:get(store, no_viable_store, Opts) of + no_viable_store -> Opts; + Store when is_list(Store) -> + % Store is already a list, apply scope normally + Opts#{ store => scope(Store, Scope) }; + Store when is_map(Store) -> + % Check if Store already has a nested 'store' key + case maps:find(store, Store) of + {ok, _NestedStores} -> + % Already has nested structure, return as-is + Opts; + error -> + % Single store map, wrap in list before scoping + % This ensures consistent behavior + Opts#{ store => scope([Store], Scope) } + end + end; scope(Store, Scope) -> filter( Store, @@ -63,7 +217,7 @@ scope(Store, Scope) -> %% default scope (local). get_store_scope(Store) -> case call_function(Store, scope, []) of - no_viable_store -> ?DEFAULT_SCOPE; + not_found -> ?DEFAULT_SCOPE; Scope -> Scope end. @@ -75,7 +229,7 @@ get_store_scope(Store) -> sort(Stores, PreferenceOrder) when is_list(PreferenceOrder) -> sort( Stores, - maps:from_list( + hb_maps:from_list( [ {Scope, -Index} || @@ -90,8 +244,8 @@ sort(Stores, PreferenceOrder) when is_list(PreferenceOrder) -> sort(Stores, ScoreMap) -> lists:sort( fun(Store1, Store2) -> - maps:get(get_store_scope(Store1), ScoreMap, 0) > - maps:get(get_store_scope(Store2), ScoreMap, 0) + hb_maps:get(get_store_scope(Store1), ScoreMap, 0) > + hb_maps:get(get_store_scope(Store2), ScoreMap, 0) end, Stores ). @@ -133,7 +287,7 @@ path(_, Path) -> path(Path). add_path(Path1, Path2) -> Path1 ++ Path2. add_path(Store, Path1, Path2) -> case call_function(Store, add_path, [Path1, Path2]) of - no_viable_store -> add_path(Path1, Path2); + not_found -> add_path(Path1, Path2); Result -> Result end. @@ -145,59 +299,189 @@ resolve(Modules, Path) -> call_function(Modules, resolve, [Path]). %% structures, so this is likely to be very slow for most stores. list(Modules, Path) -> call_function(Modules, list, [Path]). +%% @doc Match a series of keys and values against the store. Returns +%% `{ok, Matches}' if the match is successful, or `not_found' if there are no +%% messages in the store that feature all of the given key-value pairs. `Matches' +%% is given as a list of IDs. +match(Modules, Match) -> call_function(Modules, match, [Match]). + %% @doc Call a function on the first store module that succeeds. Returns its -%% result, or no_viable_store if none of the stores succeed. -call_function(X, _Function, _Args) when not is_list(X) -> - call_function([X], _Function, _Args); -call_function([], _Function, _Args) -> - no_viable_store; -call_function([{Mod, Opts} | Rest], Function, Args) -> - ?event({calling, Mod, Function, Args}), - try apply(Mod, Function, [Opts | Args]) of +%% result, or `not_found` if none of the stores succeed. If `TIME_CALLS` is set, +%% this function will also time the call and increment the appropriate event +%% counter. +-ifdef(STORE_EVENTS). +call_function(X, Function, Args) -> + {Time, Result} = timer:tc(fun() -> do_call_function(X, Function, Args) end), + ?event(store_events, + {store_call, + {function, Function}, + {args, Args}, + {primary_store, + case X of + [PrimaryStore | _] -> PrimaryStore; + _ -> X + end + }, + {time, Time}, + {result, Result} + } + ), + hb_event:increment(<<"store_duration">>, hb_util:bin(Function), #{}, Time), + hb_event:increment(<<"store">>, hb_util:bin(Function), #{}, 1), + Result. +-else. +call_function(X, Function, Args) -> + do_call_function(X, Function, Args). +-endif. +do_call_function(X, _Function, _Args) when not is_list(X) -> + do_call_function([X], _Function, _Args); +do_call_function([], _Function, _Args) -> + not_found; +do_call_function([Store = #{<<"access">> := Access} | Rest], Function, Args) -> + % If the store has an access controls, check if the function is allowed from + % the stated policies. + IsAdmissible = + lists:any( + fun(Group) -> + lists:any( + fun(F) -> F == Function end, + maps:get(Group, ?STORE_ACCESS_POLICIES, []) + ) + end, + Access + ), + case IsAdmissible of + true -> + do_call_function( + [maps:remove(<<"access">>, Store) | Rest], + Function, + Args + ); + false -> + do_call_function(Rest, Function, Args) + end; +do_call_function([Store = #{<<"store-module">> := Mod} | Rest], Function, Args) -> + % Attempt to apply the function. If it fails, try the next store. + try apply_store_function(Mod, Store, Function, Args) of not_found -> - call_function(Rest, Function, Args); + do_call_function(Rest, Function, Args); Result -> Result - catch - Class:Reason:Stacktrace -> - ?event(error, {store_call_failed, {Class, Reason, Stacktrace}}), - call_function(Rest, Function, Args) + catch _:_:_ -> do_call_function(Rest, Function, Args) + end. + +%% @doc Apply a store function, checking if the store returns a retry request or +%% errors. If it does, attempt to start the store again and retry, up to the +%% given maximum number of times. +apply_store_function(Mod, Store, Function, Args) -> + MaxAttempts = maps:get(<<"max-retries">>, Store, ?DEFAULT_RETRIES) + 1, + apply_store_function(Mod, Store, Function, Args, MaxAttempts). +apply_store_function(_Mod, _Store, _Function, _Args, 0) -> + % Too many attempts have already failed. Bail. + not_found; +apply_store_function(Mod, Store, Function, Args, AttemptsRemaining) -> + try apply(Mod, Function, [Store | Args]) of + retry -> retry(Mod, Store, Function, Args, AttemptsRemaining); + Other -> Other + catch Class:Reason:Stacktrace -> + ?event(store_error, + {store_call_failed_retrying, + #{ + store => Store, + function => Function, + args => Args, + class => Class, + reason => Reason, + stacktrace => Stacktrace + } + } + ), + retry(Mod, Store, Function, Args, AttemptsRemaining) end. +%% @doc Stop and start the store, then retry. +retry(Mod, Store, Function, Args, AttemptsRemaining) -> + % Attempt to stop the store and start it again, then retry. + try Mod:stop(Store) catch _:_ -> ignore_errors end, + set(Store, undefined), + start(Store), + apply_store_function(Mod, Store, Function, Args, AttemptsRemaining - 1). + %% @doc Call a function on all modules in the store. call_all(X, _Function, _Args) when not is_list(X) -> call_all([X], _Function, _Args); call_all([], _Function, _Args) -> ok; -call_all([{Mod, Opts} | Rest], Function, Args) -> - try - apply(Mod, Function, [Opts | Args]) +call_all([Store = #{<<"store-module">> := Mod} | Rest], Function, Args) -> + try apply_store_function(Mod, Function, Store, Args) catch Class:Reason:Stacktrace -> - ?event(error, {store_call_failed, {Class, Reason, Stacktrace}}), + ?event(warning, {store_call_failed, {Class, Reason, Stacktrace}}), ok end, call_all(Rest, Function, Args). %%% Test helpers +%% @doc Return a list of stores for testing. Additional individual functions are +%% used to generate store options for those whose drivers are not built by +%% default into all HyperBEAM distributions. test_stores() -> [ - {hb_store_rocksdb, #{ prefix => "TEST-cache-rocks" }}, - {hb_store_fs, #{ prefix => "TEST-cache-fs" }} + (hb_test_utils:test_store(hb_store_fs))#{ + <<"benchmark-scale">> => 0.001 + }, + (hb_test_utils:test_store(hb_store_lmdb))#{ + <<"benchmark-scale">> => 0.5 + }, + (hb_test_utils:test_store(hb_store_lru))#{ + <<"persistent-store">> => [ + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/lru">> + } + ] + } + ] ++ rocks_stores(). + +-ifdef(ENABLE_ROCKSDB). +rocks_stores() -> + [ + #{ + <<"store-module">> => hb_store_rocksdb, + <<"name">> => <<"cache-TEST/rocksdb">> + } ]. +-else. +rocks_stores() -> []. +-endif. generate_test_suite(Suite) -> generate_test_suite(Suite, test_stores()). generate_test_suite(Suite, Stores) -> + hb:init(), lists:map( - fun(Store = {Mod, _Opts}) -> + fun(Store = #{<<"store-module">> := Mod}) -> {foreach, - fun() -> hb_store:start(Store) end, - fun(_) -> hb_store:reset(Store) end, + fun() -> + hb_store:start(Store) + end, + fun(_) -> + hb_store:reset(Store) + % hb_store:stop(Store) + end, [ - {atom_to_list(Mod) ++ ": " ++ Desc, - fun() -> Test(#{ store => Store }) end} + { + atom_to_list(Mod) ++ ": " ++ Desc, + { + timeout, + 60, + fun() -> + TestResult = Test(Store), + TestResult + end + } + } || {Desc, Test} <- Suite ] @@ -209,31 +493,550 @@ generate_test_suite(Suite, Stores) -> %%% Tests %% @doc Test path resolution dynamics. -simple_path_resolution_test(Opts) -> - Store = hb_opts:get(store, no_viable_store, Opts), - hb_store:write(Store, "test-file", <<"test-data">>), - hb_store:make_link(Store, "test-file", "test-link"), - ?assertEqual({ok, <<"test-data">>}, hb_store:read(Store, "test-link")). +simple_path_resolution_test(Store) -> + ok = hb_store:write(Store, <<"test-file">>, <<"test-data">>), + hb_store:make_link(Store, <<"test-file">>, <<"test-link">>), + ?assertEqual({ok, <<"test-data">>}, hb_store:read(Store, <<"test-link">>)). %% @doc Ensure that we can resolve links recursively. -resursive_path_resolution_test(Opts) -> - Store = hb_opts:get(store, no_viable_store, Opts), - hb_store:write(Store, "test-file", <<"test-data">>), - hb_store:make_link(Store, "test-file", "test-link"), - hb_store:make_link(Store, "test-link", "test-link2"), - ?assertEqual({ok, <<"test-data">>}, hb_store:read(Store, "test-link2")). +resursive_path_resolution_test(Store) -> + hb_store:write(Store, <<"test-file">>, <<"test-data">>), + hb_store:make_link(Store, <<"test-file">>, <<"test-link">>), + hb_store:make_link(Store, <<"test-link">>, <<"test-link2">>), + ?assertEqual({ok, <<"test-data">>}, hb_store:read(Store, <<"test-link2">>)). %% @doc Ensure that we can resolve links through a directory. -hierarchical_path_resolution_test(Opts) -> - Store = hb_opts:get(store, no_viable_store, Opts), - hb_store:make_group(Store, "test-dir1"), - hb_store:write(Store, ["test-dir1", "test-file"], <<"test-data">>), - hb_store:make_link(Store, ["test-dir1"], "test-link"), - ?assertEqual({ok, <<"test-data">>}, hb_store:read(Store, ["test-link", "test-file"])). +hierarchical_path_resolution_test(Store) -> + hb_store:make_group(Store, <<"test-dir1">>), + hb_store:write(Store, [<<"test-dir1">>, <<"test-file">>], <<"test-data">>), + hb_store:make_link(Store, [<<"test-dir1">>], <<"test-link">>), + ?assertEqual( + {ok, <<"test-data">>}, + hb_store:read(Store, [<<"test-link">>, <<"test-file">>]) + ). store_suite_test_() -> - hb_store:generate_test_suite([ + generate_test_suite([ {"simple path resolution", fun simple_path_resolution_test/1}, {"resursive path resolution", fun resursive_path_resolution_test/1}, {"hierarchical path resolution", fun hierarchical_path_resolution_test/1} - ]). \ No newline at end of file + ]). + +benchmark_suite_test_() -> + generate_test_suite([ + {"benchmark key read write", fun benchmark_key_read_write/1}, + {"benchmark list", fun benchmark_list/1}, + {"benchmark message read write", fun benchmark_message_read_write/1} + ]). + +%% @doc Benchmark a store. By default, we write 10,000 keys and read 10,000 +%% keys. This can be altered by setting the `STORE_BENCH_WRITE_OPS' and +%% `STORE_BENCH_READ_OPS' macros. If the `benchmark-scale' key is set in the +%% store message, we use it to scale the number of operations for only that +%% store. This allows slower stores to be tested with fewer operations. +benchmark_key_read_write(Store = #{ <<"benchmark-scale">> := Scale }) -> + benchmark_key_read_write( + Store, + erlang:ceil(Scale * ?STORE_BENCH_WRITE_OPS), + erlang:ceil(Scale * ?STORE_BENCH_READ_OPS) + ); +benchmark_key_read_write(Store) -> + benchmark_key_read_write(Store, ?STORE_BENCH_WRITE_OPS, ?STORE_BENCH_READ_OPS). +benchmark_key_read_write(Store, WriteOps, ReadOps) -> + start(Store), + timer:sleep(100), + ?event( + {benchmarking, + {store, Store}, + {write_ops, WriteOps}, + {read_ops, ReadOps} + } + ), + % Generate random data to write and the keys to read ahead of time. + RandomData = hb_util:human_id(crypto:strong_rand_bytes(32)), + Keys = + lists:map( + fun(N) -> + << "key-", (integer_to_binary(N))/binary >> + end, + lists:seq(1, ReadOps) + ), + {WriteTime, ok} = + timer:tc( + fun() -> + lists:foreach( + fun(Key) -> ok = write(Store, Key, RandomData) end, + Keys + ) + end + ), + % Calculate write rate. + WriteRate = erlang:round(WriteOps / (WriteTime / 1000000)), + hb_format:eunit_print( + "Wrote ~s records in ~p ms (~s records/s)", + [ + hb_util:human_int(WriteOps), + WriteTime/1000, + hb_util:human_int(WriteRate) + ] + ), + % Generate keys to read ahead of time. + ReadKeys = + lists:map( + fun(_) -> + << "key-", (integer_to_binary(rand:uniform(ReadOps)))/binary >> + end, + lists:seq(1, ReadOps) + ), + % Time random reads. + {ReadTime, NotFoundCount} = + timer:tc( + fun() -> + lists:foldl( + fun(Key, Count) -> + case read(Store, Key) of + {ok, _} -> Count; + _ -> Count + 1 + end + end, + 0, + ReadKeys + ) + end + ), + % Calculate read rate. + ReadRate = erlang:round(ReadOps / (ReadTime / 1000000)), + hb_format:eunit_print( + "Read ~s records in ~p ms (~s records/s)", + [ + hb_util:human_int(ReadOps), + ReadTime/1000, + hb_util:human_int(ReadRate) + ] + ), + ?assertEqual(0, NotFoundCount, "Written keys not found in store."). + +benchmark_list(Store = #{ <<"benchmark-scale">> := Scale }) -> + benchmark_list( + Store, + erlang:ceil(Scale * ?STORE_BENCH_LIST_KEYS), + erlang:ceil(Scale * ?STORE_BENCH_LIST_OPS), + erlang:ceil(Scale * ?STORE_BENCH_LIST_GROUP_SIZE) + ); +benchmark_list(Store) -> + benchmark_list( + Store, + ?STORE_BENCH_LIST_KEYS, + ?STORE_BENCH_LIST_OPS, + ?STORE_BENCH_LIST_GROUP_SIZE + ). +benchmark_list(Store, WriteOps, ListOps, GroupSize) -> + start(Store), + timer:sleep(100), + ?event( + {benchmarking, + {store, Store}, + {keys, hb_util:human_int(WriteOps)}, + {groups, hb_util:human_int(WriteOps div GroupSize)}, + {lists, hb_util:human_int(ListOps)} + } + ), + % Generate a random message to write and the keys to read ahead of time. + Groups = + lists:map( + fun(_) -> + GroupID = hb_util:human_id(crypto:strong_rand_bytes(32)), + { + GroupID, + lists:map( + fun(M) -> + { + <<"key-", (integer_to_binary(M))/binary >>, + <<"value-", (integer_to_binary(M))/binary >> + } + end, + lists:seq(1, GroupSize) + ) + } + end, + lists:seq(1, GroupCount = WriteOps div GroupSize) + ), + hb_format:eunit_print( + "Generated ~s groups of ~s keys", + [ + hb_util:human_int(GroupCount), + hb_util:human_int(GroupSize) + ] + ), + {WriteTime, _} = + timer:tc( + fun() -> + lists:map( + fun({GroupID, KeyPairs}) -> + ok = make_group(Store, GroupID), + lists:foreach( + fun({Key, Value}) -> + ok = + write( + Store, + <>, + Value + ) + end, + KeyPairs + ) + end, + Groups + ), + % Perform one list operation to ensure that the write queue is + % flushed. + {LastGroupID, _} = lists:last(Groups), + list(Store, LastGroupID) + end + ), + % Print the results. Our write time is in microseconds, so we normalize it + % to seconds. + hb_test_utils:benchmark_print( + <<"Wrote and flushed">>, + <<"keys">>, + WriteOps, + WriteTime / 1_000_000 + ), + % Generate groups to read ahead of time. + ReadGroups = + lists:map( + fun(_) -> + lists:nth(rand:uniform(GroupCount), Groups) + end, + lists:seq(1, ListOps) + ), + % Time random reads. + {ReadTime, NotFoundCount} = + timer:tc( + fun() -> + lists:foldl( + fun({GroupID, GroupKeyValues}, Count) -> + ExpectedKeys = + [ KeyInGroup || {KeyInGroup, _} <- GroupKeyValues ], + case list(Store, GroupID) of + {ok, ListedKeys} -> + Res = + lists:all( + fun({KeyInGroup, _ExpectedValue}) -> + lists:member(KeyInGroup, ListedKeys) + end, + GroupKeyValues + ), + case Res of + true -> Count; + _ -> + ?event( + {list_group_not_found, + {group, GroupID}, + {received_keys, ListedKeys}, + {expected_keys, ExpectedKeys} + } + ), + Count + 1 + end; + _ -> + ?event( + {list_group_not_found, + {group, GroupID}, + {expected_keys, ExpectedKeys} + } + ), + Count + 1 + end + end, + 0, + ReadGroups + ) + end + ), + % Print the results. + hb_test_utils:benchmark_print( + <<"Listed">>, + <<"groups">>, + ListOps, + ReadTime / 1_000_000 + ), + ?assertEqual(0, NotFoundCount, "Groups listed in correctly."). + +benchmark_message_read_write(Store = #{ <<"benchmark-scale">> := Scale }) -> + benchmark_message_read_write( + Store, + erlang:ceil(Scale * ?BENCH_MSG_WRITE_OPS), + erlang:ceil(Scale * ?BENCH_MSG_READ_OPS) + ); +benchmark_message_read_write(Store) -> + benchmark_message_read_write(Store, ?BENCH_MSG_WRITE_OPS, ?BENCH_MSG_READ_OPS). +benchmark_message_read_write(Store, WriteOps, ReadOps) -> + start(Store), + Opts = #{ store => Store, priv_wallet => hb:wallet() }, + TestDataSize = ?BENCH_MSG_DATA_SIZE * 8, % in _bits_ + timer:sleep(100), + ?event( + {benchmarking, + {store, Store}, + {write_ops, WriteOps}, + {read_ops, ReadOps} + } + ), + % Generate a random message to write and the keys to read ahead of time. + Msgs = + lists:map( + fun(N) -> + #{ + <<"process">> => hb_util:human_id(crypto:strong_rand_bytes(32)), + <<"slot">> => N, + <<"message">> => + hb_message:commit( + #{ + <<"body">> => <<"test", 0:TestDataSize, N:32>> + }, + Opts + ) + } + end, + lists:seq(1, WriteOps) + ), + hb_format:eunit_print( + "Generated ~s messages (size ~s bits)", + [ + hb_util:human_int(WriteOps), + hb_util:human_int(TestDataSize) + ] + ), + {WriteTime, MsgPairs} = + timer:tc( + fun() -> + lists:map( + fun(Msg) -> + {hb_util:ok(hb_cache:write(Msg, Opts)), Msg} + end, + Msgs + ) + end + ), + % Print the results. Our write time is in microseconds, so we normalize it + % to seconds. + hb_test_utils:benchmark_print( + <<"Wrote">>, + <<"messages">>, + WriteOps, + WriteTime / 1_000_000 + ), + % Generate keys to read ahead of time. + ReadKeys = + lists:map( + fun(_) -> + lists:nth(rand:uniform(length(MsgPairs)), MsgPairs) + end, + lists:seq(1, ReadOps) + ), + % Time random reads. + {ReadTime, NotFoundCount} = + timer:tc( + fun() -> + lists:foldl( + fun({MsgID, Msg}, Count) -> + case hb_cache:read(MsgID, Opts) of + {ok, Msg1} -> + case hb_cache:ensure_all_loaded(Msg1, Opts) of + Msg -> Count; + _ -> Count + 1 + end; + _ -> Count + 1 + end + end, + 0, + ReadKeys + ) + end + ), + % Print the results. + hb_test_utils:benchmark_print( + <<"Read">>, + <<"messages">>, + ReadOps, + ReadTime / 1_000_000 + ), + ?assertEqual(0, NotFoundCount, "Written keys not found in store."). + +%%% Access Control Tests + +%% @doc Test that read-only stores allow read operations but block write operations +read_only_access_test() -> + TestStore = hb_test_utils:test_store(hb_store_fs, <<"access-read-only">>), + ReadOnlyStore = TestStore#{<<"access">> => [<<"read">>]}, + WriteStore = hb_test_utils:test_store(hb_store_fs, <<"access-write">>), + StoreList = [ReadOnlyStore, WriteStore], + TestKey = <<"test-key">>, + TestValue = <<"test-value">>, + start(StoreList), + ?event(testing, {read_only_test_started}), + WriteResponse = write(StoreList, TestKey, TestValue), + ?assertEqual(ok, WriteResponse), + ?event(testing, {write_used_fallback_store, WriteResponse}), + ReadResponse = read(StoreList, TestKey), + ?assertEqual({ok, TestValue}, ReadResponse), + ?event(testing, {read_succeeded, ReadResponse}), + ReadOnlyStoreState = read([ReadOnlyStore], TestKey), + WriteStoreState = read([WriteStore], TestKey), + ?event(testing, { + store_state, {read_only, ReadOnlyStoreState},{ write, WriteStoreState} + }), + ?assertEqual(not_found, ReadOnlyStoreState), + ?assertEqual({ok, TestValue}, WriteStoreState). + +%% @doc Test that write-only stores allow write operations but block read operations +write_only_access_test() -> + WriteOnlyStore = + (hb_test_utils:test_store(hb_store_fs, <<"access-write-only">>))#{ + <<"access">> => [<<"write">>] + }, + ReadStore = hb_test_utils:test_store(hb_store_fs, <<"access-read-fallback">>), + StoreList = [WriteOnlyStore, ReadStore], + TestKey = <<"write-test-key">>, + TestValue = <<"write-test-value">>, + start(StoreList), + ?event(testing, {write_only_test_started}), + ?assertEqual(ok, write(StoreList, TestKey, TestValue)), + ?event(testing, {write_succeeded_on_write_only}), + ReadStoreState = read(StoreList, TestKey), + ?assertEqual(not_found, ReadStoreState), + ?event(testing, {read_skipped_write_only_store, ReadStoreState}), + WriteOnlyStoreNoAccess = maps:remove(<<"access">>, WriteOnlyStore), + ReadStoreNoAccess = read([WriteOnlyStoreNoAccess], TestKey), + ?event(testing, {store, ReadStoreNoAccess}), + ?assertEqual({ok, TestValue}, ReadStoreNoAccess). + +%% @doc Test admin-only stores for start/stop/reset operations +admin_only_access_test() -> + AdminOnlyStore = + (hb_test_utils:test_store(hb_store_fs, <<"access-admin-only">>))#{ + <<"access">> => [<<"admin">>, <<"read">>, <<"write">>] + }, + StoreList = [AdminOnlyStore], + TestKey = <<"admin-test-key">>, + TestValue = <<"admin-test-value">>, + start(StoreList), + ?assertEqual(ok, write(StoreList, TestKey, TestValue)), + ?assertEqual({ok, TestValue}, read(StoreList, TestKey)), + reset(StoreList), + ?assertEqual(ok, start(StoreList)), + ?assertEqual(not_found, read(StoreList, TestKey)). + +%% @doc Test multiple access permissions +multi_access_permissions_test() -> + ReadWriteStore = + (hb_test_utils:test_store(hb_store_fs, <<"access-read-write">>))#{ + <<"access">> => [<<"read">>, <<"write">>] + }, + AdminStore = + (hb_test_utils:test_store(hb_store_fs, <<"access-admin-fallback">>))#{ + <<"access">> => [<<"admin">>] + }, + StoreList = [ReadWriteStore, AdminStore], + TestKey = <<"multi-access-key">>, + TestValue = <<"multi-access-value">>, + start(StoreList), + ?event(testing, {multi_access_test_started}), + ?assertEqual(ok, write(StoreList, TestKey, TestValue)), + ?event(testing, {write_succeeded_on_read_write_store}), + ?assertEqual({ok, TestValue}, read(StoreList, TestKey)), + ?event(testing, {read_succeeded_on_read_write_store}), + reset(StoreList), + ?assertEqual(ok, start(StoreList)), + ?assertEqual(not_found, read(StoreList, TestKey)). + +%% @doc Test access control with a list of stores. +store_access_list_test() -> + % Chain: Read-only -> Write-only -> Unrestricted + ReadOnlyStore = + (hb_test_utils:test_store(hb_store_fs, <<"chain-read-only">>))#{ + <<"access">> => [<<"read">>] + }, + WriteOnlyStore = + (hb_test_utils:test_store(hb_store_fs, <<"chain-write-only">>))#{ + <<"access">> => [<<"write">>] + }, + UnrestrictedStore = + hb_test_utils:test_store(hb_store_fs, <<"chain-unrestricted">>), + StoreChain = [ReadOnlyStore, WriteOnlyStore, UnrestrictedStore], + TestKey = <<"chain-test-key">>, + TestValue = <<"chain-test-value">>, + start(StoreChain), + ?event(testing, {fallback_chain_test_started, length(StoreChain)}), + ?assertEqual(ok, write(StoreChain, TestKey, TestValue)), + ?event(testing, {write_used_second_store_in_chain}), + ?assertEqual(not_found, read(StoreChain, TestKey)), + ?event(testing, {read_fell_through_entire_chain}), + WriteOnlyNoAccess = maps:remove(<<"access">>, WriteOnlyStore), + ?assertEqual({ok, TestValue}, read([WriteOnlyNoAccess], TestKey)). + +%% @doc Test invalid access permissions are ignored +invalid_access_permissions_test() -> + InvalidAccessStore = + (hb_test_utils:test_store(hb_store_fs, <<"access-invalid">>))#{ + <<"access">> => [<<"invalid-policy">>, <<"nonexistent-policy">>] + }, + FallbackStore = hb_test_utils:test_store(hb_store_fs, <<"access-fallback">>), + StoreList = [InvalidAccessStore, FallbackStore], + TestKey = <<"invalid-access-key">>, + TestValue = <<"invalid-access-value">>, + start(StoreList), + ?event(testing, {invalid_access_test_started}), + ?assertEqual(ok, write(StoreList, TestKey, TestValue)), + ?event(testing, {write_used_fallback_store}), + ?assertEqual({ok, TestValue}, read(StoreList, TestKey)), + ?event(testing, {read_used_fallback_store}), + InvalidStoreNoAccess = maps:remove(<<"access">>, InvalidAccessStore), + start([InvalidStoreNoAccess]), + ?assertEqual(not_found, read([InvalidStoreNoAccess], TestKey)). + +%% @doc Test list operations with access control +list_access_control_test() -> + ReadOnlyStore = + (hb_test_utils:test_store(hb_store_fs, <<"list-read-only">>))#{ + <<"access">> => [<<"read">>] + }, + WriteStore = hb_test_utils:test_store(hb_store_fs, <<"list-write">>), + StoreList = [ReadOnlyStore, WriteStore], + ListGroup = <<"list-test-group">>, + TestKey = <<"list-test-key">>, + TestValue = <<"list-test-value">>, + start(StoreList), + ?event(testing, {list_access_test_started}), + GroupResult = make_group(StoreList, ListGroup), + ?assertEqual(ok, GroupResult), + ?event(testing, {group_created, GroupResult}), + WriteResponse = write(StoreList, [ListGroup, TestKey], TestValue), + ?assertEqual(ok, WriteResponse), + ListResult = list(StoreList, ListGroup), + ListValue = read(StoreList, [ListGroup, TestKey]), + ?event(testing, {list_result, ListResult, ListValue}), + ?assertEqual({ok,[TestKey]}, ListResult), + ?assertEqual({ok,TestValue}, ListValue). + +%% @doc Test make_link operations with write access +make_link_access_test() -> + WriteOnlyStore = + (hb_test_utils:test_store(hb_store_fs, <<"link-write-only">>))#{ + <<"access">> => [<<"write">>,<<"read">>] + }, + FallbackStore = hb_test_utils:test_store(hb_store_fs, <<"link-fallback">>), + StoreList = [WriteOnlyStore, FallbackStore], + SourceKey = <<"link-source">>, + TargetKey = <<"link-target">>, + TestValue = <<"link-test-value">>, + start(StoreList), + ?event(testing, {make_link_access_test_started}), + ?assertEqual(ok, write(StoreList, TargetKey, TestValue)), + LinkResult = make_link(StoreList, TargetKey, SourceKey), + ?event(testing, {make_link_result, LinkResult}), + ReadResult = read(StoreList, SourceKey), + ?event(testing, {read_linked_value, ReadResult}), + ?assertEqual({ok, TestValue}, ReadResult), + ?assertEqual(ok, LinkResult). \ No newline at end of file diff --git a/src/hb_store_fs.erl b/src/hb_store_fs.erl index af4eedec3..ac2bf8849 100644 --- a/src/hb_store_fs.erl +++ b/src/hb_store_fs.erl @@ -1,56 +1,80 @@ +%%% @doc A key-value store implementation, following the `hb_store' behavior +%%% and interface. This implementation utilizes the node's local file system as +%%% its storage mechanism, offering an alternative to other store's that require +%%% the compilation of additional libraries in order to function. +%%% +%%% As this store implementation operates using Erlang's native `file' and +%%% `filelib' mechanisms, it largely inherits its performance characteristics +%%% from those of the underlying OS/filesystem drivers. Certain filesystems can +%%% be quite performant for the types of workload that HyperBEAM AO-Core execution +%%% requires (many reads and writes to explicit keys, few directory 'listing' or +%%% search operations), awhile others perform suboptimally. +%%% +%%% Additionally, thisstore implementation offers the ability for simple +%%% integration of HyperBEAM with other non-volatile storage media: `hb_store_fs' +%%% will interact with any service that implements the host operating system's +%%% native filesystem API. By mounting devices via `FUSE' (etc), HyperBEAM is +%%% able to interact with a large number of existing storage systems (for example, +%%% S3-compatible cloud storage APIs, etc). -module(hb_store_fs). -behavior(hb_store). --export([start/1, stop/1, reset/1, scope/1]). +-export([start/1, stop/1, reset/1, scope/0, scope/1]). -export([type/2, read/2, write/3, list/2]). -export([make_group/2, make_link/3, resolve/2]). -include_lib("kernel/include/file.hrl"). -include("include/hb.hrl"). -%%% A key-value store abstraction, such that the underlying implementation -%%% can be swapped out easily. The default implementation is a file-based -%%% store. - -start(#{ prefix := DataDir }) -> +%% @doc Initialize the file system store with the given data directory. +start(#{ <<"name">> := DataDir }) -> ok = filelib:ensure_dir(DataDir). -stop(#{ prefix := _DataDir }) -> +%% @doc Stop the file system store. Currently a no-op. +stop(#{ <<"name">> := _DataDir }) -> ok. %% @doc The file-based store is always local, for now. In the future, we may %% want to allow that an FS store is shared across a cluster and thus remote. -scope(_) -> local. +scope() -> local. +scope(#{ <<"scope">> := Scope }) -> Scope; +scope(_) -> scope(). -reset(#{ prefix := DataDir }) -> - os:cmd("rm -Rf " ++ DataDir), - ok = filelib:ensure_dir(DataDir), +%% @doc Reset the store by completely removing its directory and recreating it. +reset(#{ <<"name">> := DataDir }) -> + % Use pattern that completely removes directory then recreates it + os:cmd(binary_to_list(<< "rm -Rf ", DataDir/binary >>)), ?event({reset_store, {path, DataDir}}). %% @doc Read a key from the store, following symlinks as needed. read(Opts, Key) -> read(add_prefix(Opts, resolve(Opts, Key))). read(Path) -> - ?event({read, Path}), - case file:read_file_info(Path) of - {ok, #file_info{type = regular}} -> - {ok, _} = file:read_file(Path); - _ -> - case file:read_link(Path) of - {ok, Link} -> - ?event({link_found, Path, Link}), - read(Link); - _ -> - not_found - end - end. + ?event({read, Path}), + case file:read_file_info(Path) of + {ok, #file_info{type = regular}} -> + {ok, _} = file:read_file(Path); + _ -> + case file:read_link(Path) of + {ok, Link} -> + ?event({link_found, Path, Link}), + read(Link); + _ -> + not_found + end + end. +%% @doc Write a value to the specified path in the store. write(Opts, PathComponents, Value) -> Path = add_prefix(Opts, PathComponents), ?event({writing, Path, byte_size(Value)}), filelib:ensure_dir(Path), ok = file:write_file(Path, Value). +%% @doc List contents of a directory in the store. list(Opts, Path) -> - file:list_dir(add_prefix(Opts, Path)). + case file:list_dir(add_prefix(Opts, Path)) of + {ok, Files} -> {ok, lists:map(fun hb_util:bin/1, Files)}; + {error, _} -> not_found + end. %% @doc Replace links in a path successively, returning the final path. %% Each element of the path is resolved in turn, with the result of each @@ -63,22 +87,31 @@ list(Opts, Path) -> %% %% will resolve "a/b/c" to "Correct data". resolve(Opts, RawPath) -> - Res = resolve(Opts, "", hb_path:term_to_path_parts(hb_store:join(RawPath))), + Res = resolve(Opts, "", hb_path:term_to_path_parts(hb_store:join(RawPath), Opts)), ?event({resolved, RawPath, Res}), Res. resolve(_, CurrPath, []) -> hb_store:join(CurrPath); resolve(Opts, CurrPath, [Next|Rest]) -> PathPart = hb_store:join([CurrPath, Next]), - ?event({resolving, {accumulated_path, CurrPath}, {next_segment, Next}, {generated_partial_path_to_test, PathPart}}), + ?event( + {resolving, + {accumulated_path, CurrPath}, + {next_segment, Next}, + {generated_partial_path_to_test, PathPart} + } + ), case file:read_link(add_prefix(Opts, PathPart)) of {ok, RawLink} -> Link = remove_prefix(Opts, RawLink), resolve(Opts, Link, Rest); + {error, enoent} -> + not_found; _ -> resolve(Opts, PathPart, Rest) end. +%% @doc Determine the type of a key in the store. type(Opts, Key) -> type(add_prefix(Opts, Key)). type(Path) -> @@ -95,26 +128,67 @@ type(Path) -> end end. -make_group(#{ prefix := DataDir }, Path) -> - P = hb_store:join([DataDir, Path]), +%% @doc Create a directory (group) in the store. +make_group(Opts = #{ <<"name">> := _DataDir }, Path) -> + P = add_prefix(Opts, Path), ?event({making_group, P}), - ok = filelib:ensure_dir(P). + % We need to ensure that the parent directory exists, so that we can + % make the group. + filelib:ensure_dir(P), + case file:make_dir(P) of + ok -> ok; + {error, eexist} -> ok + end. +%% @doc Create a symlink, handling the case where the link would point to itself. make_link(_, Link, Link) -> ok; make_link(Opts, Existing, New) -> ?event({symlink, - add_prefix(Opts, Existing), - P2 = add_prefix(Opts, New)}), + add_prefix(Opts, Existing), + P2 = add_prefix(Opts, New)}), filelib:ensure_dir(P2), - file:make_symlink( - add_prefix(Opts, Existing), - add_prefix(Opts, New) - ). + case file:make_symlink(add_prefix(Opts, Existing), N = add_prefix(Opts, New)) of + ok -> ok; + {error, eexist} -> + file:delete(N), + R = file:make_symlink(add_prefix(Opts, Existing), N), + ?event(debug_fs, + {symlink_recreated, + {existing, Existing}, + {new, New}, + {result, R} + } + ), + R + end. %% @doc Add the directory prefix to a path. -add_prefix(#{ prefix := Prefix }, Path) -> - hb_store:join([Prefix, Path]). +add_prefix(#{ <<"name">> := Prefix }, Path) -> + ?event({add_prefix, Prefix, Path}), + % Check if the prefix is an absolute path + IsAbsolute = is_binary(Prefix) andalso binary:first(Prefix) =:= $/ orelse + is_list(Prefix) andalso hd(Prefix) =:= $/, + % Join the paths + JoinedPath = hb_store:join([Prefix, Path]), + % If the prefix was absolute, ensure the joined path is also absolute + case IsAbsolute of + true -> + case is_binary(JoinedPath) of + true -> + case binary:first(JoinedPath) of + $/ -> JoinedPath; + _ -> <<"/", JoinedPath/binary>> + end; + false -> + case JoinedPath of + [$/ | _] -> JoinedPath; + _ -> [$/ | JoinedPath] + end + end; + false -> + JoinedPath + end. %% @doc Remove the directory prefix from a path. -remove_prefix(#{ prefix := Prefix }, Path) -> +remove_prefix(#{ <<"name">> := Prefix }, Path) -> hb_util:remove_common(Path, Prefix). \ No newline at end of file diff --git a/src/hb_store_gateway.erl b/src/hb_store_gateway.erl new file mode 100644 index 000000000..86cb24624 --- /dev/null +++ b/src/hb_store_gateway.erl @@ -0,0 +1,357 @@ +%%% @doc A store module that reads data from the nodes Arweave gateway and +%%% GraphQL routes, additionally including additional store-specific routes. +-module(hb_store_gateway). +-export([scope/1, type/2, read/2, resolve/2, list/2]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc The scope of a GraphQL store is always remote, due to performance. +scope(_) -> remote. +resolve(_, Key) -> Key. + +list(StoreOpts, Key) -> + ?event(store_gateway, executing_list), + case read(StoreOpts, Key) of + not_found -> not_found; + {ok, Message} -> {ok, hb_maps:keys(Message, StoreOpts)} + end. + +%% @doc Get the type of the data at the given key. We potentially cache the +%% result, so that we don't have to read the data from the GraphQL route +%% multiple times. +type(StoreOpts, Key) -> + ?event(store_gateway, executing_type), + case read(StoreOpts, Key) of + not_found -> not_found; + {ok, Data} -> + ?event({type, hb_private:reset(hb_message:uncommitted(Data, StoreOpts))}), + IsFlat = lists:all( + fun({_, Value}) -> not is_map(Value) end, + hb_maps:to_list( + hb_private:reset( + hb_message:uncommitted(Data, StoreOpts) + ), + StoreOpts + ) + ), + if + IsFlat -> simple; + true -> composite + end + end. + +%% @doc Read the data at the given key from the GraphQL route. Will only attempt +%% to read the data if the key is an ID. +read(StoreOpts, Key) -> + case hb_path:term_to_path_parts(Key, StoreOpts) of + [ID] when ?IS_ID(ID) -> + ?event({read, StoreOpts, Key}), + case hb_gateway_client:read(Key, StoreOpts) of + {error, _} -> + ?event(store_gateway, {read_not_found, {key, ID}}), + not_found; + {ok, Message} -> + ?event(store_gateway, {read_found, {key, ID}}), + try hb_store_remote_node:maybe_cache(StoreOpts, Message) + catch _:_ -> ignored end, + {ok, Message} + end; + _ -> + ?event({ignoring_non_id, Key}), + not_found + end. + +%%% Tests + +%% @doc Store is accessible via the default options. +graphql_as_store_test_() -> + hb_http_server:start_node(#{}), + {timeout, 10, fun() -> + hb_http_server:start_node(#{}), + ?assertMatch( + {ok, #{ <<"app-name">> := <<"aos">> }}, + hb_store:read( + [#{ <<"store-module">> => hb_store_gateway }], + <<"BOogk_XAI3bvNWnxNxwxmvOfglZt17o4MOVAdPNZ_ew">> + ) + ) + end}. + +%% @doc Stored messages are accessible via `hb_cache' accesses. +graphql_from_cache_test() -> + hb_http_server:start_node(#{}), + Opts = + #{ + store => + [ + #{ + <<"store-module">> => hb_store_gateway + } + ] + }, + ?assertMatch( + {ok, #{ <<"app-name">> := <<"aos">> }}, + hb_cache:read( + <<"BOogk_XAI3bvNWnxNxwxmvOfglZt17o4MOVAdPNZ_ew">>, + Opts + ) + ). + +manual_local_cache_test() -> + hb_http_server:start_node(#{}), + Local = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/gw-local-cache">> + }, + hb_store:reset(Local), + Gateway = #{ + <<"store-module">> => hb_store_gateway, + <<"local-store">> => Local + }, + {ok, FromRemote} = + hb_cache:read( + <<"BOogk_XAI3bvNWnxNxwxmvOfglZt17o4MOVAdPNZ_ew">>, + #{ store => [Gateway] } + ), + ?event({writing_recvd_to_local, FromRemote}), + {ok, _} = hb_cache:write(FromRemote, #{ store => [Local] }), + {ok, Read} = + hb_cache:read( + <<"BOogk_XAI3bvNWnxNxwxmvOfglZt17o4MOVAdPNZ_ew">>, + #{ store => [Local] } + ), + ?event({read_from_local, Read}), + ?assert(hb_message:match(Read, FromRemote)). + +%% @doc Ensure that saving to the gateway store works. +cache_read_message_test() -> + hb_http_server:start_node(#{}), + Local = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/1">> + }, + hb_store:reset(Local), + WriteOpts = #{ + store => + [ + #{ <<"store-module">> => hb_store_gateway, + <<"local-store">> => [Local] + } + ] + }, + {ok, Written} = + hb_cache:read( + <<"BOogk_XAI3bvNWnxNxwxmvOfglZt17o4MOVAdPNZ_ew">>, + WriteOpts + ), + {ok, Read} = + hb_cache:read( + <<"BOogk_XAI3bvNWnxNxwxmvOfglZt17o4MOVAdPNZ_ew">>, + #{ store => [Local] } + ), + ?assert(hb_message:match(Read, Written)). + +%% @doc Routes can be specified in the options, overriding the default routes. +%% We test this by inversion: If the above cache read test works, then we know +%% that the default routes allow access to the item. If the test below were to +%% produce the same result, despite an empty 'only' route list, then we would +%% know that the module is not respecting the route list. +specific_route_test() -> + hb_http_server:start_node(#{}), + Opts = #{ + store => + [ + #{ <<"store-module">> => hb_store_gateway, + <<"routes">> => [], + <<"only">> => local + } + ] + }, + ?assertMatch( + not_found, + hb_cache:read( + <<"BOogk_XAI3bvNWnxNxwxmvOfglZt17o4MOVAdPNZ_ew">>, + Opts + ) + ). + +%% @doc Test that the default node config allows for data to be accessed. +external_http_access_test() -> + Node = hb_http_server:start_node( + #{ + cache_control => <<"cache">>, + store => + [ + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + }, + #{ <<"store-module">> => hb_store_gateway } + ] + } + ), + ?assertMatch( + {ok, #{ <<"data-protocol">> := <<"ao">> }}, + hb_http:get( + Node, + <<"p45HPD-ENkLS7Ykqrx6p_DYGbmeHDeeF8LJ09N2K53g">>, + #{} + ) + ). + +%% Ensure that we can get data from the gateway and execute upon it. +% resolve_on_gateway_test_() -> +% {timeout, 10, fun() -> +% TestProc = <<"p45HPD-ENkLS7Ykqrx6p_DYGbmeHDeeF8LJ09N2K53g">>, +% EmptyStore = #{ +% <<"store-module">> => hb_store_fs, +% <<"name">> => <<"cache-TEST">> +% }, +% hb_store:reset(EmptyStore), +% hb_http_server:start_node(#{}), +% Opts = #{ +% store => +% [ +% #{ +% <<"store-module">> => hb_store_gateway, +% <<"store">> => false +% }, +% EmptyStore +% ], +% cache_control => <<"cache">> +% }, +% ?assertMatch( +% {ok, #{ <<"type">> := <<"Process">> }}, +% hb_cache:read(TestProc, Opts) +% ), +% % TestProc is an AO Legacynet process: No device tag, so we start by resolving +% % only an explicit key. +% ?assertMatch( +% {ok, <<"Process">>}, +% hb_ao:resolve(TestProc, <<"type">>, Opts) +% ), +% % Next, we resolve the schedule key on the message, as a `process@1.0' +% % message. +% {ok, X} = +% hb_ao:resolve( +% {as, <<"process@1.0">>, TestProc}, +% <<"schedule">>, +% Opts +% ), +% ?assertMatch(#{ <<"assignments">> := _ }, X) +% end}. + +%% @doc Test to verify store opts is being set for Data-Protocol ao +store_opts_test() -> + Opts = #{ + cache_control => <<"cache">>, + store => + [ + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST">> + }, + #{ + <<"store-module">> => hb_store_gateway, + <<"local-store">> => false, + <<"subindex">> => [ + #{ + <<"name">> => <<"Data-Protocol">>, + <<"value">> => <<"ao">> + } + ] + } + ] + }, + Node = hb_http_server:start_node(Opts), + {ok, Res} = + hb_http:get( + Node, + <<"myb2p8_TSM0KSgBMoG-nu6TLuqWwPmdZM5V2QSUeNmM">>, + #{} + ), + ?event(debug_gateway, {res, Res}), + ?assertEqual(<<"Hello World">>, hb_ao:get(<<"data">>, Res)). + +%% @doc Test that items retreived from the gateway store are verifiable. +verifiability_test() -> + hb_http_server:start_node(#{}), + {ok, Message} = + hb_cache:read( + <<"BOogk_XAI3bvNWnxNxwxmvOfglZt17o4MOVAdPNZ_ew">>, + #{ + store => + [ + #{ + <<"store-module">> => hb_store_gateway + } + ] + } + ), + % Ensure that the message is verifiable after being converted to + % httpsig@1.0 and back to structured@1.0. + HTTPSig = + hb_message:convert( + Message, + <<"httpsig@1.0">>, + <<"structured@1.0">>, + #{} + ), + ?assert(hb_message:verify(HTTPSig)), + Structured = + hb_message:convert( + HTTPSig, + <<"structured@1.0">>, + <<"httpsig@1.0">>, + #{} + ), + ?event({verifying, {structured, Structured}, {original, Message}}), + ?assert(hb_message:verify(Structured)). + +%% @doc Test that another HyperBEAM node offering the `~query@1.0' device can +%% be used as a store. +remote_hyperbeam_node_ans104_test() -> + ServerOpts = + #{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store() + }, + Server = hb_http_server:start_node(ServerOpts), + Msg = + hb_message:commit( + #{ + <<"hello">> => <<"world">> + }, + ServerOpts, + #{ <<"commitment-device">> => <<"ans104@1.0">> } + ), + {ok, ID} = hb_cache:write(Msg, ServerOpts), + {ok, ReadMsg} = hb_cache:read(ID, ServerOpts), + ?assert(hb_message:verify(ReadMsg)), + ClientOpts = + #{ + store => + [ + #{ + <<"store-module">> => hb_store_gateway, + <<"node">> => Server + }, + hb_test_utils:test_store() + ], + routes => [ + #{ + % Routes for GraphQL requests to use the remote server's + % GraphQL API. + <<"template">> => <<"/graphql">>, + <<"nodes">> => + [ + #{ + <<"prefix">> => <> + } + ] + } + ] + }, + {ok, Msg2} = hb_cache:read(ID, ClientOpts), + ?assert(hb_message:verify(Msg2)), + ?assert(hb_message:match(Msg, Msg2)). \ No newline at end of file diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl new file mode 100644 index 000000000..dd3101bed --- /dev/null +++ b/src/hb_store_lmdb.erl @@ -0,0 +1,1086 @@ +%% @doc An LMDB (Lightning Memory Database) implementation of the HyperBeam store interface. +%% +%% This module provides a persistent key-value store backend using LMDB, which is a +%% high-performance embedded transactional database. The implementation follows a +%% singleton pattern where each database environment gets its own dedicated server +%% process to manage transactions and coordinate writes. +%% +%% Key features include: +%%
    +%%
  • Asynchronous writes with batched transactions for performance
    • +%%
  • Automatic link resolution for creating symbolic references between keys
    • +%%
  • Group support for organizing hierarchical data structures
    • +%%
  • Prefix-based key listing for directory-like navigation
    • +%%
  • Process-local caching of database handles for efficiency
    • +%%
+%% +%% The module implements a dual-flush strategy: writes are accumulated in memory +%% and flushed either after an idle timeout or when explicitly requested during +%% read operations that encounter cache misses. +-module(hb_store_lmdb). + +%% Public API exports +-export([start/1, stop/1, scope/0, scope/1, reset/1]). +-export([read/2, write/3, list/2, match/2]). +-export([make_group/2, make_link/3, type/2]). +-export([path/2, add_path/3, resolve/2]). + +%% Test framework and project includes +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% Configuration constants with reasonable defaults +-define(DEFAULT_SIZE, 16 * 1024 * 1024 * 1024). % 16GB default database size +-define(CONNECT_TIMEOUT, 6000). % Timeout for server communication +-define(DEFAULT_IDLE_FLUSH_TIME, 5). % Idle server time before auto-flush +-define(DEFAULT_MAX_FLUSH_TIME, 50). % Maximum time between flushes +-define(MAX_REDIRECTS, 1000). % Only resolve 1000 links to data +-define(MAX_PENDING_WRITES, 400). % Force flush after x pending +-define(FOLD_YIELD_INTERVAL, 100). % Yield every x keys + +%% @doc Start the LMDB storage system for a given database configuration. +%% +%% This function initializes or connects to an existing LMDB database instance. +%% It uses a singleton pattern, so multiple calls with the same configuration +%% will return the same server process. The server process manages the LMDB +%% environment and coordinates all database operations. +%% +%% The StoreOpts map must contain a "prefix" key specifying the +%% database directory path. Also the required configuration includes "capacity" +%% for the maximum database size and flush timing parameters. +%% +%% @param StoreOpts A map containing database configuration options +%% @returns {ok, ServerPid} on success, {error, Reason} on failure +start(Opts = #{ <<"name">> := DataDir }) -> + % Ensure the directory exists before opening LMDB environment + DataDirPath = hb_util:list(DataDir), + ok = filelib:ensure_dir(filename:join(DataDirPath, "dummy")), + % Create the LMDB environment with specified size limit + {ok, Env} = + elmdb:env_open( + DataDirPath, + [ + {map_size, maps:get(<<"capacity">>, Opts, ?DEFAULT_SIZE)}, + no_mem_init, no_sync + ] + ), + {ok, DBInstance} = elmdb:db_open(Env, [create]), + % Store both environment and DB instance in persistent_term for later cleanup + StoreKey = {lmdb, ?MODULE, DataDir}, + persistent_term:put(StoreKey, {Env, DBInstance, DataDir}), + {ok, #{ <<"env">> => Env, <<"db">> => DBInstance }}; +start(_) -> + {error, {badarg, <<"StoreOpts must be a map">>}}. + +%% @doc Determine whether a key represents a simple value or composite group. +%% +%% This function reads the value associated with a key and examines its content +%% to classify the entry type. Keys storing the literal binary "group" are +%% considered composite (directory-like) entries, while all other values are +%% treated as simple key-value pairs. +%% +%% This classification is used by higher-level HyperBeam components to understand +%% the structure of stored data and provide appropriate navigation interfaces. +%% +%% @param Opts Database configuration map +%% @param Key The key to examine +%% @returns 'composite' for group entries, 'simple' for regular values +-spec type(map(), binary()) -> composite | simple | not_found. +type(Opts, Key) -> + case read_direct(Opts, Key) of + {ok, Value} -> + case is_link(Value) of + {true, Link} -> + % This is a link, check the target's type + type(Opts, Link); + false -> + case Value of + <<"group">> -> + composite; + _ -> + simple + end + end; + not_found -> not_found + end. + +%% @doc Write a key-value pair to the database asynchronously. +%% +%% This function sends a write request to the database server process and returns +%% immediately without waiting for the write to be committed to disk. The server +%% accumulates writes in a transaction that is periodically flushed based on +%% timing constraints or explicit flush requests. +%% +%% The asynchronous nature provides better performance for write-heavy workloads +%% while the batching strategy ensures data consistency and reduces I/O overhead. +%% However, recent writes may not be immediately visible to readers until the +%% next flush occurs. +%% +%% @param Opts Database configuration map +%% @param Path Binary path to write +%% @param Value Binary value to store +%% @returns 'ok' immediately (write happens asynchronously) +-spec write(map(), binary() | list(), binary()) -> ok. +write(Opts, PathParts, Value) when is_list(PathParts) -> + % Convert to binary + PathBin = to_path(PathParts), + write(Opts, PathBin, Value); +write(Opts, Path, Value) -> + #{ <<"db">> := DBInstance } = find_env(Opts), + ?event({elmdb_write, {db, DBInstance}, {path, Path}, {value, Value}}), + case elmdb:put(DBInstance, Path, Value) of + ok -> ok; + {error, Type, Description} -> + ?event( + error, + {lmdb_error, + {type, Type}, + {description, Description} + } + ), + retry + end. + +%% @doc Read a value from the database by key, with automatic link resolution. +%% +%% This function attempts to read a value directly from the committed database. +%% If the key is not found, it triggers a flush operation to ensure any pending +%% writes are committed before retrying the read. +%% +%% The function automatically handles link resolution: if a stored value begins +%% with the "link:" prefix, it extracts the target key and recursively reads +%% from that location instead. This creates a symbolic link mechanism that +%% allows multiple keys to reference the same underlying data. +%% +%% When given a list of path segments, the function first attempts a direct read +%% for optimal performance. Only if the direct read fails does it perform link +%% resolution at each level of the path except the final segment, allowing path +%% traversal through symbolic links to work transparently. +%% +%% Link resolution is transparent to the caller and can chain through multiple +%% levels of indirection, though care should be taken to avoid circular references. +%% +%% @param Opts Database configuration map +%% @param Path Binary key or list of path segments to read +%% @returns {ok, Value} on success, {error, Reason} on failure +-spec read(map(), binary() | list()) -> {ok, binary()} | {error, term()}. +read(Opts, PathParts) when is_list(PathParts) -> + read(Opts, to_path(PathParts)); +read(Opts, Path) -> + % Try direct read first (fast path for non-link paths) + case read_with_links(Opts, Path) of + {ok, Value} -> + {ok, Value}; + not_found -> + try + PathParts = binary:split(Path, <<"/">>, [global]), + case resolve_path_links(Opts, PathParts) of + {ok, ResolvedPathParts} -> + ResolvedPathBin = to_path(ResolvedPathParts), + read_with_links(Opts, ResolvedPathBin); + {error, _} -> + not_found + end + catch + Class:Reason:Stacktrace -> + ?event(error, + { + resolve_path_links_failed, + {class, Class}, + {reason, Reason}, + {stacktrace, Stacktrace}, + {path, Path} + } + ), + % If link resolution fails, return not_found + not_found + end + end. + +%% @doc Helper function to check if a value is a link and extract the target. +is_link(Value) -> + LinkPrefixSize = byte_size(<<"link:">>), + case byte_size(Value) > LinkPrefixSize andalso + binary:part(Value, 0, LinkPrefixSize) =:= <<"link:">> of + true -> + Link = + binary:part( + Value, + LinkPrefixSize, + byte_size(Value) - LinkPrefixSize + ), + {true, Link}; + false -> + false + end. + +%% @doc Helper function to convert to a path +to_path(PathParts) -> + hb_util:bin(lists:join(<<"/">>, PathParts)). + +%% @doc Unified read function that handles LMDB reads with fallback to the +%% in-process pending writes, if necessary. +%% +%% Returns {ok, Value} or not_found. +read_direct(Opts, Path) -> + #{ <<"db">> := DBInstance } = find_env(Opts), + case elmdb:get(DBInstance, Path) of + {ok, Value} -> {ok, Value}; + {error, not_found} -> not_found; % Normalize error format + not_found -> not_found % Handle both old and new format + end. + +%% @doc Read a value directly from the database with link resolution. +%% This is the internal implementation that handles actual database reads. +read_with_links(Opts, Path) -> + case read_direct(Opts, Path) of + {ok, Value} -> + % Check if this value is actually a link to another key + case is_link(Value) of + {true, Link} -> + % Extract the target key and recursively resolve the link + read_with_links(Opts, Link); + false -> + % Check if this is a group marker - groups should not be + % readable as simple values + case Value of + <<"group">> -> not_found; + _ -> {ok, Value} + end + end; + not_found -> + not_found + end. + +%% @doc Resolve links in a path, checking each segment except the last. +%% Returns the resolved path where any intermediate links have been followed. +resolve_path_links(Opts, Path) -> + resolve_path_links(Opts, Path, 0). + +%% Internal helper with depth limit to prevent infinite loops +resolve_path_links(_Opts, _Path, Depth) when Depth > ?MAX_REDIRECTS -> + % Prevent infinite loops with depth limit + {error, too_many_redirects}; +resolve_path_links(_Opts, [LastSegment], _Depth) -> + % Base case: only one segment left, no link resolution needed + {ok, [LastSegment]}; +resolve_path_links(Opts, Path, Depth) -> + resolve_path_links_acc(Opts, Path, [], Depth). + +%% Internal helper that accumulates the resolved path +resolve_path_links_acc(_Opts, [], AccPath, _Depth) -> + % No more segments to process + {ok, lists:reverse(AccPath)}; +resolve_path_links_acc(_, FullPath = [<<"data">>|_], [], _Depth) -> + {ok, FullPath}; +resolve_path_links_acc(Opts, [Head | Tail], AccPath, Depth) -> + % Build the accumulated path so far + CurrentPath = lists:reverse([Head | AccPath]), + CurrentPathBin = to_path(CurrentPath), + % Check if the accumulated path (not just the segment) is a link + case read_direct(Opts, CurrentPathBin) of + {ok, Value} -> + case is_link(Value) of + {true, Link} -> + % The accumulated path is a link! Resolve it + LinkSegments = binary:split(Link, <<"/">>, [global]), + % Replace the accumulated path with the link target and + % continue with remaining segments + NewPath = LinkSegments ++ Tail, + resolve_path_links(Opts, NewPath, Depth + 1); + false -> + % Not a link, continue accumulating + resolve_path_links_acc(Opts, Tail, [Head | AccPath], Depth) + end; + not_found -> + % Path doesn't exist as a complete link, continue accumulating + resolve_path_links_acc(Opts, Tail, [Head | AccPath], Depth) + end. + +%% @doc Return the scope of this storage backend. +%% +%% The LMDB implementation is always local-only and does not support distributed +%% operations. This function exists to satisfy the HyperBeam store interface +%% contract and inform the system about the storage backend's capabilities. +%% +%% @returns 'local' always +-spec scope() -> local. +scope() -> local. + +%% @doc Return the scope of this storage backend (ignores parameters). +%% +%% This is an alternate form of scope/0 that ignores any parameters passed to it. +%% The LMDB backend is always local regardless of configuration. +%% +%% @param _Opts Ignored parameter +%% @returns 'local' always +-spec scope(term()) -> local. +scope(_) -> scope(). + +%% @doc List all keys that start with a given prefix. +%% +%% This function provides directory-like navigation by finding all keys that +%% begin with the specified path prefix. It uses the native elmdb:list/2 function +%% to efficiently scan through the database and collect matching keys. +%% +%% The implementation returns only the immediate children of the given path, +%% not the full paths. For example, listing "colors/" will return ["red", "blue"] +%% not ["colors/red", "colors/blue"]. +%% +%% If the Path points to a link, the function resolves the link and lists +%% the contents of the target directory instead. +%% +%% This is particularly useful for implementing hierarchical data organization +%% and providing tree-like navigation interfaces in applications. +%% +%% @param StoreOpts Database configuration map +%% @param Path Binary prefix to search for +%% @returns {ok, [Key]} list of matching keys, {error, Reason} on failure +-spec list(map(), binary()) -> {ok, [binary()]} | {error, term()}. +list(Opts, Path) -> + % Check if Path is a link and resolve it if necessary + ResolvedPath = + case read_direct(Opts, Path) of + {ok, Value} -> + case is_link(Value) of + {true, Link} -> + Link; + false -> + % Not a link; use original path + Path + end; + not_found -> + Path + end, + % Ensure path ends with / for elmdb:list API + SearchPath = + case ResolvedPath of + <<>> -> <<>>; % Root path + <<"/">> -> <<>>; % Root path variant + _ -> + case binary:last(ResolvedPath) of + $/ -> ResolvedPath; + _ -> <> + end + end, + % Use native elmdb:list function + #{ <<"db">> := DBInstance } = find_env(Opts), + case elmdb:list(DBInstance, SearchPath) of + {ok, Children} -> {ok, Children}; + {error, not_found} -> {ok, []}; % Normalize new error format + not_found -> {ok, []} % Handle both old and new format + end. + +%% @doc Match a series of keys and values against the database. Returns +%% `{ok, Matches}' if the match is successful, or `not_found' if there are no +%% messages in the store that feature all of the given key-value pairs. `Matches' +%% is given as a list of IDs. +match(Opts, MatchMap) when is_map(MatchMap) -> + match(Opts, maps:to_list(MatchMap)); +match(Opts, MatchKVs) -> + #{ <<"db">> := DBInstance } = find_env(Opts), + WithPrefixes = + lists:map( + fun({Key, Path}) -> + {Key, <<"link:", Path/binary>>} + end, + MatchKVs + ), + ?event({elmdb_match, MatchKVs}), + case elmdb:match(DBInstance, WithPrefixes) of + {ok, Matches} -> + ?event({elmdb_matched, Matches}), + {ok, Matches}; + {error, not_found} -> not_found; + not_found -> not_found + end. + + +%% @doc Create a group entry that can contain other keys hierarchically. +%% +%% Groups in the HyperBeam system represent composite entries that can contain +%% child elements, similar to directories in a filesystem. This function creates +%% a group by storing the special value "group" at the specified key. +%% +%% The group mechanism allows applications to organize data hierarchically and +%% provides semantic meaning that can be used by navigation and visualization +%% tools to present appropriate user interfaces. +%% +%% Groups can be identified later using the type/2 function, which will return +%% 'composite' for group entries versus 'simple' for regular key-value pairs. +%% +%% @param Opts Database configuration map +%% @param GroupName Binary name for the group +%% @returns Result of the write operation +-spec make_group(map(), binary()) -> ok | {error, term()}. +make_group(Opts, GroupName) when is_map(Opts), is_binary(GroupName) -> + write(Opts, GroupName, <<"group">>); +make_group(_,_) -> + {error, {badarg, <<"StoreOps must be map and GroupName must be a binary">>}}. + +%% @doc Ensure all parent groups exist for a given path. +%% +%% This function creates the necessary parent groups for a path, similar to +%% how filesystem stores use ensure_dir. For example, if the path is +%% "a/b/c/file", it will ensure groups "a", "a/b", and "a/b/c" exist. +%% +%% @param Opts Database configuration map +%% @param Path The path whose parents should exist +%% @returns ok +-spec ensure_parent_groups(map(), binary()) -> ok. +ensure_parent_groups(Opts, Path) -> + PathParts = binary:split(Path, <<"/">>, [global]), + case PathParts of + [_] -> + % Single segment, no parents to create + ok; + _ -> + % Multiple segments, create parent groups + ParentParts = lists:droplast(PathParts), + create_parent_groups(Opts, [], ParentParts) + end. + +%% @doc Helper function to recursively create parent groups. +create_parent_groups(_Opts, _Current, []) -> + ok; +create_parent_groups(Opts, Current, [Next | Rest]) -> + NewCurrent = Current ++ [Next], + GroupPath = to_path(NewCurrent), + % Only create group if it doesn't already exist. + case read_direct(Opts, GroupPath) of + not_found -> + make_group(Opts, GroupPath); + {ok, _} -> + % Already exists, skip + ok + end, + create_parent_groups(Opts, NewCurrent, Rest). + +%% @doc Create a symbolic link from a new key to an existing key. +%% +%% This function implements a symbolic link mechanism by storing a special +%% "link:" prefixed value at the new key location. When the new key is read, +%% the system will automatically resolve the link and return the value from +%% the target key instead. +%% +%% Links provide a way to create aliases, shortcuts, or alternative access +%% paths to the same underlying data without duplicating storage. They can +%% be chained together to create complex reference structures, though care +%% should be taken to avoid circular references. +%% +%% The link resolution happens transparently during read operations, making +%% links invisible to most application code while providing powerful +%% organizational capabilities. +%% +%% @param StoreOpts Database configuration map +%% @param Existing The key that already exists and contains the target value +%% @param New The new key that should link to the existing key +%% @returns Result of the write operation +-spec make_link(map(), binary() | list(), binary()) -> ok. +make_link(Opts, Existing, New) when is_list(Existing) -> + ExistingBin = to_path(Existing), + make_link(Opts, ExistingBin, New); +make_link(Opts, Existing, New) -> + ExistingBin = hb_util:bin(Existing), + % Ensure parent groups exist for the new link path (like filesystem ensure_dir) + ensure_parent_groups(Opts, New), + write(Opts, New, <<"link:", ExistingBin/binary>>). + +%% @doc Transform a path into the store's canonical form. +%% For LMDB, paths are simply joined with "/" separators. +path(_Opts, PathParts) when is_list(PathParts) -> + to_path(PathParts); +path(_Opts, Path) when is_binary(Path) -> + Path. + +%% @doc Add two path components together. +%% For LMDB, this concatenates the path lists. +add_path(_Opts, Path1, Path2) when is_list(Path1), is_list(Path2) -> + Path1 ++ Path2; +add_path(Opts, Path1, Path2) when is_binary(Path1), is_binary(Path2) -> + % Convert binaries to lists, concatenate, then convert back + Parts1 = binary:split(Path1, <<"/">>, [global]), + Parts2 = binary:split(Path2, <<"/">>, [global]), + path(Opts, Parts1 ++ Parts2); +add_path(Opts, Path1, Path2) when is_list(Path1), is_binary(Path2) -> + Parts2 = binary:split(Path2, <<"/">>, [global]), + path(Opts, Path1 ++ Parts2); +add_path(Opts, Path1, Path2) when is_binary(Path1), is_list(Path2) -> + Parts1 = binary:split(Path1, <<"/">>, [global]), + path(Opts, Parts1 ++ Path2). + +%% @doc Resolve a path by following any symbolic links. +%% +%% For LMDB, we handle links through our own "link:" prefix mechanism. +%% This function resolves link chains in paths, similar to filesystem symlink resolution. +%% It's used by the cache to resolve paths before type checking and reading. +%% +%% @param StoreOpts Database configuration map +%% @param Path The path to resolve (binary or list) +%% @returns The resolved path as a binary +-spec resolve(map(), binary() | list()) -> binary(). +resolve(Opts, Path) when is_binary(Path) -> + resolve(Opts, binary:split(Path, <<"/">>, [global])); +resolve(Opts, PathParts) when is_list(PathParts) -> + % Handle list paths by resolving directly and converting to binary + case resolve_path_links(Opts, PathParts) of + {ok, ResolvedParts} -> + to_path(ResolvedParts); + {error, _} -> + % If resolution fails, return original path as binary + to_path(PathParts) + end; +resolve(_,_) -> not_found. + +%% @doc Retrieve or create the LMDB environment handle for a database. +find_env(Opts) -> hb_store:find(Opts). + +%% Shutdown LMDB environment and cleanup resources +stop(#{ <<"store-module">> := ?MODULE, <<"name">> := DataDir }) -> + StoreKey = {lmdb, ?MODULE, DataDir}, + close_environment(StoreKey, DataDir); +stop(_InvalidStoreOpts) -> + ok. + +%% Close environment using persistent_term lookup with fallback +close_environment(StoreKey, DataDir) -> + case safe_get_persistent_term(StoreKey) of + {ok, {Env, DBInstance}} -> + close_and_cleanup(Env, DBInstance, StoreKey, DataDir); + not_found -> + ?event({lmdb_stop_not_found_in_persistent_term, DataDir}), + safe_close_by_name(DataDir) + end, + ok. + +%% Get environment and DB instance from persistent_term without exceptions +safe_get_persistent_term(Key) -> + case persistent_term:get(Key, undefined) of + {Env, DBInstance, _DataDir} -> {ok, {Env, DBInstance}}; + {Env, _DataDir} -> {ok, {Env, undefined}}; % Backwards compatibility + _ -> not_found + end. + +%% Close DB instance and environment, then cleanup persistent_term entry +close_and_cleanup(Env, DBInstance, StoreKey, DataDir) -> + % Close DB instance first if it exists + DBCloseResult = safe_close_db(DBInstance), + ?event({db_close_result, DBCloseResult}), + % Then close the environment + EnvCloseResult = safe_close_env(Env), + persistent_term:erase(StoreKey), + case EnvCloseResult of + ok -> ?event({lmdb_stop_success, DataDir}); + {error, Reason} -> ?event({lmdb_stop_error, Reason}) + end. + +%% Close DB instance with error capture +safe_close_db(undefined) -> + ok; % No DB instance to close +safe_close_db(DBInstance) -> + try + elmdb:db_close(DBInstance) + catch + error:Reason -> {error, Reason} + end. + +%% Close environment handle with error capture +safe_close_env(Env) -> + try + elmdb:env_close(Env) + catch + error:Reason -> {error, Reason} + end. + +%% Fallback close by name with error suppression +safe_close_by_name(DataDir) -> + try + elmdb:env_close_by_name(binary_to_list(DataDir)) + catch + error:_ -> ok + end. + +%% @doc Completely delete the database directory and all its contents. +%% +%% This is a destructive operation that removes all data from the specified +%% database. It first performs a graceful shutdown to ensure data consistency, +%% then uses the system shell to recursively delete the entire database +%% directory structure. +%% +%% This function is primarily intended for testing and development scenarios +%% where you need to start with a completely clean database state. It should +%% be used with extreme caution in production environments. +%% +%% @param StoreOpts Database configuration map containing the directory prefix +%% @returns 'ok' when deletion is complete +reset(Opts) -> + case maps:get(<<"name">>, Opts, undefined) of + undefined -> + % No prefix specified, nothing to reset + ok; + DataDir -> + % Stop the store and remove the database. + % stop(Opts), + os:cmd(binary_to_list(<< "rm -Rf ", DataDir/binary >>)), + ok + end. + +%% @doc Test suite demonstrating basic store operations. +%% +%% The following functions implement unit tests using EUnit to verify that +%% the LMDB store implementation correctly handles various scenarios including +%% basic read/write operations, hierarchical listing, group creation, link +%% resolution, and type detection. + +%% @doc Basic store test - verifies fundamental read/write functionality. +%% +%% This test creates a temporary database, writes a key-value pair, reads it +%% back to verify correctness, and cleans up by stopping the database. It +%% serves as a sanity check that the basic storage mechanism is working. +basic_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-1">> + }, + reset(StoreOpts), + Res = write(StoreOpts, <<"Hello">>, <<"World2">>), + ?assertEqual(ok, Res), + {ok, Value} = read(StoreOpts, <<"Hello">>), + ?assertEqual(Value, <<"World2">>), + ok = stop(StoreOpts). + +%% @doc List test - verifies prefix-based key listing functionality. +%% +%% This test creates several keys with hierarchical names and verifies that +%% the list operation correctly returns only keys matching a specific prefix. +%% It demonstrates the directory-like navigation capabilities of the store. +list_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-2">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + ?assertEqual(list(StoreOpts, <<"colors">>), {ok, []}), + % Create immediate children under colors/ + write(StoreOpts, <<"colors/red">>, <<"1">>), + write(StoreOpts, <<"colors/blue">>, <<"2">>), + write(StoreOpts, <<"colors/green">>, <<"3">>), + % Create nested directories under colors/ - these should show up as immediate children + write(StoreOpts, <<"colors/multi/foo">>, <<"4">>), + write(StoreOpts, <<"colors/multi/bar">>, <<"5">>), + write(StoreOpts, <<"colors/primary/red">>, <<"6">>), + write(StoreOpts, <<"colors/primary/blue">>, <<"7">>), + write(StoreOpts, <<"colors/nested/deep/value">>, <<"8">>), + % Create other top-level directories + write(StoreOpts, <<"foo/bar">>, <<"baz">>), + write(StoreOpts, <<"beep/boop">>, <<"bam">>), + read(StoreOpts, <<"colors">>), + % Test listing colors/ - should return immediate children only + {ok, ListResult} = list(StoreOpts, <<"colors">>), + ?event({list_result, ListResult}), + % Expected: red, blue, green (files) + multi, primary, nested (directories) + % Should NOT include deeply nested items like foo, bar, deep, value + ExpectedChildren = [<<"blue">>, <<"green">>, <<"multi">>, <<"nested">>, <<"primary">>, <<"red">>], + ?assert(lists:all(fun(Key) -> lists:member(Key, ExpectedChildren) end, ListResult)), + % Test listing a nested directory - should only show immediate children + {ok, NestedListResult} = list(StoreOpts, <<"colors/multi">>), + ?event({nested_list_result, NestedListResult}), + ExpectedNestedChildren = [<<"bar">>, <<"foo">>], + ?assert(lists:all(fun(Key) -> lists:member(Key, ExpectedNestedChildren) end, NestedListResult)), + % Test listing a deeper nested directory + {ok, DeepListResult} = list(StoreOpts, <<"colors/nested">>), + ?event({deep_list_result, DeepListResult}), + ExpectedDeepChildren = [<<"deep">>], + ?assert(lists:all(fun(Key) -> lists:member(Key, ExpectedDeepChildren) end, DeepListResult)), + ok = stop(StoreOpts). + +%% @doc Group test - verifies group creation and type detection. +%% +%% This test creates a group entry and verifies that it is correctly identified +%% as a composite type and cannot be read directly (like filesystem directories). +group_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store3">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + make_group(StoreOpts, <<"colors">>), + % Groups should be detected as composite types + ?assertEqual(composite, type(StoreOpts, <<"colors">>)), + % Groups should not be readable directly (like directories in filesystem) + ?assertEqual(not_found, read(StoreOpts, <<"colors">>)). + +%% @doc Link test - verifies symbolic link creation and resolution. +%% +%% This test creates a regular key-value pair, creates a link pointing to it, +%% and verifies that reading from the link location returns the original value. +%% This demonstrates the transparent link resolution mechanism. +link_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store3">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + write(StoreOpts, <<"foo/bar/baz">>, <<"Bam">>), + make_link(StoreOpts, <<"foo/bar/baz">>, <<"foo/beep/baz">>), + {ok, Result} = read(StoreOpts, <<"foo/beep/baz">>), + ?event({ result, Result}), + ?assertEqual(<<"Bam">>, Result). + +link_fragment_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store3">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + write(StoreOpts, [<<"data">>, <<"bar">>, <<"baz">>], <<"Bam">>), + make_link(StoreOpts, [<<"data">>, <<"bar">>], <<"my-link">>), + {ok, Result} = read(StoreOpts, [<<"my-link">>, <<"baz">>]), + ?event({ result, Result}), + ?assertEqual(<<"Bam">>, Result). + +%% @doc Type test - verifies type detection for both simple and composite entries. +%% +%% This test creates both a group (composite) entry and a regular (simple) entry, +%% then verifies that the type detection function correctly identifies each one. +%% This demonstrates the semantic classification system used by the store. +type_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-6">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + make_group(StoreOpts, <<"assets">>), + Type = type(StoreOpts, <<"assets">>), + ?event({type, Type}), + ?assertEqual(composite, Type), + write(StoreOpts, <<"assets/1">>, <<"bam">>), + Type2 = type(StoreOpts, <<"assets/1">>), + ?event({type2, Type2}), + ?assertEqual(simple, Type2). + +%% @doc Link key list test - verifies symbolic link creation using structured key paths. +%% +%% This test demonstrates the store's ability to handle complex key structures +%% represented as lists of binary segments, and verifies that symbolic links +%% work correctly when the target key is specified as a list rather than a +%% flat binary string. +%% +%% The test creates a hierarchical key structure using a list format (which +%% presumably gets converted to a path-like binary internally), creates a +%% symbolic link pointing to that structured key, and verifies that link +%% resolution works transparently to return the original value. +%% +%% This is particularly important for applications that organize data in +%% hierarchical structures where keys represent nested paths or categories, +%% and need to create shortcuts or aliases to deeply nested data. +link_key_list_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-7">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + write(StoreOpts, [ <<"parent">>, <<"key">> ], <<"value">>), + make_link(StoreOpts, [ <<"parent">>, <<"key">> ], <<"my-link">>), + {ok, Result} = read(StoreOpts, <<"my-link">>), + ?event({result, Result}), + ?assertEqual(<<"value">>, Result). + +%% @doc Path traversal link test - verifies link resolution during path traversal. +%% +%% This test verifies that when reading a path as a list, intermediate path +%% segments that are links get resolved correctly. For example, if "link" +%% is a symbolic link to "group", then reading ["link", "key"] should +%% resolve to reading ["group", "key"]. +%% +%% This functionality enables transparent redirection at the directory level, +%% allowing reorganization of hierarchical data without breaking existing +%% access patterns. +path_traversal_link_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-8">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + % Create the actual data at group/key + write(StoreOpts, [<<"group">>, <<"key">>], <<"target-value">>), + % Create a link from "link" to "group" + make_link(StoreOpts, <<"group">>, <<"link">>), + % Reading via the link path should resolve to the target value + {ok, Result} = read(StoreOpts, [<<"link">>, <<"key">>]), + ?event({path_traversal_result, Result}), + ?assertEqual(<<"target-value">>, Result), + ok = stop(StoreOpts). + +%% @doc Test that matches the exact hb_store hierarchical test pattern +exact_hb_store_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-exact">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + % Follow exact same pattern as hb_store test + ?event(step1_make_group), + make_group(StoreOpts, <<"test-dir1">>), + ?event(step2_write_file), + write(StoreOpts, [<<"test-dir1">>, <<"test-file">>], <<"test-data">>), + ?event(step3_make_link), + make_link(StoreOpts, [<<"test-dir1">>], <<"test-link">>), + % Debug: test that the link behaves like the target (groups are unreadable) + ?event(step4_check_link), + LinkResult = read(StoreOpts, <<"test-link">>), + ?event({link_result, LinkResult}), + % Since test-dir1 is a group and groups are unreadable, the link should also be unreadable + ?assertEqual(not_found, LinkResult), + % Debug: test intermediate steps + ?event(step5_test_direct_read), + DirectResult = read(StoreOpts, <<"test-dir1/test-file">>), + ?event({direct_result, DirectResult}), + % This should work: reading via the link path + ?event(step6_test_link_read), + Result = read(StoreOpts, [<<"test-link">>, <<"test-file">>]), + ?event({final_result, Result}), + ?assertEqual({ok, <<"test-data">>}, Result), + ok = stop(StoreOpts). + +%% @doc Test cache-style usage through hb_store interface +cache_style_test() -> + hb:init(), + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-cache-style">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + % Start the store + hb_store:start(StoreOpts), + % Test writing through hb_store interface + ok = hb_store:write(StoreOpts, <<"test-key">>, <<"test-value">>), + % Test reading through hb_store interface + Result = hb_store:read(StoreOpts, <<"test-key">>), + ?event({cache_style_read_result, Result}), + ?assertEqual({ok, <<"test-value">>}, Result), + hb_store:stop(StoreOpts). + +%% @doc Test nested map storage with cache-like linking behavior +%% +%% This test demonstrates how to store a nested map structure where: +%% 1. Each value is stored at data/{hash_of_value} +%% 2. Links are created to compose the values back into the original map structure +%% 3. Reading the composed structure reconstructs the original nested map +nested_map_cache_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-nested-cache">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + % Clean up any previous test data + reset(StoreOpts), + % Original nested map structure + OriginalMap = #{ + <<"target">> => <<"Foo">>, + <<"commitments">> => #{ + <<"key1">> => #{ + <<"alg">> => <<"rsa-pss-512">>, + <<"committer">> => <<"unique-id">> + }, + <<"key2">> => #{ + <<"alg">> => <<"hmac">>, + <<"commiter">> => <<"unique-id-2">> + } + }, + <<"other-key">> => #{ + <<"other-key-key">> => <<"other-key-value">> + } + }, + ?event({original_map, OriginalMap}), + % Step 1: Store each leaf value at data/{hash} + TargetValue = <<"Foo">>, + TargetHash = base64:encode(crypto:hash(sha256, TargetValue)), + write(StoreOpts, <<"data/", TargetHash/binary>>, TargetValue), + AlgValue1 = <<"rsa-pss-512">>, + AlgHash1 = base64:encode(crypto:hash(sha256, AlgValue1)), + write(StoreOpts, <<"data/", AlgHash1/binary>>, AlgValue1), + CommitterValue1 = <<"unique-id">>, + CommitterHash1 = base64:encode(crypto:hash(sha256, CommitterValue1)), + write(StoreOpts, <<"data/", CommitterHash1/binary>>, CommitterValue1), + AlgValue2 = <<"hmac">>, + AlgHash2 = base64:encode(crypto:hash(sha256, AlgValue2)), + write(StoreOpts, <<"data/", AlgHash2/binary>>, AlgValue2), + CommitterValue2 = <<"unique-id-2">>, + CommitterHash2 = base64:encode(crypto:hash(sha256, CommitterValue2)), + write(StoreOpts, <<"data/", CommitterHash2/binary>>, CommitterValue2), + OtherKeyValue = <<"other-key-value">>, + OtherKeyHash = base64:encode(crypto:hash(sha256, OtherKeyValue)), + write(StoreOpts, <<"data/", OtherKeyHash/binary>>, OtherKeyValue), + % Step 2: Create the nested structure with groups and links + % Create the root group + make_group(StoreOpts, <<"root">>), + % Create links for the root level keys + make_link(StoreOpts, <<"data/", TargetHash/binary>>, <<"root/target">>), + % Create the commitments subgroup + make_group(StoreOpts, <<"root/commitments">>), + % Create the key1 subgroup within commitments + make_group(StoreOpts, <<"root/commitments/key1">>), + make_link(StoreOpts, <<"data/", AlgHash1/binary>>, <<"root/commitments/key1/alg">>), + make_link(StoreOpts, <<"data/", CommitterHash1/binary>>, <<"root/commitments/key1/committer">>), + % Create the key2 subgroup within commitments + make_group(StoreOpts, <<"root/commitments/key2">>), + make_link(StoreOpts, <<"data/", AlgHash2/binary>>, <<"root/commitments/key2/alg">>), + make_link(StoreOpts, <<"data/", CommitterHash2/binary>>, <<"root/commitments/key2/commiter">>), + % Create the other-key subgroup + make_group(StoreOpts, <<"root/other-key">>), + make_link(StoreOpts, <<"data/", OtherKeyHash/binary>>, <<"root/other-key/other-key-key">>), + % Step 3: Test reading the structure back + % Verify the root is a composite + ?assertEqual(composite, type(StoreOpts, <<"root">>)), + % List the root contents + {ok, RootKeys} = list(StoreOpts, <<"root">>), + ?event({root_keys, RootKeys}), + ExpectedRootKeys = [<<"commitments">>, <<"other-key">>, <<"target">>], + ?assert(lists:all(fun(Key) -> lists:member(Key, ExpectedRootKeys) end, RootKeys)), + % Read the target directly + {ok, TargetValueRead} = read(StoreOpts, <<"root/target">>), + ?assertEqual(<<"Foo">>, TargetValueRead), + % Verify commitments is a composite + ?assertEqual(composite, type(StoreOpts, <<"root/commitments">>)), + % Verify other-key is a composite + ?assertEqual(composite, type(StoreOpts, <<"root/other-key">>)), + % Step 4: Test programmatic reconstruction of the nested map + ReconstructedMap = reconstruct_map(StoreOpts, <<"root">>), + ?event({reconstructed_map, ReconstructedMap}), + % Verify the reconstructed map matches the original structure + ?assert(hb_message:match(OriginalMap, ReconstructedMap)), + stop(StoreOpts). + +%% Helper function to recursively reconstruct a map from the store +reconstruct_map(StoreOpts, Path) -> + case type(StoreOpts, Path) of + composite -> + % This is a group, reconstruct it as a map + {ok, ImmediateChildren} = list(StoreOpts, Path), + % The list function now correctly returns only immediate children + ?event({path, Path, immediate_children, ImmediateChildren}), + maps:from_list([ + {Key, reconstruct_map(StoreOpts, <>)} + || Key <- ImmediateChildren + ]); + simple -> + % This is a simple value, read it directly + {ok, Value} = read(StoreOpts, Path), + Value; + not_found -> + % Path doesn't exist + undefined + end. + +%% @doc Debug test to understand cache linking behavior +cache_debug_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/cache-debug">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + % Simulate what the cache does: + % 1. Create a group for message ID + MessageID = <<"test_message_123">>, + make_group(StoreOpts, MessageID), + % 2. Store a value at data/hash + Value = <<"test_value">>, + ValueHash = base64:encode(crypto:hash(sha256, Value)), + DataPath = <<"data/", ValueHash/binary>>, + write(StoreOpts, DataPath, Value), + % 3. Calculate a key hashpath (simplified version) + KeyHashPath = <>, + % 4. Create link from data path to key hash path + make_link(StoreOpts, DataPath, KeyHashPath), + % 5. Test what the cache would see: + ?event(debug_cache_test, {step, check_message_type}), + MsgType = type(StoreOpts, MessageID), + ?event(debug_cache_test, {message_type, MsgType}), + ?event(debug_cache_test, {step, list_message_contents}), + {ok, Subkeys} = list(StoreOpts, MessageID), + ?event(debug_cache_test, {message_subkeys, Subkeys}), + ?event(debug_cache_test, {step, read_key_hashpath}), + KeyHashResult = read(StoreOpts, KeyHashPath), + ?event(debug_cache_test, {key_hash_read_result, KeyHashResult}), + % 6. Test with path as list (what cache does): + ?event(debug_cache_test, {step, read_path_as_list}), + PathAsList = [MessageID, <<"key_hash_abc">>], + PathAsListResult = read(StoreOpts, PathAsList), + ?event(debug_cache_test, {path_as_list_result, PathAsListResult}), + stop(StoreOpts). + +%% @doc Isolated test focusing on the exact cache issue +isolated_type_debug_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/isolated-debug">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + % Create the exact scenario from user's description: + % 1. A message ID with nested structure + MessageID = <<"message123">>, + make_group(StoreOpts, MessageID), + % 2. Create nested groups for "commitments" and "other-test-key" + CommitmentsPath = <>, + OtherKeyPath = <>, + ?event(isolated_debug, {creating_nested_groups, CommitmentsPath, OtherKeyPath}), + make_group(StoreOpts, CommitmentsPath), + make_group(StoreOpts, OtherKeyPath), + % 3. Add some actual data within those groups + write(StoreOpts, <>, <<"signature_data_1">>), + write(StoreOpts, <>, <<"nested_value">>), + % 4. Test type detection on the nested paths + ?event(isolated_debug, {testing_main_message_type}), + MainType = type(StoreOpts, MessageID), + ?event(isolated_debug, {main_message_type, MainType}), + ?event(isolated_debug, {testing_commitments_type}), + CommitmentsType = type(StoreOpts, CommitmentsPath), + ?event(isolated_debug, {commitments_type, CommitmentsType}), + ?event(isolated_debug, {testing_other_key_type}), + OtherKeyType = type(StoreOpts, OtherKeyPath), + ?event(isolated_debug, {other_key_type, OtherKeyType}), + % 5. Test what happens when reading these nested paths + ?event(isolated_debug, {reading_commitments_directly}), + CommitmentsResult = read(StoreOpts, CommitmentsPath), + ?event(isolated_debug, {commitments_read_result, CommitmentsResult}), + ?event(isolated_debug, {reading_other_key_directly}), + OtherKeyResult = read(StoreOpts, OtherKeyPath), + ?event(isolated_debug, {other_key_read_result, OtherKeyResult}), + stop(StoreOpts). + +%% @doc Test that list function resolves links correctly +list_with_link_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-list-link">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + % Create a group with some children + make_group(StoreOpts, <<"real-group">>), + write(StoreOpts, <<"real-group/child1">>, <<"value1">>), + write(StoreOpts, <<"real-group/child2">>, <<"value2">>), + write(StoreOpts, <<"real-group/child3">>, <<"value3">>), + % Create a link to the group + make_link(StoreOpts, <<"real-group">>, <<"link-to-group">>), + % List the real group to verify expected children + {ok, RealGroupChildren} = list(StoreOpts, <<"real-group">>), + ?event({real_group_children, RealGroupChildren}), + ExpectedChildren = [<<"child1">>, <<"child2">>, <<"child3">>], + ?assertEqual(ExpectedChildren, lists:sort(RealGroupChildren)), + % List via the link - should return the same children + {ok, LinkChildren} = list(StoreOpts, <<"link-to-group">>), + ?event({link_children, LinkChildren}), + ?assertEqual(ExpectedChildren, lists:sort(LinkChildren)), + stop(StoreOpts). \ No newline at end of file diff --git a/src/hb_store_lru.erl b/src/hb_store_lru.erl new file mode 100644 index 000000000..d68aaa8f7 --- /dev/null +++ b/src/hb_store_lru.erl @@ -0,0 +1,854 @@ +%%% @doc An in-memory store implementation, following the `hb_store` behavior +%%% and interface. This implementation uses a least-recently-used cache first, +%%% and offloads evicted data to a specified non-volatile store over time. +%%% +%%% This cache is registered under `{in_memory, HTTPServerID}`, in `hb_name` +%%% so that all processes that are executing using the HTTP server’s Opts +%%% can find it quickly. +%%% +%%% The least-recently-used strategy (first is the most recent used, last is the +%%% least recently used) is implemented by keeping track of the order and bytes +%%% on ets tables: +%%% - A cache table containing all the entries along with the value size and +%%% key index. +%%% - A cache indexing table containing all the index pointing to the keys. The +%%% IDs are then sorted to ease the eviction policy. +%%% - A cache statistics table containing all the information about the cache +%%% size, capacity, and indexing. +-module(hb_store_lru). +-export([start/1, stop/1, reset/1, scope/1]). +-export([write/3, read/2, list/2, type/2, make_link/3, make_group/2, resolve/2]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%%% @doc The default capacity is used when no capacity is provided in the store +%%% options. +-define(DEFAULT_LRU_CAPACITY, 4_000_000_000). + +%% @doc Maximum number of retries when fetching cache entries that aren't +%% immediately found due to timing issues in concurrent operations. +-define(RETRY_THRESHOLD, 2). + +%% @doc Start the LRU cache. +start(StoreOpts = #{ <<"name">> := Name }) -> + ?event(cache_lru, {starting_lru_server, Name}), + From = self(), + spawn( + fun() -> + State = init(From, StoreOpts), + server_loop(State, StoreOpts) + end + ), + receive + {ok, InstanceMessage} -> {ok, InstanceMessage} + end. + +%% @doc Create the `ets' tables for the LRU cache: +%% - The cache of data itself (public, with read concurrency enabled) +%% - A set for the LRU's stats. +%% - An ordered set for the cache's index. +init(From, StoreOpts) -> + % Start the persistent store. + case hb_maps:get(<<"persistent-store">>, StoreOpts, no_store) of + no_store -> ok; + Store -> hb_store:start(Store) + end, + % Create LRU tables + CacheTable = ets:new(hb_cache_lru, [ + set, + protected, + {read_concurrency, true} + ]), + CacheStatsTable = ets:new(hb_cache_lru_stats, [set]), + CacheIndexTable = ets:new(hb_cache_lru_index, [ordered_set]), + From ! {ok, #{ <<"pid">> => self(), <<"cache-table">> => CacheTable }}, + #{ + cache_table => CacheTable, + stats_table => CacheStatsTable, + index_table => CacheIndexTable + }. + +%% @doc Stop the LRU in memory by offloading the keys in the ETS tables +%% before exiting the process. +stop(Opts) -> + ?event(cache_lru, {stopping_lru_server, Opts}), + #{ <<"pid">> := CacheServer } = hb_store:find(Opts), + CacheServer ! {stop, self(), Ref = make_ref()}, + receive + {ok, Ref} -> ok + end. + +%% @doc The LRU store is always local, for now. +scope(_) -> local. + +%% @doc Reset the store by completely cleaning the ETS tables and +%% delegate the reset to the underlying offloading store. +reset(Opts) -> + #{ <<"pid">> := CacheServer } = hb_store:find(Opts), + CacheServer ! {reset, self(), Ref = make_ref()}, + receive + {ok, Ref} -> + ?event({reset_store, {in_memory, CacheServer}}), + case get_persistent_store(Opts) of + no_store -> + ok; + Store -> + hb_store:reset(Store) + end + end. + +server_loop(State = + #{cache_table := CacheTable, + stats_table := StatsTable, + index_table := IndexTable}, + Opts) -> + receive + {sync, From} -> + From ! {ok, self()}, + server_loop(State, Opts); + {get_cache_table, From} -> + From ! CacheTable; + {put, Key, Value, From, Ref} -> + put_cache_entry(State, Key, Value, Opts), + ?event(debug_lru, {put, {key, Key}, {value, Value}}), + From ! {ok, Ref}; + {link, Existing, New, From, Ref} -> + link_cache_entry(State, Existing, New, Opts), + From ! {ok, Ref}; + {make_group, Key, From, Ref} -> + ?event(debug_lru, {make_group, Key}), + ensure_dir(State, Key), + From ! {ok, Ref}; + {update_recent, Key, Entry, From, Ref} -> + update_recently_used(State, Key, Entry), + From ! {ok, Ref}; + {reset, From, Ref} -> + ets:delete_all_objects(CacheTable), + ets:delete_all_objects(StatsTable), + ets:delete_all_objects(IndexTable), + From ! {ok, Ref}; + {stop, From, Ref} -> + evict_all_entries(State, Opts), + From ! {ok, Ref}, + exit(self(), ok) + end, + server_loop(State, Opts). + +%% @doc Force the caller to wait until the server has fully processed all +%% messages in its mailbox, up to the initiation of the call. +sync(Server) -> + Server ! {sync, self()}, + receive + {ok, Server} -> ok + end. + +%% @doc Write an entry in the cache. +%% +%% After writing, the LRU is updated by moving the key in the most-recently-used +%% key to cycle and re-prioritize cache entry. +write(Opts, RawKey, Value) -> + Key = hb_store:join(RawKey), + #{ <<"pid">> := CacheServer } = hb_store:find(Opts), + CacheServer ! {put, Key, Value, self(), Ref = make_ref()}, + receive + {ok, Ref} -> ok + end. + +%% @doc Retrieve value in the cache from the given key. +%% Because the cache uses LRU, the key is moved on the most recent used key to +%% cycle and re-prioritize cache entry. +read(Opts, RawKey) -> + #{ <<"pid">> := Server } = hb_store:find(Opts), + Key = resolve(Opts, RawKey), + case fetch_cache_with_retry(Opts, Key) of + nil -> + case get_persistent_store(Opts) of + no_store -> + not_found; + PersistentStore -> + % FIXME: It might happens some links can be in LRU while data on + % the permanent store and resolve doesn't produce the same key. + ResolvedKey = case RawKey == Key of + true -> + hb_store:resolve(PersistentStore, RawKey); + false -> + Key + end, + hb_store:read(PersistentStore, ResolvedKey) + end; + {raw, Entry = #{value := Value}} -> + Server ! {update_recent, Key, Entry, self(), Ref = make_ref()}, + receive + {ok, Ref} -> {ok, Value} + end; + {link, Link} -> + ?event({link_found, RawKey, Link}), + read(Opts, Link); + Unexpected -> + ?event({unexpected_result, {unexpected, Unexpected}}), + not_found + end. + +resolve(Opts, Key) -> + Res = resolve(Opts, "", hb_path:term_to_path_parts(hb_store:join(Key), Opts)), + ?event({resolved, Key, Res}), + Res. + +resolve(_, CurrPath, []) -> + hb_store:join(CurrPath); +resolve(Opts, CurrPath, [Next|Rest]) -> + PathPart = hb_store:join([CurrPath, Next]), + ?event( + {resolving, + {accumulated_path, CurrPath}, + {next_segment, Next}, + {generated_partial_path_to_test, PathPart} + } + ), + case fetch_cache_with_retry(Opts, PathPart) of + {link, Link} -> + resolve(Opts, Link, Rest); + _ -> + resolve(Opts, PathPart, Rest) + end. + +%% @doc Make a link from a key to another in the store. +make_link(_, Link, Link) -> + ok; +make_link(Opts, RawExisting, New) -> + #{ <<"pid">> := Server } = hb_store:find(Opts), + ExistingKeyBin = convert_if_list(RawExisting), + NewKeyBin = convert_if_list(New), + case fetch_cache_with_retry(Opts, ExistingKeyBin) of + nil -> + case get_persistent_store(Opts) of + no_store -> + not_found; + Store -> + hb_store:make_link(Store, ExistingKeyBin, NewKeyBin) + end; + _ -> + Server ! {link, ExistingKeyBin, NewKeyBin, self(), Ref = make_ref()}, + receive + {ok, Ref} -> + ok + end + end. + +%% @doc List all the keys registered. +list(Opts, Path) -> + PersistentKeys = + case get_persistent_store(Opts) of + no_store -> + not_found; + Store -> + ResolvedPath = hb_store:resolve(Store, Path), + case hb_store:list(Store, ResolvedPath) of + {ok, Keys} -> Keys; + not_found -> not_found + end + end, + case {ets_keys(Opts, Path), PersistentKeys} of + {not_found, not_found} -> + not_found; + {InMemoryKeys, not_found} -> + {ok, InMemoryKeys}; + {not_found, PersistentKeys} -> + {ok, PersistentKeys}; + {InMemoryKeys, PersistentKeys} -> + {ok, hb_util:unique(InMemoryKeys ++ PersistentKeys)} + end. + +%% @doc List all of the keys in the store for a given path, supporting a special +%% case for the root. +ets_keys(Opts, <<"">>) -> ets_keys(Opts, <<"/">>); +ets_keys(Opts, <<"/">>) -> + #{ <<"cache-table">> := Table } = hb_store:find(Opts), + table_keys(Table, undefined); +ets_keys(Opts, Path) -> + case fetch_cache_with_retry(Opts, Path) of + {group, Set} -> + sets:to_list(Set); + {link, Link} -> + list(Opts, Link); + {raw, #{value := Value}} when is_map(Value) -> + maps:keys(Value); + {raw, #{value := Value}} when is_list(Value) -> + Value; + nil -> + not_found + end. + +%% @doc Determine the type of a key in the store. +type(Opts, Key) -> + case fetch_cache_with_retry(Opts, Key) of + nil -> + case get_persistent_store(Opts) of + no_store -> + not_found; + Store -> + ResolvedKey = hb_store:resolve(Store, Key), + hb_store:type(Store, ResolvedKey) + end; + {raw, _} -> + simple; + {link, NewKey} -> + type(Opts, NewKey); + {group, _Item} -> + composite + end. + +%% @doc Create a directory inside the store. +make_group(Opts, Key) -> + #{ <<"pid">> := Server } = hb_store:find(Opts), + Server ! {make_group, Key, self(), Ref = make_ref()}, + receive + {ok, Ref} -> + ok + end. + +table_keys(TableName) -> + table_keys(TableName, undefined). + +table_keys(TableName, Prefix) -> + FirstKey = ets:first(TableName), + table_keys(TableName, FirstKey, Prefix, []). + +table_keys(_TableName, '$end_of_table', _Prefix, Acc) -> + Acc; +table_keys(TableName, CurrentKey, Prefix, Acc) -> + NextKey = ets:next(TableName, CurrentKey), + case Prefix of + undefined -> + table_keys(TableName, NextKey, Prefix, [CurrentKey | Acc]); + _ -> + PrefixParts = hb_path:term_to_path_parts(Prefix), + Key = hb_path:term_to_path_parts(CurrentKey), + case lists:prefix(PrefixParts, Key) of + true -> + Extracted = lists:nthtail(length(PrefixParts), Key), + table_keys( + TableName, + NextKey, + Prefix, + [hb_path:to_binary(Extracted) | Acc] + ); + false -> + table_keys(TableName, NextKey, Prefix, Acc) + end + end. + +get_cache_entry(#{cache_table := Table}, Key) -> + get_cache_entry(Table, Key); +get_cache_entry(Table, Key) -> + case ets:lookup(Table, Key) of + [] -> + nil; + [{_, Entry}] -> + Entry + end. + +fetch_cache_with_retry(Opts, Key) -> + fetch_cache_with_retry(Opts, Key, 1). + +fetch_cache_with_retry(Opts, Key, Retries) -> + #{<<"cache-table">> := Table, <<"pid">> := Server} = hb_store:find(Opts), + case get_cache_entry(Table, Key) of + nil -> + case Retries < ?RETRY_THRESHOLD of + true -> + sync(Server), + fetch_cache_with_retry(Opts, Key, Retries + 1); + false -> + nil + end; + Entry -> + Entry + end. + +put_cache_entry(State, Key, Value, Opts) -> + ValueSize = erlang:external_size(Value), + CacheSize = cache_size(State), + ?event(cache_lru, {putting_entry, {size, ValueSize}, {opts, Opts}, {cache_size, CacheSize}}), + Capacity = hb_maps:get(<<"capacity">>, Opts, ?DEFAULT_LRU_CAPACITY), + case get_cache_entry(State, Key) of + nil -> + % For new entries, we check if the size will the fit the full + % capacity (even by evicting keys). + FitInCache = ValueSize =< Capacity, + case FitInCache of + false -> + case get_persistent_store(Opts) of + no_store -> + skip; + _ -> + Group = handle_group(State, Key, Opts#{mode => offload}), + offload_to_store(Key, Value, [], Group, Opts) + end; + true -> + ?event(cache_lru, {assign_entry, Key, Value}), + Group = handle_group(State, Key, Opts), + assign_new_entry( + State, + Key, + Value, + ValueSize, + Capacity, + Group, + Opts + ) + end; + Entry -> + ?event(cache_lru, {replace_entry, Key, Value}), + replace_entry(State, Key, Value, ValueSize, Entry) + end. + +handle_group(State, Key, Opts) -> + case filename:dirname(hb_store:join(Key)) of + <<".">> -> undefined ; + BaseDir -> + case maps:get(mode, Opts, undefined) of + offload -> + Store = get_persistent_store(Opts), + ?event(cache_lru, {create_group, BaseDir}), + hb_store:make_group(Store, BaseDir), + BaseDir; + undefined -> + ensure_dir(State, BaseDir), + {group, Entry} = get_cache_entry(State, BaseDir), + BaseName = filename:basename(Key), + NewGroup = append_key_to_group(BaseName, Entry), + add_cache_entry(State, BaseDir, {group, NewGroup}), + BaseDir + end + end. +ensure_dir(State, Path) -> + PathParts = hb_path:term_to_path_parts(Path), + [First | Rest] = PathParts, + Result = ensure_dir(State, First, Rest), + Result. + +ensure_dir(State, CurrentPath, []) -> + maybe_create_dir(State, CurrentPath, nil); +ensure_dir(State, CurrentPath, [Next]) -> + maybe_create_dir(State, CurrentPath, Next), + ensure_dir(State, hb_store:join([CurrentPath, Next]), []); +ensure_dir(State, CurrentPath, [Next | Rest]) -> + maybe_create_dir(State, CurrentPath, Next), + ensure_dir(State, hb_store:join([CurrentPath, Next]), Rest). + +maybe_create_dir(State, DirPath, Value) -> + CurrentValueSet = + case get_cache_entry(State, DirPath) of + nil -> + sets:new(); + {group, CurrentValue} -> + CurrentValue + end, + NewValueSet = + case Value of + nil -> + CurrentValueSet; + _ -> + sets:add_element(Value, CurrentValueSet) + end, + ?event(cache_lru, {create_group, DirPath, sets:to_list(NewValueSet)}), + add_cache_entry(State, DirPath, {group, NewValueSet}). + +append_key_to_group(Key, Group) -> + BaseName = filename:basename(Key), + sets:add_element(BaseName, Group). + +assign_new_entry(State, Key, Value, ValueSize, Capacity, Group, Opts) -> + case cache_size(State) + ValueSize >= Capacity of + true -> + ?event(cache_lru, eviction_required), + evict_oldest_entry(State, ValueSize, Opts); + false -> + ok + end, + ID = get_index_id(State), + add_cache_index(State, ID, Key), + add_cache_entry( + State, + Key, + {raw, + #{ + value => Value, + id => ID, + size => ValueSize, + group => Group + } + } + ), + increase_cache_size(State, ValueSize). + +cache_size(#{stats_table := Table}) -> + case ets:lookup(Table, size) of + [{_, Size}] -> + Size; + _ -> + 0 + end. + +get_index_id(#{stats_table := StatsTable}) -> + ets:update_counter(StatsTable, id, {2, 1}, {0, 0}). + +add_cache_entry(#{cache_table := Table}, Key, Value) -> + ets:insert(Table, {Key, Value}). + +add_cache_index(#{index_table := Table}, ID, Key) -> + ets:insert(Table, {ID, Key}). + +link_cache_entry(State = #{cache_table := Table}, Existing, New, Opts) -> + ?event(cache_lru, {link, Existing, New}), + % Remove the link from the previous linked entry + clean_old_link(Table, New), + _ = handle_group(State, New, Opts), + ets:insert(Table, {New, {link, Existing}}), + % Add links to the linked entry + case ets:lookup(Table, Existing) of + [{_, {raw, Entry}}] -> + NewLinks = + case Entry of + #{links := ExistingLinks} -> + [New | ExistingLinks]; + _ -> + [New] + end, + ets:insert(Table, {Existing, {raw, Entry#{links => NewLinks}}}); + _ -> + ignore + end. + +% @doc Remove the link association for the the old linked data to the given key +clean_old_link(Table, Link) -> + case ets:lookup(Table, Link) of + [{_, {link, PreviousEntry}}] -> + ?event(cache_lru, {removing_previous_link, + {link, Link}, + {previous_entry, PreviousEntry} + }), + case ets:lookup(Table, PreviousEntry) of + [{_, {raw, OldEntry}}] -> + Links = sets:from_list(maps:get(links, OldEntry, [])), + UpdatedLinks = sets:del_element(Link, Links), + UpdatedEntry = maps:put( + links, + sets:to_list(UpdatedLinks), + OldEntry + ), + ets:insert(Table, {PreviousEntry, {raw, UpdatedEntry}}); + _ -> + skip + end; + _ -> skip + end. + +increase_cache_size(#{stats_table := StatsTable}, ValueSize) -> + ets:update_counter(StatsTable, size, {2, ValueSize}, {0, 0}). + +evict_oldest_entry(State, ValueSize, Opts) -> + evict_oldest_entry(State, ValueSize, 0, Opts). + +evict_oldest_entry(_State, ValueSize, FreeSize, _Opts) when FreeSize >= ValueSize -> + ok; +evict_oldest_entry(State, ValueSize, FreeSize, Opts) -> + case cache_tail_key(State) of + nil -> + ok; + TailKey -> + Entry = #{ + size := ReclaimedSize, + id := ID, + value := TailValue, + group := Group + } = case get_cache_entry(State, TailKey) of + nil -> + % Raises a runtime error as this represents + % a non-recoverable error. This would signifies a + % inconsistency between the index and the cache table. + erlang:error(cache_entry_not_found, [TailKey]); + {raw, RawEntry} -> + RawEntry + end, + ?event(cache_lru, {evict, TailKey, claiming_size, ReclaimedSize}), + delete_cache_index(State, ID), + delete_cache_entry(State, TailKey), + decrease_cache_size(State, ReclaimedSize), + Links = maps:get(links, Entry, []), + case Group of + undefined -> + ignore; + _ -> + {group, GroupSet} = get_cache_entry(State, Group), + BaseName = filename:basename(TailKey), + UpdatedGroupSet = sets:del_element(BaseName, GroupSet), + case sets:size(UpdatedGroupSet) of + 0 -> + delete_cache_entry(State, Group); + _ -> + add_cache_entry( + State, + Group, + {group, UpdatedGroupSet} + ) + end + end, + offload_to_store(TailKey, TailValue, Links, Group, Opts), + evict_oldest_entry( + State, + ValueSize, + FreeSize + ReclaimedSize, + Opts + ) + end. + +evict_all_entries(#{cache_table := Table}, Opts) -> + lists:foreach( + fun(Key) -> + [{_, {raw, Entry}}] = ets:lookup(Table, Key), + #{ value := Value, group := Group } = Entry, + Links = maps:get(links, Entry, []), + offload_to_store(Key, Value, Links, Group, Opts) + end, + table_keys(Table) + ). + +offload_to_store(TailKey, TailValue, Links, Group, Opts) -> + ?event(lru_offload, {offloading_to_store, Opts}), + FoundStore = get_persistent_store(Opts), + ?event(lru_offload, {found_store, FoundStore}), + case FoundStore of + no_store -> + ok; + Store -> + case Group of + undefined -> + ignore; + _ -> + hb_store:make_group(Store, Group) + end, + case hb_store:write(Store, TailKey, TailValue) of + ok -> + lists:foreach( + fun(Link) -> + ResolvedPath = resolve(Opts, Link), + hb_store:make_link(Store, ResolvedPath, Link) + end, + Links + ), + ?event(cache_lru, {offloaded_key, TailKey}), + ok; + Err -> + ?event(warning, {error_offloading_to_local_cache, Err}), + {error, Err} + end + end. + +cache_tail_key(#{index_table := Table}) -> + case ets:first(Table) of + '$end_of_table' -> + nil; + FirstID -> + [{_, Key}] = ets:lookup(Table, FirstID), + Key + end. + +delete_cache_index(#{index_table := IndexTable}, ID) -> + ets:delete(IndexTable, ID). + +delete_cache_entry(#{cache_table := Table}, Key) -> + ets:delete(Table, Key), + ?event(cache_lru, {deleted, Key}). + +decrease_cache_size(#{stats_table := Table}, Size) -> + ets:update_counter(Table, size, {2, -Size, 0, 0}). + +replace_entry(State, Key, Value, ValueSize, {raw, OldEntry = #{ value := OldValue}}) when Value =/= OldValue -> + % Update entry and move the keys in the front of the cache + % as the most used Key + ?event(debug_lru, {replace_entry, + {key, Key}, + {value, Value}, + {explicit, OldEntry} + }), + #{size := PreviousSize} = OldEntry, + NewEntry = OldEntry#{value := Value, size := ValueSize}, + add_cache_entry(State, Key, {raw, NewEntry}), + update_recently_used(State, Key, NewEntry), + update_cache_size(State, PreviousSize, ValueSize); +replace_entry(_State, _Key, _Value, _ValueSize, {raw, _}) -> ok; +replace_entry(_State, _Key, _Value, _ValueSize, {Type, _}) -> + % Link or group should be handle directly with `make_link` or `make_group` + % This aim of this function is to be used along with direct data insertion. + throw({error, can_only_replace_raw_entry, {type, Type}}). +update_recently_used(State, Key, Entry) -> + % Acquire a new ID + NewID = get_index_id(State), + % Update the entry's ID + add_cache_entry(State, Key, {raw, Entry#{id := NewID}}), + #{id := PreviousID} = Entry, + % Delete previous ID to priorize the new NewID + delete_cache_index(State, PreviousID), + add_cache_index(State, NewID, Key). + +update_cache_size(#{stats_table := Table}, PreviousSize, NewSize) -> + ets:update_counter(Table, size, [{2, -PreviousSize}, {2, NewSize}]). + +get_persistent_store(Opts) -> + hb_maps:get( + <<"persistent-store">>, + Opts, + no_store + ). + +convert_if_list(Value) when is_list(Value) -> + join(Value); % Perform the conversion if it's a list +convert_if_list(Value) -> + Value. + +join(Key) when is_list(Key) -> + KeyList = hb_store:join(Key), + maybe_convert_to_binary(KeyList); +join(Key) when is_binary(Key) -> Key. + +maybe_convert_to_binary(Value) when is_list(Value) -> + list_to_binary(Value); +maybe_convert_to_binary(Value) when is_binary(Value) -> + Value. + +%%% Tests + +%% @doc Generate a set of options for testing. The default is to use an `fs` +%% store as the persistent backing. +test_opts(PersistentStore) -> + test_opts(PersistentStore, 1000000). +test_opts(PersistentStore, Capacity) -> + % Set the server ID to a random address. + BaseStore = #{ + <<"name">> => hb_util:human_id(crypto:strong_rand_bytes(32)), + <<"capacity">> => Capacity, + <<"store-module">> => hb_store_lru + }, + case PersistentStore of + no_store -> + BaseStore#{ <<"persistent-store">> => no_store }; + default -> + DefaultStore = [ + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/lru">> + } + ], + hb_store:reset(DefaultStore), + BaseStore#{ <<"persistent-store">> => DefaultStore }; + _ -> + hb_store:reset(PersistentStore), + BaseStore#{ <<"persistent-store">> => PersistentStore } + end. + +unknown_value_test() -> + ?assertEqual(not_found, read(test_opts(default), <<"key1">>)). + +cache_term_test() -> + StoreOpts = test_opts(default), + write(StoreOpts, <<"key1">>, <<"Hello">>), + ?assertEqual({ok, <<"Hello">>}, read(StoreOpts, <<"key1">>)). + +evict_oldest_items_test() -> + StoreOpts = test_opts(no_store, 500), + Binary = crypto:strong_rand_bytes(200), + write(StoreOpts, <<"key1">>, Binary), + write(StoreOpts, <<"key2">>, Binary), + read(StoreOpts, <<"key1">>), + write(StoreOpts, <<"key3">>, Binary), + ?assertEqual({ok, Binary}, read(StoreOpts, <<"key1">>)), + ?assertEqual(not_found, read(StoreOpts, <<"key2">>)). + +evict_items_with_insufficient_space_test() -> + StoreOpts = test_opts(no_store, 500), + Binary = crypto:strong_rand_bytes(200), + write(StoreOpts, <<"key1">>, Binary), + write(StoreOpts, <<"key2">>, Binary), + write(StoreOpts, <<"key3">>, crypto:strong_rand_bytes(400)), + ?assertEqual(not_found, read(StoreOpts, <<"key1">>)), + ?assertEqual(not_found, read(StoreOpts, <<"key2">>)). + +evict_but_able_to_read_from_fs_store_test() -> + StoreOpts = test_opts(default, 500), + Binary = crypto:strong_rand_bytes(200), + write(StoreOpts, <<"key1">>, Binary), + write(StoreOpts, <<"key2">>, Binary), + read(StoreOpts, <<"key1">>), + write(StoreOpts, <<"key3">>, Binary), + ?assertEqual({ok, Binary}, read(StoreOpts, <<"key1">>)), + ?assertEqual({ok, Binary}, read(StoreOpts, <<"key2">>)), + % Directly offloads if the data is more than the LRU capacity + write(StoreOpts, <<"sub/key">>, crypto:strong_rand_bytes(600)), + ?assertMatch({ok, _}, read(StoreOpts, <<"sub/key">>)). + +stop_test() -> + StoreOpts = test_opts(default, 500), + Binary = crypto:strong_rand_bytes(200), + write(StoreOpts, <<"key1">>, Binary), + write(StoreOpts, <<"key2">>, Binary), + #{ <<"pid">> := ServerPID } = hb_store:find(StoreOpts), + ok = stop(StoreOpts), + ?assertEqual(false, is_process_alive(ServerPID)), + PersistentStore = hb_maps:get(<<"persistent-store">>, StoreOpts), + ?assertEqual({ok, Binary}, hb_store:read(PersistentStore, <<"key1">>)), + ?assertEqual({ok, Binary}, hb_store:read(PersistentStore, <<"key2">>)). + +reset_test() -> + StoreOpts = test_opts(default), + write(StoreOpts, <<"key1">>, <<"Hello">>), + write(StoreOpts, <<"key2">>, <<"Hi">>), + reset(StoreOpts), + ?assertEqual(not_found, read(StoreOpts, <<"key1">>)), + #{ <<"cache-table">> := Table } = hb_store:find(StoreOpts), + ?assertEqual([], ets:tab2list(Table)). + +list_test() -> + StoreOpts = test_opts(default, 500), + Binary = crypto:strong_rand_bytes(200), + make_group(StoreOpts, <<"sub">>), + write(StoreOpts, <<"hello">>, <<"world">>), + write(StoreOpts, <<"sub/key1">>, Binary), + write(StoreOpts, <<"sub/key2">>, Binary), + {ok, Keys1} = list(StoreOpts, <<"sub">>), + ?assertEqual([<<"key1">>, <<"key2">>], lists:sort(Keys1)), + write(StoreOpts, <<"sub/key3">>, Binary), + {ok, Keys2} = list(StoreOpts, <<"sub">>), + ?assertEqual( + [<<"key1">>, <<"key2">>, <<"key3">>], + lists:sort(Keys2) + ), + write(StoreOpts, <<"sub/inner/key1">>, Binary), + {ok, Keys3} = list(StoreOpts, <<"sub">>), + ?assertEqual([<<"inner">>, <<"key1">>, <<"key2">>, <<"key3">>], + lists:sort(Keys3)), + write(StoreOpts, <<"complex">>, #{<<"a">> => 10, <<"b">> => Binary}), + ?assertEqual({ok, [<<"a">>, <<"b">>]}, list(StoreOpts, <<"complex">>)). + +type_test() -> + StoreOpts = test_opts(default, 500), + Binary = crypto:strong_rand_bytes(200), + write(StoreOpts, <<"key1">>, Binary), + ?assertEqual(simple, type(StoreOpts, <<"key1">>)), + write(StoreOpts, <<"sub/key1">>, Binary), + ?assertEqual(composite, type(StoreOpts, <<"sub">>)), + make_link(StoreOpts, <<"key1">>, <<"keylink">>), + ?assertEqual(simple, type(StoreOpts, <<"keylink">>)). + +replace_link_test() -> + StoreOpts = test_opts(default), + write(StoreOpts, <<"key1">>, <<"Hello">>), + make_link(StoreOpts, <<"key1">>, <<"keylink">>), + ?assertEqual({ok, <<"Hello">>}, read(StoreOpts, <<"keylink">>)), + write(StoreOpts, <<"key2">>, <<"Hello2">>), + make_link(StoreOpts, <<"key2">>, <<"keylink">>), + ?assertEqual({ok, <<"Hello2">>}, read(StoreOpts, <<"keylink">>)), + #{ <<"cache-table">> := Table } = hb_store:find(StoreOpts), + {raw, #{links := Links }}= get_cache_entry(Table, <<"key1">>), + ?assertEqual([], Links). \ No newline at end of file diff --git a/src/hb_store_opts.erl b/src/hb_store_opts.erl new file mode 100644 index 000000000..fdb3c2b5c --- /dev/null +++ b/src/hb_store_opts.erl @@ -0,0 +1,262 @@ +%%% @doc A module responsible for applying default configuration to store options. +%%% +%%% This module takes store options and store defaults and returns a new list +%%% of stores with default properties applied based on the store-module type. +%%% Supports recursive application to nested store configurations. +-module(hb_store_opts). +-export([apply/2]). +-compile({no_auto_import,[apply/2]}). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +%% @doc Apply store defaults to store options. +%% Takes StoreOpts (list of store configuration maps) and Defaults (map of defaults) +%% and returns a new list with defaults applied where appropriate. + +apply(StoreOpts, Defaults) when is_list(StoreOpts), is_map(Defaults) -> + lists:map( + fun(StoreOpt) -> + apply_defaults_to_store(StoreOpt, Defaults) + end, + StoreOpts + ). + +%% @doc Apply defaults to a single store configuration. +apply_defaults_to_store(StoreOpt, Defaults) when is_map(StoreOpt), is_map(Defaults) -> + UpdatedStore = apply_defaults_by_module_type(StoreOpt, Defaults), + apply_defaults_to_substores(UpdatedStore, Defaults). + +%% @doc Apply defaults based on store-module. +apply_defaults_by_module_type(StoreOpt, Defaults) -> + case maps:get(<<"store-module">>, StoreOpt, undefined) of + hb_store_lmdb -> + apply_type_defaults(StoreOpt, <<"lmdb">>, Defaults); + hb_store_fs -> + apply_type_defaults(StoreOpt, <<"fs">>, Defaults); + hb_store_rocksdb -> + apply_type_defaults(StoreOpt, <<"rocksdb">>, Defaults); + hb_store_gateway -> + apply_type_defaults(StoreOpt, <<"gateway">>, Defaults); + _ -> + StoreOpt + end. + +%% @doc Apply type-specific defaults to a store. +apply_type_defaults(StoreOpt, TypeKey, Defaults) -> + case maps:get(TypeKey, Defaults, #{}) of + TypeDefaults when is_map(TypeDefaults) -> + maps:merge(TypeDefaults, StoreOpt); + _ -> + StoreOpt + end. + +%% @doc Apply defaults to sub-stores recursively. +apply_defaults_to_substores(StoreOpt, Defaults) -> + case maps:get(<<"store">>, StoreOpt, undefined) of + SubStores when is_list(SubStores) -> + UpdatedSubStores = + lists:map( + fun(SubStore) -> + apply_defaults_to_store(SubStore, Defaults) + end, + SubStores + ), + maps:put(<<"store">>, UpdatedSubStores, StoreOpt); + _ -> + StoreOpt + end. + +%% EUnit tests + +basic_apply_test() -> + StoreOpts = + [ + #{ + <<"name">> => <<"cache-mainnet/lmdb">>, + <<"store-module">> => hb_store_lmdb + } + ], + Defaults = + #{ + <<"lmdb">> => #{ + <<"capacity">> => 1073741824 + } + }, + Expected = + [ + #{ + <<"name">> => <<"cache-mainnet/lmdb">>, + <<"store-module">> => hb_store_lmdb, + <<"capacity">> => 1073741824 + } + ], + Result = apply(StoreOpts, Defaults), + ?assertEqual(Expected, Result). + +empty_defaults_test() -> + StoreOpts = + [ + #{ + <<"name">> => <<"cache-mainnet/lmdb">>, + <<"store-module">> => hb_store_lmdb + } + ], + Defaults = #{}, + Expected = + [ + #{ + <<"name">> => <<"cache-mainnet/lmdb">>, + <<"store-module">> => hb_store_lmdb + } + ], + Result = apply(StoreOpts, Defaults), + ?assertEqual(Expected, Result). + +empty_store_opts_test() -> + StoreOpts = [], + Defaults = + #{ + <<"lmdb">> => #{ + <<"capacity">> => 1073741824 + } + }, + Expected = [], + Result = apply(StoreOpts, Defaults), + ?assertEqual(Expected, Result). + +nested_stores_test() -> + StoreOpts = + [ + #{ + <<"store-module">> => hb_store_gateway, + <<"store">> => [ + #{ + <<"name">> => <<"cache-mainnet/lmdb">>, + <<"store-module">> => hb_store_lmdb + } + ] + } + ], + Defaults = + #{ + <<"lmdb">> => #{ + <<"capacity">> => 1073741824 + } + }, + Expected = + [ + #{ + <<"store-module">> => hb_store_gateway, + <<"store">> => [ + #{ + <<"name">> => <<"cache-mainnet/lmdb">>, + <<"store-module">> => hb_store_lmdb, + <<"capacity">> => 1073741824 + } + ] + } + ], + Result = apply(StoreOpts, Defaults), + ?assertEqual(Expected, Result). + +%% @doc Integration test to verify that capacity is properly set for hb_store_lmdb +%% This test verifies that the capacity value is correctly applied and accessible +%% to the hb_store_lmdb module before environment creation. +lmdb_capacity_integration_test() -> + CustomCapacity = 5000, + StoreOpts = + [ + #{ + <<"name">> => <<"test-lmdb">>, + <<"store-module">> => hb_store_lmdb + } + ], + Defaults = + #{ + <<"lmdb">> => #{ + <<"capacity">> => CustomCapacity + } + }, + [UpdatedStoreOpt] = apply(StoreOpts, Defaults), + ?assertEqual(CustomCapacity, maps:get(<<"capacity">>, UpdatedStoreOpt)), + ?assertEqual(<<"test-lmdb">>, maps:get(<<"name">>, UpdatedStoreOpt)), + ?assertEqual(hb_store_lmdb, maps:get(<<"store-module">>, UpdatedStoreOpt)), + ?assertNotEqual(16 * 1024 * 1024 * 1024, maps:get(<<"capacity">>, UpdatedStoreOpt)), + MultipleStoreOpts = + [ + #{ + <<"name">> => <<"test-lmdb-1">>, + <<"store-module">> => hb_store_lmdb + }, + #{ + <<"name">> => <<"test-lmdb-2">>, + <<"store-module">> => hb_store_lmdb + }, + #{ + <<"name">> => <<"test-fs">>, + <<"store-module">> => hb_store_fs + } + ], + UpdatedMultipleStoreOpts = apply(MultipleStoreOpts, Defaults), + [LmdbStore1, LmdbStore2, FsStore] = UpdatedMultipleStoreOpts, + ?assertEqual(CustomCapacity, maps:get(<<"capacity">>, LmdbStore1)), + ?assertEqual(CustomCapacity, maps:get(<<"capacity">>, LmdbStore2)), + ?assertEqual(false, maps:is_key(<<"capacity">>, FsStore)), + ?event({integration_test_passed, {lmdb_capacity, CustomCapacity}, {note, "correctly applied to store options"}}). + +%% @doc Full integration test simulating the hb_http_server flow +%% This test verifies that the complete flow from config loading to store defaults +%% application works correctly, simulating what happens in hb_http_server:start/0 +full_integration_flow_test() -> + LoadedConfig = #{ + <<"store_defaults">> => #{ + <<"lmdb">> => #{ + <<"capacity">> => 5000 + } + } + }, + DefaultStoreOpts = [ + #{ + <<"name">> => <<"cache-mainnet/lmdb">>, + <<"store-module">> => hb_store_lmdb + }, + #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-mainnet">> + }, + #{ + <<"store-module">> => hb_store_gateway, + <<"subindex">> => [ + #{ + <<"name">> => <<"Data-Protocol">>, + <<"value">> => <<"ao">> + } + ], + <<"store">> => [ + #{ + <<"store-module">> => hb_store_lmdb, + <<"name">> => <<"cache-mainnet/lmdb">> + } + ] + } + ], + MergedConfig = maps:merge( + #{<<"store">> => DefaultStoreOpts}, + LoadedConfig + ), + StoreOpts = maps:get(<<"store">>, MergedConfig), + StoreDefaults = maps:get(<<"store_defaults">>, MergedConfig, #{}), + UpdatedStoreOpts = apply(StoreOpts, StoreDefaults), + [LmdbStore, FsStore, GatewayStore] = UpdatedStoreOpts, + ?assertEqual(5000, maps:get(<<"capacity">>, LmdbStore)), + ?assertEqual(<<"cache-mainnet/lmdb">>, maps:get(<<"name">>, LmdbStore)), + ?assertEqual(hb_store_lmdb, maps:get(<<"store-module">>, LmdbStore)), + ?assertEqual(false, maps:is_key(<<"capacity">>, FsStore)), + ?assertEqual(hb_store_fs, maps:get(<<"store-module">>, FsStore)), + ?assertEqual(hb_store_gateway, maps:get(<<"store-module">>, GatewayStore)), + NestedStores = maps:get(<<"store">>, GatewayStore), + [NestedLmdbStore] = NestedStores, + ?assertEqual(5000, maps:get(<<"capacity">>, NestedLmdbStore)), + ?assertEqual(hb_store_lmdb, maps:get(<<"store-module">>, NestedLmdbStore)), + ?assertEqual(3, length(UpdatedStoreOpts)), + ?event({full_integration_test_passed, store_defaults_correctly_applied_through_complete_flow}). \ No newline at end of file diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index 4ed66594f..e5b5b1cf9 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -1,40 +1,211 @@ --module(hb_store_remote_node). --export([scope/1, type/2, read/2, resolve/2]). --include("include/hb.hrl"). - -%%% A store module that reads data from another AO node. +%%% @doc A store module that reads data from another AO node. %%% Notably, this store only provides the _read_ side of the store interface. -%%% The write side could be added, returning an attestation that the data has +%%% The write side could be added, returning an commitment that the data has %%% been written to the remote node. In that case, the node would probably want %%% to upload it to an Arweave bundler to ensure persistence, too. +-module(hb_store_remote_node). +-export([scope/1, type/2, read/2, write/3, make_link/3, resolve/2]). +%%% Public utilities. +-export([maybe_cache/2, maybe_cache/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). -scope(_) -> remote. +%% @doc Return the scope of this store. +%% +%% For the remote store, the scope is always `remote'. +%% +%% @param StoreOpts A message with the store options (ignored). +%% @returns remote. +scope(_StoreOpts) -> + remote. -resolve(#{ node := Node }, Key) -> - ?event({resolving_to_self, Node, Key}), +%% @doc Resolve a key path in the remote store. +%% +%% For the remote node store, the key is returned as-is. +%% +%% @param Data A map containing node configuration. +%% @param Key The key to resolve. +%% @returns The resolved key. +resolve(#{ <<"node">> := Node }, Key) -> + ?event({remote_resolve, {node, Node}, {key, Key}}), Key. -type(Opts = #{ node := Node }, Key) -> - ?no_prod("No need to get the whole message in order to get its type..."), - ?event({remote_type, Node, Key}), +%% @doc Determine the type of value at a given key. +%% +%% Remote nodes support only the `simple' type or `not_found'. +%% +%% @param Opts A map of options (including node configuration). +%% @param Key The key whose value type is determined. +%% @returns simple if found, or not_found otherwise. +type(Opts = #{ <<"node">> := Node }, Key) -> + ?event({remote_type, {node, Node}, {key, Key}}), case read(Opts, Key) of not_found -> not_found; - #tx { data = Map } when is_map(Map) -> composite; _ -> simple end. -read(Opts, Key) when is_binary(Key) -> - read(Opts, binary_to_list(Key)); -read(Opts = #{ node := Node }, Key) -> - Path = Node ++ "/data?Subpath=" ++ uri_string:quote(hb_store:join(Key)), - ?event({reading, Key, Path, Opts}), - case hb_http:get_binary(Path) of - {ok, Bundle} -> - case lists:keyfind(<<"Status">>, 1, Bundle#tx.tags) of - {<<"Status">>, <<"404">>} -> - not_found; - _ -> - {ok, Bundle} +%% @doc Read a key from the remote node. +%% +%% Makes an HTTP GET request to the remote node and returns the +%% committed message. +%% +%% @param Opts A map of options (including node configuration). +%% @param Key The key to read. +%% @returns {ok, Msg} on success or not_found if the key is missing. +read(Opts = #{ <<"node">> := Node }, Key) -> + ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), + HTTPRes = + hb_http:get( + Node, + #{ <<"path">> => <<"/~cache@1.0/read">>, <<"target">> => Key }, + Opts + ), + case HTTPRes of + {ok, Res} -> + % returning the whole response to get the test-key + {ok, Msg} = hb_message:with_only_committed(Res, Opts), + ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), + maybe_cache(Opts, Msg, [Key]), + {ok, Msg}; + {error, _Err} -> + ?event(store_remote_node, {read_not_found, {key, Key}}), + not_found + end. + +%% @doc Cache the data if the cache is enabled. The `local-store' option may +%% either be `false' or a store definition to use as the local cache. Additional +%% paths may be provided that should be linked to the data. +maybe_cache(StoreOpts, Data) -> + maybe_cache(StoreOpts, Data, []). +maybe_cache(StoreOpts, Data, Links) -> + ?event({maybe_cache, StoreOpts, Data}), + % Check if the local store is in our store options. + case hb_maps:get(<<"local-store">>, StoreOpts, false, StoreOpts) of + false -> + skipped; + Store -> + case hb_cache:write(Data, #{ store => Store }) of + {ok, RootPath} -> + % Remove the base path from the links. + LinksWithoutRootPath = + lists:filter( + fun(Link) -> Link /= RootPath end, + Links + ), + ?event(store_remote_node, cached_received), + LinkResults = + lists:filter( + fun(Link) -> + hb_store:make_link(Store, RootPath, Link) == false + end, + LinksWithoutRootPath + ), + ?event(store_remote_node, + {linked_cached, + {failed_links, LinkResults} + } + ), + case LinkResults of + [] -> ok; + _ -> {failed_links, LinkResults} + end; + {error, Err} -> + ?event(store_remote_node, error_on_local_cache_write), + ?event(warning, {error_caching_remote_node_data, Err}), + {error, Err} + end + end. + +%% @doc Write a key to the remote node. +%% +%% Constructs an HTTP POST write request. If a wallet is provided, +%% the message is signed. Returns {ok, Path} on HTTP 200, or +%% {error, Reason} on failure. +%% +%% @param Opts A map of options (including node configuration). +%% @param Key The key to write. +%% @param Value The value to store. +%% @returns {ok, Path} on success or {error, Reason} on failure. +write(Opts = #{ <<"node">> := Node }, Key, Value) -> + ?event({write, {node, Node}, {key, Key}, {value, Value}}), + WriteMsg = #{ + <<"path">> => <<"/~cache@1.0/write">>, + <<"method">> => <<"POST">>, + <<"body">> => Value + }, + SignedMsg = hb_message:commit(WriteMsg, Opts), + ?event({write, {signed, SignedMsg}}), + case hb_http:post(Node, SignedMsg, Opts) of + {ok, Response} -> + Status = hb_ao:get(<<"status">>, Response, 0, #{}), + ?event(store_remote_node, {write_completed, {response, Response}}), + case Status of + 200 -> ok; + _ -> {error, {unexpected_status, Status}} + end; + {error, Err} -> + ?event({write, {error, Err}}), + {error, Err} + end. + +%% @doc Link a source to a destination in the remote node. +%% +%% Constructs an HTTP POST link request. If a wallet is provided, +%% the message is signed. Returns {ok, Path} on HTTP 200, or +%% {error, Reason} on failure. +make_link(Opts = #{ <<"node">> := Node }, Source, Destination) -> + ?event({make_remote_link, {node, Node}, {source, Source}, + {destination, Destination}}), + LinkMsg = #{ + <<"path">> => <<"/~cache@1.0/link">>, + <<"method">> => <<"POST">>, + <<"source">> => Source, + <<"destination">> => Destination + }, + SignedMsg = hb_message:commit(LinkMsg, Opts), + ?event({make_remote_link, {signed, SignedMsg}}), + case hb_http:post(Node, SignedMsg, Opts) of + {ok, Response} -> + Status = hb_ao:get(<<"status">>, Response, 0, #{}), + ?event(store_remote_node, {make_link_completed, {response, Response}}), + case Status of + 200 -> ok; + _ -> {error, {unexpected_status, Status}} end; - Error -> Error - end. \ No newline at end of file + {error, Err} -> + ?event(store_remote_node, {make_link_error, {error, Err}}), + {error, Err} + end. + +%%%-------------------------------------------------------------------- +%%% Tests +%%%-------------------------------------------------------------------- + +%% @doc Test that we can create a store, write a random message to it, then +%% start a remote node with that store, and read the message from it. +read_test() -> + rand:seed(default), + LocalStore = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-mainnet">> + }, + hb_store:reset(LocalStore), + M = #{ <<"test-key">> => Rand = rand:uniform(1337) }, + ID = hb_message:id(M), + {ok, ID} = + hb_cache:write( + M, + #{ store => LocalStore } + ), + ?event({wrote, ID}), + Node = + hb_http_server:start_node( + #{ + store => LocalStore + } + ), + RemoteStore = [ + #{ <<"store-module">> => hb_store_remote_node, <<"node">> => Node } + ], + {ok, RetrievedMsg} = hb_cache:read(ID, #{ store => RemoteStore }), + ?assertMatch(#{ <<"test-key">> := Rand }, hb_cache:ensure_all_loaded(RetrievedMsg)). \ No newline at end of file diff --git a/src/hb_store_rocksdb.erl b/src/hb_store_rocksdb.erl index 7fc631db2..e461a38c6 100644 --- a/src/hb_store_rocksdb.erl +++ b/src/hb_store_rocksdb.erl @@ -2,41 +2,65 @@ %%% @doc A process wrapper over rocksdb storage. Replicates functionality of the %%% hb_fs_store module. %%% -%%% The data is stored in two Column Families: -%%% 1. Default - for raw data (e.g. message records) -%%% 2. Meta - for meta information -%%% `(<<"raw">>/<<"link">>/<<"composite">> or <<"group">>)' +%%% Encodes the item types with the help of prefixes, see `encode_value/2' +%%% and `decode_value/1' %%% @end %%%----------------------------------------------------------------------------- -module(hb_store_rocksdb). -behaviour(gen_server). -behaviour(hb_store). --export([start/1, start_link/1, stop/1, scope/1]). --export([read/2, write/3, list/2, reset/1]). +-export([enabled/0, start/1, start_link/1, stop/1, scope/1]). +-export([read/2, write/3, list/2, reset/1, list/0]). -export([make_link/3, make_group/2, type/2, add_path/3, path/2, resolve/2]). -export([init/1, terminate/2, handle_cast/2, handle_info/2, handle_call/3]). -export([code_change/3]). --include("src/include/hb.hrl"). +-include("include/hb.hrl"). -define(TIMEOUT, 5000). -type key() :: binary() | list(). -type value() :: binary() | list(). -start_link({hb_store_rocksdb, #{ prefix := Dir}}) -> +-type value_type() :: link | raw | group. + +%% @doc Returns whether the RocksDB store is enabled. +-ifdef(ENABLE_ROCKSDB). +enabled() -> true. +-else. +enabled() -> false. +-endif. + +-ifdef(ENABLE_ROCKSDB). +%% @doc Start the RocksDB store. +start_link(#{ <<"store-module">> := hb_store_rocksdb, <<"name">> := Dir}) -> + ?event(rocksdb, {starting, Dir}), + application:ensure_all_started(rocksdb), gen_server:start_link({local, ?MODULE}, ?MODULE, Dir, []); start_link(Stores) when is_list(Stores) -> - case lists:keyfind(hb_store_rocksdb, 1, Stores) of - Store = {hb_store_rocksdb, _} -> - start_link(Store); + RocksStores = + [ + Store + || + Store = #{ <<"store-module">> := Module } <- Stores, + Module =:= hb_store_rocksdb + ], + case RocksStores of + [Store] -> start_link(Store); _ -> ignore end; start_link(Store) -> - ?event(error, {invalid_store_config, Store}), + ?event(rocksdb, {invalid_store_config, Store}), ignore. +-else. +start_link(_Opts) -> ignore. + +-endif. + +start(Opts = #{ <<"store-module">> := hb_store_rocksdb, <<"name">> := _Dir}) -> + start_link(Opts); start(Opts) -> - start_link([{hb_store_rocksdb, Opts}]). + start_link(Opts). -spec stop(any()) -> ok. stop(_Opts) -> @@ -59,18 +83,20 @@ path(_Opts, Path) -> Opts :: map(), Key :: key() | list(), Result :: {ok, value()} | not_found | {error, {corruption, string()}} | {error, any()}. -read(Opts, RawKey) -> - Key = join(RawKey), - case meta(Key) of +read(Opts, RawPath) -> + ?event({read, RawPath}), + Path = resolve(Opts, RawPath), + case do_read(Opts, Path) of not_found -> - case resolve(Opts, Key) of - not_found -> - not_found; - ResolvedPath -> - gen_server:call(?MODULE, {read, join(ResolvedPath)}, ?TIMEOUT) - end; - _Result -> - gen_server:call(?MODULE, {read, Key}, ?TIMEOUT) + not_found; + {error, _Reason} = Err -> Err; + {ok, {raw, Result}} -> + {ok, Result}; + {ok, {link, Link}} -> + ?event({link_found, Path, Link}), + read(Opts, Link); + {ok, {group, _Result}} -> + not_found end. %% @doc Write given Key and Value to the database @@ -79,104 +105,120 @@ read(Opts, RawKey) -> Key :: key(), Value :: value(), Result :: ok | {error, any()}. -write(_Opts, RawKey, Value) -> - Key = join(RawKey), - gen_server:call(?MODULE, {write, Key, Value}, ?TIMEOUT). - -%% @doc Return meta information about the given Key --spec meta(key()) -> {ok, binary()} | not_found. -meta(Key) -> - gen_server:call(?MODULE, {meta, join(Key)}, ?TIMEOUT). - -%% @doc List key/values stored in the storage so far. -%% *Note*: This function is slow, and probably should not be used on -%% production. Right now it's used for debugging purposes. -%% -%% This can't work as it works for FS store, especially for large sets -%% of data. +write(Opts, RawKey, Value) -> + Key = hb_store:join(RawKey), + EncodedValue = encode_value(raw, Value), + ?event({writing, Key, byte_size(EncodedValue)}), + do_write(Opts, Key, EncodedValue). + +%% @doc Returns the full list of items stored under the given path. Where the path +%% child items is relevant to the path of parentItem. (Same as in `hb_store_fs'). -spec list(Opts, Path) -> Result when Opts :: any(), Path :: any(), - Result :: [string()]. -list(_Opts, Path) -> - Result = gen_server:call(?MODULE, {list, hb_store:join(Path)}, ?TIMEOUT), - {ok, Result}. + Result :: {ok, [string()]} | {error, term()}. + +list(Opts, Path) -> + case do_read(Opts, Path) of + not_found -> {error, not_found}; + {error, _Reason} = Err -> + ?event(rocksdb, {could_not_list_folder, Err}), + Err; + {ok, {group, Value}} -> + {ok, sets:to_list(Value)}; + {ok, {link, LinkedPath}} -> + list(Opts, LinkedPath); + Reason -> + ?event(rocksdb, {could_not_list_folder, Reason}), + {ok, []} + end. %% @doc Replace links in a path with the target of the link. -spec resolve(Opts, Path) -> Result when Opts :: any(), Path :: binary() | list(), Result :: not_found | string(). -resolve(_Opts, RawKey) -> - Key = hb_store:join(RawKey), - Path = filename:split(Key), - - case do_resolve("", Path) of - not_found -> not_found; - <<"">> -> ""; - Result when is_list(Result) -> Result; - % converting back to list, so hb_cache can remove common prefix - BinResult -> binary_to_list(BinResult) +resolve(Opts, Path) -> + PathList = hb_path:term_to_path_parts(hb_store:join(Path)), + + ResolvedPath = do_resolve(Opts, "", PathList), + ResolvedPath. + +do_resolve(_Opts, FinalPath, []) -> + FinalPath; +do_resolve(Opts, CurrentPath, [CurrentPath | Rest]) -> + do_resolve(Opts, CurrentPath, Rest); +do_resolve(Opts, CurrentPath, [Next | Rest]) -> + PathPart = hb_store:join([CurrentPath, Next]), + case do_read(Opts, PathPart) of + not_found -> do_resolve(Opts, PathPart, Rest); + {error, _Reason} = Err -> Err; + {ok, {link, LinkValue}} -> + do_resolve(Opts, LinkValue, Rest); + {ok, _OtherType} -> do_resolve(Opts, PathPart, Rest) end. -%% @doc Helper function that is useful when it's required to get a direct data -%% under the given key, as it is, without following links -read_no_follow(Key) -> - gen_server:call(?MODULE, {read_no_follow, join(Key)}, ?TIMEOUT). - %% @doc Get type of the current item -spec type(Opts, Key) -> Result when Opts :: map(), Key :: binary(), Result :: composite | simple | not_found. -type(_Opts, RawKey) -> +type(Opts, RawKey) -> Key = hb_store:join(RawKey), - case meta(Key) of - not_found -> - not_found; - {ok, <<"composite">>} -> - composite; - {ok, <<"group">>} -> - composite; - {ok, <<"link">>} -> - simple; - {ok, _} -> - simple + case do_read(Opts, Key) of + not_found -> not_found; + {ok, {raw, _Item}} -> simple; + {ok, {link, NewKey}} -> type(Opts, NewKey); + {ok, {group, _Item}} -> composite end. %% @doc Creates group under the given path. -%% Creates an entry in the database and store `<<"group">>' as a type in -%% the meta family. -make_group(_Opts, Path) -> - BinPath = join(Path), - gen_server:call(?MODULE, {make_group, BinPath}, ?TIMEOUT). +-spec make_group(Opts, Key) -> Result when + Opts :: any(), + Key :: binary(), + Result :: ok | {error, already_added}. +make_group(#{ <<"name">> := _DataDir }, Key) -> + gen_server:call(?MODULE, {make_group, Key}, ?TIMEOUT); +make_group(_Opts, Key) -> + gen_server:call(?MODULE, {make_group, Key}, ?TIMEOUT). -spec make_link(any(), key(), key()) -> ok. make_link(_, Key1, Key1) -> ok; -make_link(_Opts, Existing, New) -> +make_link(Opts, Existing, New) -> ExistingBin = convert_if_list(Existing), NewBin = convert_if_list(New), - gen_server:call(?MODULE, {make_link, ExistingBin, NewBin}, ?TIMEOUT). + + % Create: NewValue -> ExistingBin + case do_read(Opts, NewBin) of + not_found -> + do_write(Opts, NewBin, encode_value(link, ExistingBin)); + _ -> + ok + end. %% @doc Add two path components together. // is not used add_path(_Opts, Path1, Path2) -> Path1 ++ Path2. +%% @doc List all items registered in rocksdb store. Should be used only +%% for testing/debugging, as the underlying operation is doing full traversal +%% on the KV storage, and is slow. +list() -> + gen_server:call(?MODULE, list, ?TIMEOUT). + %%%============================================================================= %%% Gen server callbacks %%%============================================================================= init(Dir) -> filelib:ensure_dir(Dir), case open_rockdb(Dir) of - {ok, DBHandle, [DefaultH, MetaH]} -> + {ok, DBHandle} -> State = #{ db_handle => DBHandle, - dir => Dir, - data_family => DefaultH, - meta_family => MetaH + dir => Dir }, {ok, State}; {error, Reason} -> @@ -189,44 +231,47 @@ handle_cast(_Request, State) -> handle_info(_Info, State) -> {noreply, State}. -handle_call({write, Key, Value}, _From, State) -> - ok = write_meta(State, Key, <<"raw">>, #{}), - Result = write_data(State, Key, Value, #{}), - {reply, Result, State}; -handle_call({make_group, Key}, _From, State) -> - ok = write_meta(State, Key, <<"group">>, #{}), - Result = write_data(State, Key, <<"group">>, #{}), - {reply, Result, State}; -handle_call({make_link, Key1, Key2}, _From, State) -> - ok = write_meta(State, Key2, <<"link">>, #{}), - Result = write_data(State, Key2, Key1, #{}), - {reply, Result, State}; -handle_call({read, Key}, _From, DBInfo) -> - Result = get(DBInfo, Key, #{}), - {reply, Result, DBInfo}; -handle_call({read_no_follow, BaseKey}, _From, State) -> - Result = get_data(State, BaseKey, #{}), - {reply, Result, State}; -handle_call({meta, Key}, _From, State) -> - Result = get_meta(State, Key, #{}), - {reply, Result, State}; -handle_call(reset, _From, #{db_handle := DBHandle, dir := Dir}) -> +handle_call(Request, From, #{ db_handle := undefined, dir := Dir } = State) -> + % Re-initialize the DB handle if it's not set. + {ok, DBHandle} = open_rockdb(Dir), + handle_call(Request, From, State#{db_handle => DBHandle}); +handle_call({do_write, Key, Value}, _From, #{db_handle := DBHandle} = State) -> + BaseName = filename:basename(Key), + rocksdb:put(DBHandle, Key, Value, #{}), + case filename:dirname(Key) of + <<".">> -> + ignore; + BaseDir -> + ensure_dir(DBHandle, BaseDir), + {ok, RawDirContent} = rocksdb:get(DBHandle, BaseDir, #{}), + NewDirContent = maybe_append_key_to_group(BaseName, RawDirContent), + ok = rocksdb:put(DBHandle, BaseDir, NewDirContent, #{}) + end, + {reply, ok, State}; +handle_call({do_read, Key}, _From, #{db_handle := DBHandle} = State) -> + Response = + case rocksdb:get(DBHandle, Key, #{}) of + {ok, Result} -> + {Type, Value} = decode_value(Result), + {ok, {Type, Value}}; + not_found -> + not_found; + {error, _Reason} = Err -> + Err + end, + {reply, Response, State}; +handle_call(reset, _From, State = #{db_handle := DBHandle, dir := Dir}) -> ok = rocksdb:close(DBHandle), - ok = rocksdb:destroy(Dir, []), - - {ok, NewDBHandle, [DefaultH, MetaH]} = open_rockdb(Dir), - NewState = #{ - db_handle => NewDBHandle, - dir => Dir, - data_family => DefaultH, - meta_family => MetaH - }, - - {reply, ok, NewState}; -handle_call({list, Path}, _From, State = #{db_handle := DBHandle}) -> + ok = rocksdb:destroy(DirStr = ensure_list(Dir), []), + os:cmd(binary_to_list(<< "rm -Rf ", (list_to_binary(DirStr))/binary >>)), + {reply, ok, State#{ db_handle := undefined }}; +handle_call(list, _From, State = #{db_handle := DBHandle}) -> {ok, Iterator} = rocksdb:iterator(DBHandle, []), - Items = collect(Iterator, Path), + Items = collect(Iterator), {reply, Items, State}; +handle_call({make_group, Path}, _From, #{db_handle := DBHandle} = State) -> + Result = ensure_dir(DBHandle, Path), + {reply, Result, State}; handle_call(_Request, _From, State) -> {reply, handle_call_unrecognized_message, State}. @@ -239,6 +284,62 @@ code_change(_OldVsn, State, _Extra) -> %%%============================================================================= %%% Private %%%============================================================================= +%% @doc Write given Key and Value to the database +-spec do_write(Opts, Key, Value) -> Result when + Opts :: map(), + Key :: key(), + Value :: value(), + Result :: ok | {error, any()}. +do_write(_Opts, Key, Value) -> + gen_server:call(?MODULE, {do_write, Key, Value}, ?TIMEOUT). + +do_read(_Opts, Key) -> + gen_server:call(?MODULE, {do_read, Key}, ?TIMEOUT). + +-spec encode_value(value_type(), binary()) -> binary(). +encode_value(link, Value) -> <<1, Value/binary>>; +encode_value(raw, Value) -> <<2, Value/binary>>; +encode_value(group, Value) -> <<3, (term_to_binary(Value))/binary>>. + +-spec decode_value(binary()) -> {value_type(), binary()}. +decode_value(<<1, Value/binary>>) -> {link, Value}; +decode_value(<<2, Value/binary>>) -> {raw, Value}; +decode_value(<<3, Value/binary>>) -> {group, binary_to_term(Value)}. + +ensure_dir(DBHandle, BaseDir) -> + PathParts = hb_path:term_to_path_parts(BaseDir), + [First | Rest] = PathParts, + Result = ensure_dir(DBHandle, First, Rest), + Result. +ensure_dir(DBHandle, CurrentPath, []) -> + maybe_create_dir(DBHandle, CurrentPath, nil), + ok; +ensure_dir(DBHandle, CurrentPath, [Next]) -> + maybe_create_dir(DBHandle, CurrentPath, Next), + ensure_dir(DBHandle, hb_store:join([CurrentPath, Next]), []); +ensure_dir(DBHandle, CurrentPath, [Next | Rest]) -> + maybe_create_dir(DBHandle, CurrentPath, Next), + ensure_dir(DBHandle, hb_store:join([CurrentPath, Next]), Rest). + +maybe_create_dir(DBHandle, DirPath, Value) -> + CurrentValueSet = + case rocksdb:get(DBHandle, DirPath, #{}) of + not_found -> sets:new(); + {ok, CurrentValue} -> + {group, DecodedOldValue} = decode_value(CurrentValue), + DecodedOldValue + end, + NewValueSet = + case Value of + nil -> CurrentValueSet; + _ -> sets:add_element(Value, CurrentValueSet) + end, + rocksdb:put(DBHandle, DirPath, encode_value(group, NewValueSet), #{}). + +open_rockdb(RawDir) -> + filelib:ensure_dir(Dir = ensure_list(RawDir)), + Options = [{create_if_missing, true}], + rocksdb:open(Dir, Options). % Helper function to convert lists to binaries convert_if_list(Value) when is_list(Value) -> @@ -246,116 +347,64 @@ convert_if_list(Value) when is_list(Value) -> convert_if_list(Value) -> Value. % Leave unchanged if it's not a list -open_rockdb(Dir) -> - ColumnFamilies = [{"default", []}, {"meta", []}], - Options = [{create_if_missing, true}, {create_missing_column_families, true}], - rocksdb:open_with_cf(Dir, Options, ColumnFamilies). - -get_data(#{data_family := F, db_handle := Handle}, Key, Opts) -> - rocksdb:get(Handle, F, Key, Opts). - -get_meta(#{meta_family := F, db_handle := Handle}, Key, Opts) -> - rocksdb:get(Handle, F, Key, Opts). - -write_meta(DBInfo, Key, Value, Opts) -> - #{meta_family := ColumnFamily, db_handle := Handle} = DBInfo, - rocksdb:put(Handle, ColumnFamily, Key, Value, Opts). - -write_data(DBInfo, Key, Value, Opts) -> - #{data_family := ColumnFamily, db_handle := Handle} = DBInfo, - rocksdb:put(Handle, ColumnFamily, Key, Value, Opts). - -% Note: this function is not yet optimized for tail recursion, -% which might be needed if we expect big amount of nested links -get(DBInfo, Key, Opts) -> - case get_data(DBInfo, Key, Opts) of - {ok, Value} -> - case get_meta(DBInfo, Key, Opts) of - {ok, <<"link">>} -> - % Automatically follow the link - get(DBInfo, Value, Opts); - {ok, <<"raw">>} -> - {ok, Value}; - _OtherMeta -> - {ok, Value} - end; - not_found -> - not_found - end. +%% @doc Ensure that the given filename is a list, not a binary. +ensure_list(Value) when is_binary(Value) -> binary_to_list(Value); +ensure_list(Value) -> Value. -collect(Iterator, Path) -> +maybe_convert_to_binary(Value) when is_list(Value) -> + list_to_binary(Value); +maybe_convert_to_binary(Value) when is_binary(Value) -> + Value. + +join(Key) when is_list(Key) -> + KeyList = hb_store:join(Key), + maybe_convert_to_binary(KeyList); +join(Key) when is_binary(Key) -> Key. + +collect(Iterator) -> case rocksdb:iterator_move(Iterator, <<>>) of {error, invalid_iterator} -> []; - {ok, Key, _Value} -> - collect(Iterator, Path, maybe_add_key(Key, Path, [])) + {ok, Key, Value} -> + DecodedValue = decode_value(Value), + collect(Iterator, [{Key, DecodedValue}]) end. - -collect(Iterator, Path, Acc) -> +collect(Iterator, Acc) -> case rocksdb:iterator_move(Iterator, next) of - {ok, Key, _Value} -> + {ok, Key, Value} -> % Continue iterating, accumulating the key-value pair in the list - NewAcc = maybe_add_key(Key, Path, Acc), - collect(Iterator, Path, NewAcc); + DecodedValue = decode_value(Value), + collect(Iterator, [{Key, DecodedValue} | Acc]); {error, invalid_iterator} -> % Reached the end of the iterator, return the accumulated list lists:reverse(Acc) end. -maybe_add_key(Key, Prefix, Keys) -> - case re:split(Key, Prefix, [{return, binary}]) of - [Key] -> Keys; % The split did not really split anything - [<<>>, <<"/", Suffix/binary>>] -> [erlang:binary_to_list(Suffix) | Keys]; - [<<>>, Suffix] -> [erlang:binary_to_list(Suffix) | Keys]; - _ -> Keys +maybe_append_key_to_group(Key, CurrentDirContents) -> + case decode_value(CurrentDirContents) of + {group, GroupSet} -> + BaseName = filename:basename(Key), + NewGroupSet = sets:add_element(BaseName, GroupSet), + encode_value(group, NewGroupSet); + _ -> + CurrentDirContents end. - - -maybe_convert_to_binary(Value) when is_list(Value) -> - list_to_binary(Value); -maybe_convert_to_binary(Value) when is_binary(Value) -> - Value. - -do_resolve(CurrPath, []) -> - CurrPath; -do_resolve(CurrPath, [LookupKey | Rest]) -> - LookupPath = hb_store:join([CurrPath, LookupKey]), - - NewCurrentPath = - case meta(LookupPath) of - {ok, <<"link">>} -> - read_no_follow(LookupPath); - {ok, <<"raw">>} -> - {ok, LookupPath}; - {ok, <<"group">>} -> - do_resolve(LookupPath, Rest); - not_found -> - do_resolve(LookupPath, Rest) - end, - case NewCurrentPath of - not_found -> - list_to_binary(CurrPath); - {ok, Path} -> - do_resolve(Path, Rest); - Result -> - maybe_convert_to_binary(Result) - end. - -join(Key) when is_list(Key) -> - KeyList = hb_store:join(Key), - maybe_convert_to_binary(KeyList); -join(Key) when is_binary(Key) -> Key. %%%============================================================================= %%% Tests %%%============================================================================= +-ifdef(ENABLE_ROCKSDB). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). get_or_start_server() -> - Store = lists:keyfind(hb_store_rocksdb, 1, hb_store:test_stores()), - case start_link(Store) of + % Store = lists:keyfind(hb_store_rocksdb2, 1, hb_store:test_stores()), + Opts = #{ + <<"store-module">> => hb_store_rocksdb, + <<"name">> => <<"cache-TEST/rocksdb">> + }, + case start_link(Opts) of {ok, Pid} -> Pid; {error, {already_started, Pid}} -> @@ -368,16 +417,16 @@ write_read_test_() -> Pid = get_or_start_server(), unlink(Pid) end, - fun(_) -> hb_store_rocksdb:reset([]) end, [ + fun(_) -> reset([]) end, + [ {"can read/write data", fun() -> ok = write(#{}, <<"test_key">>, <<"test_value">>), - {ok, Value} = read(ignored_options, <<"test_key">>), + {ok, Value} = read(#{}, <<"test_key">>), ?assertEqual(<<"test_value">>, Value) end}, {"returns not_found for non existing keys", fun() -> Value = read(#{}, <<"non_existing">>), - ?assertEqual(not_found, Value) end}, {"follows links", fun() -> @@ -396,23 +445,29 @@ api_test_() -> unlink(Pid) end, fun(_) -> reset([]) end, [ - {"list/2 lists keys under given path", fun() -> - ok = write(#{}, <<"messages/key1">>, <<>>), - ok = write(#{}, <<"messages/key2">>, <<>>), - ok = write(#{}, <<"other_path/key3">>, <<>>), + {"write/3 can automatically create folders", fun() -> + ok = write(#{}, <<"messages/key1">>, <<"val1">>), + ok = write(#{}, <<"messages/key2">>, <<"val2">>), + {ok, Items} = list(#{}, <<"messages">>), - ?assertEqual(["key1", "key2"], Items) + ?assertEqual( + lists:sort([<<"key1">>, <<"key2">>]), + lists:sort(Items) + ), + {ok, Item} = read(#{}, <<"messages/key1">>), + ?assertEqual(<<"val1">>, Item) end}, - {"list/2 resolves given path before listing", fun() -> - ok = write(#{}, <<"process/slot/1">>, <<>>), - ok = write(#{}, <<"process/slot/2">>, <<>>), - ok = write(#{}, <<"messages/key">>, <<>>), - {ok, Items} = list(#{}, ["process", "slot"]), - ?assertEqual(["1", "2"], Items) + {"list/2 lists keys under given path", fun() -> + ok = write(#{}, <<"messages/key1">>, <<"val1">>), + ok = write(#{}, <<"messages/key2">>, <<"val2">>), + ok = write(#{}, <<"other_path/key3">>, <<"val3">>), + {ok, Items} = list(#{}, <<"messages">>), + ?assertEqual( + lists:sort([<<"key1">>, <<"key2">>]), lists:sort(Items) + ) end}, {"list/2 when database is empty", fun() -> - {ok, Items} = list(#{}, <<"process/slot">>), - ?assertEqual([], Items) + ?assertEqual({error, not_found}, list(#{}, <<"process/slot">>)) end}, {"make_link/3 creates a link to actual data", fun() -> ok = write(ignored_options, <<"key1">>, <<"test_value">>), @@ -421,12 +476,6 @@ api_test_() -> ?assertEqual(<<"test_value">>, Value) end}, - {"make_group/2 creates a group", fun() -> - ok = make_group(#{}, <<"folder_path">>), - - {ok, Value} = meta(<<"folder_path">>), - ?assertEqual(<<"group">>, Value) - end}, {"make_link/3 does not create links if keys are same", fun() -> ok = make_link([], <<"key1">>, <<"key1">>), ?assertEqual(not_found, read(#{}, <<"key1">>)) @@ -451,10 +500,15 @@ api_test_() -> end }, { - "type/2 treats links as simple items", + "type/2 resolves links before checking real type of the following item", fun() -> - make_link(#{}, <<"ExistingKey">>, <<"NewKey">>), - ?assertEqual(simple, type(#{}, <<"NewKey">>)) + ok = write(#{}, <<"messages/key1">>, <<"val1">>), + ok = write(#{}, <<"messages/key2">>, <<"val2">>), + + make_link(#{}, <<"messages">>, <<"CompositeKey">>), + make_link(#{}, <<"messages/key2">>, <<"SimpleKey">>), + ?assertEqual(composite, type(#{}, <<"CompositeKey">>)), + ?assertEqual(simple, type(#{}, <<"SimpleKey">>)) end }, { @@ -465,168 +519,91 @@ api_test_() -> end }, { - "resolve/2 resolutions for computed folder", + "resolve/2 resolves raw/groups items", fun() -> - % ├── computed - % │ └── 7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw - % │ ├── 76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc -> messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc - % │ ├── 8ZSzLqadFI0DyaUMEMvEcM9N5zkWqLU2lu7XhVejLGE -> messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc - % │ ├── LbfBoMI7xNYpBFv1Fsl2FSa8QYA2k9NtbzQTOKlN2TE -> messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg - % │ ├── Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg -> messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg - % │ ├── slot - % │ │ ├── 0 -> messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg - % │ │ └── 1 -> messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc - % ├── messages - % │ ├── 76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc [raw] - % │ ├── 8ZSzLqadFI0DyaUMEMvEcM9N5zkWqLU2lu7XhVejLGE -> messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc - % │ ├── LbfBoMI7xNYpBFv1Fsl2FSa8QYA2k9NtbzQTOKlN2TE -> messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg - % │ └── Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg [raw] - - % Resolution examples: - % ["computed","7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw", "LbfBoMI7xNYpBFv1Fsl2FSa8QYA2k9NtbzQTOKlN2TE"] -> "messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg" - % ["computed","7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw", ["slot", "1"]] -> "messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc" - - % Create raw items in messages - write(#{}, <<"messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc/item">>, <<"Value">>), - write(#{}, <<"messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg/item">>, <<"Value">>), - - % Create symbolic links in messages - make_link( - #{}, - <<"messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc">>, - <<"messages/8ZSzLqadFI0DyaUMEMvEcM9N5zkWqLU2lu7XhVejLGE">> - ), - make_link( - #{}, - <<"messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg">>, - <<"messages/LbfBoMI7xNYpBFv1Fsl2FSa8QYA2k9NtbzQTOKlN2TE">> - ), + write(#{}, <<"top_level/level1/item1">>, <<"1">>), + write(#{}, <<"top_level/level1/item2">>, <<"1">>), + write(#{}, <<"top_level/level1/item3">>, <<"1">>), - % Create subdirectory in computed - make_group(#{}, <<"computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw">>), + ?assertEqual( + <<"top_level/level1/item3">>, + resolve(#{}, <<"top_level/level1/item3">>) + ) + end + }, + { + "resolve/2 follows links", + fun() -> + write(#{}, <<"data/the_data_item">>, <<"the_data">>), + make_link(#{}, <<"data/the_data_item">>, <<"top_level/level1/item">>), - % Create symbolic links in computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw - make_link( - #{}, - <<"messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc">>, - <<"computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc">> - ), - make_link( - #{}, - <<"messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc">>, - <<"computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw/8ZSzLqadFI0DyaUMEMvEcM9N5zkWqLU2lu7XhVejLGE">> - ), - make_link( - #{}, - <<"messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg">>, - <<"computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw/LbfBoMI7xNYpBFv1Fsl2FSa8QYA2k9NtbzQTOKlN2TE">> - ), - make_link( - #{}, - <<"messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg">>, - <<"computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg">> - ), + ?assertEqual( + <<"data/the_data_item">>, + resolve(#{}, <<"top_level/level1/item">>) + ) + end + }, + { + "make_group/2 creates a folder", + fun() -> + ?assertEqual(ok, make_group(#{}, <<"messages">>)), - % Create subdirectory computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw/slot - make_group(#{}, <<"computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw/slot">>), + ?assertEqual( + list(#{}, <<"messages">>), + {ok, []} + ) + end + }, + { + "make_group/2 does not override folder contents", + fun() -> + write(#{}, <<"messages/id">>, <<"1">>), + write(#{}, <<"messages/commitments">>, <<"2">>), - % Create symbolic links in computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw/slot - make_link( - #{}, - <<"messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg">>, - <<"computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw/slot/0">> - ), - make_link( - #{}, - <<"messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc">>, - <<"computed/7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw/slot/1">> - ), + ?assertEqual(ok, make_group(#{}, <<"messages">>)), - % Test ?assertEqual( - "messages/Vsf2Eto5iQ9fghmH5RsUm4b9h0fb_CCYTVTjnHEDGQg", - resolve(#{}, [ - "computed", "7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw", "LbfBoMI7xNYpBFv1Fsl2FSa8QYA2k9NtbzQTOKlN2TE" - ]) + list(#{}, <<"messages">>), + {ok, [<<"id">>, <<"commitments">>]} + ) + end + }, + { + "make_group/2 making deep nested groups", + fun() -> + make_group(#{}, <<"messages/ids/items">>), + ?assertEqual( + {ok, [<<"ids">>]}, + list(#{}, <<"messages">>) ), - ?assertEqual( - "messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc", - resolve(#{}, ["computed", "7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw", ["slot", "1"]]) + {ok, [<<"items">>]}, + list(#{}, <<"messages/ids">>) ), - ?assertEqual( - "messages/76vSvK1yAlcGTPLyP7xEVUG7kiBxQrvTxhQor_KC8Wc", - resolve(#{}, ["computed", "7bi8NdEPLJwcD5ADWQ5PIoDlBpBWSw-9N7VXYe25Lvw", "slot", "1"]) + {ok, []}, + list(#{}, <<"messages/ids/items">>) ) end }, - {"resolving interlinked item paths", fun() -> - % messages - % ├── csZNlQe-ehlhmCU8shC3vjhrW2qsaMAsQzs-ALjokOc [raw] - % ├── -7ZAg8BW_itF-f9y4L0cY0xfz34iZBZ6jlDa9Tb23ME - % │ ├── item - % │ └── level1_key -> messages/UsxVZBMaIbe15LPrzqImczl7fhUPdmK3ANhGjuHkxGo - % ├── UsxVZBMaIbe15LPrzqImczl7fhUPdmK3ANhGjuHkxGo - % │ ├── item - % │ └── level2_key -> messages/FGQgh1nQBwqi_kx7wpEIMAT2ltRsbieoZBHaUBK8riE - % └── FGQgh1nQBwqi_kx7wpEIMAT2ltRsbieoZBHaUBK8riE - % ├── item - % └── level3_key -> messages/csZNlQe-ehlhmCU8shC3vjhrW2qsaMAsQzs-ALjokOc - % - - % resolve("messages", "-7ZAg8BW_itF-f9y4L0cY0xfz34iZBZ6jlDa9Tb23ME", ["level1_key", "level2_key", "level3_key"]) - % -> "messages/csZNlQe-ehlhmCU8shC3vjhrW2qsaMAsQzs-ALjokOc" - - % Create the raw item in messages - write(#{}, <<"messages/csZNlQe-ehlhmCU8shC3vjhrW2qsaMAsQzs-ALjokOc/item">>, <<"Value">>), - - % Create the subdirectory under messages - make_group(#{}, <<"messages/-7ZAg8BW_itF-f9y4L0cY0xfz34iZBZ6jlDa9Tb23ME">>), - - % Add item in the subdirectory - write(#{}, <<"messages/-7ZAg8BW_itF-f9y4L0cY0xfz34iZBZ6jlDa9Tb23ME/item">>, <<"Value">>), - - % Create symbolic link to level1_key - make_link( - #{}, - <<"messages/UsxVZBMaIbe15LPrzqImczl7fhUPdmK3ANhGjuHkxGo">>, - <<"messages/-7ZAg8BW_itF-f9y4L0cY0xfz34iZBZ6jlDa9Tb23ME/level1_key">> - ), - - % Create group for messages/UsxVZBMaIbe15LPrzqImczl7fhUPdmK3ANhGjuHkxGo - make_group(#{}, <<"messages/UsxVZBMaIbe15LPrzqImczl7fhUPdmK3ANhGjuHkxGo">>), - - % Add item in messages/UsxVZBMaIbe15LPrzqImczl7fhUPdmK3ANhGjuHkxGo - write(#{}, <<"messages/UsxVZBMaIbe15LPrzqImczl7fhUPdmK3ANhGjuHkxGo/item">>, <<"Value">>), - - % Create symbolic link to level2_key - make_link( - #{}, - <<"messages/FGQgh1nQBwqi_kx7wpEIMAT2ltRsbieoZBHaUBK8riE">>, - <<"messages/UsxVZBMaIbe15LPrzqImczl7fhUPdmK3ANhGjuHkxGo/level2_key">> - ), - - % Create group for messages/FGQgh1nQBwqi_kx7wpEIMAT2ltRsbieoZBHaUBK8riE - make_group(#{}, <<"messages/FGQgh1nQBwqi_kx7wpEIMAT2ltRsbieoZBHaUBK8riE">>), - - % Add item in messages/FGQgh1nQBwqi_kx7wpEIMAT2ltRsbieoZBHaUBK8riE - write(#{}, <<"messages/FGQgh1nQBwqi_kx7wpEIMAT2ltRsbieoZBHaUBK8riE/item">>, <<"Value">>), - - % Create symbolic link to level3_key - make_link( - #{}, - <<"messages/csZNlQe-ehlhmCU8shC3vjhrW2qsaMAsQzs-ALjokOc">>, - <<"messages/FGQgh1nQBwqi_kx7wpEIMAT2ltRsbieoZBHaUBK8riE/level3_key">> - ), - - ?assertEqual( - "messages/csZNlQe-ehlhmCU8shC3vjhrW2qsaMAsQzs-ALjokOc", - resolve(#{}, [ - "messages", "-7ZAg8BW_itF-f9y4L0cY0xfz34iZBZ6jlDa9Tb23ME", ["level1_key", "level2_key", "level3_key"] - ]) - ) - end} + { + "write/3 automatically does deep groups", + fun() -> + write(#{}, <<"messages/ids/item1">>, <<"1">>), + write(#{}, <<"messages/ids/item2">>, <<"2">>), + ?assertEqual( + {ok, [<<"ids">>]}, + list(#{}, <<"messages">>) + ), + ?assertEqual( + {ok, [<<"item2">>, <<"item1">>]}, + list(#{}, <<"messages/ids">>) + ), + ?assertEqual(read(#{}, <<"messages/ids/item1">>),{ok, <<"1">>}), + ?assertEqual(read(#{}, <<"messages/ids/item2">>), {ok, <<"2">>}) + end + } ]}. +-endif. -endif. \ No newline at end of file diff --git a/src/hb_http_structured_fields.erl b/src/hb_structured_fields.erl similarity index 81% rename from src/hb_http_structured_fields.erl rename to src/hb_structured_fields.erl index 59f0556a1..7fc202639 100644 --- a/src/hb_http_structured_fields.erl +++ b/src/hb_structured_fields.erl @@ -1,13 +1,6 @@ --module(hb_http_structured_fields). - --export([parse_dictionary/1, parse_item/1, parse_list/1, parse_bare_item/1]). --export([dictionary/1, item/1, list/1, bare_item/1]). --export([to_dictionary/1, to_list/1, to_item/1, to_item/2]). - --include_lib("eunit/include/eunit.hrl"). - --include("include/hb_http.hrl"). - +%%% @doc A module for parsing and converting between Erlang and HTTP Structured +%%% Fields, as described in RFC-9651. +%%% %%% The mapping between Erlang and structured headers types is as follow: %%% %%% List: list() @@ -23,6 +16,13 @@ %%% Token: {token, binary()} %%% Byte sequence: {binary, binary()} %%% Boolean: boolean() +-module(hb_structured_fields). +-export([parse_dictionary/1, parse_item/1, parse_list/1, parse_bare_item/1]). +-export([parse_binary/1]). +-export([dictionary/1, item/1, list/1, bare_item/1, from_bare_item/1]). +-export([to_dictionary/1, to_list/1, to_item/1, to_item/2]). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb_http.hrl"). -type sh_list() :: [sh_item() | sh_inner_list()]. -type sh_inner_list() :: {list, [sh_item()], sh_params()}. @@ -45,9 +45,7 @@ (C =:= $z) ). -%% Mapping - -% Dictionary +%% @doc Convert a map to a dictionary. to_dictionary(Map) when is_map(Map) -> to_dictionary(maps:to_list(Map)); to_dictionary(Pairs) when is_list(Pairs) -> @@ -63,7 +61,7 @@ to_dictionary(Dict, [{Name, Value} | Rest]) -> E -> E end. -% Item +%% @doc Convert an item to a dictionary. to_item({item, Kind, Params}) when is_list(Params) -> {ok, {item, to_bare_item(Kind), [to_param(Pair) || Pair <- Params] }}; to_item(Item) -> @@ -71,7 +69,7 @@ to_item(Item) -> to_item(Item, Params) when is_list(Params) -> to_item({ item, to_bare_item(Item), Params}). -% List +%% @doc Convert a list to an SF term. to_list(List) when is_list(List) -> to_list([], List). to_list(Acc, []) -> @@ -83,15 +81,13 @@ to_list(Acc, [ItemOrInner | Rest]) -> E -> E end. -% Inner List +%% @doc Convert an inner list to an SF term. to_inner_list({list, Inner, Params}) when is_list(Inner) andalso is_list(Params) -> {ok, {list, [to_inner_item(I) || I <-- Inner], [to_param(Pair) || Pair <- Params]}}; to_inner_list(Inner) -> to_inner_list(Inner, []). - to_inner_list(Inner, Params) when is_list(Inner) andalso is_list(Params) -> to_inner_list([], Inner, Params). - to_inner_list(Inner, [], Params) when is_list(Params) -> {ok, {list, lists:reverse(Inner), [to_param(Param) || Param <- Params]}}; to_inner_list(_List, [Item | _Rest], _Params) when is_list(Item) orelse is_map(Item) -> @@ -102,6 +98,7 @@ to_inner_list(Inner, [Item | Rest], Params) -> E -> E end. +%% @doc Convert an Erlang term to an SF `item' or `inner_list'. to_item_or_inner_list(ItemOrInner) -> case ItemOrInner of Map when is_map(Map) -> {too_deep, Map}; @@ -111,6 +108,7 @@ to_item_or_inner_list(ItemOrInner) -> Inner when is_list(Inner) -> to_inner_list(Inner) end. +%% @doc Convert an Erlang term to an SF `item'. to_inner_item(Item) when is_list(Item) -> {too_deep, Item}; to_inner_item(Item) -> @@ -119,12 +117,12 @@ to_inner_item(Item) -> E -> E end. -% Parameters +%% @doc Convert an Erlang term to an SF `parameter'. to_param({Name, Value}) -> NormalizedName = key_to_binary(Name), {NormalizedName, to_bare_item(Value)}. -% Bare Items +%% @doc Convert an Erlang term to an SF `bare_item'. to_bare_item(BareItem) -> case BareItem of % Assume tuple is already parsed @@ -141,15 +139,40 @@ to_bare_item(BareItem) -> S when is_binary(S) or is_list(S) -> {string, iolist_to_binary(S)} end. +%% @doc Convert an SF `bare_item' to an Erlang term. +from_bare_item(BareItem) -> + case BareItem of + I when is_integer(I) -> I; + B when is_boolean(B) -> B; + D = {decimal, _} -> + list_to_float( + binary_to_list( + iolist_to_binary( + bare_item(D) + ) + ) + ); + {string, S} -> S; + {token, T} -> + try binary_to_existing_atom(T) of + Atom -> Atom + catch + error:badarg -> T + end; + {binary, B} -> B + end. + +%% @doc Convert an Erlang term to a binary key. key_to_binary(Key) when is_atom(Key) -> atom_to_binary(Key); key_to_binary(Key) -> iolist_to_binary(Key). -%% Parsing. - +%% @doc Parse a binary SF dictionary. -spec parse_dictionary(binary()) -> sh_dictionary(). parse_dictionary(<<>>) -> []; -parse_dictionary(<>) when ?IS_LC_ALPHA(C) or (C =:= $*) -> +parse_dictionary(<>) when ?IS_ALPHA(C) + or ?IS_DIGIT(C) or (C =:= $*) or (C =:= $%) or (C =:= $_) or (C =:= $-) + or (C =:= $.) -> parse_dict_key(R, [], <>). parse_dict_key(<<$=, $(, R0/bits>>, Acc, K) -> @@ -159,9 +182,8 @@ parse_dict_key(<<$=, R0/bits>>, Acc, K) -> {Item, R} = parse_item1(R0), parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, Item})); parse_dict_key(<>, Acc, K) when - ?IS_LC_ALPHA(C) or ?IS_DIGIT(C) or - (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) --> + ?IS_ALPHA(C) or ?IS_DIGIT(C) or + (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) or (C =:= $%) -> parse_dict_key(R, Acc, <>); parse_dict_key(<<$;, R0/bits>>, Acc, K) -> {Params, R} = parse_before_param(R0, []), @@ -169,6 +191,7 @@ parse_dict_key(<<$;, R0/bits>>, Acc, K) -> parse_dict_key(R, Acc, K) -> parse_dict_before_sep(R, lists:keystore(K, 1, Acc, {K, {item, true, []}})). +%% @doc Parse a binary SF dictionary before a separator. parse_dict_before_sep(<<$\s, R/bits>>, Acc) -> parse_dict_before_sep(R, Acc); parse_dict_before_sep(<<$\t, R/bits>>, Acc) -> @@ -178,13 +201,17 @@ parse_dict_before_sep(<>, Acc) when C =:= $, -> parse_dict_before_sep(<<>>, Acc) -> Acc. +%% @doc Parse a binary SF dictionary before a member. parse_dict_before_member(<<$\s, R/bits>>, Acc) -> parse_dict_before_member(R, Acc); parse_dict_before_member(<<$\t, R/bits>>, Acc) -> parse_dict_before_member(R, Acc); -parse_dict_before_member(<>, Acc) when ?IS_LC_ALPHA(C) or (C =:= $*) -> +parse_dict_before_member(<>, Acc) + when ?IS_ALPHA(C) or ?IS_DIGIT(C) or (C =:= $*) + or (C =:= $%) or (C =:= $_) or (C =:= $-) -> parse_dict_key(R, Acc, <>). +%% @doc Parse a binary SF item to an SF `item'. -spec parse_item(binary()) -> sh_item(). parse_item(Bin) -> {Item, <<>>} = parse_item1(Bin), @@ -199,12 +226,14 @@ parse_item1(Bin) -> {{item, Item, []}, Rest} end. +%% @doc Parse a binary SF list. -spec parse_list(binary()) -> sh_list(). parse_list(<<>>) -> []; parse_list(Bin) -> parse_list_before_member(Bin, []). +%% @doc Parse a binary SF list before a member. parse_list_member(<<$(, R0/bits>>, Acc) -> {Item, R} = parse_inner_list(R0, []), parse_list_before_sep(R, [Item | Acc]); @@ -212,6 +241,7 @@ parse_list_member(R0, Acc) -> {Item, R} = parse_item1(R0), parse_list_before_sep(R, [Item | Acc]). +%% @doc Parse a binary SF list before a separator. parse_list_before_sep(<<$\s, R/bits>>, Acc) -> parse_list_before_sep(R, Acc); parse_list_before_sep(<<$\t, R/bits>>, Acc) -> @@ -221,6 +251,7 @@ parse_list_before_sep(<<$,, R/bits>>, Acc) -> parse_list_before_sep(<<>>, Acc) -> lists:reverse(Acc). +%% @doc Parse a binary SF list before a member. parse_list_before_member(<<$\s, R/bits>>, Acc) -> parse_list_before_member(R, Acc); parse_list_before_member(<<$\t, R/bits>>, Acc) -> @@ -228,7 +259,7 @@ parse_list_before_member(<<$\t, R/bits>>, Acc) -> parse_list_before_member(R, Acc) -> parse_list_member(R, Acc). -%% Internal. +%%% Internal functions. parse_inner_list(<<$\s, R/bits>>, Acc) -> parse_inner_list(R, Acc); @@ -258,25 +289,31 @@ parse_param(<<$=, R0/bits>>, Acc, K) -> end; parse_param(<>, Acc, K) when ?IS_LC_ALPHA(C) or ?IS_DIGIT(C) or - (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) --> + (C =:= $_) or (C =:= $-) or (C =:= $.) or (C =:= $*) -> parse_param(R, Acc, <>); parse_param(R, Acc, K) -> {lists:keystore(K, 1, Acc, {K, true}), R}. -%% Integer or decimal. +%% @doc Parse an integer or decimal. parse_bare_item(<<$-, R/bits>>) -> parse_number(R, 0, <<$->>); parse_bare_item(<>) when ?IS_DIGIT(C) -> parse_number(R, 1, <>); -%% String. -parse_bare_item(<<$", R/bits>>) -> parse_string(R, <<>>); -%% Token. -parse_bare_item(<>) when ?IS_ALPHA(C) or (C =:= $*) -> parse_token(R, <>); -%% Byte sequence. -parse_bare_item(<<$:, R/bits>>) -> parse_binary(R, <<>>); -%% Boolean. -parse_bare_item(<<"?0", R/bits>>) -> {false, R}; -parse_bare_item(<<"?1", R/bits>>) -> {true, R}. - +parse_bare_item(<<$", R/bits>>) -> + % Parse a string. + parse_string(R, <<>>); +parse_bare_item(<>) when ?IS_ALPHA(C) or (C =:= $*) -> + % Parse a token. + parse_token(R, <>); +parse_bare_item(<<$:, R/bits>>) -> + % Parse a byte sequence. + parse_binary(R, <<>>); +parse_bare_item(<<"?0", R/bits>>) -> + % Parse a boolean false. + {false, R}; +parse_bare_item(<<"?1", R/bits>>) -> + % Parse a boolean true. + {true, R}. + +%% @doc Parse an integer or decimal binary. parse_number(<>, L, Acc) when ?IS_DIGIT(C) -> parse_number(R, L + 1, <>); parse_number(<<$., R/bits>>, L, Acc) -> @@ -284,6 +321,7 @@ parse_number(<<$., R/bits>>, L, Acc) -> parse_number(R, L, Acc) when L =< 15 -> {binary_to_integer(Acc), R}. +%% @doc Parse a decimal binary. parse_decimal(<>, L1, L2, IntAcc, FracAcc) when ?IS_DIGIT(C) -> parse_decimal(R, L1, L2 + 1, IntAcc, <>); parse_decimal(R, L1, L2, IntAcc, FracAcc0) when L1 =< 12, L2 >= 1, L2 =< 3 -> @@ -315,6 +353,7 @@ parse_decimal(R, L1, L2, IntAcc, FracAcc0) when L1 =< 12, L2 >= 1, L2 =< 3 -> end, {{decimal, {Int * Mul + Frac, -byte_size(FracAcc)}}, R}. +%% @doc Parse a string binary. parse_string(<<$\\, $", R/bits>>, Acc) -> parse_string(R, <>); parse_string(<<$\\, $\\, R/bits>>, Acc) -> @@ -322,17 +361,20 @@ parse_string(<<$\\, $\\, R/bits>>, Acc) -> parse_string(<<$", R/bits>>, Acc) -> {{string, Acc}, R}; parse_string(<>, Acc) when - C >= 16#20, C =< 16#21; - C >= 16#23, C =< 16#5b; - C >= 16#5d, C =< 16#7e --> + C >= 16#20, C =< 16#21; + C >= 16#23, C =< 16#5b; + C >= 16#5d, C =< 16#7e -> parse_string(R, <>). +%% @doc Parse a token binary. parse_token(<>, Acc) when ?IS_TOKEN(C) or (C =:= $:) or (C =:= $/) -> parse_token(R, <>); parse_token(R, Acc) -> {{token, Acc}, R}. +%% @doc Parse a byte sequence binary. +parse_binary(Bin) when is_binary(Bin) -> + parse_binary(Bin, <<>>). parse_binary(<<$:, R/bits>>, Acc) -> {{binary, base64:decode(Acc)}, R}; parse_binary(<>, Acc) when ?IS_ALPHANUM(C) or (C =:= $+) or (C =:= $/) or (C =:= $=) -> @@ -344,20 +386,20 @@ parse_struct_hd_test_() -> lists:flatten([ begin {ok, JSON} = file:read_file(File), - Tests = jsx:decode(JSON, [return_maps]), + Tests = json:decode(JSON), [ {iolist_to_binary(io_lib:format("~s: ~s", [filename:basename(File), Name])), fun() -> %% The implementation is strict. We fail whenever we can. - CanFail = maps:get(<<"can_fail">>, Test, false), - MustFail = maps:get(<<"must_fail">>, Test, false), + CanFail = hb_maps:get(<<"can_fail">>, Test, false), + MustFail = hb_maps:get(<<"must_fail">>, Test, false), io:format( "must fail ~p~nexpected json ~0p~n", - [MustFail, maps:get(<<"expected">>, Test, undefined)] + [MustFail, hb_maps:get(<<"expected">>, Test, undefined)] ), Expected = case MustFail of true -> undefined; - false -> expected_to_term(maps:get(<<"expected">>, Test)) + false -> expected_to_term(hb_maps:get(<<"expected">>, Test)) end, io:format("expected term: ~0p", [Expected]), Raw = raw_to_binary(Raw0), @@ -390,7 +432,8 @@ parse_struct_hd_test_() -> } <- Tests ] end - || File <- Files + || + File <- Files ]). %% The tests JSON use arrays for almost everything. Identifying @@ -402,14 +445,11 @@ parse_struct_hd_test_() -> %% inner-list: [[ [items...], params]] %% item: [bare, params] -%% Item. expected_to_term([Bare, []]) when - is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) --> + is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) -> {item, e2tb(Bare), []}; expected_to_term([Bare, Params = [[<<_/bits>>, _] | _]]) when - is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) --> + is_boolean(Bare); is_number(Bare); is_binary(Bare); is_map(Bare) -> {item, e2tb(Bare), e2tp(Params)}; %% Empty list or dictionary. expected_to_term([]) -> @@ -423,14 +463,14 @@ expected_to_term(Dict = [[<<_/bits>>, V] | _]) when V =/= [] -> e2t(Dict); %% Outer list. expected_to_term(List) when is_list(List) -> - [e2t(E) || E <- List]. + [ e2t(E) || E <- List ]. %% Dictionary. e2t(Dict = [[<<_/bits>>, _] | _]) -> [{K, e2t(V)} || [K, V] <- Dict]; %% Inner list. e2t([List, Params]) when is_list(List) -> - {list, [e2t(E) || E <- List], e2tp(Params)}; + {list, [ e2t(E) || E <- List ], e2tp(Params)}; %% Item. e2t([Bare, Params]) -> {item, e2tb(Bare), e2tp(Params)}. @@ -487,13 +527,17 @@ trim_ws_end(Value, N) -> dictionary(Map) when is_map(Map) -> dictionary(maps:to_list(Map)); dictionary(KVList) when is_list(KVList) -> - lists:join(<<", ">>, [ - case Value of - true -> Key; - _ -> [Key, $=, item_or_inner_list(Value)] - end - || {Key, Value} <- KVList - ]). + lists:join( + <<", ">>, + [ + case Value of + true -> Key; + _ -> [Key, $=, item_or_inner_list(Value)] + end + || + {Key, Value} <- KVList + ] + ). -spec item(sh_item()) -> iolist(). item({item, BareItem, Params}) -> @@ -603,41 +647,11 @@ params(Params) -> {Key, true} -> [$;, Key]; {Key, Value} -> [$;, Key, $=, bare_item(Value)] end - || Param <- Params + || + Param <- Params ]. --ifdef(TEST). -struct_hd_identity_test_() -> - Files = filelib:wildcard("deps/structured-header-tests/*.json"), - lists:flatten([ - begin - {ok, JSON} = file:read_file(File), - Tests = jsx:decode(JSON, [return_maps]), - [ - {iolist_to_binary(io_lib:format("~s: ~s", [filename:basename(File), Name])), fun() -> - io:format("expected json ~0p~n", [Expected0]), - Expected = expected_to_term(Expected0), - io:format("expected term: ~0p", [Expected]), - case HeaderType of - <<"dictionary">> -> - Expected = parse_dictionary(iolist_to_binary(dictionary(Expected))); - <<"item">> -> - Expected = parse_item(iolist_to_binary(item(Expected))); - <<"list">> -> - Expected = parse_list(iolist_to_binary(list(Expected))) - end - end} - || #{ - <<"name">> := Name, - <<"header_type">> := HeaderType, - %% We only run tests that must not fail. - <<"expected">> := Expected0 - } <- Tests - ] - end - || File <- Files - ]). --endif. +%%% Tests to_dictionary_test() -> {ok, SfDictionary} = to_dictionary(#{ @@ -661,7 +675,12 @@ to_dictionary_test() -> lists:keyfind(<<"fizz">>, 1, SfDictionary) ), ?assertEqual( - {<<"item-with">>, {item, {string,<<"params">>}, [{<<"first">>, {token,<<"param">>}}, {<<"another">>, true}]}}, + {<<"item-with">>, + {item, + {string,<<"params">>}, + [{<<"first">>, {token,<<"param">>}}, {<<"another">>, true}] + } + }, lists:keyfind(<<"item-with">>, 1, SfDictionary) ), ?assertEqual( @@ -681,15 +700,37 @@ to_dictionary_test() -> lists:keyfind(<<"empty">>, 1, SfDictionary) ), ?assertEqual( - {<<"inner">>, {list , [{item, {string, <<"a">>}, []}, {item, {token, <<"b">>}, []}, {item, true, []}, {item, 3, []}], []}}, + { + <<"inner">>, + { + list, + [ + {item, {string, <<"a">>}, []}, + {item, {token, <<"b">>}, []}, + {item, true, []}, + {item, 3, []} + ], + [] + } + }, lists:keyfind(<<"inner">>, 1, SfDictionary) ), ?assertEqual( - {<<"inner_with_params">>, {list , [{item, 1, []}, {item, 2, []}], [{<<"first">>, {token, <<"param">>}}]}}, + {<<"inner_with_params">>, + {list, + [{item, 1, []}, {item, 2, []}], + [{<<"first">>, {token, <<"param">>}}] + } + }, lists:keyfind(<<"inner_with_params">>, 1, SfDictionary) ), ?assertEqual( - {<<"inner_inner_params">>, {list, [{item, 1, [{<<"heres">>, {string, <<"one">>}}]}, {item, 2, []}], []}}, + {<<"inner_inner_params">>, + {list, + [{item, 1, [{<<"heres">>, {string, <<"one">>}}]}, {item, 2, []}], + [] + } + }, lists:keyfind(<<"inner_inner_params">>, 1, SfDictionary) ), dictionary(SfDictionary). @@ -717,13 +758,22 @@ to_item_test() -> to_list_test() -> ?assertEqual( - to_list([1,2,<<"three">>, [4, <<"five">>], {list, [6, <<"seven">>], [{first, param}]}]), + to_list( + [1, 2, <<"three">>, [4, <<"five">>], + {list, [6, <<"seven">>], + [{<<"first">>, {token, <<"param">>}}] + } + ] + ), {ok, [ {item, 1, []}, {item, 2, []}, {item, {string, <<"three">>}, []}, {list, [{ item, 4, []}, {item, {string, <<"five">>}, []}], []}, - {list, [{ item, 6, []}, {item, {string, <<"seven">>}, []}], [{<<"first">>, {token, <<"param">>}}]} + {list, + [{ item, 6, []}, {item, {string, <<"seven">>}, []}], + [{<<"first">>, {token, <<"param">>}}] + } ]} ), ok. diff --git a/src/hb_sup.erl b/src/hb_sup.erl index c4636cc11..28b0176eb 100644 --- a/src/hb_sup.erl +++ b/src/hb_sup.erl @@ -2,7 +2,7 @@ -behaviour(supervisor). -export([start_link/0, start_link/1, init/1]). -define(SERVER, ?MODULE). --include("src/include/hb.hrl"). +-include("include/hb.hrl"). start_link() -> start_link(#{}). @@ -25,12 +25,12 @@ init(Opts) -> StoreChildren = store_children(hb_opts:get(store, [], Opts)), GunChild = #{ - id => ar_http, - start => {ar_http, start_link, [Opts]}, + id => hb_http_client, + start => {hb_http_client, start_link, [Opts]}, restart => permanent, shutdown => 5000, type => worker, - modules => [ar_http] + modules => [hb_http_client] }, {ok, {SupFlags, [GunChild | StoreChildren]}}. @@ -38,7 +38,7 @@ init(Opts) -> store_children(Store) when not is_list(Store) -> store_children([Store]); store_children([]) -> []; -store_children([RocksDBOpts = {hb_store_rocksdb, _} | Rest]) -> +store_children([RocksDBOpts = #{ <<"store-module">> := hb_store_rocksdb } | Rest]) -> [ #{ id => hb_store_rocksdb, diff --git a/src/hb_test.erl b/src/hb_test.erl deleted file mode 100644 index 6b8a5c95e..000000000 --- a/src/hb_test.erl +++ /dev/null @@ -1,156 +0,0 @@ --module(hb_test). --export([init/0, generate_test_data/1, run/2]). --hb_debug(print). - --include("include/hb.hrl"). --include_lib("eunit/include/eunit.hrl"). - -init() -> - application:ensure_all_started(hb), - ok. - -run(Proc, Msg) -> - run(Proc, Msg, #{}). -run(Proc, Msg, _Opts) -> - hb_cache:write(hb_opts:get(store), Msg), - hb_cache:write(hb_opts:get(store), Proc), - Scheduler = dev_scheduler_registry:find(hb_util:id(Proc, signed), true), - Assignment = dev_scheduler_server:schedule(Scheduler, Msg), - hb_process:result( - hb_util:id(Proc, signed), - hb_util:id(Assignment, unsigned), - hb_opts:get(store), - hb:wallet() - ). - -%%% TESTS - -% simple_stack_test() -> -% init(), -% {Proc, Msg} = generate_test_data(<<"return 42">>), -% {ok, Result} = run(Proc, Msg, #{ on_idle => terminate }), -% #tx { data = <<"42">> } = maps:get(<<"/Data">>, Result), -% ok. - -% full_push_test_() -> -% {timeout, 150, ?_assert(full_push_test())}. - -% full_push_test() -> -% init(), -% ?event(full_push_test_started), -% {_, Msg} = generate_test_data(ping_ping_script()), -% hb_cache:write(hb_opts:get(store), Msg), -% hb_client:push(Msg, #{ tracing => none }), -% ok. - -% simple_load_test() -> -% init(), -% ?event(scheduling_many_items), -% Messages = 30, -% Msg = generate_test_data(ping_ping_script()), -% hb_cache:write(hb_opts:get(store), Msg), -% Start = hb:now(), -% Assignments = lists:map( -% fun(_) -> hb_client:schedule(Msg) end, -% lists:seq(1, Messages) -% ), -% Scheduled = hb:now(), -% {ok, LastAssignment} = lists:last(Assignments), -% ?event({scheduling_many_items_done_s, ((Scheduled - Start) / Messages) / 1000}), -% hb_client:compute(LastAssignment, Msg), -% Computed = hb:now(), -% ?event({compute_time_s, ((Computed - Scheduled) / Messages) / 1000}), -% ?event({total_time_s, ((Computed - Start) / Messages) / 1000}), -% ?event({processed_messages, Messages}). - -default_test_img(Wallet) -> - Store = hb_opts:get(store), - {ok, Module} = file:read_file("test/aos-2-pure-xs.wasm"), - hb_cache:write( - Store, - Img = ar_bundles:sign_item( - #tx { - tags = [ - {<<"Protocol">>, <<"ao">>}, - {<<"Variant">>, <<"ao.tn.2">>}, - {<<"Type">>, <<"Image">>} - ], - data = Module - }, - Wallet - ) - ), - Img. - - -default_test_devices(Wallet, Opts) -> - ID = ar_wallet:to_address(Wallet), - Img = maps:get(image, Opts), - Quorum = maps:get(quorum, Opts, 2), - [ - {<<"Protocol">>, <<"ao">>}, - {<<"Variant">>, <<"ao.tn.2">>}, - {<<"Type">>, <<"Process">>}, - {<<"Device">>, <<"Stack">>}, - {<<"Device.1">>, <<"Scheduler">>}, - {<<"Location">>, hb_util:id(ID)}, - {<<"Device.2">>, <<"PODA">>}, - {<<"Quorum">>, integer_to_binary(Quorum)} - ] ++ - [ - {<<"Authority">>, Addr} || - Addr <- maps:keys(maps:get(compute, hb_opts:get(nodes))), - Addr =/= '_' - ] ++ - [ - {<<"Device.3">>, <<"JSON-Interface">>}, - {<<"Device.5">>, <<"VFS">>}, - {<<"Device.6">>, <<"WASM64-pure">>}, - {<<"Module">>, <<"aos-2-pure">>}, - {<<"Image">>, hb_util:id(Img)}, - {<<"Device.7">>, <<"Cron">>}, - {<<"Time">>, <<"100-Milliseconds">>}, - {<<"Device.8">>, <<"Multipass">>}, - {<<"Passes">>, <<"3">>} - ]. - -% ping_ping_script() -> -% << -% "\n" -% "Handlers.add(\"Ping\", function(m) Send({ Target = ao.id, Action = \"Ping\" }); print(\"Sent Ping\"); end)\n" -% "Send({ Target = ao.id, Action = \"Ping\" })\n" -% >>. - -generate_test_data(Script) -> - generate_test_data(Script, hb:wallet()). -generate_test_data(Script, Wallet) -> - Img = default_test_img(Wallet), - generate_test_data(Script, Wallet, #{image => Img}). -generate_test_data(Script, Wallet, Opts) -> - Devs = default_test_devices(Wallet, Opts), - generate_test_data(Script, Wallet, Opts, Devs). -generate_test_data(Script, Wallet, _Opts, Devs) -> - Store = hb_opts:get(store), - hb_cache:write( - Store, - SignedProcess = ar_bundles:sign_item( - #tx{ tags = Devs }, - Wallet - ) - ), - Msg = ar_bundles:sign_item( - #tx{ - target = ar_bundles:id(SignedProcess, signed), - tags = [ - {<<"Protocol">>, <<"ao">>}, - {<<"Variant">>, <<"ao.tn.2">>}, - {<<"Type">>, <<"Message">>}, - {<<"Action">>, <<"Eval">>} - ], - data = Script - }, - Wallet - ), - hb_cache:write(Store, Msg), - ?event({test_data_written, {proc, hb_util:id(SignedProcess, signed)}, {msg, hb_util:id(Msg, unsigned)}}), - {SignedProcess, Msg}. \ No newline at end of file diff --git a/src/hb_test_utils.erl b/src/hb_test_utils.erl new file mode 100644 index 000000000..4b360a02b --- /dev/null +++ b/src/hb_test_utils.erl @@ -0,0 +1,256 @@ +%%% @doc Simple utilities for testing HyperBEAM. Includes functions for +%%% generating isolated (fresh) test stores, running suites of tests with +%%% differing options, as well as executing and reporting benchmarks. +-module(hb_test_utils). +-export([suite_with_opts/2, run/4, assert_throws/4]). +-export([test_store/0, test_store/1, test_store/2]). +-export([benchmark/1, benchmark/2, benchmark/3, benchmark_iterations/2]). +-export([benchmark_print/2, benchmark_print/3, benchmark_print/4]). +-export([compare_events/3, compare_events/4, compare_events/5]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The number of seconds to run a benchmark for when no time is specified. +-define(DEFAULT_BENCHMARK_TIME, 1). + +%% @doc Generate a new, unique test store as an isolated context for an execution. +test_store() -> + test_store(maps:get(<<"store-module">>, hd(hb_opts:get(store)))). +test_store(Mod) -> + test_store(Mod, <<"default">>). +test_store(Mod, Tag) -> + TestDir = + << + "cache-TEST/run-", + Tag/binary, "-", + (integer_to_binary(erlang:system_time(millisecond)))/binary + >>, + % Wait a tiny interval to ensure that any further tests will get their own + % directory. + timer:sleep(1), + filelib:ensure_dir(binary_to_list(TestDir)), + #{ <<"store-module">> => Mod, <<"name">> => TestDir }. + +%% @doc Run each test in a suite with each set of options. Start and reset +%% the store(s) for each test. Expects suites to be a list of tuples with +%% the test name, description, and test function. +%% The list of `Opts' should contain maps with the `name' and `opts' keys. +%% Each element may also contain a `skip' key with a list of test names to skip. +%% They can also contain a `desc' key with a description of the options. +suite_with_opts(Suite, OptsList) -> + lists:filtermap( + fun(OptSpec = #{ name := _Name, opts := Opts, desc := ODesc}) -> + Store = hb_opts:get(store, hb_opts:get(store), Opts), + Skip = hb_maps:get(skip, OptSpec, [], Opts), + case satisfies_requirements(OptSpec) of + true -> + {true, {foreach, + fun() -> + ?event({starting, Store}), + % Create and set a random server ID for the test + % process. + hb_http_server:set_proc_server_id( + hb_util:human_id(crypto:strong_rand_bytes(32)) + ), + hb_store:reset(Store), + hb_store:start(Store) + end, + fun(_) -> + hb_store:reset(Store), + ok + end, + [ + { + hb_util:list(ODesc) + ++ ": " + ++ hb_util:list(TestDesc), + fun() -> Test(Opts) end} + || + {TestAtom, TestDesc, Test} <- Suite, + not lists:member(TestAtom, Skip) + ] + }}; + false -> false + end + end, + OptsList + ). + +%% @doc Determine if the environment satisfies the given test requirements. +%% Requirements is a list of atoms, each corresponding to a module that must +%% return true if it exposes an `enabled/0' function. +satisfies_requirements(Requirements) when is_map(Requirements) -> + satisfies_requirements(hb_maps:get(requires, Requirements, [])); +satisfies_requirements(Requirements) -> + lists:all( + fun(Req) -> + case hb_features:enabled(Req) of + true -> true; + false -> + case code:is_loaded(Req) of + false -> false; + {file, _} -> + case erlang:function_exported(Req, enabled, 0) of + true -> Req:enabled(); + false -> true + end + end + end + end, + Requirements + ). + +%% @doc Find the options from a list of options by name. +opts_from_list(OptsName, OptsList) -> + hd([ O || #{ name := OName, opts := O } <- OptsList, OName == OptsName ]). + +%% Run a single test with a given set of options. +run(Name, OptsName, Suite, OptsList) -> + {_, _, Test} = lists:keyfind(Name, 1, Suite), + Test(opts_from_list(OptsName, OptsList)). + +%% @doc Compares the events generated by executing a test/function with two +%% different sets of options. +compare_events(Fun, Opts1, Opts2) -> + hb_store:reset(hb_opts:get(store, hb_opts:get(store), Opts1)), + hb_store:write( + hb_opts:get(store, hb_opts:get(store), Opts1), + <<"test">>, + <<"test">> + ), + {EventsSample1, _Res2} = hb_event:diff( + fun() -> + Fun(Opts1) + end + ), + hb_store:reset(hb_opts:get(store, hb_opts:get(store), Opts1)), + hb_store:reset(hb_opts:get(store, hb_opts:get(store), Opts2)), + {EventsSample2, _Res} = hb_event:diff( + fun() -> + Fun(Opts2) + end + ), + hb_store:reset(hb_opts:get(store, hb_opts:get(store), Opts2)), + EventsDiff = hb_message:diff(EventsSample1, EventsSample2, #{}), + ?event( + debug_perf, + {events, + {sample1, EventsSample1}, + {sample2, EventsSample2}, + {events_diff, EventsDiff} + } + ), + EventsDiff. +compare_events(Fun, OptsName1, OptsName2, OptsList) -> + compare_events( + Fun, + opts_from_list(OptsName1, OptsList), + opts_from_list(OptsName2, OptsList) + ). +compare_events(Name, OptsName1, OptsName2, Suite, OptsList) -> + {_, _, Test} = lists:keyfind(Name, 1, Suite), + compare_events( + Test, + opts_from_list(OptsName1, OptsList), + opts_from_list(OptsName2, OptsList) + ). + +%% @doc Assert that a function throws an expected exception. Needed to work around some +%% limitations in ?assertException (e.g. no way to attach an error message to the failure) +assert_throws(Fun, Args, ExpectedException, Label) -> + Error = try + apply(Fun, Args), + failed_to_throw + catch + error:ExpectedException -> expected_exception; + ExpectedException -> expected_exception; + error:Other -> {wrong_exception, Other}; + Other -> {wrong_exception, Other} + end, + ?assertEqual(expected_exception, Error, Label). + +%% @doc Run a function as many times as possible in a given amount of time. +benchmark(Fun) -> + benchmark(Fun, ?DEFAULT_BENCHMARK_TIME). +benchmark(Fun, TLen) -> + T0 = erlang:system_time(millisecond), + hb_util:until( + fun() -> erlang:system_time(millisecond) - T0 > (TLen * 1000) end, + Fun, + 0 + ). + +%% @doc Return the amount of time required to execute N iterations of a function +%% as a fraction of a second. +benchmark_iterations(Fun, N) -> + {Time, _} = timer:tc( + fun() -> + lists:foreach( + fun(I) -> Fun(I) end, + lists:seq(1, N) + ) + end + ), + Time / 1_000_000. + +%% @doc Run multiple instances of a function in parallel for a given amount of time. +benchmark(Fun, TLen, Procs) -> + Parent = self(), + receive _ -> worker_synchronized end, + StartWorker = + fun(_) -> + Ref = make_ref(), + spawn_link(fun() -> + Count = benchmark(Fun, TLen), + Parent ! {work_complete, Ref, Count} + end), + Ref + end, + CollectRes = + fun(R) -> + receive + {work_complete, R, Count} -> + %?event(benchmark, {work_complete, R, Count}), + Count + end + end, + Refs = lists:map(StartWorker, lists:seq(1, Procs)), + lists:sum(lists:map(CollectRes, Refs)). + +%% @doc Print benchmark results in a human-readable format that EUnit writes to +%% the console. Takes a `verb` as a string and an `iterations` count (returned +%% by the benchmark function), as well as optionally a `noun` to refer to the +%% objects in the benchmark, and a `time` in seconds. If `time' is not +%% provided, it defaults to the value of `?DEFAULT_BENCHMARK_TIME'. +benchmark_print(Verb, Iterations) -> + benchmark_print(Verb, Iterations, ?DEFAULT_BENCHMARK_TIME). +benchmark_print(Verb, Iterations, Time) when is_integer(Iterations) -> + hb_format:eunit_print( + "~s ~s in ~s (~s/s)", + [ + Verb, + hb_util:human_int(Iterations), + format_time(Time), + hb_util:human_int(Iterations / Time) + ] + ); +benchmark_print(Verb, Noun, Iterations) -> + benchmark_print(Verb, Noun, Iterations, ?DEFAULT_BENCHMARK_TIME). +benchmark_print(Verb, Noun, Iterations, Time) -> + hb_format:eunit_print( + "~s ~s ~s in ~s (~s ~s/s)", + [ + Verb, + hb_util:human_int(Iterations), + Noun, + format_time(Time), + hb_util:human_int(Iterations / Time), + Noun + ] + ). + +%% @doc Format a time in human-readable format. Takes arguments in seconds. +format_time(Time) when is_integer(Time) -> + hb_util:human_int(Time) ++ "s"; +format_time(Time) -> + hb_util:human_int(Time * 1000) ++ "ms". \ No newline at end of file diff --git a/src/hb_tracer.erl b/src/hb_tracer.erl new file mode 100644 index 000000000..4b4521ecb --- /dev/null +++ b/src/hb_tracer.erl @@ -0,0 +1,131 @@ +%%% @doc A module for tracing the flow of requests through the system. +%%% This allows for tracking the lifecycle of a request from HTTP receipt through processing and response. + +-module(hb_tracer). + +-export([start_trace/0, record_step/2, get_trace/1, format_error_trace/1]). + +-include("include/hb.hrl"). + +%%% @doc Start a new tracer acting as queue of events registered. +start_trace() -> + Trace = #{steps => queue:new()}, + TracePID = spawn(fun() -> trace_loop(Trace) end), + ?event(trace, {trace_started, TracePID}), + TracePID. + +trace_loop(Trace) -> + receive + {record_step, Step} -> + Steps = maps:get(steps, Trace), + NewTrace = Trace#{steps => queue:in(Step, Steps)}, + ?event(trace, {step_recorded, Step}), + trace_loop(NewTrace); + {get_trace, From} -> + % Convert queue to list for the response + TraceWithList = + Trace#{steps => + queue:to_list( + maps:get(steps, Trace))}, + From ! {trace, TraceWithList}, + trace_loop(Trace) + end. + +%%% @doc Register a new step into a tracer +record_step(TracePID, Step) -> + TracePID ! {record_step, Step}. + +%%% @doc Exports the complete queue of events +get_trace(TracePID) -> + TracePID ! {get_trace, self()}, + receive + {trace, Trace} -> + Trace + after 5000 -> + ?event(trace, {trace_timeout, TracePID}), + {trace, #{}} + end. + +%%% @doc Format a trace for error in a user-friendly emoji oriented output +format_error_trace(Trace) -> + Steps = maps:get(steps, Trace, []), + TraceMap = + lists:foldl(fun(TraceItem, Acc) -> + case TraceItem of + {http, {parsed_singleton, _ReqSingleton, _}} -> + maps:put(request_parsing, true, Acc); + {ao_core, {stage, Stage, _Task}} -> + maps:put(resolve_stage, Stage, Acc); + {ao_result, + {load_device_failed, _, _, _, _, {exec_exception, Exception}, _, _}} -> + maps:put(error, Exception, Acc); + {ao_result, + {exec_failed, + _, + _, + _, + {func, Fun}, + _, + {exec_exception, Error}, + _, + _}} -> + maps:put(error, {Fun, Error}, Acc); + _ -> Acc + end + end, + #{}, + Steps), + % Build the trace message + TraceStrings = <<"Oops! Something went wrong. Here's the rundown:">>, + % Add parsing status + ParsingTrace = + case maps:get(request_parsing, TraceMap, false) of + false -> + Emoji = failure_emoji(), + <>; + true -> + Emoji = checkmark_emoji(), + <> + end, + % Add stage information + StageTrace = + case maps:get(resolve_stage, TraceMap, undefined) of + undefined -> + ParsingTrace; + Stage -> + StageEmoji = stage_to_emoji(Stage), + try << ParsingTrace/binary, "\n", StageEmoji/binary, + " Resolved steps of your execution" >> + catch + error:badarg -> + iolist_to_binary(io_lib:format("~p", [ParsingTrace])) + end + end, + % Add error information + case maps:get(error, TraceMap, undefined) of + undefined -> + StageTrace; + {Fun, Reason} -> + FailureEmoji = failure_emoji(), + ErrMsg = list_to_binary(io_lib:format("~p -> ~p", [Fun, Reason])), + <>; + Error -> + FailureEmoji = failure_emoji(), + <> + end. + +checkmark_emoji() -> + % Unicode for checkmark + <<"\xE2\x9C\x85">>. % \xE2\x9C\x85 is the checkmark emoji in UTF-8 + +failure_emoji() -> + % Unicode for failure emoji + <<"\xE2\x9D\x8C">>. % \xE2\x9D\x8C is the failure emoji in UTF-8 + +% Helper function to convert stage number to emoji +stage_to_emoji(Stage) when Stage >= 1, Stage =< 9 -> + % Unicode for circled numbers 1-9 + StageEmoji = Stage + 48, + <>; +stage_to_emoji(_) -> + "". diff --git a/src/hb_util.erl b/src/hb_util.erl index 18f6fc6ef..b9122ff46 100644 --- a/src/hb_util.erl +++ b/src/hb_util.erl @@ -1,21 +1,98 @@ %% @doc A collection of utility functions for building with HyperBEAM. -module(hb_util). --export([id/1, id/2, native_id/1, human_id/1, short_id/1, human_int/1]). +-export([int/1, float/1, atom/1, bin/1, list/1, map/1]). +-export([ceil_int/2, floor_int/2]). +-export([id/1, id/2, native_id/1, human_id/1, human_int/1, to_hex/1]). +-export([key_to_atom/1, key_to_atom/2, binary_to_addresses/1]). -export([encode/1, decode/1, safe_encode/1, safe_decode/1]). -export([find_value/2, find_value/3]). --export([number/1, list_to_numbered_map/1, message_to_numbered_list/1]). +-export([deep_merge/3, deep_set/4, deep_get/3, deep_get/4]). +-export([number/1, list_to_numbered_message/1]). +-export([find_target_path/2, template_matches/3]). +-export([is_ordered_list/2, message_to_ordered_list/1, message_to_ordered_list/2]). +-export([numbered_keys_to_list/2]). +-export([is_string_list/1, list_replace/3, list_without/2, list_with/2]). +-export([to_sorted_list/1, to_sorted_list/2, to_sorted_keys/1, to_sorted_keys/2]). -export([hd/1, hd/2, hd/3]). -export([remove_common/2, to_lower/1]). -export([maybe_throw/2]). --export([format_indented/2, format_indented/3, format_binary/1]). --export([format_map/1, format_map/2, remove_trailing_noise/2]). --export([debug_print/4, debug_fmt/1, eunit_print/2]). --export([print_trace/4, trace_macro_helper/5, print_trace_short/4]). --export([ok/1, ok/2]). --export([format_trace_short/1]). --export([count/2, mean/1, stddev/1, variance/1]). +-export([is_hb_module/1, is_hb_module/2, all_hb_modules/0]). +-export([ok/1, ok/2, until/1, until/2, until/3]). +-export([count/2, mean/1, stddev/1, variance/1, weighted_random/1]). +-export([unique/1]). +-export([split_depth_string_aware/2, split_depth_string_aware_single/2]). +-export([split_escaped_single/2]). +-export([check_size/2, check_value/2, check_type/2, ok_or_throw/3]). +-export([all_atoms/0, binary_is_atom/1]). +-export([lower_case_key_map/2]). -include("include/hb.hrl"). + +%%% Simple type coercion functions, useful for quickly turning inputs from the +%%% HTTP API into the correct types for the HyperBEAM runtime, if they are not +%%% annotated by the user. + +%% @doc Coerce a string to an integer. +int(Str) when is_binary(Str) -> + list_to_integer(binary_to_list(Str)); +int(Str) when is_list(Str) -> + list_to_integer(Str); +int(Int) when is_integer(Int) -> + Int. + +%% @doc Coerce a string to a float. +float(Str) when is_binary(Str) -> + list_to_float(binary_to_list(Str)); +float(Str) when is_list(Str) -> + list_to_float(Str); +float(Float) when is_float(Float) -> + Float; +float(Int) when is_integer(Int) -> + Int / 1. + +%% @doc Coerce a string to an atom. +atom(Str) when is_binary(Str) -> + list_to_existing_atom(binary_to_list(Str)); +atom(Str) when is_list(Str) -> + list_to_existing_atom(Str); +atom(Atom) when is_atom(Atom) -> + Atom. + +%% @doc Coerce a value to a binary. +bin(Value) when is_atom(Value) -> + atom_to_binary(Value, utf8); +bin(Value) when is_integer(Value) -> + integer_to_binary(Value); +bin(Value) when is_float(Value) -> + float_to_binary(Value, [{decimals, 10}, compact]); +bin(Value) when is_list(Value) -> + list_to_binary(Value); +bin(Value) when is_binary(Value) -> + Value. + +%% @doc Coerce a value to a string list. +list(Value) when is_binary(Value) -> + binary_to_list(Value); +list(Value) when is_list(Value) -> Value; +list(Value) when is_atom(Value) -> atom_to_list(Value). + +%% @doc Ensure that a value is a map. Only supports maps and lists of key-value +%% pairs. +map(Value) when is_list(Value) -> + maps:from_list(Value); +map(Value) when is_map(Value) -> + Value. + +%% @doc: rounds IntValue up to the nearest multiple of Nearest. +%% Rounds up even if IntValue is already a multiple of Nearest. +ceil_int(IntValue, Nearest) -> + IntValue - (IntValue rem Nearest) + Nearest. + +%% @doc: rounds IntValue down to the nearest multiple of Nearest. +%% Doesn't change IntValue if it's already a multiple of Nearest. +floor_int(IntValue, Nearest) -> + IntValue - (IntValue rem Nearest). + %% @doc Unwrap a tuple of the form `{ok, Value}', or throw/return, depending on %% the value of the `error_strategy' option. ok(Value) -> ok(Value, #{}). @@ -26,16 +103,33 @@ ok(Other, Opts) -> _ -> {unexpected, Other} end. +%% @doc Utility function to wait for a condition to be true. Optionally, +%% you can pass a function that will be called with the current count of +%% iterations, returning an integer that will be added to the count. Once the +%% condition is true, the function will return the count. +until(Condition) -> + until(Condition, 0). +until(Condition, Count) -> + until(Condition, fun() -> receive after 100 -> 1 end end, Count). +until(Condition, Fun, Count) -> + case Condition() of + false -> + case apply(Fun, hb_ao:truncate_args(Fun, [Count])) of + {count, AddToCount} -> + until(Condition, Fun, Count + AddToCount); + _ -> + until(Condition, Fun, Count + 1) + end; + true -> Count + end. + %% @doc Return the human-readable form of an ID of a message when given either %% a message explicitly, raw encoded ID, or an Erlang Arweave `tx' record. id(Item) -> id(Item, unsigned). id(TX, Type) when is_record(TX, tx) -> encode(ar_bundles:id(TX, Type)); id(Map, Type) when is_map(Map) -> - case Type of - unsigned -> hb_converge:get(unsigned_id, Map); - signed -> encode(hb_converge:get(id, Map)) - end; + hb_message:id(Map, Type); id(Bin, _) when is_binary(Bin) andalso byte_size(Bin) == 43 -> Bin; id(Bin, _) when is_binary(Bin) andalso byte_size(Bin) == 32 -> @@ -43,57 +137,75 @@ id(Bin, _) when is_binary(Bin) andalso byte_size(Bin) == 32 -> id(Data, Type) when is_list(Data) -> id(list_to_binary(Data), Type). -%% @doc Convert a string to a lowercase. -to_lower(Str) when is_list(Str) -> - string:to_lower(Str); -to_lower(Bin) when is_binary(Bin) -> - list_to_binary(to_lower(binary_to_list(Bin))). +%% @doc Convert a binary to a lowercase. +to_lower(Str) -> + string:lowercase(Str). + +%% @doc Is the given term a string list? +is_string_list(MaybeString) -> + lists:all(fun is_integer/1, MaybeString). + +%% @doc Given a map or KVList, return a deterministically sorted list of its +%% key-value pairs. +to_sorted_list(Msg) -> + to_sorted_list(Msg, #{}). +to_sorted_list(Msg, Opts) when is_map(Msg) -> + to_sorted_list(hb_maps:to_list(Msg, Opts), Opts); +to_sorted_list(Msg = [{_Key, _} | _], _Opts) when is_list(Msg) -> + lists:sort(fun({Key1, _}, {Key2, _}) -> Key1 < Key2 end, Msg); +to_sorted_list(Msg, _Opts) when is_list(Msg) -> + lists:sort(fun(Key1, Key2) -> Key1 < Key2 end, Msg). + +%% @doc Given a map or KVList, return a deterministically ordered list of its keys. +to_sorted_keys(Msg) -> + to_sorted_keys(Msg, #{}). +to_sorted_keys(Msg, Opts) when is_map(Msg) -> + to_sorted_keys(hb_maps:keys(Msg, Opts), Opts); +to_sorted_keys(Msg, _Opts) when is_list(Msg) -> + lists:sort(fun(Key1, Key2) -> Key1 < Key2 end, Msg). + +%% @doc Convert keys in a map to atoms, lowering `-' to `_'. +key_to_atom(Key) -> key_to_atom(Key, existing). +key_to_atom(Key, _Mode) when is_atom(Key) -> Key; +key_to_atom(Key, Mode) -> + WithoutDashes = to_lower(binary:replace(Key, <<"-">>, <<"_">>, [global])), + case Mode of + new_atoms -> binary_to_atom(WithoutDashes, utf8); + _ -> binary_to_existing_atom(WithoutDashes, utf8) + end. %% @doc Convert a human readable ID to a native binary ID. If the ID is already %% a native binary ID, it is returned as is. native_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 43 -> decode(Bin); native_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 32 -> - Bin. + Bin; +native_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 42 -> + Bin; +native_id(Wallet = {_Priv, _Pub}) -> + native_id(ar_wallet:to_address(Wallet)). %% @doc Convert a native binary ID to a human readable ID. If the ID is already -%% a human readable ID, it is returned as is. +%% a human readable ID, it is returned as is. If it is an ethereum address, it +%% is returned as is. human_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 32 -> encode(Bin); human_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 43 -> - Bin. + Bin; +human_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 42 -> + Bin; +human_id(Wallet = {_Priv, _Pub}) -> + human_id(ar_wallet:to_address(Wallet)). -%% @doc Return a short ID for the different types of IDs used in Converge. -short_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 32 -> - short_id(human_id(Bin)); -short_id(Bin) when is_binary(Bin) andalso byte_size(Bin) == 43 -> - << FirstTag:5/binary, _:33/binary, LastTag:5/binary >> = Bin, - << FirstTag/binary, "..", LastTag/binary >>; -short_id(Bin) when byte_size(Bin) > 43 andalso byte_size(Bin) < 100 -> - case binary:split(Bin, <<"/">>, [trim_all, global]) of - [First, Second] when byte_size(Second) == 43 -> - FirstEnc = short_id(First), - SecondEnc = short_id(Second), - << FirstEnc/binary, "/", SecondEnc/binary >>; - [First, Key] -> - FirstEnc = short_id(First), - << FirstEnc/binary, "/", Key/binary >>; - _ -> - Bin - end; -short_id(<< "/", SingleElemHashpath/binary >>) -> - Enc = short_id(SingleElemHashpath), - << "/", Enc/binary >>; -short_id(Key) when byte_size(Key) < 43 -> Key; -short_id(_) -> undefined. - -%% @doc Determine whether a binary is human-readable. -is_human_binary(Bin) when is_binary(Bin) -> - case unicode:characters_to_binary(Bin) of - {error, _, _} -> false; - _ -> true - end; -is_human_binary(_) -> false. + +%% @doc Add `,' characters to a number every 3 digits to make it human readable. +human_int(Float) when is_float(Float) -> + human_int(erlang:round(Float)); +human_int(Int) -> + lists:reverse(add_commas(lists:reverse(integer_to_list(Int)))). + +add_commas([A,B,C,Z|Rest]) -> [A,B,C,$,|add_commas([Z|Rest])]; +add_commas(List) -> List. %% @doc Encode a binary to URL safe base64 binary string. encode(Bin) -> @@ -117,10 +229,96 @@ safe_decode(E) -> D = decode(E), {ok, D} catch - _:_ -> - {error, invalid} + _:_ -> {error, invalid} + end. + +%% @doc Convert a binary to a hex string. Do not use this for anything other than +%% generating a lower-case, non-special character id. It should not become part of +%% the core protocol. We use b64u for efficient encoding. +to_hex(Bin) when is_binary(Bin) -> + to_lower( + iolist_to_binary( + [io_lib:format("~2.16.0B", [X]) || X <- binary_to_list(Bin)] + ) + ). + +%% @doc Deep merge two maps, recursively merging nested maps. +deep_merge(Map1, Map2, Opts) when is_map(Map1), is_map(Map2) -> + hb_maps:fold( + fun(Key, Value2, AccMap) -> + case deep_get(Key, AccMap, Opts) of + Value1 when is_map(Value1), is_map(Value2) -> + % Both values are maps, recursively merge them + deep_set(Key, deep_merge(Value1, Value2, Opts), AccMap, Opts); + _ -> + % Either the key doesn't exist in Map1 or at least one of + % the values isn't a map. Simply use the value from Map2 + deep_set(Key, Value2, AccMap, Opts) + end + end, + Map1, + Map2, + Opts + ). + +%% @doc Set a deep value in a message by its path, _assuming all messages are +%% `device: message@1.0`_. +deep_set(_Path, undefined, Msg, _Opts) -> Msg; +deep_set(Path, Value, Msg, Opts) when not is_list(Path) -> + deep_set(hb_path:term_to_path_parts(Path, Opts), Value, Msg, Opts); +deep_set([Key], unset, Msg, Opts) -> + hb_maps:remove(Key, Msg, Opts); +deep_set([Key], Value, Msg, Opts) -> + case hb_maps:get(Key, Msg, not_found, Opts) of + ExistingMap when is_map(ExistingMap) andalso is_map(Value) -> + % If both are maps, merge them + Msg#{ Key => hb_maps:merge(ExistingMap, Value, Opts) }; + _ -> + Msg#{ Key => Value } + end; +deep_set([Key|Rest], Value, Map, Opts) -> + SubMap = hb_maps:get(Key, Map, #{}, Opts), + hb_maps:put(Key, deep_set(Rest, Value, SubMap, Opts), Map, Opts). + +%% @doc Get a deep value from a message. +deep_get(Path, Msg, Opts) -> deep_get(Path, Msg, not_found, Opts). +deep_get(Path, Msg, Default, Opts) when not is_list(Path) -> + deep_get(hb_path:term_to_path_parts(Path, Opts), Msg, Default, Opts); +deep_get([Key], Msg, Default, Opts) -> + case hb_maps:find(Key, Msg, Opts) of + {ok, Value} -> Value; + error -> Default + end; +deep_get([Key|Rest], Msg, Default, Opts) -> + case hb_maps:find(Key, Msg, Opts) of + {ok, DeepMsg} when is_map(DeepMsg) -> + deep_get(Rest, DeepMsg, Default, Opts); + error -> Default end. +%% @doc Find the target path to route for a request message. +find_target_path(Msg, Opts) -> + case hb_ao:get(<<"route-path">>, Msg, not_found, Opts) of + not_found -> + ?event({find_target_path, {msg, Msg}, not_found}), + hb_ao:get(<<"path">>, Msg, no_path, Opts); + RoutePath -> RoutePath + end. + +%% @doc Check if a message matches a given template. +%% Templates can be either: +%% - A map: Uses structural matching against the message +%% - A binary regex: Matches against the message's target path +%% Returns true/false for map templates, or regex match result for binary templates. +template_matches(ToMatch, Template, _Opts) when is_map(Template) -> + case hb_message:match(Template, ToMatch, primary) of + {value_mismatch, _Key, _Val1, _Val2} -> false; + Match -> Match + end; +template_matches(ToMatch, Regex, Opts) when is_binary(Regex) -> + MsgPath = find_target_path(ToMatch, Opts), + hb_path:regex_matches(MsgPath, Regex). + %% @doc Label a list of elements with a number. number(List) -> lists:map( @@ -129,71 +327,164 @@ number(List) -> ). %% @doc Convert a list of elements to a map with numbered keys. -list_to_numbered_map(List) -> - maps:from_list(number(List)). +list_to_numbered_message(Msg) when is_map(Msg) -> + case is_ordered_list(Msg, #{}) of + true -> Msg; + false -> + throw({cannot_convert_to_numbered_message, Msg}) + end; +list_to_numbered_message(List) -> + hb_maps:from_list(number(List)). + +%% @doc Determine if the message given is an ordered list, starting from 1. +is_ordered_list(Msg, _Opts) when is_list(Msg) -> true; +is_ordered_list(Msg, Opts) -> + is_ordered_list(1, hb_ao:normalize_keys(Msg, Opts), Opts). +is_ordered_list(_, Msg, _Opts) when map_size(Msg) == 0 -> true; +is_ordered_list(N, Msg, _Opts) -> + case maps:get(NormKey = hb_ao:normalize_key(N), Msg, not_found) of + not_found -> false; + _ -> + is_ordered_list( + N + 1, + maps:without([NormKey], Msg), + _Opts + ) + end. + +%% @doc Replace a key in a list with a new value. +list_replace(List, Key, Value) -> + lists:foldr( + fun(Elem, Acc) -> + case Elem of + Key when is_list(Value) -> Value ++ Acc; + Key -> [Value | Acc]; + _ -> [Elem | Acc] + end + end, + [], + List + ). + +%% @doc Take a list and return a list of unique elements. The function is +%% order-preserving. +unique(List) -> + Unique = + lists:foldl( + fun(Item, Acc) -> + case lists:member(Item, Acc) of + true -> Acc; + false -> [Item | Acc] + end + end, + [], + List + ), + lists:reverse(Unique). + +%% @doc Returns the intersection of two lists, with stable ordering. +list_with(List1, List2) -> + lists:filter(fun(Item) -> lists:member(Item, List2) end, List1). + +%% @doc Remove all occurrences of all items in the first list from the second list. +list_without(List1, List2) -> + lists:filter(fun(Item) -> not lists:member(Item, List1) end, List2). %% @doc Take a message with numbered keys and convert it to a list of tuples -%% with the associated key as an integer and a value. Optionally, it takes a -%% standard map of HyperBEAM runtime options. -message_to_numbered_list(Message) -> - message_to_numbered_list(Message, #{}). -message_to_numbered_list(Message, Opts) -> - {ok, Keys} = hb_converge:keys(Message, Opts), - KeyValList = - lists:filtermap( - fun(Key) -> - case string:to_integer(Key) of - {Int, ""} -> - { - true, - {Int, hb_converge:get(Key, Message, Opts)} - }; +%% with the associated key as an integer. Optionally, it takes a standard +%% message of HyperBEAM runtime options. +message_to_ordered_list(Message) -> + message_to_ordered_list(Message, #{}). +message_to_ordered_list(Message, _Opts) when ?IS_EMPTY_MESSAGE(Message) -> + []; +message_to_ordered_list(List, _Opts) when is_list(List) -> + List; +message_to_ordered_list(Message, Opts) -> + NormMessage = hb_ao:normalize_keys(Message, Opts), + Keys = hb_maps:keys(NormMessage, Opts) -- [<<"priv">>, <<"commitments">>], + SortedKeys = + lists:map( + fun hb_ao:normalize_key/1, + lists:sort(lists:map(fun int/1, Keys)) + ), + message_to_ordered_list(NormMessage, SortedKeys, erlang:hd(SortedKeys), Opts). +message_to_ordered_list(_Message, [], _Key, _Opts) -> + []; +message_to_ordered_list(Message, [Key|Keys], Key, Opts) -> + case hb_maps:get(Key, Message, undefined, Opts#{ hashpath => ignore }) of + undefined -> + throw( + {missing_key, + {key, Key}, + {remaining_keys, Keys}, + {message, Message} + } + ); + Value -> + [ + Value + | + message_to_ordered_list( + Message, + Keys, + hb_ao:normalize_key(int(Key) + 1), + Opts + ) + ] + end; +message_to_ordered_list(Message, [Key|_Keys], ExpectedKey, _Opts) -> + throw({missing_key, {expected, ExpectedKey, {next, Key}, {message, Message}}}). + +%% @doc Convert a message with numbered keys and others to a sorted list with only +%% the numbered values. +numbered_keys_to_list(Message, Opts) -> + OnlyNumbered = + hb_maps:filter( + fun(Key, _Value) -> + try int(hb_ao:normalize_key(Key)) of + IntKey when is_integer(IntKey) -> true; _ -> false + catch _:_ -> false end end, - Keys + Message, + Opts ), - lists:sort(KeyValList). + message_to_ordered_list(OnlyNumbered, Opts). %% @doc Get the first element (the lowest integer key >= 1) of a numbered map. %% Optionally, it takes a specifier of whether to return the key or the value, %% as well as a standard map of HyperBEAM runtime options. -%% -%% If `error_strategy' is `throw', raise an exception if no integer keys are -%% found. If `error_strategy' is `any', return `undefined' if no integer keys -%% are found. By default, the function does not pass a `throw' execution -%% strategy to `hb_converge:to_key/2', such that non-integer keys present in the -%% message will not lead to an exception. hd(Message) -> hd(Message, value). hd(Message, ReturnType) -> hd(Message, ReturnType, #{ error_strategy => throw }). hd(Message, ReturnType, Opts) -> - {ok, Keys} = hb_converge:resolve(Message, keys, #{}), - hd(Message, Keys, 1, ReturnType, Opts). + hd(Message, hb_ao:keys(Message, Opts), 1, ReturnType, Opts). hd(_Map, [], _Index, _ReturnType, #{ error_strategy := throw }) -> throw(no_integer_keys); hd(_Map, [], _Index, _ReturnType, _Opts) -> undefined; hd(Message, [Key|Rest], Index, ReturnType, Opts) -> - case hb_converge:to_key(Key, Opts#{ error_strategy => return }) of + case hb_ao:normalize_key(Key, Opts#{ error_strategy => return }) of undefined -> hd(Message, Rest, Index + 1, ReturnType, Opts); Key -> case ReturnType of key -> Key; - value -> hb_converge:resolve(Message, Key, #{}) + value -> hb_ao:resolve(Message, Key, #{}) end end. %% @doc Find the value associated with a key in parsed a JSON structure list. find_value(Key, List) -> find_value(Key, List, undefined). - -find_value(Key, Map, Default) when is_map(Map) -> - case maps:find(Key, Map) of +find_value(Key, Map, Default) -> + find_value(Key, Map, Default, #{}). +find_value(Key, Map, Default, Opts) when is_map(Map) -> + case hb_maps:find(Key, Map, Opts) of {ok, Value} -> Value; error -> Default end; -find_value(Key, List, Default) -> +find_value(Key, List, Default, _Opts) -> case lists:keyfind(Key, 1, List) of {Key, Val} -> Val; false -> Default @@ -216,256 +507,21 @@ remove_common(Rest, _) -> Rest. %% @doc Throw an exception if the Opts map has an `error_strategy' key with the %% value `throw'. Otherwise, return the value. maybe_throw(Val, Opts) -> - case hb_converge:get(error_strategy, Opts) of + case hb_ao:get(error_strategy, Opts) of throw -> throw(Val); _ -> Val end. -%% @doc Print a message to the standard error stream, prefixed by the amount -%% of time that has elapsed since the last call to this function. -debug_print(X, Mod, Func, LineNum) -> - Now = erlang:system_time(millisecond), - Last = erlang:put(last_debug_print, Now), - TSDiff = case Last of undefined -> 0; _ -> Now - Last end, - io:format(standard_error, "=== HB DEBUG ===[~pms in ~p @ ~s]==>~n~s~n", - [ - TSDiff, self(), - format_debug_trace(Mod, Func, LineNum), - debug_fmt(X, 0) - ]), - X. - -%% @doc Generate the appropriate level of trace for a given call. -format_debug_trace(Mod, Func, Line) -> - case hb_opts:get(debug_print_trace, false, #{}) of - short -> - format_trace_short(get_trace()); - false -> - io_lib:format("~p:~w ~p", [Mod, Line, Func]) - end. - -%% @doc Convert a term to a string for debugging print purposes. -debug_fmt(X) -> debug_fmt(X, 0). -debug_fmt(X, Indent) -> - try do_debug_fmt(X, Indent) - catch _:_ -> - case hb_opts:get(mode, prod) of - prod -> - format_indented("[!PRINT FAIL!]", Indent); - _ -> - format_indented("[PRINT FAIL:] ~80p", [X], Indent) - end - end. - -do_debug_fmt(Wallet = {{rsa, _PublicExpnt}, _Priv, _Pub}, Indent) -> - format_address(Wallet, Indent); -do_debug_fmt({_, Wallet = {{rsa, _PublicExpnt}, _Priv, _Pub}}, Indent) -> - format_address(Wallet, Indent); -do_debug_fmt({explicit, X}, Indent) -> - format_indented("~p", [X], Indent); -do_debug_fmt({X, Y}, Indent) when is_atom(X) and is_atom(Y) -> - format_indented("~p: ~p", [X, Y], Indent); -do_debug_fmt({X, Y}, Indent) when is_record(Y, tx) -> - format_indented("~p: [TX item]~n~s", - [X, ar_bundles:format(Y, Indent + 1)], - Indent - ); -do_debug_fmt({X, Y}, Indent) when is_map(Y) -> - Formatted = hb_util:format_map(Y, Indent + 1), - HasNewline = lists:member($\n, Formatted), - format_indented("~p~s", - [ - X, - case HasNewline of - true -> " ==>" ++ Formatted; - false -> ": " ++ Formatted - end - ], - Indent - ); -do_debug_fmt({X, Y}, Indent) -> - format_indented("~s: ~s", [debug_fmt(X, Indent), debug_fmt(Y, Indent)], Indent); -do_debug_fmt(Map, Indent) when is_map(Map) -> - hb_util:format_map(Map, Indent); -do_debug_fmt(Tuple, Indent) when is_tuple(Tuple) -> - format_tuple(Tuple, Indent); -do_debug_fmt(X, Indent) when is_binary(X) -> - format_indented("~s", [format_binary(X)], Indent); -do_debug_fmt(Str = [X | _], Indent) when is_integer(X) andalso X >= 32 andalso X < 127 -> - format_indented("~s", [Str], Indent); -do_debug_fmt(X, Indent) -> - format_indented("~80p", [X], Indent). - -%% @doc If the user attempts to print a wallet, format it as an address. -format_address(Wallet, Indent) -> - format_indented(human_id(ar_wallet:to_address(Wallet)), Indent). - -%% @doc Helper function to format tuples with arity greater than 2. -format_tuple(Tuple, Indent) -> - to_lines(lists:map( - fun(Elem) -> - debug_fmt(Elem, Indent) - end, - tuple_to_list(Tuple) - )). - -to_lines(Elems) -> - remove_trailing_noise(do_to_lines(Elems)). -do_to_lines([]) -> []; -do_to_lines(In =[RawElem | Rest]) -> - Elem = lists:flatten(RawElem), - case lists:member($\n, Elem) of - true -> lists:flatten(lists:join("\n", In)); - false -> Elem ++ ", " ++ do_to_lines(Rest) - end. - -remove_trailing_noise(Str) -> - remove_trailing_noise(Str, " \n,"). -remove_trailing_noise(Str, Noise) -> - case lists:member(lists:last(Str), Noise) of - true -> - remove_trailing_noise(lists:droplast(Str), Noise); - false -> Str - end. - -%% @doc Format a string with an indentation level. -format_indented(Str, Indent) -> format_indented(Str, "", Indent). -format_indented(RawStr, Fmt, Ind) -> - IndentSpaces = hb_opts:get(debug_print_indent), - lists:droplast( - lists:flatten( - io_lib:format( - [$\s || _ <- lists:seq(1, Ind * IndentSpaces)] ++ - lists:flatten(RawStr) ++ "\n", - Fmt - ) - ) - ). - -%% @doc Format a binary as a short string suitable for printing. -format_binary(Bin) -> - case short_id(Bin) of - undefined -> - MaxBinPrint = hb_opts:get(debug_print_binary_max), - Printable = - binary:part( - Bin, - 0, - case byte_size(Bin) of - X when X < MaxBinPrint -> X; - _ -> MaxBinPrint - end - ), - PrintSegment = - case is_human_binary(Printable) of - true -> Printable; - false -> encode(Printable) - end, - lists:flatten( - [ - "\"", - [PrintSegment], - case Printable == Bin of - true -> "\""; - false -> - io_lib:format("...\" <~s bytes>", [human_int(byte_size(Bin))]) - end - ] - ); - ShortID -> - lists:flatten(io_lib:format("~s", [ShortID])) - end. - -%% @doc Add `,` characters to a number every 3 digits to make it human readable. -human_int(Int) -> - lists:reverse(add_commas(lists:reverse(integer_to_list(Int)))). - -add_commas([A,B,C,Z|Rest]) -> [A,B,C,$,|add_commas([Z|Rest])]; -add_commas(List) -> List. - -%% @doc Format a map as either a single line or a multi-line string depending -%% on the value of the `debug_print_map_line_threshold' runtime option. -format_map(Map) -> format_map(Map, 0). -format_map(Map, Indent) -> - MaxLen = hb_opts:get(debug_print_map_line_threshold), - SimpleFmt = io_lib:format("~p", [Map]), - case lists:flatlength(SimpleFmt) of - Len when Len > MaxLen -> - "\n" ++ lists:flatten(hb_message:format(Map, Indent)); - _ -> SimpleFmt - end. - -%% @doc Format and print an indented string to standard error. -eunit_print(FmtStr, FmtArgs) -> - io:format( - standard_error, - "~n~s ", - [hb_util:format_indented(FmtStr ++ "...", FmtArgs, 4)] - ). - -%% @doc Print the trace of the current stack, up to the first non-hyperbeam -%% module. Prints each stack frame on a new line, until it finds a frame that -%% does not start with a prefix in the `stack_print_prefixes' hb_opts. -%% Optionally, you may call this function with a custom label and caller info, -%% which will be used instead of the default. -print_trace(Stack, CallMod, CallFunc, CallLine) -> - print_trace(Stack, "HB TRACE", - lists:flatten(io_lib:format("[~s:~w ~p]", - [CallMod, CallLine, CallFunc]) - )). - -print_trace(Stack, Label, CallerInfo) -> - io:format(standard_error, "=== ~s ===~s==>~n~s", - [ - Label, CallerInfo, - lists:flatten( - format_trace( - Stack, - hb_opts:get(stack_print_prefixes, [], #{}) - ) - ) - ]). - -%% @doc Format a stack trace as a list of strings, one for each stack frame. -%% Each stack frame is formatted if it matches the `stack_print_prefixes' -%% option. At the first frame that does not match a prefix in the -%% `stack_print_prefixes' option, the rest of the stack is not formatted. -format_trace([], _) -> []; -format_trace([Item|Rest], Prefixes) -> - case element(1, Item) of - Atom when is_atom(Atom) -> - case trace_is_relevant(Atom, Prefixes) of - true -> - [ - format_trace(Item, Prefixes) | - format_trace(Rest, Prefixes) - ]; - false -> [] - end; - _ -> [] - end; -format_trace({Func, ArityOrTerm, Extras}, Prefixes) -> - format_trace({no_module, Func, ArityOrTerm, Extras}, Prefixes); -format_trace({Mod, Func, ArityOrTerm, Extras}, _Prefixes) -> - ExtraMap = maps:from_list(Extras), - format_indented( - "~p:~p/~p [~s]~n", - [ - Mod, Func, ArityOrTerm, - case maps:get(line, ExtraMap, undefined) of - undefined -> "No details"; - Line -> - maps:get(file, ExtraMap) - ++ ":" ++ integer_to_list(Line) - end - ], - 1 - ). - -%% @doc Is the trace formatted string relevant to HyperBEAM? -trace_is_relevant(Atom, Prefixes) when is_atom(Atom) -> - trace_is_relevant(atom_to_list(Atom), Prefixes); -trace_is_relevant(Str, Prefixes) -> +%% @doc Is the given module part of HyperBEAM? +is_hb_module(Atom) -> + is_hb_module(Atom, hb_opts:get(stack_print_prefixes, [], #{})). +is_hb_module(Atom, Prefixes) when is_atom(Atom) -> + is_hb_module(atom_to_list(Atom), Prefixes); +is_hb_module("hb_event" ++ _, _) -> + % Explicitly exclude hb_event from the stack trace, as it is always included, + % creating noise in the output. + false; +is_hb_module(Str, Prefixes) -> case string:tokens(Str, "_") of [Pre|_] -> lists:member(Pre, Prefixes); @@ -473,66 +529,9 @@ trace_is_relevant(Str, Prefixes) -> false end. -%% @doc Print a trace to the standard error stream. -print_trace_short(Trace, Mod, Func, Line) -> - io:format(standard_error, "=== [ HB SHORT TRACE ~p:~w ~p ] ==> ~s~n", - [ - Mod, Line, Func, - format_trace_short(Trace) - ] - ). - -%% @doc Format a trace to a short string. -format_trace_short(Trace) -> - lists:join( - " / ", - lists:reverse(format_trace_short( - hb_opts:get(short_trace_len, 3, #{}), - false, - Trace, - hb_opts:get(stack_print_prefixes, [], #{}) - )) - ). -format_trace_short(_Max, _Latch, [], _Prefixes) -> []; -format_trace_short(0, _Latch, _Trace, _Prefixes) -> []; -format_trace_short(Max, Latch, [Item|Rest], Prefixes) -> - Formatted = format_trace_short(Max, Latch, Item, Prefixes), - case {Latch, trace_is_relevant(Formatted, Prefixes)} of - {false, true} -> - [Formatted | format_trace_short(Max - 1, true, Rest, Prefixes)]; - {false, false} -> - format_trace_short(Max, false, Rest, Prefixes); - {true, true} -> - [Formatted | format_trace_short(Max - 1, true, Rest, Prefixes)]; - {true, false} -> [] - end; -format_trace_short(Max, Latch, {Func, ArityOrTerm, Extras}, Prefixes) -> - format_trace_short( - Max, Latch, {no_module, Func, ArityOrTerm, Extras}, Prefixes - ); -format_trace_short(_, _Latch, {Mod, _, _, [{file, _}, {line, Line}|_]}, _) -> - lists:flatten(io_lib:format("~p:~p", [Mod, Line])); -format_trace_short(_, _Latch, {Mod, Func, _ArityOrTerm, _Extras}, _Prefixes) -> - lists:flatten(io_lib:format("~p:~p", [Mod, Func])). - -%% @doc Utility function to help macro `?trace/0' remove the first frame of the -%% stack trace. -trace_macro_helper(Fun, {_, {_, Stack}}, Mod, Func, Line) -> - Fun(Stack, Mod, Func, Line). - -%% @doc Get the trace of the current process. -get_trace() -> - case catch error(debugging_print) of - {_, {_, Stack}} -> - normalize_trace(Stack); - _ -> [] - end. - -%% @doc Remove all calls from this module from the top of a trace. -normalize_trace([]) -> []; -normalize_trace([{Mod, _, _, _}|Rest]) when Mod == ?MODULE -> - normalize_trace(Rest); -normalize_trace(Trace) -> Trace. +%% @doc Get all loaded modules that are loaded and are part of HyperBEAM. +all_hb_modules() -> + lists:filter(fun(Module) -> is_hb_module(Module) end, erlang:loaded()). %%% Statistics @@ -547,4 +546,177 @@ stddev(List) -> variance(List) -> Mean = mean(List), - lists:sum([ math:pow(X - Mean, 2) || X <- List ]) / length(List). \ No newline at end of file + lists:sum([ math:pow(X - Mean, 2) || X <- List ]) / length(List). + +%% @doc Shuffle a list. +shuffle(List) -> + [ Y || {_, Y} <- lists:sort([ {rand:uniform(), X} || X <- List]) ]. + +%% @doc Return a random element from a list, weighted by the values in the list. +weighted_random(List) -> + TotalWeight = lists:sum([ Weight || {_, Weight} <- List ]), + Normalized = [ {Item, Weight / TotalWeight} || {Item, Weight} <- List ], + Shuffled = shuffle(Normalized), + pick_weighted(Shuffled, rand:uniform()). + +%% @doc Pick a random element from a list, weighted by the values in the list. +pick_weighted([], _) -> + error(empty_list); +pick_weighted([{Item, Weight}|_Rest], Remaining) when Remaining < Weight -> + Item; +pick_weighted([{_Item, Weight}|Rest], Remaining) -> + pick_weighted(Rest, Remaining - Weight). + +%% @doc Serialize the given list of addresses to a binary, using the structured +%% fields format. +addresses_to_binary(List) when is_list(List) -> + try + iolist_to_binary( + hb_structured_fields:list( + [ + {item, {string, hb_util:human_id(Addr)}, []} + || + Addr <- List + ] + ) + ) + catch + _:_ -> + error({cannot_parse_list, List}) + end. + +%% @doc Parse a list from a binary. First attempts to parse the binary as a +%% structured-fields list, and if that fails, it attempts to parse the list as +%% a comma-separated value, stripping quotes and whitespace. +binary_to_addresses(List) when is_list(List) -> + % If the argument is already a list, return it. + binary_to_addresses(List); +binary_to_addresses(List) when is_binary(List) -> + try + Res = lists:map( + fun({item, {string, Item}, []}) -> + Item + end, + hb_structured_fields:parse_list(List) + ), + Res + catch + _:_ -> + try + binary:split( + binary:replace(List, <<"\"">>, <<"">>, [global]), + <<",">>, + [global, trim_all] + ) + catch + _:_ -> + error({cannot_parse_list, List}) + end + end. + + +%% @doc Extract all of the parts from the binary, given (a list of) separators. +split_depth_string_aware(_Sep, <<>>) -> []; +split_depth_string_aware(Sep, Bin) -> + {_MatchedSep, Part, Rest} = split_depth_string_aware_single(Sep, Bin), + [Part | split_depth_string_aware(Sep, Rest)]. + +%% @doc Parse a binary, extracting a part until a separator is found, while +%% honoring nesting characters. +split_depth_string_aware_single(Sep, Bin) when not is_list(Sep) -> + split_depth_string_aware_single([Sep], Bin); +split_depth_string_aware_single(Seps, Bin) -> + split_depth_string_aware_single(Seps, Bin, 0, <<>>). +split_depth_string_aware_single(_Seps, <<>>, _Depth, CurrAcc) -> + {no_match, CurrAcc, <<>>}; +split_depth_string_aware_single(Seps, << $\", Rest/binary>>, Depth, CurrAcc) -> + {QuotedStr, AfterStr} = split_escaped_single($\", Rest), + split_depth_string_aware_single( + Seps, + AfterStr, + Depth, + << CurrAcc/binary, "\"", QuotedStr/binary, "\"">> + ); +split_depth_string_aware_single(Seps, << $\(, Rest/binary>>, Depth, CurrAcc) -> + %% Increase depth + split_depth_string_aware_single(Seps, Rest, Depth + 1, << CurrAcc/binary, "(" >>); +split_depth_string_aware_single(Seps, << $\), Rest/binary>>, Depth, Acc) when Depth > 0 -> + %% Decrease depth + split_depth_string_aware_single(Seps, Rest, Depth - 1, << Acc/binary, ")">>); +split_depth_string_aware_single(Seps, <>, Depth, CurrAcc) -> + case Depth == 0 andalso lists:member(C, Seps) of + true -> {C, CurrAcc, Rest}; + false -> + split_depth_string_aware_single( + Seps, + Rest, + Depth, + << CurrAcc/binary, C:8/integer >> + ) + end. + +%% @doc Read a binary until a separator is found without a preceding backslash. +split_escaped_single(Sep, Bin) -> + split_escaped_single(Sep, Bin, []). +split_escaped_single(_Sep, <<>>, Acc) -> + {hb_util:bin(lists:reverse(Acc)), <<>>}; +split_escaped_single(Sep, <<"\\", Char:8/integer, Rest/binary>>, Acc) -> + split_escaped_single(Sep, Rest, [Char, $\\ | Acc]); +split_escaped_single(Sep, <>, Acc) -> + {hb_util:bin(lists:reverse(Acc)), Rest}; +split_escaped_single(Sep, <>, Acc) -> + split_escaped_single(Sep, Rest, [C | Acc]). + +%% @doc Force that a binary is either empty or the given number of bytes. +check_size(Bin, {range, Start, End}) -> + check_type(Bin, binary) + andalso byte_size(Bin) >= Start + andalso byte_size(Bin) =< End; +check_size(Bin, Sizes) -> + check_type(Bin, binary) + andalso lists:member(byte_size(Bin), Sizes). + +check_value(Value, ExpectedValues) -> + lists:member(Value, ExpectedValues). + +%% @doc Ensure that a value is of the given type. +check_type(Value, binary) -> is_binary(Value); +check_type(Value, integer) -> is_integer(Value); +check_type(Value, list) -> is_list(Value); +check_type(Value, map) -> is_map(Value); +check_type(Value, tx) -> is_record(Value, tx); +check_type(Value, message) -> + is_record(Value, tx) or is_map(Value) or is_list(Value); +check_type(_Value, _) -> false. + +%% @doc Throw an error if the given value is not ok. +ok_or_throw(_, true, _) -> true; +ok_or_throw(_TX, false, Error) -> + throw(Error). + +%% @doc List the loaded atoms in the Erlang VM. +all_atoms() -> all_atoms(0). +all_atoms(N) -> + case atom_from_int(N) of + not_found -> []; + A -> [A | all_atoms(N+1)] + end. + +%% @doc Find the atom with the given integer reference. +atom_from_int(Int) -> + case catch binary_to_term(<<131,75,Int:24>>) of + A -> A; + _ -> not_found + end. + +%% @doc Check if a given binary is already an atom. +binary_is_atom(X) -> + lists:member(X, lists:map(fun hb_util:bin/1, all_atoms())). + +lower_case_key_map(Map, Opts) -> + hb_maps:fold(fun + (K, V, Acc) when is_map(V) -> + maps:put(hb_util:to_lower(K), lower_case_key_map(V, Opts), Acc); + (K, V, Acc) -> + maps:put(hb_util:to_lower(K), V, Acc) + end, #{}, Map, Opts). \ No newline at end of file diff --git a/src/hb_volume.erl b/src/hb_volume.erl new file mode 100644 index 000000000..182217334 --- /dev/null +++ b/src/hb_volume.erl @@ -0,0 +1,853 @@ +-module(hb_volume). +-moduledoc """ +Module for managing physical disks and volumes, providing operations +for partitioning, formatting, mounting, and managing encrypted volumes. +""". +-export([list_partitions/0, create_partition/2]). +-export([format_disk/2, mount_disk/4, change_node_store/2]). +-export([check_for_device/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-doc """ +List available partitions in the system. +@returns {ok, Map} where Map contains the partition information, + or {error, Reason} if the operation fails. +""". +-spec list_partitions() -> {ok, map()} | {error, binary()}. +list_partitions() -> + ?event(debug_volume, {list_partitions, entry, starting}), + % Get the partition information using fdisk -l + ?event(debug_volume, {list_partitions, executing_fdisk, command}), + case os:cmd("sudo fdisk -l") of + [] -> + % Empty output indicates an error + Reason = <<"Failed to list partitions: no output">>, + ?event(debug_volume, {list_partitions, fdisk_error, no_output}), + {error, Reason}; + Output -> + ?event(debug_volume, {list_partitions, fdisk_success, parsing}), + + % Split output into lines + Lines = string:split(Output, "\n", all), + + % Process the output to group information by disk + {_, DiskData} = lists:foldl( + fun process_disk_line/2, + {undefined, []}, + Lines + ), + % Process each disk's data to extract all information + DiskObjects = lists:filtermap( + fun(DiskEntry) -> + Device = maps:get(<<"device">>, DiskEntry), + DiskLines = lists:reverse(maps:get(<<"data">>, DiskEntry)), + DiskInfo = parse_disk_info(Device, DiskLines), + {true, DiskInfo} + end, + DiskData + ), + % Return the partition information + ?event(debug_volume, + {list_partitions, success, + {disk_count, length(DiskObjects)} + } + ), + {ok, #{ + <<"status">> => 200, + <<"content-type">> => <<"application/json">>, + <<"body">> => hb_json:encode(#{<<"disks">> => DiskObjects}) + }} + end. + +%%% Helper functions for list_partitions +% Process a line of fdisk output to group by disk +process_disk_line(Line, {CurrentDisk, Acc}) -> + % Match for a new disk entry + DiskPattern = "^Disk (/dev/(?!ram)\\S+):", + case re:run(Line, DiskPattern, [{capture, [1], binary}]) of + {match, [Device]} -> + % Start a new disk entry + NewDisk = #{ + <<"device">> => Device, + <<"data">> => [Line] + }, + {NewDisk, [NewDisk | Acc]}; + _ when CurrentDisk =:= undefined -> + % Not a disk line and no current disk + {undefined, Acc}; + _ -> + % Add line to current disk's data + CurrentData = maps:get(<<"data">>, CurrentDisk), + UpdatedDisk = CurrentDisk#{ + <<"data">> => [Line | CurrentData] + }, + % Update the list with the modified disk entry + UpdatedAcc = [UpdatedDisk | lists:delete(CurrentDisk, Acc)], + {UpdatedDisk, UpdatedAcc} + end. + +% Parse detailed disk information from fdisk output lines +parse_disk_info(Device, Lines) -> + % Initialize with device ID + DiskInfo = #{<<"device">> => Device}, + % Process each line to extract information + lists:foldl( + fun parse_disk_line/2, + DiskInfo, + Lines + ). + +% Parse a single line of disk information +parse_disk_line(Line, Info) -> + % Extract disk size and bytes + SizePattern = "^Disk .+: ([0-9.]+ [KMGT]iB), ([0-9]+) bytes, ([0-9]+) sectors", + case re:run(Line, SizePattern, [{capture, [1, 2, 3], binary}]) of + {match, [Size, Bytes, Sectors]} -> + Info#{ + <<"size">> => Size, + <<"bytes">> => binary_to_integer(Bytes), + <<"sectors">> => binary_to_integer(Sectors) + }; + _ -> + parse_disk_model_line(Line, Info) + end. + +% Parse disk model information +parse_disk_model_line(Line, Info) -> + % Extract disk model + ModelPattern = "^Disk model: (.+)\\s*$", + case re:run(Line, ModelPattern, [{capture, [1], binary}]) of + {match, [Model]} -> + Info#{<<"model">> => string:trim(Model)}; + _ -> + parse_disk_units_line(Line, Info) + end. + +% Parse disk units information +parse_disk_units_line(Line, Info) -> + % Extract units information + UnitsPattern = "^Units: (.+)$", + case re:run(Line, UnitsPattern, [{capture, [1], binary}]) of + {match, [Units]} -> + Info#{<<"units">> => Units}; + _ -> + parse_sector_size_line(Line, Info) + end. + +% Parse sector size information +parse_sector_size_line(Line, Info) -> + % Extract sector size + SectorPattern = "^Sector size \\(logical/physical\\): ([^/]+)/(.+)$", + case re:run(Line, SectorPattern, [{capture, [1, 2], binary}]) of + {match, [LogicalSize, PhysicalSize]} -> + Info#{ + <<"sector_size">> => #{ + <<"logical">> => string:trim(LogicalSize), + <<"physical">> => string:trim(PhysicalSize) + } + }; + _ -> + parse_io_size_line(Line, Info) + end. + +% Parse I/O size information +parse_io_size_line(Line, Info) -> + % Extract I/O size + IOPattern = "^I/O size \\(minimum/optimal\\): ([^/]+)/(.+)$", + case re:run(Line, IOPattern, [{capture, [1, 2], binary}]) of + {match, [MinSize, OptSize]} -> + Info#{ + <<"io_size">> => #{ + <<"minimum">> => string:trim(MinSize), + <<"optimal">> => string:trim(OptSize) + } + }; + _ -> + Info + end. + +-doc """ +Create a partition on a disk device. +@param Device The path to the device, e.g. "/dev/sdb". +@param PartType The partition type to create, defaults to "ext4". +@returns {ok, Map} on success where Map includes status and partition + information, or {error, Reason} if the operation fails. +""". +-spec create_partition(Device :: binary(), PartType :: binary()) -> + {ok, map()} | {error, binary()}. +create_partition(undefined, _PartType) -> + ?event(debug_volume, {create_partition, error, device_undefined}), + {error, <<"Device path not specified">>}; +create_partition(Device, PartType) -> + ?event(debug_volume, + {create_partition, entry, + {device, Device, part_type, PartType} + } + ), + % Create a GPT partition table + DeviceStr = binary_to_list(Device), + MklabelCmd = "sudo parted " ++ DeviceStr ++ " mklabel gpt", + ?event(debug_volume, + {create_partition, creating_gpt_label, + {device, Device} + } + ), + ?event(debug_volume, + {create_partition, executing_mklabel, + {command, MklabelCmd} + } + ), + case safe_exec(MklabelCmd) of + {ok, Result} -> + ?event(debug_volume, + {create_partition, gpt_label_success, + {result, Result} + } + ), + create_actual_partition(Device, PartType); + {error, ErrorMsg} -> + ?event(debug_volume, + {create_partition, gpt_label_error, + {error, ErrorMsg} + } + ), + {error, ErrorMsg} + end. + +% Create the actual partition after making the GPT label +create_actual_partition(Device, PartType) -> + ?event(debug_volume, + {create_actual_partition, entry, + {device, Device, part_type, PartType} + } + ), + DeviceStr = binary_to_list(Device), + PartTypeStr = binary_to_list(PartType), + % Build the parted command to create the partition + MkpartCmd = + "sudo parted -a optimal " ++ DeviceStr ++ + " mkpart primary " ++ PartTypeStr ++ " 0% 100%", + ?event(debug_volume, + {create_actual_partition, executing_mkpart, + {command, MkpartCmd} + } + ), + case safe_exec(MkpartCmd) of + {ok, Result} -> + ?event(debug_volume, + {create_actual_partition, mkpart_success, + {result, Result} + } + ), + get_partition_info(Device); + {error, ErrorMsg} -> + ?event(debug_volume, + {create_actual_partition, mkpart_error, + {error, ErrorMsg} + } + ), + {error, ErrorMsg} + end. + +% Get the partition information after creating a partition +get_partition_info(Device) -> + ?event(debug_volume, {get_partition_info, entry, {device, Device}}), + DeviceStr = binary_to_list(Device), + % Print partition information + PrintCmd = "sudo parted " ++ DeviceStr ++ " print", + ?event(debug_volume, + {get_partition_info, executing_print, {command, PrintCmd}} + ), + PartitionInfo = os:cmd(PrintCmd), + ?event(debug_volume, + {get_partition_info, success, partition_created, + {result, PartitionInfo} + } + ), + {ok, #{ + <<"status">> => 200, + <<"message">> => <<"Partition created successfully.">>, + <<"device_path">> => Device, + <<"partition_info">> => list_to_binary(PartitionInfo) + }}. + +-doc """ +Format a disk or partition with LUKS encryption. +@param Partition The path to the partition, e.g. "/dev/sdc1". +@param EncKey The encryption key to use for LUKS. +@returns {ok, Map} on success where Map includes the status and + confirmation message, or {error, Reason} if the operation fails. +""". +-spec format_disk(Partition :: binary(), EncKey :: binary()) -> + {ok, map()} | {error, binary()}. +format_disk(undefined, _EncKey) -> + ?event(debug_volume, {format_disk, error, partition_undefined}), + {error, <<"Partition path not specified">>}; +format_disk(_Partition, undefined) -> + ?event(debug_volume, {format_disk, error, key_undefined}), + {error, <<"Encryption key not specified">>}; +format_disk(Partition, EncKey) -> + ?event(debug_volume, + {format_disk, entry, + { + partition, Partition, + key_present, true + } + } + ), + PartitionStr = binary_to_list(Partition), + ?event(debug_volume, {format_disk, creating_secure_key_file, starting}), + with_secure_key_file(EncKey, fun(KeyFile) -> + FormatCmd = + "sudo cryptsetup luksFormat --batch-mode " ++ + "--key-file " ++ KeyFile ++ " " ++ PartitionStr, + ?event(debug_volume, + {format_disk, executing_luks_format, {command, FormatCmd}} + ), + case safe_exec(FormatCmd, ["failed"]) of + {ok, Result} -> + ?event(debug_volume, + {format_disk, luks_format_success, completed, + {result, Result} + } + ), + {ok, #{ + <<"status">> => 200, + <<"message">> => + <<"Partition formatted with LUKS encryption " + "successfully.">> + }}; + {error, ErrorMsg} -> + ?event(debug_volume, + {format_disk, luks_format_error, ErrorMsg} + ), + {error, ErrorMsg} + end + end). + +-doc """ +Mount a LUKS-encrypted disk. +@param Partition The path to the partition, e.g. "/dev/sdc1". +@param EncKey The encryption key for LUKS. +@param MountPoint The directory where the disk should be mounted. +@param VolumeName The name to use for the decrypted LUKS volume. +@returns {ok, Map} on success where Map includes the status and + confirmation message, or {error, Reason} if the operation fails. +""". +-spec mount_disk( + Partition :: binary(), + EncKey :: binary(), + MountPoint :: binary(), + VolumeName :: binary() +) -> {ok, map()} | {error, binary()}. +mount_disk(undefined, _EncKey, _MountPoint, _VolumeName) -> + ?event(debug_volume, {mount_disk, error, partition_undefined}), + {error, <<"Partition path not specified">>}; +mount_disk(_Partition, undefined, _MountPoint, _VolumeName) -> + ?event(debug_volume, {mount_disk, error, key_undefined}), + {error, <<"Encryption key not specified">>}; +mount_disk(_Partition, _EncKey, undefined, _VolumeName) -> + ?event(debug_volume, {mount_disk, error, mount_point_undefined}), + {error, <<"Mount point not specified">>}; +mount_disk(Partition, EncKey, MountPoint, VolumeName) -> + ?event(debug_volume, + {mount_disk, entry, + { + partition, Partition, + mount_point, MountPoint, + volume_name, VolumeName} + } + ), + PartitionStr = binary_to_list(Partition), + VolumeNameStr = binary_to_list(VolumeName), + ?event(debug_volume, {mount_disk, opening_luks_volume, starting}), + with_secure_key_file(EncKey, fun(KeyFile) -> + OpenCmd = + "sudo cryptsetup luksOpen --key-file " ++ KeyFile ++ + " " ++ PartitionStr ++ " " ++ VolumeNameStr, + ?event(debug_volume, {mount_disk, executing_luks_open, {command, OpenCmd}}), + case safe_exec(OpenCmd, ["failed"]) of + {ok, Result} -> + ?event(debug_volume, + {mount_disk, luks_open_success, proceeding_to_mount, + {result, Result} + } + ), + mount_opened_volume(Partition, MountPoint, VolumeName); + {error, ErrorMsg} -> + ?event(debug_volume, {mount_disk, luks_open_error, ErrorMsg}), + {error, ErrorMsg} + end + end). + +% Mount an already opened LUKS volume +mount_opened_volume(Partition, MountPoint, VolumeName) -> + ?event(debug_volume, + {mount_opened_volume, entry, + { + partition, Partition, + mount_point, MountPoint, + volume_name, VolumeName + } + } + ), + % Create mount point if it doesn't exist + MountPointStr = binary_to_list(MountPoint), + ?event(debug_volume, + {mount_opened_volume, creating_mount_point, MountPoint} + ), + os:cmd("sudo mkdir -p " ++ MountPointStr), + % Check if filesystem exists on the opened LUKS volume + VolumeNameStr = binary_to_list(VolumeName), + DeviceMapperPath = "/dev/mapper/" ++ VolumeNameStr, + % Check filesystem type + FSCheckCmd = "sudo blkid " ++ DeviceMapperPath, + ?event(debug_volume, + {mount_opened_volume, checking_filesystem, {command, FSCheckCmd}} + ), + FSCheckResult = os:cmd(FSCheckCmd), + ?event(debug_volume, + {mount_opened_volume, filesystem_check_result, FSCheckResult} + ), + % Create filesystem if none exists + case string:find(FSCheckResult, "TYPE=") of + nomatch -> + % No filesystem found, create ext4 + ?event(debug_volume, + {mount_opened_volume, creating_filesystem, ext4} + ), + MkfsCmd = "sudo mkfs.ext4 -F " ++ DeviceMapperPath, + ?event(debug_volume, + {mount_opened_volume, executing_mkfs, {command, MkfsCmd}} + ), + MkfsResult = os:cmd(MkfsCmd), + ?event(debug_volume, + {mount_opened_volume, mkfs_result, MkfsResult} + ); + _ -> + ?event(debug_volume, + {mount_opened_volume, filesystem_exists, skipping_creation} + ) + end, + % Mount the unlocked LUKS volume + MountCmd = "sudo mount " ++ DeviceMapperPath ++ " " ++ MountPointStr, + ?event(debug_volume, + {mount_opened_volume, executing_mount, + {command, MountCmd} + } + ), + case safe_exec(MountCmd, ["failed"]) of + {ok, Result} -> + ?event(debug_volume, + {mount_opened_volume, mount_success, + creating_info, {result, Result} + } + ), + create_mount_info(Partition, MountPoint, VolumeName); + {error, ErrorMsg} -> + ?event(debug_volume, + {mount_opened_volume, mount_error, + {error, ErrorMsg, closing_luks} + } + ), + % Close the LUKS volume if mounting failed + os:cmd("sudo cryptsetup luksClose " ++ VolumeNameStr), + {error, ErrorMsg} + end. + +% Create mount info response +create_mount_info(Partition, MountPoint, VolumeName) -> + ?event(debug_volume, + {create_mount_info, success, + { + partition, Partition, + mount_point, MountPoint, + volume_name, VolumeName + } + } + ), + {ok, #{ + <<"status">> => 200, + <<"message">> => + <<"Encrypted partition mounted successfully.">>, + <<"mount_point">> => MountPoint, + <<"mount_info">> => #{ + partition => Partition, + mount_point => MountPoint, + volume_name => VolumeName + } + }}. + +-doc """ +Change the node's data store location to the mounted encrypted disk. +@param StorePath The new path for the store directory. +@param CurrentStore The current store configuration. +@returns {ok, Map} on success where Map includes the status and + confirmation message, or {error, Reason} if the operation fails. +""". +-spec change_node_store(StorePath :: binary(), + CurrentStore :: list()) -> + {ok, map()} | {error, binary()}. +change_node_store(undefined, _CurrentStore) -> + ?event(debug_volume, {change_node_store, error, store_path_undefined}), + {error, <<"Store path not specified">>}; +change_node_store(StorePath, CurrentStore) -> + ?event(debug_volume, + {change_node_store, entry, + {store_path, StorePath, current_store, CurrentStore} + } + ), + % Create the store directory if it doesn't exist + StorePathStr = binary_to_list(StorePath), + ?event(debug_volume, {change_node_store, creating_directory, StorePath}), + os:cmd("sudo mkdir -p " ++ StorePathStr), + % Update the store configuration with the new path + ?event(debug_volume, + {change_node_store, updating_config, + {current_store, CurrentStore} + } + ), + NewStore = update_store_config(CurrentStore, StorePath), + % Return the result + ?event(debug_volume, + {change_node_store, success, {new_store_config, NewStore}} + ), + {ok, #{ + <<"status">> => 200, + <<"message">> => + <<"Node store updated to use encrypted disk.">>, + <<"store_path">> => StorePath, + <<"store">> => NewStore + }}. + +%%% Helper functions +%% Execute system command with error checking +safe_exec(Command) -> + safe_exec(Command, ["Error", "failed", "bad", "error"]). + +safe_exec(Command, ErrorKeywords) -> + Result = os:cmd(Command), + case check_command_errors(Result, ErrorKeywords) of + ok -> {ok, Result}; + error -> {error, list_to_binary(Result)} + end. + +%% Check if command result contains error indicators +check_command_errors(Result, Keywords) -> + case lists:any(fun(Keyword) -> + string:find(Result, Keyword) =/= nomatch + end, Keywords) of + true -> error; + false -> ok + end. + +%% Secure key file management with automatic cleanup +with_secure_key_file(EncKey, Fun) -> + ?event(debug_volume, {with_secure_key_file, entry, creating_temp_file}), + os:cmd("sudo mkdir -p /root/tmp"), + % Get process ID and create filename + PID = os:getpid(), + ?event(debug_volume, {with_secure_key_file, process_id, PID}), + KeyFile = "/root/tmp/luks_key_" ++ PID, + ?event(debug_volume, {with_secure_key_file, key_file_path, KeyFile}), + % Check if directory was created successfully + DirCheck = os:cmd("ls -la /root/tmp/"), + ?event(debug_volume, {with_secure_key_file, directory_check, DirCheck}), + try + % Convert EncKey to binary using hb_util + BinaryEncKey = case EncKey of + % Handle RSA wallet tuples - extract private key or use hash + {{rsa, _}, PrivKey, _PubKey} when is_binary(PrivKey) -> + % Use first 32 bytes of private key for AES-256 + case byte_size(PrivKey) of + Size when Size >= 32 -> + binary:part(PrivKey, 0, 32); + _ -> + % If private key is too short, hash it to get 32 bytes + crypto:hash(sha256, PrivKey) + end; + % Handle other complex terms + _ when not is_binary(EncKey) andalso not is_list(EncKey) -> + try + hb_util:bin(EncKey) + catch + _:_ -> + % Fallback to term_to_binary and hash to get consistent + % key size + crypto:hash(sha256, term_to_binary(EncKey)) + end; + % Simple cases handled by hb_util:bin + _ -> + hb_util:bin(EncKey) + end, + WriteResult = file:write_file(KeyFile, BinaryEncKey, [raw]), + ?event(debug_volume, + {with_secure_key_file, write_result, WriteResult} + ), + % Check if file was created + FileExists = filelib:is_regular(KeyFile), + ?event(debug_volume, + {with_secure_key_file, file_exists_check, FileExists} + ), + % If file exists, get its info + case FileExists of + true -> + FileInfo = file:read_file_info(KeyFile), + ?event(debug_volume, + {with_secure_key_file, file_info, FileInfo} + ); + false -> + ?event(debug_volume, + {with_secure_key_file, file_not_found, KeyFile} + ) + end, + % Execute function with key file path + ?event(debug_volume, + {with_secure_key_file, executing_function, with_key_file} + ), + Result = Fun(KeyFile), + % Always clean up the key file + ?event(debug_volume, + {with_secure_key_file, cleanup, shredding_key_file} + ), + os:cmd("sudo shred -u " ++ KeyFile), + ?event(debug_volume, {with_secure_key_file, success, completed}), + Result + catch + Class:Reason:Stacktrace -> + ?event(debug_volume, + {with_secure_key_file, exception, + {class, Class, reason, Reason, cleanup, starting} + } + ), + % Ensure cleanup even if function fails + os:cmd("sudo shred -u " ++ KeyFile), + ?event(debug_volume, + {with_secure_key_file, exception_cleanup, completed} + ), + erlang:raise(Class, Reason, Stacktrace) + end. + +% Update the store configuration with a new base path +-spec update_store_config(StoreConfig :: term(), + NewPath :: binary()) -> term(). +update_store_config(StoreConfig, NewPath) when is_list(StoreConfig) -> + % For a list, update each element + [update_store_config(Item, NewPath) || Item <- StoreConfig]; +update_store_config( + #{<<"store-module">> := Module} = StoreConfig, + NewPath +) when is_map(StoreConfig) -> + % Handle various store module types differently + case Module of + hb_store_fs -> + % For filesystem store, prefix the existing path with the new path + ExistingPath = maps:get(<<"name">>, StoreConfig, <<"">>), + NewName = <>, + ?event(debug_volume, {fs, StoreConfig, NewPath, NewName}), + StoreConfig#{<<"name">> => NewName}; + hb_store_lmdb -> + ExistingPath = maps:get(<<"name">>, StoreConfig, <<"">>), + NewName = <>, + ?event(debug_volume, {migrate_start, ExistingPath, NewName}), + safe_stop_lmdb_store(StoreConfig), + ?event(debug_volume, {using_existing_store, NewName}), + FinalConfig = StoreConfig#{<<"name">> => NewName}, + safe_start_lmdb_store(FinalConfig), + FinalConfig; + hb_store_rocksdb -> + StoreConfig; + hb_store_gateway -> + % For gateway store, recursively update nested store configs + NestedStore = maps:get(<<"store">>, StoreConfig, []), + StoreConfig#{ + <<"store">> => update_store_config(NestedStore, NewPath) + }; + _ -> + % For any other store type, update the prefix + % StoreConfig#{<<"name">> => NewPath} + ?event(debug_volume, {other, StoreConfig, NewPath}), + StoreConfig + end; +update_store_config({Type, _OldPath, Opts}, NewPath) -> + % For tuple format with options + {Type, NewPath, Opts}; +update_store_config({Type, _OldPath}, NewPath) -> + % For tuple format without options + {Type, NewPath}; +update_store_config(StoreConfig, _NewPath) -> + % Return unchanged for any other format + StoreConfig. + +%% Safely stop LMDB store with error handling +safe_stop_lmdb_store(StoreConfig) -> + ?event(debug_volume, {stopping_current_store, StoreConfig}), + try + hb_store_lmdb:stop(StoreConfig) + catch + error:StopReason -> + ?event(debug_volume, {stop_error, StopReason}) + end. + +%% Safely start LMDB store +safe_start_lmdb_store(StoreConfig) -> + NewName = maps:get(<<"name">>, StoreConfig), + ?event(debug_volume, {starting_new_store, NewName}), + hb_store_lmdb:start(StoreConfig). + +-doc """ +Check if a device exists on the system. +@param Device The path to the device to check (binary). +@returns true if the device exists, false otherwise. +""". +-spec check_for_device(Device :: binary()) -> boolean(). +check_for_device(Device) -> + ?event(debug_volume, {check_for_device, entry, {device, Device}}), + Command = + io_lib:format( + "ls -l ~s 2>/dev/null || echo 'not_found'", + [binary_to_list(Device)] + ), + ?event(debug_volume, {check_for_device, executing_command, ls_check}), + Result = os:cmd(Command), + DeviceExists = string:find(Result, "not_found") =:= nomatch, + ?event(debug_volume, + {check_for_device, result, + {device, Device, exists, DeviceExists} + } + ), + DeviceExists. + +%%% Unit Tests +%% Test helper function error checking +check_command_errors_test() -> + % Test successful case - no errors + ?assertEqual( + ok, + check_command_errors( + "Success: operation completed", + ["Error", "failed"] + ) + ), + % Test error detection + ?assertEqual( + error, + check_command_errors( + "Error: something went wrong", + ["Error", "failed"] + ) + ), + ?assertEqual( + error, + check_command_errors( + "Operation failed", + ["Error", "failed"] + ) + ), + % Test case sensitivity + ?assertEqual( + ok, + check_command_errors( + "error (lowercase)", + ["Error", "failed"] + ) + ), + % Test multiple keywords + ?assertEqual( + error, + check_command_errors( + "Command failed with Error", + ["Error", "failed"] + ) + ). + +%% Test store configuration updates for different types +update_store_config_test() -> + % Test filesystem store + FSStore = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache">> + }, + NewPath = <<"/encrypted/mount">>, + Updated = update_store_config(FSStore, NewPath), + Expected = FSStore#{<<"name">> => <<"/encrypted/mount/cache">>}, + ?assertEqual(Expected, Updated), + % Test list of stores + StoreList = [FSStore, #{<<"store-module">> => hb_store_gateway}], + UpdatedList = update_store_config(StoreList, NewPath), + ?assertEqual(2, length(UpdatedList)), + % Test tuple format + TupleStore = {fs, <<"old_path">>, []}, + UpdatedTuple = update_store_config(TupleStore, NewPath), + ?assertEqual({fs, NewPath, []}, UpdatedTuple). + +%% Test secure key file management +with_secure_key_file_test() -> + TestKey = <<"test_encryption_key_123">>, + % Create a safe test version that doesn't use /root/tmp + TestWithSecureKeyFile = fun(EncKey, Fun) -> + % Use /tmp instead of /root/tmp for testing + TmpDir = "/tmp", + KeyFile = TmpDir ++ "/test_luks_key_" ++ os:getpid(), + try + % Write key to temporary file + file:write_file(KeyFile, EncKey, [raw]), + % Execute function with key file path + Result = Fun(KeyFile), + % Clean up the key file + file:delete(KeyFile), + Result + catch + Class:Reason:Stacktrace -> + % Ensure cleanup even if function fails + file:delete(KeyFile), + erlang:raise(Class, Reason, Stacktrace) + end + end, + % Test successful execution + Result = TestWithSecureKeyFile(TestKey, fun(KeyFile) -> + % Verify key file was created and contains the key + ?assert(filelib:is_regular(KeyFile)), + {ok, FileContent} = file:read_file(KeyFile), + ?assertEqual(TestKey, FileContent), + {ok, <<"success">>} + end), + ?assertEqual({ok, <<"success">>}, Result), + % Test exception handling and cleanup + TestException = fun() -> + TestWithSecureKeyFile(TestKey, fun(KeyFile) -> + ?assert(filelib:is_regular(KeyFile)), + error(test_error) + end) + end, + ?assertError(test_error, TestException()). + +%% Test device checking with mocked commands +check_for_device_test() -> + % This test would need mocking of os:cmd to be fully testable + % For now, test with /dev/null which should always exist + ?assertEqual(true, check_for_device(<<"/dev/null">>)), + % Test non-existent device + ?assertEqual( + false, + check_for_device(<<"/dev/nonexistent_device_123">>) + ). + +%% Test safe command execution with mocked results +safe_exec_mock_test() -> + % We can't easily mock os:cmd, but we can test the error checking logic + % This is covered by check_command_errors_test above + % Test with default error keywords + TestResult1 = + check_command_errors( + "Operation completed successfully", + ["Error", "failed"] + ), + ?assertEqual(ok, TestResult1), + TestResult2 = + check_command_errors( + "Error: disk not found", + ["Error", "failed"] + ), + ?assertEqual(error, TestResult2). \ No newline at end of file diff --git a/src/html/cacheviz@1.0/graph.html b/src/html/cacheviz@1.0/graph.html new file mode 100644 index 000000000..908ed48ba --- /dev/null +++ b/src/html/cacheviz@1.0/graph.html @@ -0,0 +1,321 @@ + + + + + + HyperBEAM Cache Graph + + + + + + + +
+
+ HyperBEAM Cache Graph +
+ +
+ + + + +
+ + +
+ +
+ + +
+
+ Nodes: + 0 +
+
+ Links: + 0 +
+
+
+ Simple +
+
+
+ Composite +
+
+
+ +
+
+
Loading graph data...
+
FPS: 0
+ + + + + +
+ + + + + + + + + + + \ No newline at end of file diff --git a/src/html/cacheviz@1.0/graph.js b/src/html/cacheviz@1.0/graph.js new file mode 100644 index 000000000..3d438f27d --- /dev/null +++ b/src/html/cacheviz@1.0/graph.js @@ -0,0 +1,3767 @@ +/** + * HyperBEAM Cache Graph Renderer - Modular Version + * A 2D force-directed graph visualization for the HyperBEAM cache system + */ + +/** + * Utility function to create a circular texture for node rendering + * @param {number} size - Size of the texture in pixels + * @param {number|string} color - Color of the circle (hex) + * @param {boolean} border - Whether to add a border + * @param {number|string} borderColor - Color of the border (hex) + * @returns {THREE.Texture} The generated texture + */ +function createCircleTexture(size = 64, color = 0xffffff, border = false, borderColor = 0x000000) { + // Create a canvas element + const canvas = document.createElement('canvas'); + canvas.width = size; + canvas.height = size; + const context = canvas.getContext('2d'); + + // Clear canvas with transparent background + context.clearRect(0, 0, size, size); + + // Convert color to string format if it's a number + const fillColor = typeof color === 'number' ? '#' + color.toString(16).padStart(6, '0') : color; + const strokeColor = typeof borderColor === 'number' ? '#' + borderColor.toString(16).padStart(6, '0') : borderColor; + + // Draw a circle + const radius = size / 2 - 2; + context.beginPath(); + context.arc(size / 2, size / 2, radius, 0, 2 * Math.PI, false); + context.fillStyle = fillColor; + context.fill(); + + // Add border if requested + if (border) { + context.lineWidth = 1; + context.strokeStyle = strokeColor; + context.stroke(); + } + + // Create a texture from the canvas + const texture = new THREE.CanvasTexture(canvas); + texture.needsUpdate = true; + + return texture; +} + +/** + * ThemeManager - Handles configuration and visual styling + */ +class ThemeManager { + constructor() { + this.config = { + // Node styling + nodeSize: { + simple: 6, + composite: 8 + }, + // Color scheme + colors: { + background: 0xf9f9f9, + simpleNode: 0x6495ED, // Light blue + compositeNode: 0xF08080, // Light coral + highlight: 0xFFA500, // Orange for highlighting + selectedNode: 0xFF5500, // Orange-red for selected node + neighborNode: 0x4CAF50, // Green for neighbor nodes + link: 0xcccccc, // Light gray for links + activeLink: 0x333333, // Dark gray for active links + hover: 0xfafa33 // Warm orange/yellow for hover + }, + // Display options + showLabels: true, + physicsEnabled: true, + // Physics settings + defaultDistance: 150, + highConnectionThreshold: 10, + // Camera settings + zoomLevel: { + default: 1.0, + focused: 2.5 + }, + // Z-positions for layering + zPos: { + line: 0, + node: 5, + label: 10 + } + }; + } + + /** + * Get the color for a node based on its type and state + * @param {string} nodeType - The type of node ('simple' or 'composite') + * @param {string} state - The state of the node ('default', 'selected', 'neighbor', 'hover') + * @returns {number} The color as a hex number + */ + getNodeColor(nodeType, state = 'default') { + switch(state) { + case 'selected': + return this.config.colors.selectedNode; + case 'neighbor': + return this.config.colors.neighborNode; + case 'hover': + return this.config.colors.hover; + default: + return nodeType === 'simple' ? + this.config.colors.simpleNode : + this.config.colors.compositeNode; + } + } + + /** + * Get the color for a link based on its state + * @param {string} state - The state of the link ('default' or 'active') + * @returns {number} The color as a hex number + */ + getLinkColor(state = 'default') { + return state === 'active' ? + this.config.colors.activeLink : + this.config.colors.link; + } + + /** + * Get the size for a node based on its type + * @param {string} nodeType - The type of node ('simple' or 'composite') + * @returns {number} The node size + */ + getNodeSize(nodeType) { + return nodeType === 'simple' ? + this.config.nodeSize.simple : + this.config.nodeSize.composite; + } + + /** + * Toggle label visibility + * @returns {boolean} The new label visibility state + */ + toggleLabels() { + this.config.showLabels = !this.config.showLabels; + return this.config.showLabels; + } + + /** + * Toggle physics simulation + * @returns {boolean} The new physics enabled state + */ + togglePhysics() { + this.config.physicsEnabled = !this.config.physicsEnabled; + return this.config.physicsEnabled; + } +} + +/** + * SceneManager - Handles Three.js scene, camera, and rendering + */ +class SceneManager { + constructor(container, themeManager) { + this.container = container; + this.themeManager = themeManager; + + // Three.js components + this.scene = null; + this.camera = null; + this.renderer = null; + this.controls = null; + this.raycaster = new THREE.Raycaster(); + this.mouse = new THREE.Vector2(); + + // Performance optimization + this.frustum = new THREE.Frustum(); + this.projScreenMatrix = new THREE.Matrix4(); + this.tmpVector = new THREE.Vector3(); + this.enableFrustumCulling = true; + this.frustumCullingDistance = 1250; // Beyond this distance, apply visibility culling + + // Initialize the scene + this.initScene(); + } + + /** + * Initialize the Three.js scene and renderer + */ + initScene() { + const width = this.container.clientWidth; + const height = this.container.clientHeight; + + // Create scene with background color + this.scene = new THREE.Scene(); + this.scene.background = new THREE.Color(this.themeManager.config.colors.background); + + // Create perspective camera with large clipping plane to prevent culling + const aspectRatio = width / height; + this.camera = new THREE.PerspectiveCamera( + 40, // Narrower field of view for less distortion + aspectRatio, + 0.1, + 15000 // Increased far clipping plane + ); + this.camera.position.z = 1000; + + // Create renderer + this.renderer = new THREE.WebGLRenderer({ + antialias: true, + alpha: true + }); + this.renderer.setSize(width, height); + this.renderer.setClearColor(this.themeManager.config.colors.background, 1); + this.renderer.sortObjects = true; // Enable sorting for proper z-ordering + this.container.appendChild(this.renderer.domElement); + + // Configure raycaster for better point detection + this.raycaster = new THREE.Raycaster(); + this.raycaster.params.Points.threshold = 10; // Increase threshold for easier point selection + + // Add orbit controls limited to 2D movement with perspective camera + this.controls = new THREE.OrbitControls(this.camera, this.renderer.domElement); + this.controls.enableDamping = true; + this.controls.dampingFactor = 0.1; + this.controls.enableRotate = false; // Disable 3D rotation + this.controls.screenSpacePanning = true; + + // Set zoom limits - constrain camera between 750 and 15000 on z-axis + this.controls.minDistance = 750; + this.controls.maxDistance = 15000; + + // Handle window resize + window.addEventListener('resize', () => this.onWindowResize()); + } + + /** + * Handle window resize events + */ + onWindowResize() { + const width = this.container.clientWidth; + const height = this.container.clientHeight; + + // Update perspective camera aspect ratio + this.camera.aspect = width / height; + this.camera.updateProjectionMatrix(); + + // Update renderer + this.renderer.setSize(width, height); + } + + /** + * Reset the camera view + */ + resetView() { + // Reset camera position for perspective camera + this.camera.position.set(0, 0, 1000); + this.controls.target.set(0, 0, 0); + this.camera.updateProjectionMatrix(); + this.controls.update(); + } + + /** + * Focus camera on a specific position with smooth animation + * @param {THREE.Vector3} position - The position to focus on + */ + focusCamera(position) { + const duration = 500; // milliseconds + const startTime = Date.now(); + + // Save starting values + const startPosition = this.camera.position.clone(); + const startTarget = this.controls.target.clone(); + + // Define a fixed Z-offset for viewing the target + const zOffset = 600; + + // Animation function + const animateCamera = () => { + const elapsed = Date.now() - startTime; + const progress = Math.min(elapsed / duration, 1); + + // Ease function - ease out cubic + const easeProgress = 1 - Math.pow(1 - progress, 3); + + // Only update the target x and y position, keeping rotation consistent + this.controls.target.x = startTarget.x + (position.x - startTarget.x) * easeProgress; + this.controls.target.y = startTarget.y + (position.y - startTarget.y) * easeProgress; + // Keep z at the same value to maintain default camera angle + + // Move camera x and y to match target + this.camera.position.x = startPosition.x + (position.x - startPosition.x) * easeProgress; + this.camera.position.y = startPosition.y + (position.y - startPosition.y) * easeProgress; + + // Adjust Z with fixed offset + const targetZ = position.z + zOffset; + this.camera.position.z = startPosition.z + (targetZ - startPosition.z) * easeProgress; + + this.camera.updateProjectionMatrix(); + this.controls.update(); + + if (progress < 1) { + requestAnimationFrame(animateCamera); + } + }; + + animateCamera(); + } + + /** + * Update the mouse position for raycasting + * @param {MouseEvent} event - The mouse event + */ + updateMousePosition(event) { + const rect = this.renderer.domElement.getBoundingClientRect(); + this.mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1; + this.mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1; + } + + /** + * Get objects intersecting with the current mouse position + * @returns {Array} Array of intersected objects + */ + getIntersectedObjects() { + this.raycaster.setFromCamera(this.mouse, this.camera); + return this.raycaster.intersectObjects(this.scene.children, true); + } + + /** + * Update the scene (called in animation loop) + */ + update() { + // Update controls + if (this.controls) { + this.controls.update(); + + // Enforce camera z-position limits + if (this.camera.position.z < 750) { + this.camera.position.z = 750; + } else if (this.camera.position.z > 15000) { + this.camera.position.z = 15000; + } + } + + // Apply frustum culling for distant objects + if (this.enableFrustumCulling) { + this.updateFrustumCulling(); + } + + // Render the scene + this.renderer.render(this.scene, this.camera); + } + + /** + * Update frustum and apply visibility culling for better performance + */ + updateFrustumCulling() { + // Update the frustum + this.projScreenMatrix.multiplyMatrices( + this.camera.projectionMatrix, + this.camera.matrixWorldInverse + ); + this.frustum.setFromProjectionMatrix(this.projScreenMatrix); + + // If we have a reference to the controller, access the dataManager + if (this.graphController && this.graphController.dataManager) { + const dataManager = this.graphController.dataManager; + + // Process all nodes + dataManager.graphObjects.nodes.forEach(node => { + if (!node.object) return; + + // Get distance from camera + this.tmpVector.copy(node.object.position); + const distance = this.tmpVector.distanceTo(this.camera.position); + + // If the node is beyond our threshold, check if it's in the frustum + if (distance > this.frustumCullingDistance) { + // Check if the node is in the frustum + const isVisible = this.frustum.containsPoint(node.object.position); + + // Only update visibility if necessary to avoid unnecessary matrix updates + if (node.object.visible !== isVisible) { + node.object.visible = isVisible; + + // Also update label visibility if it exists + if (node.labelObject) { + node.labelObject.visible = isVisible && this.themeManager.config.showLabels; + } + } + } else if (!node.object.visible) { + // If node is within threshold distance but not visible, make visible + node.object.visible = true; + if (node.labelObject) { + node.labelObject.visible = this.themeManager.config.showLabels; + } + } + }); + + // Optional: Process links for better culling + // Only show links if both endpoints are visible + dataManager.graphObjects.links.forEach(link => { + if (!link.line) return; + + const sourceNode = dataManager.graphObjects.nodes.get(link.sourceId); + const targetNode = dataManager.graphObjects.nodes.get(link.targetId); + + if (sourceNode && targetNode && sourceNode.object && targetNode.object) { + const sourceDist = sourceNode.object.position.distanceTo(this.camera.position); + const targetDist = targetNode.object.position.distanceTo(this.camera.position); + + // If both nodes are distant, check if they're visible + if (sourceDist > this.frustumCullingDistance && targetDist > this.frustumCullingDistance) { + const sourceVisible = sourceNode.object.visible; + const targetVisible = targetNode.object.visible; + + // Only show link if both endpoints are visible + link.line.visible = sourceVisible && targetVisible; + + // Update label visibility if needed + if (link.labelObject) { + link.labelObject.visible = sourceVisible && targetVisible && + this.themeManager.config.showLabels && + dataManager.activeLinks.has(`${link.sourceId}-${link.targetId}`); + } + } else if (!link.line.visible) { + // If at least one endpoint is close, show the link + link.line.visible = true; + } + } + }); + } + } + + /** + * Add an object to the scene + * @param {THREE.Object3D} object - The object to add + */ + addToScene(object) { + this.scene.add(object); + } + + /** + * Remove an object from the scene + * @param {THREE.Object3D} object - The object to remove + */ + removeFromScene(object) { + this.scene.remove(object); + } +} + +/** + * DataManager - Handles graph data loading and processing + */ +class DataManager { + constructor() { + // Graph data + this.graphData = { nodes: [], links: [] }; + this.graphObjects = { nodes: new Map(), links: new Map() }; + + // State tracking + this.selectedNode = null; + this.neighborNodes = new Set(); + this.activeLinks = new Set(); + this.hoveredNode = null; + } + + /** + * Load graph data from the server + * @returns {Promise} Promise that resolves when data is loaded + */ + loadData() { + return new Promise((resolve, reject) => { + let url = window.location; + console.log(url); + const params = window.location.search; + console.log('params', params); + fetch('~cacheviz@1.0/json' + params) + .then(response => { + if (!response.ok) { + throw new Error(`HTTP error ${response.status}`); + } + return response.json(); + }) + .then(data => { + // Clear existing data + this.clearData(); + + // Validate data + if (!this.validateData(data)) { + reject(new Error('Invalid data format')); + return; + } + + this.graphData = data; + resolve(data); + }) + .catch(error => { + console.error('Error loading graph data:', error); + reject(error); + }); + }); + } + + /** + * Validate the graph data structure + * @param {Object} data - The data to validate + * @returns {boolean} Whether the data is valid + */ + validateData(data) { + // Check if we have valid data + if (!data || !data.nodes || !data.links || + !Array.isArray(data.nodes) || !Array.isArray(data.links)) { + return false; + } + + // Check if we have any nodes + if (data.nodes.length === 0) { + return false; + } + + return true; + } + + /** + * Clear all graph data + */ + clearData() { + this.graphData = { nodes: [], links: [] }; + this.graphObjects.nodes.clear(); + this.graphObjects.links.clear(); + + // Reset state + this.selectedNode = null; + this.hoveredNode = null; + this.neighborNodes.clear(); + this.activeLinks.clear(); + } + + /** + * Determine node type based on ID pattern + * @param {string} nodeId - The node ID + * @returns {string} The node type ('simple' or 'composite') + */ + determineNodeType(nodeId) { + const pathParts = nodeId.split('/').filter(p => p.length > 0); + return (pathParts.length <= 1 && !nodeId.endsWith('/')) ? 'simple' : 'composite'; + } + + /** + * Search for nodes matching a term + * @param {string} searchTerm - The term to search for + * @returns {Array} Array of matching node IDs + */ + searchNodes(searchTerm) { + if (!searchTerm) return []; + + const searchLower = searchTerm.toLowerCase(); + + // Find matching nodes + return this.graphData.nodes + .filter(node => + (node.id && node.id.toLowerCase().includes(searchLower)) || + (node.label && node.label.toLowerCase().includes(searchLower)) + ) + .map(node => node.id); + } + + /** + * Get nodes connected to a starting node up to a specified depth + * @param {string} startNodeId - The ID of the starting node + * @param {number} maxDepth - Maximum depth/distance to traverse + * @returns {Object} Object containing connected nodes and links + */ + getConnectedSubgraph(startNodeId, maxDepth = 1) { + const connectedNodes = new Map(); + const connectedLinks = new Set(); + const queue = [{id: startNodeId, depth: 0}]; + const visited = new Set([startNodeId]); + + // First make sure we have the start node + const startNode = this.graphData.nodes.find(n => n.id === startNodeId); + if (!startNode) return {nodes: [], links: []}; + + connectedNodes.set(startNodeId, startNode); + + // BFS to find connected nodes up to maxDepth + while (queue.length > 0) { + const {id, depth} = queue.shift(); + + if (depth >= maxDepth) continue; + + // Find all links connected to this node + this.graphData.links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + if (sourceId === id || targetId === id) { + const linkId = `${sourceId}-${targetId}`; + + // If we've already processed this link, skip it + if (connectedLinks.has(linkId)) return; + + connectedLinks.add(linkId); + + // Get the ID of the node on the other end of the link + const otherId = sourceId === id ? targetId : sourceId; + + // If we haven't visited this node yet, add it to the queue + if (!visited.has(otherId)) { + visited.add(otherId); + const otherNode = this.graphData.nodes.find(n => n.id === otherId); + if (otherNode) { + connectedNodes.set(otherId, otherNode); + queue.push({id: otherId, depth: depth + 1}); + } + } + } + }); + } + + return { + nodes: Array.from(connectedNodes.values()), + links: this.graphData.links.filter(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + return connectedNodes.has(sourceId) && connectedNodes.has(targetId); + }) + }; + } + + /** + * Store a node object reference + * @param {string} nodeId - The node ID + * @param {Object} nodeData - The node data + */ + storeNodeObject(nodeId, nodeData) { + this.graphObjects.nodes.set(nodeId, nodeData); + } + + /** + * Store a link object reference + * @param {string} linkId - The link ID (format: "sourceId-targetId") + * @param {Object} linkData - The link data + */ + storeLinkObject(linkId, linkData) { + this.graphObjects.links.set(linkId, linkData); + } + + /** + * Get links connected to a node + * @param {string} nodeId - The node ID + * @returns {Array} Array of connected links + */ + getConnectedLinks(nodeId) { + return this.graphData.links.filter(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + return sourceId === nodeId || targetId === nodeId; + }); + } + + /** + * Track selected node + * @param {string} nodeId - The selected node ID + */ + setSelectedNode(nodeId) { + this.selectedNode = nodeId; + + // Find and track connected nodes and links + if (nodeId) { + // Clear previous + this.neighborNodes.clear(); + this.activeLinks.clear(); + + // Find connected links + this.graphData.links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + if (sourceId === nodeId || targetId === nodeId) { + // This is a connected link + const otherNodeId = sourceId === nodeId ? targetId : sourceId; + this.neighborNodes.add(otherNodeId); + + // Track active link + const linkKey = `${sourceId}-${targetId}`; + this.activeLinks.add(linkKey); + } + }); + } + } + + /** + * Clear selected node + */ + clearSelectedNode() { + this.selectedNode = null; + this.neighborNodes.clear(); + this.activeLinks.clear(); + } + + /** + * Track hovered node + * @param {string} nodeId - The hovered node ID + */ + setHoveredNode(nodeId) { + this.hoveredNode = nodeId; + } + + /** + * Clear hovered node + */ + clearHoveredNode() { + this.hoveredNode = null; + } +} + +/** + * GraphObjectManager - Creates and manages visual objects for nodes and links + */ +class GraphObjectManager { + constructor(sceneManager, dataManager, themeManager) { + this.sceneManager = sceneManager; + this.dataManager = dataManager; + this.themeManager = themeManager; + } + + /** + * Create a visual node object + * @param {Object} node - The node data + * @returns {Object} The created node object with visual elements + */ + createNodeObject(node) { + // Determine node type if not set + if (!node.type) { + node.type = this.dataManager.determineNodeType(node.id); + } + + // Add to NodeCloud for efficient rendering + if (this.graphController && this.graphController.nodeCloud) { + // Add to node cloud + const nodeIndex = this.graphController.nodeCloud.addNode(node); + + // Create a virtual object for compatibility + // This is needed because other code expects a THREE.Object3D + const virtualObject = { + position: new THREE.Vector3(node.x || 0, node.y || 0, this.themeManager.config.zPos.node), + visible: true, + userData: { id: node.id, type: node.type, label: node.label } + }; + + // Store virtual object reference + node.object = virtualObject; + node.nodeCloudIndex = nodeIndex; + } + + // Create label if enabled + let labelObject = null; + if (this.themeManager.config.showLabels) { + labelObject = this.createLabel(node); + } + + // Store label reference + node.labelObject = labelObject; + + // Store in dataManager + this.dataManager.storeNodeObject(node.id, node); + + // Add to spatial grid if simulation manager is available + if (this.graphController && this.graphController.simulationManager) { + this.graphController.simulationManager.addNodeToSpatialGrid(node); + } + + return node; + } + + /** + * Create a visual link object + * @param {Object} link - The link data + * @returns {Object} The created link object with visual elements + */ + createLinkObject(link) { + // Get node IDs + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + // Get node objects + const sourceNode = this.dataManager.graphObjects.nodes.get(sourceId); + const targetNode = this.dataManager.graphObjects.nodes.get(targetId); + + if (!sourceNode || !targetNode) { + return null; + } + + // Create line geometry + const points = [ + new THREE.Vector3(sourceNode.x, sourceNode.y, this.themeManager.config.zPos.line), + new THREE.Vector3(targetNode.x, targetNode.y, this.themeManager.config.zPos.line) + ]; + + // Create material and line + const material = new THREE.LineDashedMaterial({ + color: this.themeManager.getLinkColor(), + dashSize: 2.5, + gapSize: 1.5, + transparent: true, + opacity: 0.6, + linewidth: 1, + depthWrite: false + }); + + const geometry = new THREE.BufferGeometry().setFromPoints(points); + const line = new THREE.Line(geometry, material); + + // Important: Disable frustum culling to ensure lines are always visible + line.frustumCulled = false; + line.renderOrder = 0; // Ensure lines render before nodes + + this.sceneManager.addToScene(line); + + // Store connection info + link.sourceId = sourceId; + link.targetId = targetId; + link.line = line; + + // Create label if needed + let labelObject = null; + if (this.themeManager.config.showLabels && link.label) { + labelObject = this.createLinkLabel(link, sourceNode, targetNode); + } + + link.labelObject = labelObject; + + // Store reference + const linkId = `${sourceId}-${targetId}`; + this.dataManager.storeLinkObject(linkId, link); + + return link; + } + + /** + * Truncate text and add ellipsis in the middle + * @param {string} text - The text to truncate + * @returns {string} Truncated text with ellipsis + */ + truncateWithEllipsis(text) { + // Show only if longer than 15 characters (6 + 3 + 6) + if (!text || text.length <= 15) { + return text; + } + + // Take exactly 6 chars from start and 6 from end + return text.substring(0, 6) + '...' + text.substring(text.length - 6); + } + + /** + * Create a label for a node + * @param {Object} node - The node data + * @returns {Object} The created label object + */ + createLabel(node) { + // Get display text and truncate it if needed + const displayText = this.truncateWithEllipsis(node.label || node.id); + const label = new SpriteText(displayText); + + // Improved text rendering settings + label.fontFace = 'Arial, Helvetica, sans-serif'; + label.fontSize = 32; + label.fontWeight = '600'; + label.strokeWidth = 0; // No stroke for sharper text + label.color = '#000000'; + label.backgroundColor = 'rgba(255,255,255,0.95)'; + label.padding = 3; + label.textHeight = 5; // Increased for better resolution with larger text + label.borderWidth = 0; // No border for sharper edges + + // Position above node with pixel-perfect positioning + const offset_val = 100; + const isSimple = node.type === 'simple'; + const offset = isSimple ? + this.themeManager.config.nodeSize.simple + offset_val : + this.themeManager.config.nodeSize.composite + offset_val; + + // Round to whole pixels to avoid subpixel rendering + const x = Math.round(node.x || 0); + const y = Math.round((node.y || 0) + offset); + const z = this.themeManager.config.zPos.label; + + label.position.set(x, y, z); + label.renderOrder = 20; + + this.sceneManager.addToScene(label); + + return label; + } + + /** + * Create a label for a link + * @param {Object} link - The link data + * @param {Object} sourceNode - The source node + * @param {Object} targetNode - The target node + * @returns {Object} The created label object + */ + createLinkLabel(link, sourceNode, targetNode) { + const midPoint = new THREE.Vector3( + (sourceNode.x + targetNode.x) / 2, + (sourceNode.y + targetNode.y) / 2, + this.themeManager.config.zPos.label + ); + + let label_text = link.label; + if (link.label.length == 43) { + label_text = this.truncateWithEllipsis(link.label); + } + + const label = new SpriteText(label_text); + + // Improved text rendering settings + label.fontFace = 'Arial, Helvetica, sans-serif'; + label.fontSize = 32; + label.fontWeight = '600'; + label.strokeWidth = 0; // No stroke for sharper text + label.color = '#000000'; + label.backgroundColor = 'rgba(255,255,255,0.95)'; + label.padding = 3; + label.textHeight = 4; // Better resolution for link labels + label.borderWidth = 0; // No border for sharper edges + + // Round to whole pixels to avoid subpixel rendering + midPoint.x = Math.round(midPoint.x); + midPoint.y = Math.round(midPoint.y); + + label.position.copy(midPoint); + label.renderOrder = 20; + + // Hide link labels by default - only show when node is selected + label.visible = false; + + this.sceneManager.addToScene(label); + + return label; + } + + /** + * Update the position of a node object + * @param {Object} node - The node object to update + */ + updateNodePosition(node) { + if (!node) return; + + if (this.graphController && this.graphController.nodeCloud) { + // Update position in the NodeCloud + this.graphController.nodeCloud.updateNodePosition(node.id, node.x || 0, node.y || 0); + + // Also update the virtual object for compatibility + if (node.object && node.object.position) { + node.object.position.x = node.x || 0; + node.object.position.y = node.y || 0; + } + } + + // Update label position if it exists + if (node.labelObject) { + const offset_val = 6; // Same value as in createLabel + const isSimple = node.type === 'simple'; + const offset = isSimple ? + this.themeManager.config.nodeSize.simple + offset_val : + this.themeManager.config.nodeSize.composite + offset_val; + + node.labelObject.position.x = node.x || 0; + node.labelObject.position.y = (node.y || 0) + offset; + } + } + + /** + * Update the position of a link object + * @param {Object} link - The link object to update + */ + updateLinkPosition(link) { + if (!link.line) return; + + const sourceId = link.sourceId || (typeof link.source === 'object' ? link.source.id : link.source); + const targetId = link.targetId || (typeof link.target === 'object' ? link.target.id : link.target); + + const sourceNode = this.dataManager.graphObjects.nodes.get(sourceId); + const targetNode = this.dataManager.graphObjects.nodes.get(targetId); + + if (sourceNode && targetNode) { + // Update the link line geometry + const points = [ + new THREE.Vector3(sourceNode.x || 0, sourceNode.y || 0, this.themeManager.config.zPos.line), + new THREE.Vector3(targetNode.x || 0, targetNode.y || 0, this.themeManager.config.zPos.line) + ]; + + // Update the line geometry + link.line.geometry.setFromPoints(points); + + // Ensure frustum culling remains disabled after updates + link.line.frustumCulled = false; + + // Update the link label position if it exists + if (link.labelObject) { + const midPoint = new THREE.Vector3( + (sourceNode.x + targetNode.x) / 2, + (sourceNode.y + targetNode.y) / 2, + this.themeManager.config.zPos.label + ); + link.labelObject.position.copy(midPoint); + } + } + } + + /** + * Update node colors based on selection and hover state + * @param {string} nodeId - The ID of the node to update + */ + updateNodeColors(nodeId) { + const node = this.dataManager.graphObjects.nodes.get(nodeId); + if (!node) return; + + let state = 'default'; + + // Determine state based on selection and hover + if (nodeId === this.dataManager.selectedNode) { + state = 'selected'; + } else if (this.dataManager.neighborNodes.has(nodeId)) { + state = 'neighbor'; + } else if (nodeId === this.dataManager.hoveredNode) { + state = 'hover'; + } + + if (this.graphController && this.graphController.nodeCloud) { + // Update the color in the node cloud + this.graphController.nodeCloud.updateNodeColor(nodeId, node.type, state); + } + } + + /** + * Update link colors based on selection state + * @param {string} linkId - The ID of the link to update (format: "sourceId-targetId") + */ + updateLinkColors(linkId) { + const link = this.dataManager.graphObjects.links.get(linkId); + if (!link || !link.line) return; + + // Determine if this is an active link + const isActive = this.dataManager.activeLinks.has(linkId); + + // Set color and opacity based on active state + link.line.material.color.set(this.themeManager.getLinkColor(isActive ? 'active' : 'default')); + link.line.material.opacity = isActive ? 1.0 : 0.6; + } + + /** + * Remove a node and its visual elements from the scene + * @param {string} nodeId - The ID of the node to remove + */ + removeNode(nodeId) { + const node = this.dataManager.graphObjects.nodes.get(nodeId); + if (!node) return; + + // Remove from NodeCloud if available + if (this.graphController && this.graphController.nodeCloud) { + this.graphController.nodeCloud.removeNode(nodeId); + } + + // Remove label from scene + if (node.labelObject) { + this.sceneManager.removeFromScene(node.labelObject); + } + + // Remove from data manager + this.dataManager.graphObjects.nodes.delete(nodeId); + } + + /** + * Remove a link and its visual elements from the scene + * @param {string} linkId - The ID of the link to remove + */ + removeLink(linkId) { + const link = this.dataManager.graphObjects.links.get(linkId); + if (!link) return; + + // Remove line from scene + if (link.line) { + this.sceneManager.removeFromScene(link.line); + } + + // Remove label from scene + if (link.labelObject) { + this.sceneManager.removeFromScene(link.labelObject); + } + + // Remove from data manager + this.dataManager.graphObjects.links.delete(linkId); + } + + /** + * Clear all visible nodes and links from the scene + */ + clearVisibleObjects() { + // Clear NodeCloud if available + if (this.graphController && this.graphController.nodeCloud) { + this.graphController.nodeCloud.clear(); + } + + // Remove label objects from the scene + this.dataManager.graphObjects.nodes.forEach((node, id) => { + if (node.labelObject) { + this.sceneManager.removeFromScene(node.labelObject); + } + }); + + this.dataManager.graphObjects.links.forEach((link, id) => { + if (link.line) this.sceneManager.removeFromScene(link.line); + if (link.labelObject) this.sceneManager.removeFromScene(link.labelObject); + }); + + // Clear references in data manager + this.dataManager.graphObjects.nodes.clear(); + this.dataManager.graphObjects.links.clear(); + } + + /** + * Toggle visibility of all labels + * @returns {boolean} The new label visibility state + */ + toggleLabels() { + const showLabels = this.themeManager.toggleLabels(); + + // Update node labels + this.dataManager.graphObjects.nodes.forEach((node, id) => { + if (node.labelObject) { + node.labelObject.visible = showLabels; + } else if (showLabels) { + // Create label if it doesn't exist yet + node.labelObject = this.createLabel(node); + } + }); + + // Update link labels - only show for active links connected to selected node + this.dataManager.graphObjects.links.forEach((link, id) => { + if (link.labelObject) { + // Only show if labels are enabled AND this is an active link + const isActive = this.dataManager.activeLinks.has(id); + link.labelObject.visible = showLabels && isActive; + } else if (showLabels && link.label) { + // Create label if it doesn't exist yet + const sourceNode = this.dataManager.graphObjects.nodes.get(link.sourceId); + const targetNode = this.dataManager.graphObjects.nodes.get(link.targetId); + + if (sourceNode && targetNode) { + link.labelObject = this.createLinkLabel(link, sourceNode, targetNode); + // Only show if this is an active link + link.labelObject.visible = this.dataManager.activeLinks.has(id); + } + } + }); + + return showLabels; + } +} + +/** + * SpatialGrid - Simple spatial partitioning for efficient queries + */ +class SpatialGrid { + constructor(cellSize = 200) { + this.cellSize = cellSize; + this.grid = new Map(); + this.objects = new Set(); + } + + /** + * Get the cell key for a position + * @param {THREE.Vector3} position - The position to get the cell for + * @returns {string} The cell key + */ + getCellKey(position) { + const x = Math.floor(position.x / this.cellSize); + const y = Math.floor(position.y / this.cellSize); + const z = Math.floor(position.z / this.cellSize); + return `${x},${y},${z}`; + } + + /** + * Add an object to the grid + * @param {Object} object - The object to add + */ + addObject(object) { + if (!object.object || !object.object.position) return; + + const position = object.object.position; + const cellKey = this.getCellKey(position); + + // Create cell if it doesn't exist + if (!this.grid.has(cellKey)) { + this.grid.set(cellKey, new Set()); + } + + // Add to cell + this.grid.get(cellKey).add(object); + this.objects.add(object); + } + + /** + * Remove an object from the grid + * @param {Object} object - The object to remove + */ + removeObject(object) { + if (!object.object || !object.object.position) return; + + // Remove from all cells (in case it moved) + this.grid.forEach(cell => { + cell.delete(object); + }); + + this.objects.delete(object); + + // Clean up empty cells + this.grid.forEach((cell, key) => { + if (cell.size === 0) { + this.grid.delete(key); + } + }); + } + + /** + * Find objects within a radius of a position + * @param {THREE.Vector3} position - The center position + * @param {number} radius - The radius to search within + * @returns {Array} Array of objects within the radius + */ + findNearbyObjects(position, radius) { + // Calculate the cell range to check + const cellRadius = Math.ceil(radius / this.cellSize); + const centerX = Math.floor(position.x / this.cellSize); + const centerY = Math.floor(position.y / this.cellSize); + const centerZ = Math.floor(position.z / this.cellSize); + + const result = []; + const radiusSquared = radius * radius; + + // Check each cell in the range + for (let x = centerX - cellRadius; x <= centerX + cellRadius; x++) { + for (let y = centerY - cellRadius; y <= centerY + cellRadius; y++) { + for (let z = centerZ - cellRadius; z <= centerZ + cellRadius; z++) { + const cellKey = `${x},${y},${z}`; + const cell = this.grid.get(cellKey); + + if (cell) { + // Check each object in the cell + cell.forEach(object => { + if (object.object && object.object.position) { + const distSquared = position.distanceToSquared(object.object.position); + if (distSquared <= radiusSquared) { + result.push(object); + } + } + }); + } + } + } + } + + return result; + } + + /** + * Clear all objects from the grid + */ + clear() { + this.grid.clear(); + this.objects.clear(); + } + + /** + * Get the total number of objects in the grid + * @returns {number} The number of objects + */ + size() { + return this.objects.size; + } +} + +/** + * NodeCloud - Manages efficient point cloud rendering for graph nodes + */ +class NodeCloud { + constructor(scene, themeManager) { + this.scene = scene; + this.sceneManager = null; // Will be set by the EventManager + this.themeManager = themeManager; + + // Capacity tracking + this.maxNodes = 1000; // Initial capacity + this.nodeCount = 0; + + // Node tracking + this.nodeIndices = new Map(); // Maps node IDs to their index in the arrays + this.nodeTypes = new Map(); // Maps node IDs to their types + this.positions = null; + this.colors = null; + this.sizes = null; + + // Selection tracking + this.selectedIndices = new Set(); + this.neighborIndices = new Set(); + this.hoverIndex = -1; + + // Create base texture for all nodes + this.baseTexture = createCircleTexture(64, 0xffffff); + + // Initialize geometry and point cloud + this.initialize(); + } + + /** + * Initialize buffers and point cloud with initial capacity + */ + initialize() { + // Create buffer attributes with initial capacity + this.positions = new Float32Array(this.maxNodes * 3); + this.colors = new Float32Array(this.maxNodes * 3); + this.sizes = new Float32Array(this.maxNodes); + + // Create buffer geometry + this.geometry = new THREE.BufferGeometry(); + this.geometry.setAttribute('position', new THREE.BufferAttribute(this.positions, 3)); + this.geometry.setAttribute('color', new THREE.BufferAttribute(this.colors, 3)); + this.geometry.setAttribute('size', new THREE.BufferAttribute(this.sizes, 1)); + + // Set draw range to only render active nodes + this.geometry.setDrawRange(0, 0); + + // Create point material + this.material = new THREE.ShaderMaterial({ + uniforms: { + pointTexture: { value: this.baseTexture } + }, + vertexShader: ` + precision highp float; + attribute float size; + attribute vec3 color; + varying vec3 vColor; + + void main() { + vColor = color; + vec4 mvPosition = modelViewMatrix * vec4(position, 1.0); + gl_PointSize = size * (1200.0 / -mvPosition.z); + gl_Position = projectionMatrix * mvPosition; + } + `, + fragmentShader: ` + precision highp float; + uniform sampler2D pointTexture; + varying vec3 vColor; + + void main() { + vec4 texColor = texture2D(pointTexture, gl_PointCoord); + if (texColor.a < 0.5) discard; + gl_FragColor = vec4(vColor, 1.0) * texColor; + } + `, + transparent: true, + depthWrite: false, + blending: THREE.NormalBlending + }); + + // Create points + this.points = new THREE.Points(this.geometry, this.material); + this.points.frustumCulled = false; // Disable frustum culling + this.points.renderOrder = 10; + + // Add to scene + this.scene.add(this.points); + } + + /** + * Resize buffers if needed + */ + ensureCapacity(requiredNodes) { + if (requiredNodes <= this.maxNodes) return; + + // Calculate new capacity (1.5x required or 2x current, whichever is larger) + const newCapacity = Math.max(Math.ceil(requiredNodes * 1.5), this.maxNodes * 2); + + // Create new arrays + const newPositions = new Float32Array(newCapacity * 3); + const newColors = new Float32Array(newCapacity * 3); + const newSizes = new Float32Array(newCapacity); + + // Copy existing data + newPositions.set(this.positions); + newColors.set(this.colors); + newSizes.set(this.sizes); + + // Update references + this.positions = newPositions; + this.colors = newColors; + this.sizes = newSizes; + this.maxNodes = newCapacity; + + // Update buffer attributes + this.geometry.setAttribute('position', new THREE.BufferAttribute(this.positions, 3)); + this.geometry.setAttribute('color', new THREE.BufferAttribute(this.colors, 3)); + this.geometry.setAttribute('size', new THREE.BufferAttribute(this.sizes, 1)); + } + + /** + * Add a node to the point cloud + * @param {Object} node - Node data with position and type + * @returns {number} Index of the node in the point cloud + */ + addNode(node) { + // Get node ID + const nodeId = node.id; + + // Check if this node is already in the cloud + if (this.nodeIndices.has(nodeId)) { + return this.nodeIndices.get(nodeId); + } + + // Ensure we have enough capacity + this.ensureCapacity(this.nodeCount + 1); + + // Add to the end of the arrays + const index = this.nodeCount; + const i3 = index * 3; + + // Set position + this.positions[i3] = node.x || 0; + this.positions[i3 + 1] = node.y || 0; + this.positions[i3 + 2] = this.themeManager.config.zPos.node; + + // Store node type for later reference + this.nodeTypes.set(nodeId, node.type || 'simple'); + + // Set color based on node type + const color = new THREE.Color(this.themeManager.getNodeColor(node.type)); + this.colors[i3] = color.r; + this.colors[i3 + 1] = color.g; + this.colors[i3 + 2] = color.b; + + // Set size based on node type - use larger sizes to make selection easier + const baseSize = this.themeManager.getNodeSize(node.type); + this.sizes[index] = baseSize * 4; // Increase size to improve interaction + + // Track this node + this.nodeIndices.set(nodeId, index); + this.nodeCount++; + + // Update draw range + this.geometry.setDrawRange(0, this.nodeCount); + + // Mark attributes as needing update + this.geometry.attributes.position.needsUpdate = true; + this.geometry.attributes.color.needsUpdate = true; + this.geometry.attributes.size.needsUpdate = true; + + return index; + } + + /** + * Update a node's position + * @param {string} nodeId - ID of the node to update + * @param {number} x - New X position + * @param {number} y - New Y position + */ + updateNodePosition(nodeId, x, y) { + if (!this.nodeIndices.has(nodeId)) return; + + const index = this.nodeIndices.get(nodeId); + const i3 = index * 3; + + this.positions[i3] = x; + this.positions[i3 + 1] = y; + + // Mark position attribute as needing update + this.geometry.attributes.position.needsUpdate = true; + } + + /** + * Update a node's color based on state + * @param {string} nodeId - ID of the node to update + * @param {string} nodeType - Type of the node + * @param {string} state - State of the node (default, selected, neighbor, hover) + */ + updateNodeColor(nodeId, nodeType, state = 'default') { + if (!this.nodeIndices.has(nodeId)) return; + + // Store node type if provided + if (nodeType) { + this.nodeTypes.set(nodeId, nodeType); + } else { + // Use stored type if available + nodeType = this.nodeTypes.get(nodeId) || 'simple'; + } + + const index = this.nodeIndices.get(nodeId); + const i3 = index * 3; + + // Get color for this node state + const color = new THREE.Color(this.themeManager.getNodeColor(nodeType, state)); + + // Update color in buffer + this.colors[i3] = color.r; + this.colors[i3 + 1] = color.g; + this.colors[i3 + 2] = color.b; + + // Mark colors attribute as needing update + this.geometry.attributes.color.needsUpdate = true; + } + + /** + * Update colors for all nodes based on selection state + * @param {string} selectedId - ID of the selected node + * @param {Set} neighborIds - Set of neighbor node IDs + * @param {string} hoveredId - ID of the hovered node + */ + updateColors(selectedId, neighborIds, hoveredId) { + // Reset tracking sets + this.selectedIndices.clear(); + this.neighborIndices.clear(); + this.hoverIndex = -1; + + // Track indices for faster updates + if (selectedId && this.nodeIndices.has(selectedId)) { + this.selectedIndices.add(this.nodeIndices.get(selectedId)); + } + + neighborIds.forEach(id => { + if (this.nodeIndices.has(id)) { + this.neighborIndices.add(this.nodeIndices.get(id)); + } + }); + + if (hoveredId && this.nodeIndices.has(hoveredId)) { + this.hoverIndex = this.nodeIndices.get(hoveredId); + } + + // Update all node colors based on state + for (const [nodeId, index] of this.nodeIndices.entries()) { + const i3 = index * 3; + let state = 'default'; + + // Determine state based on selection and hover + if (index === this.hoverIndex) { + state = 'hover'; + } else if (this.selectedIndices.has(index)) { + state = 'selected'; + } else if (this.neighborIndices.has(index)) { + state = 'neighbor'; + } + + // Get node type from our stored map + const nodeType = this.nodeTypes.get(nodeId) || 'simple'; + + // Get color for this state + const color = new THREE.Color(this.themeManager.getNodeColor(nodeType, state)); + + // Update color + this.colors[i3] = color.r; + this.colors[i3 + 1] = color.g; + this.colors[i3 + 2] = color.b; + } + + // Mark attributes as needing update + this.geometry.attributes.color.needsUpdate = true; + } + + /** + * Find the closest node to a mouse position + * @param {THREE.Raycaster} raycaster - The raycaster + * @param {number} threshold - Maximum distance to consider a hit (in screen space) + * @returns {string|null} ID of the closest node or null if none found + */ + findClosestNode(raycaster, threshold = 0.05) { + if (this.nodeCount === 0 || !this.positions) return null; + + // Get the camera from the scene manager or from the global controller + let camera = null; + let mousePosition = null; + + if (this.sceneManager) { + camera = this.sceneManager.camera; + mousePosition = this.sceneManager.mouse; // This is the normalized mouse position + } else if (window.lastGraphController && window.lastGraphController.sceneManager) { + camera = window.lastGraphController.sceneManager.camera; + mousePosition = window.lastGraphController.sceneManager.mouse; + } + + // If we can't get a camera or mouse position, we can't continue + if (!camera || !mousePosition) return null; + + // Find the closest node based on screen space distance + let closestDistance = Infinity; + let closestNodeId = null; + + // Check each node + for (const [nodeId, index] of this.nodeIndices.entries()) { + const i3 = index * 3; + + // Get node position + const nodePos = new THREE.Vector3( + this.positions[i3], + this.positions[i3 + 1], + this.positions[i3 + 2] + ); + + // Project to screen space + const screenPos = nodePos.clone().project(camera); + + // Calculate 2D distance in screen space between mouse and node + const dx = screenPos.x - mousePosition.x; + const dy = screenPos.y - mousePosition.y; + const distance = Math.sqrt(dx * dx + dy * dy); + + // Get node size and adjust threshold based on size + const nodeSize = this.sizes[index]; + const adjustedThreshold = threshold * (1 + (nodeSize / 10)); + + // If this node is closer than the current closest and within threshold, update + if (distance < closestDistance && distance < adjustedThreshold) { + closestDistance = distance; + closestNodeId = nodeId; + } + } + + return closestNodeId; + } + + /** + * Remove a node from the point cloud + * @param {string} nodeId - ID of the node to remove + */ + removeNode(nodeId) { + if (!this.nodeIndices.has(nodeId)) return; + + const indexToRemove = this.nodeIndices.get(nodeId); + + // Only perform complex removal if not the last node + if (indexToRemove !== this.nodeCount - 1) { + // Move the last node to this position + const lastIndex = this.nodeCount - 1; + const lastI3 = lastIndex * 3; + const removeI3 = indexToRemove * 3; + + // Copy position + this.positions[removeI3] = this.positions[lastI3]; + this.positions[removeI3 + 1] = this.positions[lastI3 + 1]; + this.positions[removeI3 + 2] = this.positions[lastI3 + 2]; + + // Copy color + this.colors[removeI3] = this.colors[lastI3]; + this.colors[removeI3 + 1] = this.colors[lastI3 + 1]; + this.colors[removeI3 + 2] = this.colors[lastI3 + 2]; + + // Copy size + this.sizes[indexToRemove] = this.sizes[lastIndex]; + + // Find which node was at the last position + let lastNodeId = null; + for (const [id, index] of this.nodeIndices.entries()) { + if (index === lastIndex) { + lastNodeId = id; + break; + } + } + + // Update the moved node's index + if (lastNodeId) { + this.nodeIndices.set(lastNodeId, indexToRemove); + } + } + + // Remove the node from tracking + this.nodeIndices.delete(nodeId); + this.nodeCount--; + + // Update draw range + this.geometry.setDrawRange(0, this.nodeCount); + + // Mark attributes as needing update + this.geometry.attributes.position.needsUpdate = true; + this.geometry.attributes.color.needsUpdate = true; + this.geometry.attributes.size.needsUpdate = true; + } + + /** + * Clear all nodes from the point cloud + */ + clear() { + this.nodeIndices.clear(); + this.nodeTypes.clear(); + this.nodeCount = 0; + this.geometry.setDrawRange(0, 0); + + // Reset state tracking + this.selectedIndices.clear(); + this.neighborIndices.clear(); + this.hoverIndex = -1; + } + + /** + * Update all positions from the node objects + * @param {Map} nodes - Map of nodes with current positions + */ + updateAllPositions(nodes) { + // Update all node positions from data + for (const [nodeId, node] of nodes.entries()) { + if (this.nodeIndices.has(nodeId) && node.x !== undefined && node.y !== undefined) { + const index = this.nodeIndices.get(nodeId); + const i3 = index * 3; + + this.positions[i3] = node.x; + this.positions[i3 + 1] = node.y; + } + } + + // Mark position attribute as needing update + this.geometry.attributes.position.needsUpdate = true; + } +} + +/** + * SimulationManager - Manages the D3 force-directed simulation + */ +class SimulationManager { + constructor(dataManager, graphObjectManager, themeManager) { + this.dataManager = dataManager; + this.graphObjectManager = graphObjectManager; + this.themeManager = themeManager; + + this.simulation = null; + this.isRunning = false; + + // Initialize spatial partitioning + this.spatialGrid = null; + this.useSpatialIndex = true; + this.gridUpdateInterval = 30; // Update grid every 30 frames + this.gridUpdateCounter = 0; + this.gridCellSize = 200; // Size of each grid cell + + // Initialize the simulation + this.initSimulation(); + this.initSpatialIndex(); + } + + /** + * Initialize D3 force simulation + */ + initSimulation() { + // Calculate connection counts for better distribution + const connectionCounts = this.calculateConnectionCounts(); + + // Link distance based on connection count + const linkDistance = link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + const sourceConnections = connectionCounts.get(sourceId) || 0; + const targetConnections = connectionCounts.get(targetId) || 0; + + // Scale distance based on connection count + const baseDistance = this.themeManager.config.defaultDistance; + const connectionFactor = Math.max(sourceConnections, targetConnections); + + return baseDistance * (1 + Math.log(1 + connectionFactor * 0.2)); + }; + + // Collision radius based on connection count + const collisionRadius = node => { + const connections = connectionCounts.get(node.id) || 0; + const baseRadius = 15; + + // Increase collision radius for highly connected nodes + if (connections > this.themeManager.config.highConnectionThreshold) { + return baseRadius * (1 + Math.log(connections) * 0.1); + } + return baseRadius; + }; + + // For monitoring simulation progress + this.tickCounter = 0; + this.lastLogTime = 0; + + // Create the simulation with all forces + this.simulation = d3.forceSimulation() + .force('link', d3.forceLink().id(d => d.id).distance(linkDistance)) + .force('charge', d3.forceManyBody().strength(-15)) + .force('center', d3.forceCenter(0, 0)) + .force('collision', d3.forceCollide().radius(collisionRadius)) + .force('x', d3.forceX().strength(0.001)) + .force('y', d3.forceY().strength(0.005)) + .on('tick', () => { + this.tickCounter++; + this.onSimulationTick(); + this.monitorSimulationProgress(); + }) + .on('end', () => { + console.log('Simulation reached equilibrium!'); + console.log('Final alpha:', this.simulation.alpha()); + console.log('Alpha min:', this.simulation.alphaMin()); + console.log('Alpha decay:', this.simulation.alphaDecay()); + console.log('Node count:', this.simulation.nodes().length); + console.log('Total ticks:', this.tickCounter); + this.isRunning = false; + }); + + // Adjust alpha settings for longer simulation time + // Reduce decay rate (default is ~0.0228 which is 1% cooling per tick) + this.simulation.alphaDecay(0.0228); // Slower decay (about 0.5% cooling per tick) + + // Lower minimum alpha threshold (default is 0.001) + this.simulation.alphaMin(0.001); // Lower threshold for stopping + + // Reduce velocity decay for more momentum (default is 0.4) + this.simulation.velocityDecay(0.1); + } + + /** + * Calculate connection counts for each node + * @returns {Map} Map of node IDs to connection counts + */ + calculateConnectionCounts() { + const connectionCounts = new Map(); + + // Count connections for each node + this.dataManager.graphData.links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + connectionCounts.set(sourceId, (connectionCounts.get(sourceId) || 0) + 1); + connectionCounts.set(targetId, (connectionCounts.get(targetId) || 0) + 1); + }); + + return connectionCounts; + } + + /** + * Update the simulation with current nodes and links + * @param {boolean} restart - Whether to restart the simulation + */ + updateSimulation(restart = true) { + if (!this.simulation) return; + + // Get visible nodes and links from data manager + const visibleNodes = Array.from(this.dataManager.graphObjects.nodes.values()); + const visibleLinks = Array.from(this.dataManager.graphObjects.links.values()); + + console.log(`Updating simulation with ${visibleNodes.length} nodes and ${visibleLinks.length} links`); + + // Update nodes and links in the simulation + this.simulation.nodes(visibleNodes); + this.simulation.force('link').links(visibleLinks); + + // Restart simulation if needed + if (restart && this.themeManager.config.physicsEnabled) { + // Reset tick counter and timing + this.tickCounter = 0; + this.lastLogTime = Date.now(); + + // Set a higher alpha to ensure thorough exploration of layout space + const startingAlpha = 1.0; + console.log(`Starting simulation with alpha=${startingAlpha}, alphaMin=${this.simulation.alphaMin()}, alphaDecay=${this.simulation.alphaDecay()}`); + this.simulation.alpha(startingAlpha).restart(); + this.isRunning = true; + } else { + console.log("Simulation not restarted (either restart=false or physics is disabled)"); + this.simulation.alpha(0); + this.isRunning = false; + } + } + + /** + * Toggle physics simulation on/off + * @returns {boolean} The new physics state + */ + togglePhysics() { + const physicsEnabled = this.themeManager.togglePhysics(); + + console.log(`Physics simulation ${physicsEnabled ? 'enabled' : 'disabled'}`); + + // Use updateSimulation to properly handle the physics state + this.updateSimulation(physicsEnabled); + + return physicsEnabled; + } + + /** + * Handle force simulation tick events + * Updates the positions of nodes and links in the visualization + */ + onSimulationTick() { + this.updatePositions(); + + // Update spatial grid periodically + if (this.useSpatialIndex) { + this.updateSpatialGrid(); + } + } + + /** + * Update positions of all nodes and links + */ + updatePositions() { + // Check if we have a NodeCloud available + if (this.graphController && this.graphController.nodeCloud) { + // Bulk update the NodeCloud for better performance + this.graphController.nodeCloud.updateAllPositions(this.dataManager.graphObjects.nodes); + + // Update virtual objects for compatibility with other systems + this.dataManager.graphObjects.nodes.forEach(node => { + if (node.object && node.object.position) { + node.object.position.x = node.x || 0; + node.object.position.y = node.y || 0; + } + + // Update labels separately + if (node.labelObject) { + this.graphObjectManager.updateNodePosition(node); + } + }); + } + + // Update the position of links in the scene + this.dataManager.graphObjects.links.forEach((link) => { + this.graphObjectManager.updateLinkPosition(link); + }); + } + + /** + * Monitor simulation progress with periodic logging + */ + monitorSimulationProgress() { + // Only log every 100 ticks to avoid spamming the console + if (this.tickCounter % 100 === 0) { + const now = Date.now(); + const timeSinceLastLog = now - this.lastLogTime; + this.lastLogTime = now; + + // Only print if we're still running + if (this.isRunning) { + const currentAlpha = this.simulation.alpha(); + console.log(`Simulation progress: tick=${this.tickCounter}, alpha=${currentAlpha.toFixed(6)}, ticks/second=${(100 / (timeSinceLastLog / 1000)).toFixed(1)}`); + } + } + } + + /** + * Initialize spatial index using a simple grid system + */ + initSpatialIndex() { + this.spatialGrid = new SpatialGrid(this.gridCellSize); + } + + /** + * Add a node to the spatial grid + * @param {Object} node - The node to add + */ + addNodeToSpatialGrid(node) { + if (!this.useSpatialIndex || !this.spatialGrid || !node.object) return; + + // Add node to the grid + this.spatialGrid.addObject(node); + } + + /** + * Update the spatial grid with current node positions + */ + updateSpatialGrid() { + if (!this.useSpatialIndex || !this.spatialGrid) return; + + // Only update periodically for performance + this.gridUpdateCounter++; + if (this.gridUpdateCounter < this.gridUpdateInterval) return; + this.gridUpdateCounter = 0; + + // Rebuild the grid + this.spatialGrid.clear(); + + // Add all current nodes to the grid + this.dataManager.graphObjects.nodes.forEach(node => { + if (node.object) { + this.spatialGrid.addObject(node); + } + }); + } + + /** + * Get nodes within a specific radius of a position + * @param {THREE.Vector3} position - The center position + * @param {number} radius - The radius to search within + * @returns {Array} Array of nodes within the radius + */ + getNodesInRadius(position, radius) { + // Use spatial grid for more efficient spatial query + return this.spatialGrid.findNearbyObjects(position, radius); + } +} + +/** + * UIManager - Handles UI elements and user interaction + */ +class UIManager { + constructor(container, dataManager, graphController) { + this.container = container; + this.dataManager = dataManager; + this.graphController = graphController; + + // DOM elements + this.nodeCountEl = document.getElementById('node-count'); + this.linkCountEl = document.getElementById('link-count'); + this.loadingEl = document.getElementById('loading'); + this.searchInput = document.getElementById('search-input'); + this.initialMessageEl = null; + this.nodeInfoPanel = null; + + // Search state + this.previousSearchValue = ''; + this.isUpdatingAutocomplete = false; + + // Autocomplete state + this.autocompleteList = null; + this.autocompleteSuggestions = []; + this.autocompleteSelectedIndex = -1; + + // Initialize UI elements + this.createAutocompleteUI(); + this.createNodeInfoPanel(); + this.setupEventListeners(); + } + + /** + * Set up event listeners for UI controls + */ + setupEventListeners() { + // Button event listeners + const labelsBtn = document.getElementById('toggle-labels-btn'); + if (labelsBtn) { + // Set initial state based on ThemeManager config + // The initial state should be true by default + labelsBtn.classList.add('active'); + + // Add click handler + labelsBtn.addEventListener('click', () => { + const showLabels = this.graphController.toggleLabels(); + // Toggle active class based on the returned state + labelsBtn.classList.toggle('active', showLabels); + }); + } + + document.getElementById('reset-btn')?.addEventListener('click', () => { + this.graphController.resetView(); + this.hideAutocomplete(); + this.previousSearchValue = ''; + this.searchInput.value = ''; + }); + + // Add click handler for the Load All Nodes button + document.getElementById('load-all-btn')?.addEventListener('click', () => { + this.graphController.loadAllNodes(); + }); + + // Set up search input with debounced handling + if (this.searchInput) { + let searchTimeout = null; + + // Input event for search text changes + this.searchInput.addEventListener('input', (e) => { + if (this.isUpdatingAutocomplete) return; + + const currentValue = e.target.value.trim(); + if (currentValue === this.previousSearchValue) return; + + this.previousSearchValue = currentValue; + clearTimeout(searchTimeout); + + if (currentValue.length > 0) { + this.isUpdatingAutocomplete = true; + searchTimeout = setTimeout(() => { + this.updateAutocompleteSuggestions(currentValue); + this.isUpdatingAutocomplete = false; + }, 250); + } else { + this.hideAutocomplete(); + } + }); + + // Keyboard navigation in autocomplete dropdown + this.searchInput.addEventListener('keydown', (e) => { + if (['ArrowUp', 'ArrowDown', 'Enter', 'Escape'].includes(e.key)) { + this.handleAutocompleteKeydown(e); + } + }); + + // Focus handler to show autocomplete + this.searchInput.addEventListener('focus', () => { + if (this.isUpdatingAutocomplete) return; + + const currentValue = this.searchInput.value.trim(); + if (currentValue.length > 0) { + this.previousSearchValue = currentValue; + this.isUpdatingAutocomplete = true; + this.updateAutocompleteSuggestions(currentValue); + this.isUpdatingAutocomplete = false; + } + }); + } + + // Hide autocomplete when clicking outside + document.addEventListener('click', (e) => { + if (this.autocompleteList && e.target !== this.searchInput && !this.autocompleteList.contains(e.target)) { + this.hideAutocomplete(); + } + }); + } + + /** + * Create the autocomplete UI elements + */ + createAutocompleteUI() { + if (!this.searchInput) return; + + // Create autocomplete container if it doesn't exist + if (!this.autocompleteList) { + this.autocompleteList = document.createElement('div'); + this.autocompleteList.className = 'autocomplete-items'; + this.autocompleteList.style.display = 'none'; + this.autocompleteList.style.position = 'absolute'; + this.autocompleteList.style.zIndex = '999'; + this.autocompleteList.style.maxHeight = '300px'; + this.autocompleteList.style.overflowY = 'auto'; + this.autocompleteList.style.width = '100%'; + this.autocompleteList.style.background = '#fff'; + this.autocompleteList.style.border = '1px solid #ddd'; + this.autocompleteList.style.borderRadius = '0 0 4px 4px'; + this.autocompleteList.style.boxShadow = '0 2px 4px rgba(0,0,0,0.2)'; + + // Append to parent container + const searchContainer = this.searchInput.parentNode; + searchContainer.appendChild(this.autocompleteList); + } + } + + /** + * Update autocomplete suggestions based on search term + * @param {string} searchTerm - The current search term + */ + updateAutocompleteSuggestions(searchTerm) { + if (!this.autocompleteList || !searchTerm) { + this.hideAutocomplete(); + return; + } + + // Clear previous suggestions + this.autocompleteList.innerHTML = ''; + this.autocompleteSuggestions = []; + this.autocompleteSelectedIndex = -1; + + const maxSuggestions = 10; + const searchLower = searchTerm.toLowerCase(); + + // Get all nodes that match the search term + const matchingNodes = this.dataManager.graphData.nodes + .filter(node => ( + (node.id && node.id.toLowerCase().includes(searchLower)) || + (node.label && node.label.toLowerCase().includes(searchLower)) + )) + .sort((a, b) => { + // Prioritize exact matches and matches at the beginning + const aId = a.id.toLowerCase(); + const bId = b.id.toLowerCase(); + const aLabel = (a.label || '').toLowerCase(); + const bLabel = (b.label || '').toLowerCase(); + + // Check for exact matches first + if (aId === searchLower || aLabel === searchLower) return -1; + if (bId === searchLower || bLabel === searchLower) return 1; + + // Then check for starting with search term + if (aId.startsWith(searchLower) || aLabel.startsWith(searchLower)) return -1; + if (bId.startsWith(searchLower) || bLabel.startsWith(searchLower)) return 1; + + // Fallback to alphabetical + return aId.localeCompare(bId); + }) + .slice(0, maxSuggestions); + + if (matchingNodes.length === 0) { + this.hideAutocomplete(); + return; + } + + // Save suggestions for keyboard navigation + this.autocompleteSuggestions = matchingNodes; + + // Create suggestion items + matchingNodes.forEach((node, index) => { + const item = document.createElement('div'); + item.className = 'autocomplete-item'; + item.style.padding = '8px 12px'; + item.style.cursor = 'pointer'; + item.style.borderBottom = '1px solid #f4f4f4'; + + // Highlight matching parts + const displayText = node.label || node.id; + const parts = displayText.split(new RegExp(`(${searchTerm})`, 'i')); + + parts.forEach(part => { + const span = document.createElement('span'); + span.textContent = part; + if (part.toLowerCase() === searchTerm.toLowerCase()) { + span.style.fontWeight = 'bold'; + span.style.backgroundColor = 'rgba(66, 133, 244, 0.1)'; + } + item.appendChild(span); + }); + + // Add node type indicator + const typeIndicator = document.createElement('span'); + typeIndicator.style.marginLeft = '8px'; + typeIndicator.style.padding = '2px 6px'; + typeIndicator.style.borderRadius = '10px'; + typeIndicator.style.fontSize = '0.8em'; + + // Different styling for different node types + if (node.type === 'simple') { + typeIndicator.textContent = 'item'; + typeIndicator.style.backgroundColor = 'rgba(100, 149, 237, 0.2)'; + typeIndicator.style.color = 'rgb(50, 90, 160)'; + } else { + typeIndicator.textContent = 'collection'; + typeIndicator.style.backgroundColor = 'rgba(240, 128, 128, 0.2)'; + typeIndicator.style.color = 'rgb(180, 70, 70)'; + } + + item.appendChild(typeIndicator); + + // Add hover effect + item.addEventListener('mouseover', () => { + this.autocompleteSelectedIndex = index; + this.highlightSelectedSuggestion(); + }); + + // Add click handler + item.addEventListener('click', () => { + this.searchInput.value = node.id; + this.hideAutocomplete(); + this.graphController.searchNodes(node.id); + }); + + this.autocompleteList.appendChild(item); + }); + + // Show the autocomplete list + this.autocompleteList.style.display = 'block'; + } + + /** + * Handle keyboard navigation in autocomplete list + * @param {KeyboardEvent} event - The keyboard event + */ + handleAutocompleteKeydown(event) { + // If no suggestions or hidden, do nothing special except for Enter + if (this.autocompleteSuggestions.length === 0 || + this.autocompleteList.style.display === 'none') { + if (event.key === 'Enter') { + const searchTerm = this.searchInput.value.trim(); + if (searchTerm) { + this.graphController.searchNodes(searchTerm); + this.hideAutocomplete(); + } + } + return; + } + + switch (event.key) { + case 'ArrowDown': + // Move selection down + event.preventDefault(); + this.autocompleteSelectedIndex = Math.min( + this.autocompleteSelectedIndex + 1, + this.autocompleteSuggestions.length - 1 + ); + this.highlightSelectedSuggestion(); + break; + + case 'ArrowUp': + // Move selection up + event.preventDefault(); + this.autocompleteSelectedIndex = Math.max(this.autocompleteSelectedIndex - 1, -1); + this.highlightSelectedSuggestion(); + break; + + case 'Enter': + // Select current suggestion or search with current text + event.preventDefault(); + if (this.autocompleteSelectedIndex >= 0) { + const selectedNode = this.autocompleteSuggestions[this.autocompleteSelectedIndex]; + this.searchInput.value = selectedNode.id; + this.graphController.searchNodes(selectedNode.id); + } else { + const searchTerm = this.searchInput.value.trim(); + if (searchTerm) { + this.graphController.searchNodes(searchTerm); + } + } + this.hideAutocomplete(); + break; + + case 'Escape': + // Hide autocomplete + event.preventDefault(); + this.hideAutocomplete(); + break; + } + } + + /** + * Highlight the currently selected suggestion item + */ + highlightSelectedSuggestion() { + // Remove highlight from all items + const items = this.autocompleteList.querySelectorAll('.autocomplete-item'); + items.forEach(item => { + item.style.backgroundColor = ''; + }); + + // Highlight selected item if any + if (this.autocompleteSelectedIndex >= 0 && this.autocompleteSelectedIndex < items.length) { + const selectedItem = items[this.autocompleteSelectedIndex]; + selectedItem.style.backgroundColor = 'rgba(66, 133, 244, 0.1)'; + + // Scroll into view if needed + if (selectedItem.offsetTop < this.autocompleteList.scrollTop) { + this.autocompleteList.scrollTop = selectedItem.offsetTop; + } else if (selectedItem.offsetTop + selectedItem.offsetHeight > + this.autocompleteList.scrollTop + this.autocompleteList.offsetHeight) { + this.autocompleteList.scrollTop = + selectedItem.offsetTop + selectedItem.offsetHeight - this.autocompleteList.offsetHeight; + } + } + } + + /** + * Hide the autocomplete list + */ + hideAutocomplete() { + if (this.autocompleteList) { + this.autocompleteList.style.display = 'none'; + this.autocompleteSelectedIndex = -1; + } + } + + /** + * Show or hide the loading indicator + * @param {boolean} show - Whether to show or hide the loading indicator + */ + showLoading(show) { + if (this.loadingEl) { + this.loadingEl.style.display = show ? 'block' : 'none'; + } + } + + /** + * Show an initial message in the graph area + * @param {string} message - The message to display + */ + showInitialMessage(message) { + // Create or update the message element + if (!this.initialMessageEl) { + this.initialMessageEl = document.createElement('div'); + this.initialMessageEl.style.position = 'absolute'; + this.initialMessageEl.style.top = '50%'; + this.initialMessageEl.style.left = '50%'; + this.initialMessageEl.style.transform = 'translate(-50%, -50%)'; + this.initialMessageEl.style.background = 'rgba(0, 0, 0, 0.7)'; + this.initialMessageEl.style.color = '#ffffff'; + this.initialMessageEl.style.padding = '20px'; + this.initialMessageEl.style.borderRadius = '8px'; + this.initialMessageEl.style.maxWidth = '80%'; + this.initialMessageEl.style.textAlign = 'center'; + this.initialMessageEl.style.fontSize = '18px'; + this.container.appendChild(this.initialMessageEl); + } + + this.initialMessageEl.textContent = message; + this.initialMessageEl.style.display = 'block'; + } + + /** + * Hide the initial message + */ + hideInitialMessage() { + if (this.initialMessageEl) { + this.initialMessageEl.style.display = 'none'; + } + } + + /** + * Show error message + * @param {string} message - The error message to display + */ + showError(message) { + console.error(message); + this.showInitialMessage(message); + } + + /** + * Update statistics display + */ + updateStats() { + if (this.nodeCountEl) { + this.nodeCountEl.textContent = this.dataManager.graphData.nodes.length; + } + if (this.linkCountEl) { + this.linkCountEl.textContent = this.dataManager.graphData.links.length; + } + } + + /** + * Create the node info panel + */ + createNodeInfoPanel() { + if (!this.nodeInfoPanel) { + this.nodeInfoPanel = document.createElement('div'); + this.nodeInfoPanel.className = 'node-info-panel'; + this.nodeInfoPanel.style.display = 'none'; + this.container.appendChild(this.nodeInfoPanel); + } + } + + /** + * Show node information in the info panel + * @param {string} nodeId - The ID of the node to display info for + */ + showNodeInfo(nodeId) { + if (!this.nodeInfoPanel) this.createNodeInfoPanel(); + + const node = this.dataManager.graphObjects.nodes.get(nodeId); + if (!node) return; + + // Store the current node ID for the Get Data button + this.currentNodeId = nodeId; + + // Get the node's connections + const connectedLinks = this.dataManager.getConnectedLinks(nodeId); + const connectionCount = connectedLinks.length; + + // Get any additional properties + const nodeType = node.type || 'Unknown'; + const nodeLabel = node.label || nodeId; + + // Build HTML content + let html = ` +

${nodeLabel}

+

ID: ${this.truncateWithEllipsis(nodeId)}

+

Type: ${nodeType}

+

Connections: ${connectionCount}

+ `; + + // Only show the Get Data button for composite nodes where the ID doesn't start with "data/" + if (nodeType === 'composite' && !nodeId.startsWith('data/')) { + html += ` + + + `; + } + + // Add any other properties that exist + if (node.data) { + html += `

Data: ${JSON.stringify(node.data)}

`; + } + + // Add information about connected nodes + if (connectionCount > 0) { + html += `

Connected Nodes:

`; + html += `
`; + + // Get info about connected nodes + const connectedNodes = new Map(); // Use Map to avoid duplicates + + connectedLinks.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + // Get the ID of the connected node (not the current node) + const connectedNodeId = sourceId === nodeId ? targetId : sourceId; + + // Store relationship type if available + const relationship = link.label || ''; + + // Get the connected node + const connectedNode = this.dataManager.graphObjects.nodes.get(connectedNodeId); + if (connectedNode && !connectedNodes.has(connectedNodeId)) { + connectedNodes.set(connectedNodeId, { + node: connectedNode, + relationship: relationship + }); + } + }); + + // Display connected nodes (limited to avoid overwhelming the panel) + const maxNodesToShow = 50; + let nodeCount = 0; + + connectedNodes.forEach((data, connectedNodeId) => { + if (nodeCount < maxNodesToShow) { + const connectedNode = data.node; + const relationship = data.relationship; + + const truncatedId = this.truncateWithEllipsis(connectedNodeId); + const nodeLabel = connectedNode.label || truncatedId; + const nodeType = connectedNode.type || 'Unknown'; + + html += `
`; + html += `${nodeLabel}`; + html += `
${nodeType}
`; + + if (relationship) { + html += `
+ Relationship: ${relationship}
`; + } + + html += `
`; + + nodeCount++; + } + }); + + // If there are more nodes than we're showing + if (connectedNodes.size > maxNodesToShow) { + html += `
...and ${connectedNodes.size - maxNodesToShow} more
`; + } + + html += `
`; + } + + // Set content and show panel + this.nodeInfoPanel.innerHTML = html; + this.nodeInfoPanel.style.display = 'block'; + + // Add event listener for the Get Data button + const getDataBtn = document.getElementById('get-node-data'); + if (getDataBtn) { + getDataBtn.addEventListener('click', () => this.fetchNodeData(nodeId)); + } + + // Add event listeners to connected node items + const nodeItems = this.nodeInfoPanel.querySelectorAll('.connected-node-item'); + nodeItems.forEach(item => { + // Hover effect + item.addEventListener('mouseover', () => { + item.style.backgroundColor = '#f5f5f5'; + item.style.borderLeftColor = '#4D90FE'; + }); + + item.addEventListener('mouseout', () => { + item.style.backgroundColor = ''; + item.style.borderLeftColor = '#eee'; + }); + + // Click to select the node + item.addEventListener('click', () => { + const clickedNodeId = item.getAttribute('data-node-id'); + if (clickedNodeId && this.graphController.eventManager) { + this.graphController.eventManager.selectNode(clickedNodeId); + this.graphController.eventManager.focusOnNode(clickedNodeId); + } + }); + }); + } + + /** + * Fetch data for a specific node from the server + * @param {string} nodeId - The ID of the node to fetch data for + */ + fetchNodeData(nodeId) { + // Get the result container + const resultContainer = document.getElementById('node-data-result'); + if (!resultContainer) return; + + // Show loading indicator + resultContainer.style.display = 'block'; + resultContainer.innerHTML = ` +
+
+
Loading data...
+
+ + `; + + // Construct the URL for the data endpoint + const url = `/${nodeId}`; + + // Fetch the data + fetch(url) + .then(response => { + if (!response.ok) { + throw new Error(`HTTP error ${response.status}`); + } + return response.text(); // Using text() instead of json() to handle any type of response + }) + .then(data => { + // Try to parse as JSON if possible + try { + const jsonData = JSON.parse(data); + this.displayNodeData(jsonData, resultContainer); + } catch (e) { + // If not JSON, display as text + this.displayNodeData(data, resultContainer, false); + } + }) + .catch(error => { + // Show error message + resultContainer.innerHTML = ` +
+ Error loading data: ${error.message} +
+ `; + }); + } + + /** + * Display node data in the result container + * @param {Object|string} data - The data to display + * @param {HTMLElement} container - The container to display the data in + * @param {boolean} isJson - Whether the data is JSON + */ + displayNodeData(data, container, isJson = true) { + if (isJson) { + // Format JSON for display + const formattedJson = JSON.stringify(data, null, 2); + container.innerHTML = ` +
+ ${formattedJson.replace(//g, '>')} +
+ `; + } else { + // Display as text + container.innerHTML = ` +
+ ${data.toString().replace(//g, '>')} +
+ `; + } + } + + /** + * Hide the node info panel + */ + hideNodeInfo() { + if (this.nodeInfoPanel) { + this.nodeInfoPanel.style.display = 'none'; + } + } + + /** + * Truncate text and add ellipsis in the middle + * @param {string} text - The text to truncate + * @returns {string} Truncated text with ellipsis + */ + truncateWithEllipsis(text) { + // Show only if longer than 15 characters (6 + 3 + 6) + if (!text || text.length <= 15) { + return text; + } + + // Take exactly 6 chars from start and 6 from end + return text.substring(0, 6) + '...' + text.substring(text.length - 6); + } +} + +/** + * EventManager - Handles user interaction events + */ +class EventManager { + constructor(sceneManager, dataManager, graphObjectManager) { + this.sceneManager = sceneManager; + this.dataManager = dataManager; + this.graphObjectManager = graphObjectManager; + + // Set up event listeners + this.setupEventListeners(); + } + + /** + * Set up graph-specific interaction handlers + */ + setupEventListeners() { + const renderer = this.sceneManager.renderer; + if (!renderer || !renderer.domElement) return; + + // Add click event listener for node selection + renderer.domElement.addEventListener('click', this.onMouseClick.bind(this)); + + // Add double-click event listener for camera focus + renderer.domElement.addEventListener('dblclick', this.onDoubleClick.bind(this)); + + // Add hover event listeners + renderer.domElement.addEventListener('mousemove', this.onMouseMove.bind(this)); + } + + /** + * Handle mouse click events for node selection + * @param {MouseEvent} event - The mouse event + */ + onMouseClick(event) { + // Calculate mouse position and find intersections + this.sceneManager.updateMousePosition(event); + + // Set the scene manager on the NodeCloud for camera access + this.graphController.nodeCloud.sceneManager = this.sceneManager; + + const nodeId = this.graphController.nodeCloud.findClosestNode( + this.sceneManager.raycaster, + 0.08 // Screen space threshold for clicks (0-1 normalized coordinates) + ); + + if (nodeId) { + // We clicked on a node + this.selectNode(nodeId); + } else { + // Clicked on empty space - deselect + this.deselectNode(); + } + } + + /** + * Handle double-click events for focusing on nodes + * @param {MouseEvent} event - The mouse event + */ + onDoubleClick(event) { + // Calculate mouse position and find intersections + this.sceneManager.updateMousePosition(event); + + // Set the scene manager on the NodeCloud for camera access + this.graphController.nodeCloud.sceneManager = this.sceneManager; + + const nodeId = this.graphController.nodeCloud.findClosestNode( + this.sceneManager.raycaster, + 0.08 // Screen space threshold for double-clicks (0-1 normalized coordinates) + ); + + if (nodeId) { + // Double-clicked on a node - focus camera on this node + this.focusOnNode(nodeId); + } + } + + /** + * Handle mouse movement for hover effects + * @param {MouseEvent} event - The mouse event + */ + onMouseMove(event) { + // Calculate mouse position + this.sceneManager.updateMousePosition(event); + + // Set the scene manager on the NodeCloud for camera access + this.graphController.nodeCloud.sceneManager = this.sceneManager; + + const nodeId = this.graphController.nodeCloud.findClosestNode( + this.sceneManager.raycaster, + 0.04 // Screen space threshold for hover (0-1 normalized coordinates) + ); + + if (nodeId) { + // Hovering over a node + this.hoverNode(nodeId); + } else { + // Not hovering over any node + this.unhoverNode(); + } + } + + /** + * Select a node and highlight it and its connections + * @param {string} nodeId - The ID of the node to select + */ + selectNode(nodeId) { + if (this.dataManager.selectedNode === nodeId) return; + + // Set selection in data manager + this.dataManager.setSelectedNode(nodeId); + + // Update colors in bulk + this.graphController.nodeCloud.updateColors( + nodeId, + this.dataManager.neighborNodes, + this.dataManager.hoveredNode + ); + + // Update link colors + this.updateLinkColors(); + + // First, hide all link labels + this.dataManager.graphObjects.links.forEach((link, id) => { + if (link.labelObject) { + link.labelObject.visible = false; + } + }); + + // Then show labels for active links + this.dataManager.activeLinks.forEach(linkId => { + const link = this.dataManager.graphObjects.links.get(linkId); + if (link && link.labelObject) { + link.labelObject.visible = true; + } + }); + + // Show node info panel + const graphController = this.sceneManager.graphController || + (this.graphObjectManager && this.graphObjectManager.graphController); + + if (graphController && graphController.uiManager) { + graphController.uiManager.showNodeInfo(nodeId); + } + } + + /** + * Deselect the currently selected node + */ + deselectNode() { + if (!this.dataManager.selectedNode) return; + + // Hide all link labels before clearing selection + this.dataManager.activeLinks.forEach(linkId => { + const link = this.dataManager.graphObjects.links.get(linkId); + if (link && link.labelObject) { + link.labelObject.visible = false; + } + }); + + // Clear selection in data manager + this.dataManager.clearSelectedNode(); + + // Update colors in bulk + this.graphController.nodeCloud.updateColors( + null, // No selected node + new Set(), // No neighbor nodes + this.dataManager.hoveredNode // Keep hover state + ); + + // Update link colors + this.updateLinkColors(); + + // Hide node info panel + const graphController = this.sceneManager.graphController || + (this.graphObjectManager && this.graphObjectManager.graphController); + + if (graphController && graphController.uiManager) { + graphController.uiManager.hideNodeInfo(); + } + } + + /** + * Focus the camera on a specific node + * @param {string} nodeId - The ID of the node to focus on + */ + focusOnNode(nodeId) { + const node = this.dataManager.graphObjects.nodes.get(nodeId); + if (!node || !node.object) return; + + const position = node.object.position.clone(); + this.sceneManager.focusCamera(position); + } + + /** + * Apply hover effect to a node + * @param {string} nodeId - The ID of the node to hover + */ + hoverNode(nodeId) { + // If already hovering over this node, do nothing + if (this.dataManager.hoveredNode === nodeId) return; + + // Set hover in data manager + this.dataManager.setHoveredNode(nodeId); + + // Update colors in bulk + this.graphController.nodeCloud.updateColors( + this.dataManager.selectedNode, + this.dataManager.neighborNodes, + nodeId + ); + + // Change cursor to pointer + this.sceneManager.renderer.domElement.style.cursor = 'pointer'; + } + + /** + * Remove hover effect from the currently hovered node + */ + unhoverNode() { + if (!this.dataManager.hoveredNode) return; + + // Get the node ID before clearing + const nodeId = this.dataManager.hoveredNode; + + // Clear hover in data manager + this.dataManager.clearHoveredNode(); + + // Update colors in bulk + this.graphController.nodeCloud.updateColors( + this.dataManager.selectedNode, + this.dataManager.neighborNodes, + null // No hover + ); + + // Reset cursor + this.sceneManager.renderer.domElement.style.cursor = 'auto'; + } + + /** + * Update the colors of all visible nodes based on selection state + */ + updateNodeColors() { + this.dataManager.graphObjects.nodes.forEach((node, id) => { + this.graphObjectManager.updateNodeColors(id); + }); + } + + /** + * Update the colors of all visible links based on selection state + */ + updateLinkColors() { + this.dataManager.graphObjects.links.forEach((link, id) => { + this.graphObjectManager.updateLinkColors(id); + }); + } +} + +/** + * DebugVisualizer - Generic visualization for debugging graph components + */ +class DebugVisualizer { + constructor(sceneManager, graphController) { + this.sceneManager = sceneManager; + this.graphController = graphController; + this.debugObjects = []; + this.enabled = false; + this.lastUpdateTime = 0; + this.updateInterval = 1000; // Update debug visuals every second + this.activeVisualizations = { + grid: true, + performance: true, + nodes: true + }; + this.stats = {}; + } + + /** + * Toggle debug visualization + * @param {boolean} enabled - Whether to enable or disable visualization + */ + toggle(enabled) { + this.enabled = enabled; + + // Show or hide debug UI elements + const debugPanel = document.getElementById('debug-info-panel'); + const frameGraph = document.getElementById('debug-frame-graph'); + + if (debugPanel) { + debugPanel.style.display = enabled ? 'block' : 'none'; + } + + if (frameGraph) { + frameGraph.style.display = enabled ? 'block' : 'none'; + } + + if (enabled) { + // Initialize frame history if needed + if (!this.frameHistory) { + const canvas = document.getElementById('debug-frame-canvas'); + if (canvas) { + this.frameHistory = new Array(canvas.width).fill(0); + } + } + + this.createDebugVisualization(); + } else { + this.clearDebugVisualization(); + } + } + + /** + * Toggle specific visualization types + * @param {string} type - Visualization type to toggle + */ + toggleVisualization(type) { + if (this.activeVisualizations.hasOwnProperty(type)) { + this.activeVisualizations[type] = !this.activeVisualizations[type]; + if (this.enabled) { + this.createDebugVisualization(); + } + } + } + + /** + * Create visual representation of debug data + */ + createDebugVisualization() { + this.clearDebugVisualization(); + + // Collect debug stats + this.collectStats(); + + // Create visualizations based on active settings + if (this.activeVisualizations.grid) { + this.createSpatialGridVisualization(); + } + + if (this.activeVisualizations.nodes) { + this.createNodeStatsVisualization(); + } + + if (this.activeVisualizations.performance) { + this.createPerformanceVisualization(); + } + + // Create debug panel with statistics + this.createDebugPanel(); + + this.lastUpdateTime = performance.now(); + } + + /** + * Collect statistics for debug display + */ + collectStats() { + // Clear previous stats + this.stats = { + fps: this.graphController.fps || 0, + nodeCount: 0, + visibleNodeCount: 0, + linkCount: 0, + gridStats: { + cells: 0, + objects: 0, + avgPerCell: 0 + }, + cameraPosition: { + x: 0, + y: 0, + z: 0 + }, + performanceMode: this.graphController.performanceMode + }; + + // Collect node and link stats + if (this.graphController.dataManager) { + const dataManager = this.graphController.dataManager; + + this.stats.nodeCount = dataManager.graphData.nodes.length; + this.stats.visibleNodeCount = dataManager.graphObjects.nodes.size; + this.stats.linkCount = dataManager.graphObjects.links.size; + } + + // Collect grid stats + if (this.graphController.simulationManager && + this.graphController.simulationManager.spatialGrid) { + + const grid = this.graphController.simulationManager.spatialGrid; + this.stats.gridStats.cells = grid.grid.size; + this.stats.gridStats.objects = grid.objects.size; + + if (grid.grid.size > 0 && grid.objects.size > 0) { + this.stats.gridStats.avgPerCell = + (grid.objects.size / grid.grid.size).toFixed(1); + } + } + + // Collect camera position + if (this.graphController.sceneManager && this.graphController.sceneManager.camera) { + const camera = this.graphController.sceneManager.camera; + this.stats.cameraPosition.x = camera.position.x.toFixed(1); + this.stats.cameraPosition.y = camera.position.y.toFixed(1); + this.stats.cameraPosition.z = camera.position.z.toFixed(1); + } + } + + /** + * Create spatial grid visualization + */ + createSpatialGridVisualization() { + const spatialGrid = this.graphController.simulationManager?.spatialGrid; + if (!spatialGrid) return; + + // Create wireframe boxes for each cell in the grid + spatialGrid.grid.forEach((cell, key) => { + const [x, y, z] = key.split(',').map(Number); + const cellSize = spatialGrid.cellSize; + + // Create box geometry for the cell + const geometry = new THREE.BoxGeometry(cellSize, cellSize, cellSize); + const material = new THREE.MeshBasicMaterial({ + color: 0x00ff00, + wireframe: true, + transparent: true, + opacity: 0.05 + (0.05 * Math.min(cell.size, 10)) // Brighter for more populated cells + }); + + const box = new THREE.Mesh(geometry, material); + box.position.set( + (x + 0.5) * cellSize, + (y + 0.5) * cellSize, + (z + 0.5) * cellSize + ); + + this.sceneManager.addToScene(box); + this.debugObjects.push(box); + + // Add text label showing object count in cell + if (cell.size > 0) { + const text = new SpriteText(`${cell.size}`, 12); + text.color = '#ffff00'; + text.backgroundColor = 'rgba(0,0,0,0.5)'; + text.padding = 2; + text.position.copy(box.position); + this.sceneManager.addToScene(text); + this.debugObjects.push(text); + } + }); + } + + /** + * Create node statistics visualization + */ + createNodeStatsVisualization() { + // Highlight nodes with different colors based on properties + const nodeManager = this.graphController.dataManager; + const nodeCloud = this.graphController.nodeCloud; + + if (!nodeManager || !nodeCloud || !nodeCloud.colors) return; + + // Store original colors to restore later + this.originalColors = new Float32Array(nodeCloud.colors.length); + this.originalColors.set(nodeCloud.colors); // Make a copy of all colors + + // Iterate through nodes and update colors in the buffer + nodeManager.graphObjects.nodes.forEach((node, id) => { + if (nodeCloud.nodeIndices.has(id)) { + // Get the node's index in the color buffer + const index = nodeCloud.nodeIndices.get(id); + const i3 = index * 3; + + // Get connection count + const connectedLinks = nodeManager.getConnectedLinks(id); + const connectionCount = connectedLinks.length; + + // Set color based on connection count + let color; + if (connectionCount > 10) { + color = new THREE.Color(0xff0000); // Red for highly connected + } else if (connectionCount > 5) { + color = new THREE.Color(0xff8800); // Orange for medium + } else if (connectionCount > 2) { + color = new THREE.Color(0xffff00); // Yellow for low + } else { + color = new THREE.Color(0x00ffff); // Cyan for minimal + } + + // Update the color buffer directly + nodeCloud.colors[i3] = color.r; + nodeCloud.colors[i3 + 1] = color.g; + nodeCloud.colors[i3 + 2] = color.b; + } + }); + + // Mark the color buffer as needing update + if (nodeCloud.geometry && nodeCloud.geometry.attributes.color) { + nodeCloud.geometry.attributes.color.needsUpdate = true; + } + } + + /** + * Create performance metrics visualization + */ + createPerformanceVisualization() { + // Update frame history and redraw + this.updateFrameGraph(); + } + + /** + * Update the frame rate graph + */ + updateFrameGraph() { + const canvas = document.getElementById('debug-frame-canvas'); + const fpsLabel = document.getElementById('debug-fps-label'); + + if (!canvas || !fpsLabel) return; + + // Update FPS label + fpsLabel.textContent = `${this.stats.fps} FPS`; + + // Add current FPS to history + if (!this.frameHistory) { + this.frameHistory = new Array(canvas.width).fill(0); + } + + this.frameHistory.push(this.stats.fps); + this.frameHistory.shift(); + + // Draw frame history + const ctx = canvas.getContext('2d'); + const width = canvas.width; + const height = canvas.height; + + // Clear canvas + ctx.clearRect(0, 0, width, height); + + // Calculate scale - find max FPS in history for scaling + const maxFPS = Math.max(60, ...this.frameHistory); + const scale = height / maxFPS; + + // Draw background grid + ctx.strokeStyle = '#333'; + ctx.lineWidth = 0.5; + + // Draw horizontal grid lines at 15, 30, 45, 60 FPS + [15, 30, 45, 60].forEach(fps => { + const y = height - (fps * scale); + if (y >= 0 && y <= height) { + ctx.beginPath(); + ctx.moveTo(0, y); + ctx.lineTo(width, y); + ctx.stroke(); + } + }); + + // Draw FPS graph + ctx.strokeStyle = '#4CAF50'; + ctx.lineWidth = 1.5; + ctx.beginPath(); + + // Start at bottom-left corner with 0 FPS + ctx.moveTo(0, height); + + // Draw lines for each frame sample + this.frameHistory.forEach((fps, x) => { + const y = height - (fps * scale); + ctx.lineTo(x, y); + }); + + // Finish at bottom-right corner + ctx.lineTo(width - 1, height); + ctx.closePath(); + + // Fill gradient + const gradient = ctx.createLinearGradient(0, 0, 0, height); + gradient.addColorStop(0, 'rgba(76, 175, 80, 0.7)'); + gradient.addColorStop(1, 'rgba(76, 175, 80, 0.1)'); + ctx.fillStyle = gradient; + ctx.fill(); + + // Stroke the line on top of the fill + ctx.strokeStyle = '#4CAF50'; + ctx.lineWidth = 1.5; + ctx.beginPath(); + + this.frameHistory.forEach((fps, x) => { + const y = height - (fps * scale); + if (x === 0) { + ctx.moveTo(x, y); + } else { + ctx.lineTo(x, y); + } + }); + + ctx.stroke(); + } + + /** + * Create debug panel with statistics + */ + createDebugPanel() { + // Update debug panel content using the existing HTML element + document.getElementById('debug-nodes').textContent = `${this.stats.visibleNodeCount}/${this.stats.nodeCount}`; + document.getElementById('debug-links').textContent = `${this.stats.linkCount}`; + document.getElementById('debug-cells').textContent = `${this.stats.gridStats.cells}`; + document.getElementById('debug-objects').textContent = `${this.stats.gridStats.objects}`; + document.getElementById('debug-avg-per-cell').textContent = `${this.stats.gridStats.avgPerCell}`; + + // Update camera position + document.getElementById('debug-camera-x').textContent = `${this.stats.cameraPosition.x}`; + document.getElementById('debug-camera-y').textContent = `${this.stats.cameraPosition.y}`; + document.getElementById('debug-camera-z').textContent = `${this.stats.cameraPosition.z}`; + } + + /** + * Remove all debug visualization objects + */ + clearDebugVisualization() { + // Remove all debug visualization objects from scene + this.debugObjects.forEach(obj => { + this.sceneManager.removeFromScene(obj); + }); + this.debugObjects = []; + + // Restore original node colors + if (this.originalColors && this.graphController.nodeCloud) { + const nodeCloud = this.graphController.nodeCloud; + + // Copy the original colors back to the nodeCloud color buffer + if (nodeCloud.colors && this.originalColors.length === nodeCloud.colors.length) { + nodeCloud.colors.set(this.originalColors); + + // Mark the color buffer as needing update + if (nodeCloud.geometry && nodeCloud.geometry.attributes.color) { + nodeCloud.geometry.attributes.color.needsUpdate = true; + } + } + + this.originalColors = null; + } + } + + /** + * Update debug visualization + */ + update() { + if (!this.enabled) return; + + // Update stats more frequently than full visualization refresh + this.collectStats(); + + // Update FPS graph and info panel more frequently + if (this.activeVisualizations.performance && document.getElementById('debug-frame-canvas')) { + this.updateFrameGraph(); + } + + // Update info panel if it exists + if (document.getElementById('debug-info-panel')) { + this.createDebugPanel(); // Updates panel content + } + + // Only update full visualization periodically to avoid performance impact + const now = performance.now(); + if (now - this.lastUpdateTime < this.updateInterval) return; + + // Recreate visualization + this.createDebugVisualization(); + } + + /** + * Handle keyboard shortcuts for debugging + * @param {KeyboardEvent} event - Keyboard event + */ + handleKeyPress(event) { + if (!this.enabled) return; + + switch (event.key) { + case '1': + this.toggleVisualization('grid'); + break; + case '2': + this.toggleVisualization('nodes'); + break; + case '3': + this.toggleVisualization('performance'); + break; + } + } +} + +/** + * Main controller class that coordinates all graph components + */ +class GraphController { + constructor(containerId) { + // DOM container reference + this.container = document.getElementById(containerId); + + // Initialize component managers + this.themeManager = new ThemeManager(); + this.sceneManager = new SceneManager(this.container, this.themeManager); + this.sceneManager.graphController = this; // Add reference to this controller + + this.dataManager = new DataManager(); + this.graphObjectManager = new GraphObjectManager(this.sceneManager, this.dataManager, this.themeManager); + this.graphObjectManager.graphController = this; // Add reference to this controller + + // Create the node cloud for efficient node rendering + this.nodeCloud = new NodeCloud(this.sceneManager.scene, this.themeManager); + + this.simulationManager = new SimulationManager(this.dataManager, this.graphObjectManager, this.themeManager); + this.simulationManager.graphController = this; // Add reference to this controller + + this.uiManager = new UIManager(this.container, this.dataManager, this); + this.eventManager = new EventManager(this.sceneManager, this.dataManager, this.graphObjectManager); + this.eventManager.graphController = this; // Add reference to this controller + + // Performance and debug settings + this.performanceMode = true; // Always on + this.debugMode = false; + + // Initialize generic debugger + this.debugger = new DebugVisualizer(this.sceneManager, this); + + // Initialize FPS counter + this.fpsCounter = document.getElementById('fps-counter'); + this.frameCount = 0; + this.lastTime = performance.now(); + this.fps = 0; + this.fpsUpdateInterval = 500; // Update FPS display every 500ms + + // Set up UI button handlers + this.setupButtonHandlers(); + + // Setup keyboard listeners for debug controls + document.addEventListener('keydown', this.handleKeyPress.bind(this)); + + // Store a global reference for convenience (used by NodeCloud) + window.lastGraphController = this; + + // Load data + this.loadGraphData(); + + // Always enable performance optimizations + this.enablePerformanceMode(); + + // Start animation loop + this.animate(); + } + + /** + * Set up handlers for UI buttons + */ + setupButtonHandlers() { + // Debug mode button + const debugBtn = document.getElementById('toggle-debug-btn'); + if (debugBtn) { + debugBtn.addEventListener('click', () => { + this.toggleDebugMode(); + debugBtn.classList.toggle('active', this.debugMode); + }); + } + } + + /** + * Enable performance optimizations + */ + enablePerformanceMode() { + // Enable spatial grid and frustum culling + if (this.simulationManager) { + this.simulationManager.useSpatialIndex = true; + } + if (this.sceneManager) { + this.sceneManager.enableFrustumCulling = true; + } + } + + /** + * Handle keyboard shortcuts + * @param {KeyboardEvent} event - Keyboard event + */ + handleKeyPress(event) { + // Pass to debugger if debug mode is on + if (this.debugMode && this.debugger) { + this.debugger.handleKeyPress(event); + } + } + + /** + * Toggle debug visualization mode + */ + toggleDebugMode() { + this.debugMode = !this.debugMode; + + if (this.debugger) { + this.debugger.toggle(this.debugMode); + } + + return this.debugMode; + } + + /** + * Animation loop + */ + animate() { + requestAnimationFrame(() => this.animate()); + + // Update FPS calculation + this.frameCount++; + const currentTime = performance.now(); + const elapsed = currentTime - this.lastTime; + + // Update FPS counter every interval + if (elapsed > this.fpsUpdateInterval) { + this.fps = Math.round((this.frameCount * 1000) / elapsed); + this.fpsCounter.textContent = `FPS: ${this.fps}`; + + // Reset counters + this.frameCount = 0; + this.lastTime = currentTime; + } + + // Update debug visualization if enabled + if (this.debugMode && this.debugger) { + this.debugger.update(); + } + + // Update scene + this.sceneManager.update(); + } + + /** + * Load graph data from the server + */ + loadGraphData() { + this.clearDisplay(); + // Show loading indicator + this.uiManager.showLoading(true); + + // Load data via the data manager + this.dataManager.loadData() + .then(data => { + // Initialize the force simulation with loaded data + this.simulationManager.updateSimulation(false); + + // Update statistics + this.uiManager.updateStats(); + + // Show initial message + this.uiManager.showInitialMessage("Enter a search term to display nodes"); + + // Hide loading indicator + this.uiManager.showLoading(false); + + // Render all nodes + this.loadAllNodes(); + + + console.log(data); + let id = data?.links[0]?.source.id; + if(id) { + this.eventManager.selectNode(id); + this.eventManager.focusOnNode(id); + } + console.log(id); + }) + .catch(error => { + // Show error message + this.uiManager.showError('Failed to load graph data: ' + error.message); + this.uiManager.showLoading(false); + }); + } + + /** + * Search for nodes by term and display them + * @param {string} searchTerm - The term to search for + */ + searchNodes(searchTerm) { + // Clear current display + this.clearDisplay(); + + // Hide the initial message + this.uiManager.hideInitialMessage(); + + if (!searchTerm) return; + + // Find matching nodes + const matchingNodeIds = this.dataManager.searchNodes(searchTerm); + + // If no nodes found, show a message + if (matchingNodeIds.length === 0) { + console.log(`No nodes found matching "${searchTerm}"`); + this.uiManager.showInitialMessage(`No nodes found matching "${searchTerm}"`); + return; + } + + console.log(`Found ${matchingNodeIds.length} nodes matching "${searchTerm}"`); + + // Show loading indicator during simulation + this.uiManager.showLoading(true); + + // Add each matching node and its connections with depth of 10 + const addedNodes = new Set(); + + matchingNodeIds.forEach(nodeId => { + // Use getConnectedSubgraph to get nodes and links up to depth 10 + const { nodes, links } = this.dataManager.getConnectedSubgraph(nodeId, 10); + + // Add all nodes to the scene + nodes.forEach(node => { + if (!this.dataManager.graphObjects.nodes.has(node.id)) { + this.graphObjectManager.createNodeObject(node); + addedNodes.add(node.id); + } + }); + + // Add all links to the scene + links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + const linkId = `${sourceId}-${targetId}`; + + if (!this.dataManager.graphObjects.links.has(linkId)) { + this.graphObjectManager.createLinkObject(link); + } + }); + }); + + // Update simulation and restart it properly + this.simulationManager.updateSimulation(true); + + // Center the view on the found nodes + this.centerOnNodes(Array.from(addedNodes)); + + // Hide loading indicator when view is centered + this.uiManager.showLoading(false); + } + + /** + * Clear the current display + */ + clearDisplay() { + this.graphObjectManager.clearVisibleObjects(); + } + + /** + * Center the view on a set of nodes + * @param {Array} nodeIds - Array of node IDs to center on + */ + centerOnNodes(nodeIds) { + if (!nodeIds || nodeIds.length === 0) return; + + // Calculate the center position of the specified nodes + let center = { x: 0, y: 0, z: 0 }; + let count = 0; + + nodeIds.forEach(nodeId => { + const nodeData = this.dataManager.graphObjects.nodes.get(nodeId); + if (nodeData && nodeData.object) { + center.x += nodeData.object.position.x; + center.y += nodeData.object.position.y; + center.z += nodeData.object.position.z; + count++; + } + }); + + if (count === 0) return; + + center.x /= count; + center.y /= count; + center.z /= count; + + // Set the camera target for perspective camera + this.sceneManager.controls.target.set(center.x, center.y, 0); + + // Position the perspective camera + const distance = 1000; + this.sceneManager.camera.position.set( + center.x, + center.y, + distance + ); + + // Update the camera and controls + this.sceneManager.camera.updateProjectionMatrix(); + this.sceneManager.controls.update(); + } + + /** + * Toggle label visibility + */ + toggleLabels() { + const showLabels = this.graphObjectManager.toggleLabels(); + return showLabels; + } + + /** + * Toggle physics simulation + */ + togglePhysics() { + this.simulationManager.togglePhysics(); + } + + /** + * Reset the view + */ + resetView() { + // Clear current display + this.clearDisplay(); + + // Reset camera + this.sceneManager.resetView(); + + // Show initial message + this.uiManager.showInitialMessage("Enter a search term to display nodes"); + } + + /** + * Load all nodes in the graph + */ + loadAllNodes() { + // Clear current display + this.clearDisplay(); + + // Hide the initial message + this.uiManager.hideInitialMessage(); + + // Show loading indicator + this.uiManager.showLoading(true); + + console.log(`Loading all ${this.dataManager.graphData.nodes.length} nodes`); + + // Store all added node IDs + const addedNodes = new Set(); + + // Add all nodes to the scene + this.dataManager.graphData.nodes.forEach(node => { + if (!this.dataManager.graphObjects.nodes.has(node.id)) { + this.graphObjectManager.createNodeObject(node); + addedNodes.add(node.id); + } + }); + + // Add all links between the visible nodes + this.dataManager.graphData.links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + const linkId = `${sourceId}-${targetId}`; + + // Only add links between nodes that are visible + if (addedNodes.has(sourceId) && addedNodes.has(targetId) && + !this.dataManager.graphObjects.links.has(linkId)) { + this.graphObjectManager.createLinkObject(link); + } + }); + + // Update simulation and restart it + this.simulationManager.updateSimulation(true); + + // Center the view on all nodes + this.centerOnNodes(Array.from(addedNodes)); + + // Hide loading indicator + this.uiManager.showLoading(false); + } +} + +// Initialize the application when the page loads +document.addEventListener('DOMContentLoaded', () => { + new GraphController('graph-container'); +}); \ No newline at end of file diff --git a/src/html/hyperbuddy@1.0/404.html b/src/html/hyperbuddy@1.0/404.html new file mode 100644 index 000000000..1005d1329 --- /dev/null +++ b/src/html/hyperbuddy@1.0/404.html @@ -0,0 +1,306 @@ + + + 404 - Page not found. + + + + + + +
+
+

404.

+

Page cannot be found.

+

+ This hashpath cannot be resolved on this node, yet + ... + +

+
+
+ + + + diff --git a/src/html/hyperbuddy@1.0/500.html b/src/html/hyperbuddy@1.0/500.html new file mode 100644 index 000000000..3b9e3fabf --- /dev/null +++ b/src/html/hyperbuddy@1.0/500.html @@ -0,0 +1,311 @@ + + + 500 - Oops. + + + + + + +
+
+

500.

+

Oops, your hashpath couldn't be resolved right now.

+ 

{{error}}

+
+
+ + + + diff --git a/src/html/hyperbuddy@1.0/console.html b/src/html/hyperbuddy@1.0/console.html new file mode 100644 index 000000000..98d1259c8 --- /dev/null +++ b/src/html/hyperbuddy@1.0/console.html @@ -0,0 +1,1299 @@ + + + + + + + HyperBEAM HTTP Console + + + + + + + + + + + + + + +
+

HyperBEAM Console

+
+
+
+ Connected +
+ + +
+
+ + +
+
+ +
+
+

Type 'help' to see available commands

+

Use arrow keys (↑/↓) to navigate command history

+
+
+ + +
+ HyperBEAM> +
+ +
+
+
+
+ + + + + \ No newline at end of file diff --git a/src/html/hyperbuddy@1.0/dashboard.html b/src/html/hyperbuddy@1.0/dashboard.html new file mode 100755 index 000000000..639371183 --- /dev/null +++ b/src/html/hyperbuddy@1.0/dashboard.html @@ -0,0 +1,179 @@ + + + + + + + + + HyperBEAM + + + +
+
+ +
+

Operator:

+ +
+
+
+
+
+
+ +
+
+
+

Uptime

+
+

+ - +

+ Seconds +
+
+
+

+ AO-Core Executions +

+
+

+ - +

+ Executions +
+
+
+

+ System Load +

+
+

+ - +

+ CPU Average +
+
+
+
+
+

+ Read Requests Handled +

+
+

-

+ Requests +
+
+
+

+ Write Requests Handled +

+
+

-

+ Requests +
+
+
+
+ +
+
+ + + + + +
+ +
+
+
+

Loading...

+
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+
+
+
+
+
+
+

Status:

+
+
+

Live

+
+
+
+
+
+ + + + + diff --git a/src/html/hyperbuddy@1.0/dashboard.js b/src/html/hyperbuddy@1.0/dashboard.js new file mode 100644 index 000000000..350a7db04 --- /dev/null +++ b/src/html/hyperbuddy@1.0/dashboard.js @@ -0,0 +1,69 @@ +import { showTab, copyToClipboard, initTabListeners, SimpleJsonViewer } from '/~hyperbuddy@1.0/utils.js'; +import { fetchInfo, addConsole, addGraph } from '/~hyperbuddy@1.0/devices.js'; +import { startFetchingMetrics } from '/~hyperbuddy@1.0/metrics.js'; + +/** + * Fetch and display ledger data + */ +async function loadLedger() { + const ledgerSection = document.getElementById('ledger-section'); + + try { + ledgerSection.innerHTML = '

Loading ledger data...

'; + + const response = await fetch(`${window.location.origin}/ledger~node-process@1.0/now/balance/serialize~json@1.0`); + const data = await response.json(); + + ledgerSection.innerHTML = ''; + + const viewer = new SimpleJsonViewer({ + container: ledgerSection, + data: data, + theme: 'dark', + expand: true + }); + + } catch (error) { + // Remove the ledger tab button and content if the request fails + const ledgerTabButton = document.querySelector('.tab-button[data-tab="ledger-tab"]'); + const ledgerTabContent = document.getElementById('ledger-tab'); + + if (ledgerTabButton) { + ledgerTabButton.remove(); + } + if (ledgerTabContent) { + ledgerTabContent.remove(); + } + + console.error('Failed to load ledger data:', error); + } +} + +/** + * Initialize the application + */ +function init() { + // Set up global event handlers + window.copyToClipboard = copyToClipboard; + + // Initialize tab switching + initTabListeners(); + + // Fetch initial data + fetchInfo(); + + // Start fetching metrics periodically + startFetchingMetrics(); + + // Add console iframe + addConsole(); + + // Add graph iframe + addGraph(); + + // Load ledger data + loadLedger(); +} + +// Initialize when DOM is fully loaded +document.addEventListener('DOMContentLoaded', init); \ No newline at end of file diff --git a/src/html/hyperbuddy@1.0/devices.js b/src/html/hyperbuddy@1.0/devices.js new file mode 100644 index 000000000..a844af4d2 --- /dev/null +++ b/src/html/hyperbuddy@1.0/devices.js @@ -0,0 +1,272 @@ +import { get, copyToClipboard } from '/~hyperbuddy@1.0/utils.js'; + +// Set up global message handler for console communications +window.addEventListener('message', (event) => { + if (event.data && event.data.action) { + if (event.data.action === 'expandConsole') { + expandConsole(); + } else if (event.data.action === 'exitExpandedConsole') { + exitExpandedConsole(); + } + } +}); + +/** + * Parse info data from the server response + * @param {string} text - The raw info text + * @returns {Array} - The parsed info objects + */ +function parseInfo(text) { + const lines = text + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + + let boundary = ""; + for (const line of lines) { + if (line.startsWith("--")) { + boundary = line; + break; + } + } + if (!boundary) return []; + + const parts = text.split(boundary); + const results = []; + + parts.forEach((part) => { + part = part.trim(); + if (!part || part === "--") return; + const sections = part.split("\r\n\r\n"); + if (sections.length < 2) return; + const header = sections[0].trim(); + const content = sections + .slice(1) + .join("\r\n\r\n") + .trim() + .replaceAll(`"`, ""); + if ( + content.toLowerCase().startsWith("atom") || + !content.toLowerCase().startsWith("dev") + ) + return; + const nameMatch = header.match(/name="([^"]+)"/); + if (!nameMatch) return; + const module = nameMatch[1]; + results.push({ + name: content.replaceAll(`"`, `''`), + content: module, + }); + }); + + return results; +} + +/** + * Render the info groups in the UI + * @param {Array} groups - The info groups + * @param {string} devicesStr - The devices string in JSON format + */ +function renderInfoGroups(groups, devicesStr) { + const devices = JSON.parse(devicesStr); + const container = document.getElementById("info-section-lines"); + + if (container) { + + // Clear previous content + container.innerHTML = ""; + + // Make sure it has the right class + container.className = "device-cards-container"; + + for (const [key, device] of Object.entries(devices)) { + if (key == "device") continue; + const [name, variant] = device.name.split("@"); + + // Create a card for each device + const card = document.createElement("div"); + card.classList.add("device-card"); + + // Create device name element + const deviceName = document.createElement("div"); + deviceName.classList.add("device-name"); + deviceName.textContent = name; + + // Create variant element with color based on version number + const deviceVariant = document.createElement("div"); + deviceVariant.classList.add("device-variant"); + + // Add color class based on version number + const versionNum = parseFloat(variant); + if (versionNum >= 1.0) { + deviceVariant.classList.add("device-variant-high"); + } else if (versionNum >= 0.5) { + deviceVariant.classList.add("device-variant-medium"); + } else { + deviceVariant.classList.add("device-variant-low"); + } + + deviceVariant.textContent = variant; + + // Add elements to card + card.appendChild(deviceName); + card.appendChild(deviceVariant); + + // Add card to container + container.appendChild(card); + } + } +} + +async function fetchInfo() { + try { + const operatorAddress = await get("/~meta@1.0/info/address"); + const operatorAction = document.getElementById("operator-action"); + + if (operatorAction) { + operatorAction.innerHTML = formatAddress(operatorAddress); + operatorAction.value = operatorAddress; + operatorAction.disabled = false; + + operatorAction.addEventListener("click", function () { + copyToClipboard(this); + }); + } + + const info = await get("/~meta@1.0/info"); + const deviceInfo = + await get("/~meta@1.0/info/preloaded_devices/serialize~json@1.0"); + const infoGroups = parseInfo(info); + renderInfoGroups(infoGroups, deviceInfo); + } catch (error) { + console.error("Error fetching or parsing info:", error); + } +} + +/** + * Format an address for display + * @param {string} address - The address to format + * @returns {string} - The formatted address + */ +function formatAddress(address) { + if (!address) return ""; + return address.substring(0, 6) + "..." + address.substring(36, address.length); +} + +/** + * Add the console iframe to the UI + */ +function addConsole() { + const container = document.getElementById("console-section"); + const consoleContainer = document.createElement("iframe"); + consoleContainer.style.width = "100%"; + consoleContainer.style.minHeight = "500px"; + consoleContainer.style.border = "none"; + consoleContainer.src = "/~hyperbuddy@1.0/console"; + consoleContainer.id = "console-iframe"; + container.appendChild(consoleContainer); +} + +function addGraph() { + const container = document.getElementById("cache-section"); + const cacheContainer = document.createElement("iframe"); + cacheContainer.style.width = "100%"; + cacheContainer.style.minHeight = "500px"; + cacheContainer.style.border = "none"; + cacheContainer.src = "/~hyperbuddy@1.0/graph"; + cacheContainer.id = "cache-iframe"; + container.appendChild(cacheContainer); +} + +/** + * Expand the console to take over the entire screen + */ +function expandConsole() { + const consoleIframe = document.getElementById("console-iframe"); + if (!consoleIframe) { + return; + } + + // Save original position and size + if (!consoleIframe.dataset.originalStyles) { + consoleIframe.dataset.originalStyles = JSON.stringify({ + width: consoleIframe.style.width, + height: consoleIframe.style.height, + minHeight: consoleIframe.style.minHeight, + position: consoleIframe.style.position, + top: consoleIframe.style.top, + left: consoleIframe.style.left, + zIndex: consoleIframe.style.zIndex + }); + } + + // Apply fullscreen styles directly to the iframe + consoleIframe.style.position = "fixed"; + consoleIframe.style.top = "0"; + consoleIframe.style.left = "0"; + consoleIframe.style.width = "100%"; + consoleIframe.style.height = "100%"; + consoleIframe.style.minHeight = "100%"; + consoleIframe.style.zIndex = "9999"; + + // Notify the iframe that it's now expanded + setTimeout(() => { + if (consoleIframe.contentWindow) { + consoleIframe.contentWindow.postMessage({ + consoleState: 'expanded' + }, '*'); + } + }, 100); // Short delay to ensure iframe is ready +} + +/** + * Exit the expanded console view and return to normal layout + */ +function exitExpandedConsole() { + const consoleIframe = document.getElementById("console-iframe"); + + if (!consoleIframe) { + return; + } + + // Restore original position and size + if (consoleIframe.dataset.originalStyles) { + const originalStyles = JSON.parse(consoleIframe.dataset.originalStyles); + consoleIframe.style.position = originalStyles.position; + consoleIframe.style.top = originalStyles.top; + consoleIframe.style.left = originalStyles.left; + consoleIframe.style.width = originalStyles.width; + consoleIframe.style.height = originalStyles.height; + consoleIframe.style.minHeight = originalStyles.minHeight || "500px"; + consoleIframe.style.zIndex = originalStyles.zIndex; + } else { + // Fallback if original styles weren't saved + consoleIframe.style.position = ""; + consoleIframe.style.top = ""; + consoleIframe.style.left = ""; + consoleIframe.style.width = "100%"; + consoleIframe.style.height = ""; + consoleIframe.style.minHeight = "500px"; + consoleIframe.style.zIndex = ""; + } + + // Notify the iframe that it's now in normal mode + setTimeout(() => { + if (consoleIframe.contentWindow) { + consoleIframe.contentWindow.postMessage({ + consoleState: 'normal' + }, '*'); + } + }, 100); // Short delay to ensure iframe is ready +} + +// Export functions for use in other modules +export { + parseInfo, + renderInfoGroups, + fetchInfo, + addConsole, + addGraph, + expandConsole, + exitExpandedConsole +}; \ No newline at end of file diff --git a/src/html/hyperbuddy@1.0/graph.html b/src/html/hyperbuddy@1.0/graph.html new file mode 100644 index 000000000..079054e58 --- /dev/null +++ b/src/html/hyperbuddy@1.0/graph.html @@ -0,0 +1,321 @@ + + + + + + HyperBEAM Cache Graph + + + + + + + +
+
+ HyperBEAM Cache Graph +
+ +
+ + + + +
+ + +
+ +
+ + +
+
+ Nodes: + 0 +
+
+ Links: + 0 +
+
+
+ Simple +
+
+
+ Composite +
+
+
+ +
+
+
Loading graph data...
+
FPS: 0
+ + + + + +
+ + + + + + + + + + + \ No newline at end of file diff --git a/src/html/hyperbuddy@1.0/graph.js b/src/html/hyperbuddy@1.0/graph.js new file mode 100644 index 000000000..2500e795c --- /dev/null +++ b/src/html/hyperbuddy@1.0/graph.js @@ -0,0 +1,3751 @@ +/** + * HyperBEAM Cache Graph Renderer - Modular Version + * A 2D force-directed graph visualization for the HyperBEAM cache system + */ + +/** + * Utility function to create a circular texture for node rendering + * @param {number} size - Size of the texture in pixels + * @param {number|string} color - Color of the circle (hex) + * @param {boolean} border - Whether to add a border + * @param {number|string} borderColor - Color of the border (hex) + * @returns {THREE.Texture} The generated texture + */ +function createCircleTexture(size = 64, color = 0xffffff, border = false, borderColor = 0x000000) { + // Create a canvas element + const canvas = document.createElement('canvas'); + canvas.width = size; + canvas.height = size; + const context = canvas.getContext('2d'); + + // Clear canvas with transparent background + context.clearRect(0, 0, size, size); + + // Convert color to string format if it's a number + const fillColor = typeof color === 'number' ? '#' + color.toString(16).padStart(6, '0') : color; + const strokeColor = typeof borderColor === 'number' ? '#' + borderColor.toString(16).padStart(6, '0') : borderColor; + + // Draw a circle + const radius = size / 2 - 2; + context.beginPath(); + context.arc(size / 2, size / 2, radius, 0, 2 * Math.PI, false); + context.fillStyle = fillColor; + context.fill(); + + // Add border if requested + if (border) { + context.lineWidth = 1; + context.strokeStyle = strokeColor; + context.stroke(); + } + + // Create a texture from the canvas + const texture = new THREE.CanvasTexture(canvas); + texture.needsUpdate = true; + + return texture; +} + +/** + * ThemeManager - Handles configuration and visual styling + */ +class ThemeManager { + constructor() { + this.config = { + // Node styling + nodeSize: { + simple: 6, + composite: 8 + }, + // Color scheme + colors: { + background: 0xf9f9f9, + simpleNode: 0x6495ED, // Light blue + compositeNode: 0xF08080, // Light coral + highlight: 0xFFA500, // Orange for highlighting + selectedNode: 0xFF5500, // Orange-red for selected node + neighborNode: 0x4CAF50, // Green for neighbor nodes + link: 0xcccccc, // Light gray for links + activeLink: 0x333333, // Dark gray for active links + hover: 0xfafa33 // Warm orange/yellow for hover + }, + // Display options + showLabels: true, + physicsEnabled: true, + // Physics settings + defaultDistance: 150, + highConnectionThreshold: 10, + // Camera settings + zoomLevel: { + default: 1.0, + focused: 2.5 + }, + // Z-positions for layering + zPos: { + line: 0, + node: 5, + label: 10 + } + }; + } + + /** + * Get the color for a node based on its type and state + * @param {string} nodeType - The type of node ('simple' or 'composite') + * @param {string} state - The state of the node ('default', 'selected', 'neighbor', 'hover') + * @returns {number} The color as a hex number + */ + getNodeColor(nodeType, state = 'default') { + switch(state) { + case 'selected': + return this.config.colors.selectedNode; + case 'neighbor': + return this.config.colors.neighborNode; + case 'hover': + return this.config.colors.hover; + default: + return nodeType === 'simple' ? + this.config.colors.simpleNode : + this.config.colors.compositeNode; + } + } + + /** + * Get the color for a link based on its state + * @param {string} state - The state of the link ('default' or 'active') + * @returns {number} The color as a hex number + */ + getLinkColor(state = 'default') { + return state === 'active' ? + this.config.colors.activeLink : + this.config.colors.link; + } + + /** + * Get the size for a node based on its type + * @param {string} nodeType - The type of node ('simple' or 'composite') + * @returns {number} The node size + */ + getNodeSize(nodeType) { + return nodeType === 'simple' ? + this.config.nodeSize.simple : + this.config.nodeSize.composite; + } + + /** + * Toggle label visibility + * @returns {boolean} The new label visibility state + */ + toggleLabels() { + this.config.showLabels = !this.config.showLabels; + return this.config.showLabels; + } + + /** + * Toggle physics simulation + * @returns {boolean} The new physics enabled state + */ + togglePhysics() { + this.config.physicsEnabled = !this.config.physicsEnabled; + return this.config.physicsEnabled; + } +} + +/** + * SceneManager - Handles Three.js scene, camera, and rendering + */ +class SceneManager { + constructor(container, themeManager) { + this.container = container; + this.themeManager = themeManager; + + // Three.js components + this.scene = null; + this.camera = null; + this.renderer = null; + this.controls = null; + this.raycaster = new THREE.Raycaster(); + this.mouse = new THREE.Vector2(); + + // Performance optimization + this.frustum = new THREE.Frustum(); + this.projScreenMatrix = new THREE.Matrix4(); + this.tmpVector = new THREE.Vector3(); + this.enableFrustumCulling = true; + this.frustumCullingDistance = 1250; // Beyond this distance, apply visibility culling + + // Initialize the scene + this.initScene(); + } + + /** + * Initialize the Three.js scene and renderer + */ + initScene() { + const width = this.container.clientWidth; + const height = this.container.clientHeight; + + // Create scene with background color + this.scene = new THREE.Scene(); + this.scene.background = new THREE.Color(this.themeManager.config.colors.background); + + // Create perspective camera with large clipping plane to prevent culling + const aspectRatio = width / height; + this.camera = new THREE.PerspectiveCamera( + 40, // Narrower field of view for less distortion + aspectRatio, + 0.1, + 15000 // Increased far clipping plane + ); + this.camera.position.z = 1000; + + // Create renderer + this.renderer = new THREE.WebGLRenderer({ + antialias: true, + alpha: true + }); + this.renderer.setSize(width, height); + this.renderer.setClearColor(this.themeManager.config.colors.background, 1); + this.renderer.sortObjects = true; // Enable sorting for proper z-ordering + this.container.appendChild(this.renderer.domElement); + + // Configure raycaster for better point detection + this.raycaster = new THREE.Raycaster(); + this.raycaster.params.Points.threshold = 10; // Increase threshold for easier point selection + + // Add orbit controls limited to 2D movement with perspective camera + this.controls = new THREE.OrbitControls(this.camera, this.renderer.domElement); + this.controls.enableDamping = true; + this.controls.dampingFactor = 0.1; + this.controls.enableRotate = false; // Disable 3D rotation + this.controls.screenSpacePanning = true; + + // Set zoom limits - constrain camera between 750 and 15000 on z-axis + this.controls.minDistance = 750; + this.controls.maxDistance = 15000; + + // Handle window resize + window.addEventListener('resize', () => this.onWindowResize()); + } + + /** + * Handle window resize events + */ + onWindowResize() { + const width = this.container.clientWidth; + const height = this.container.clientHeight; + + // Update perspective camera aspect ratio + this.camera.aspect = width / height; + this.camera.updateProjectionMatrix(); + + // Update renderer + this.renderer.setSize(width, height); + } + + /** + * Reset the camera view + */ + resetView() { + // Reset camera position for perspective camera + this.camera.position.set(0, 0, 1000); + this.controls.target.set(0, 0, 0); + this.camera.updateProjectionMatrix(); + this.controls.update(); + } + + /** + * Focus camera on a specific position with smooth animation + * @param {THREE.Vector3} position - The position to focus on + */ + focusCamera(position) { + const duration = 500; // milliseconds + const startTime = Date.now(); + + // Save starting values + const startPosition = this.camera.position.clone(); + const startTarget = this.controls.target.clone(); + + // Define a fixed Z-offset for viewing the target + const zOffset = 600; + + // Animation function + const animateCamera = () => { + const elapsed = Date.now() - startTime; + const progress = Math.min(elapsed / duration, 1); + + // Ease function - ease out cubic + const easeProgress = 1 - Math.pow(1 - progress, 3); + + // Only update the target x and y position, keeping rotation consistent + this.controls.target.x = startTarget.x + (position.x - startTarget.x) * easeProgress; + this.controls.target.y = startTarget.y + (position.y - startTarget.y) * easeProgress; + // Keep z at the same value to maintain default camera angle + + // Move camera x and y to match target + this.camera.position.x = startPosition.x + (position.x - startPosition.x) * easeProgress; + this.camera.position.y = startPosition.y + (position.y - startPosition.y) * easeProgress; + + // Adjust Z with fixed offset + const targetZ = position.z + zOffset; + this.camera.position.z = startPosition.z + (targetZ - startPosition.z) * easeProgress; + + this.camera.updateProjectionMatrix(); + this.controls.update(); + + if (progress < 1) { + requestAnimationFrame(animateCamera); + } + }; + + animateCamera(); + } + + /** + * Update the mouse position for raycasting + * @param {MouseEvent} event - The mouse event + */ + updateMousePosition(event) { + const rect = this.renderer.domElement.getBoundingClientRect(); + this.mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1; + this.mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1; + } + + /** + * Get objects intersecting with the current mouse position + * @returns {Array} Array of intersected objects + */ + getIntersectedObjects() { + this.raycaster.setFromCamera(this.mouse, this.camera); + return this.raycaster.intersectObjects(this.scene.children, true); + } + + /** + * Update the scene (called in animation loop) + */ + update() { + // Update controls + if (this.controls) { + this.controls.update(); + + // Enforce camera z-position limits + if (this.camera.position.z < 750) { + this.camera.position.z = 750; + } else if (this.camera.position.z > 15000) { + this.camera.position.z = 15000; + } + } + + // Apply frustum culling for distant objects + if (this.enableFrustumCulling) { + this.updateFrustumCulling(); + } + + // Render the scene + this.renderer.render(this.scene, this.camera); + } + + /** + * Update frustum and apply visibility culling for better performance + */ + updateFrustumCulling() { + // Update the frustum + this.projScreenMatrix.multiplyMatrices( + this.camera.projectionMatrix, + this.camera.matrixWorldInverse + ); + this.frustum.setFromProjectionMatrix(this.projScreenMatrix); + + // If we have a reference to the controller, access the dataManager + if (this.graphController && this.graphController.dataManager) { + const dataManager = this.graphController.dataManager; + + // Process all nodes + dataManager.graphObjects.nodes.forEach(node => { + if (!node.object) return; + + // Get distance from camera + this.tmpVector.copy(node.object.position); + const distance = this.tmpVector.distanceTo(this.camera.position); + + // If the node is beyond our threshold, check if it's in the frustum + if (distance > this.frustumCullingDistance) { + // Check if the node is in the frustum + const isVisible = this.frustum.containsPoint(node.object.position); + + // Only update visibility if necessary to avoid unnecessary matrix updates + if (node.object.visible !== isVisible) { + node.object.visible = isVisible; + + // Also update label visibility if it exists + if (node.labelObject) { + node.labelObject.visible = isVisible && this.themeManager.config.showLabels; + } + } + } else if (!node.object.visible) { + // If node is within threshold distance but not visible, make visible + node.object.visible = true; + if (node.labelObject) { + node.labelObject.visible = this.themeManager.config.showLabels; + } + } + }); + + // Optional: Process links for better culling + // Only show links if both endpoints are visible + dataManager.graphObjects.links.forEach(link => { + if (!link.line) return; + + const sourceNode = dataManager.graphObjects.nodes.get(link.sourceId); + const targetNode = dataManager.graphObjects.nodes.get(link.targetId); + + if (sourceNode && targetNode && sourceNode.object && targetNode.object) { + const sourceDist = sourceNode.object.position.distanceTo(this.camera.position); + const targetDist = targetNode.object.position.distanceTo(this.camera.position); + + // If both nodes are distant, check if they're visible + if (sourceDist > this.frustumCullingDistance && targetDist > this.frustumCullingDistance) { + const sourceVisible = sourceNode.object.visible; + const targetVisible = targetNode.object.visible; + + // Only show link if both endpoints are visible + link.line.visible = sourceVisible && targetVisible; + + // Update label visibility if needed + if (link.labelObject) { + link.labelObject.visible = sourceVisible && targetVisible && + this.themeManager.config.showLabels && + dataManager.activeLinks.has(`${link.sourceId}-${link.targetId}`); + } + } else if (!link.line.visible) { + // If at least one endpoint is close, show the link + link.line.visible = true; + } + } + }); + } + } + + /** + * Add an object to the scene + * @param {THREE.Object3D} object - The object to add + */ + addToScene(object) { + this.scene.add(object); + } + + /** + * Remove an object from the scene + * @param {THREE.Object3D} object - The object to remove + */ + removeFromScene(object) { + this.scene.remove(object); + } +} + +/** + * DataManager - Handles graph data loading and processing + */ +class DataManager { + constructor() { + // Graph data + this.graphData = { nodes: [], links: [] }; + this.graphObjects = { nodes: new Map(), links: new Map() }; + + // State tracking + this.selectedNode = null; + this.neighborNodes = new Set(); + this.activeLinks = new Set(); + this.hoveredNode = null; + } + + /** + * Load graph data from the server + * @returns {Promise} Promise that resolves when data is loaded + */ + loadData() { + return new Promise((resolve, reject) => { + fetch('/~hyperbuddy@1.0/graph-data') + .then(response => { + if (!response.ok) { + throw new Error(`HTTP error ${response.status}`); + } + return response.json(); + }) + .then(data => { + // Clear existing data + this.clearData(); + + // Validate data + if (!this.validateData(data)) { + reject(new Error('Invalid data format')); + return; + } + + this.graphData = data; + resolve(data); + }) + .catch(error => { + console.error('Error loading graph data:', error); + reject(error); + }); + }); + } + + /** + * Validate the graph data structure + * @param {Object} data - The data to validate + * @returns {boolean} Whether the data is valid + */ + validateData(data) { + // Check if we have valid data + if (!data || !data.nodes || !data.links || + !Array.isArray(data.nodes) || !Array.isArray(data.links)) { + return false; + } + + // Check if we have any nodes + if (data.nodes.length === 0) { + return false; + } + + return true; + } + + /** + * Clear all graph data + */ + clearData() { + this.graphData = { nodes: [], links: [] }; + this.graphObjects.nodes.clear(); + this.graphObjects.links.clear(); + + // Reset state + this.selectedNode = null; + this.hoveredNode = null; + this.neighborNodes.clear(); + this.activeLinks.clear(); + } + + /** + * Determine node type based on ID pattern + * @param {string} nodeId - The node ID + * @returns {string} The node type ('simple' or 'composite') + */ + determineNodeType(nodeId) { + const pathParts = nodeId.split('/').filter(p => p.length > 0); + return (pathParts.length <= 1 && !nodeId.endsWith('/')) ? 'simple' : 'composite'; + } + + /** + * Search for nodes matching a term + * @param {string} searchTerm - The term to search for + * @returns {Array} Array of matching node IDs + */ + searchNodes(searchTerm) { + if (!searchTerm) return []; + + const searchLower = searchTerm.toLowerCase(); + + // Find matching nodes + return this.graphData.nodes + .filter(node => + (node.id && node.id.toLowerCase().includes(searchLower)) || + (node.label && node.label.toLowerCase().includes(searchLower)) + ) + .map(node => node.id); + } + + /** + * Get nodes connected to a starting node up to a specified depth + * @param {string} startNodeId - The ID of the starting node + * @param {number} maxDepth - Maximum depth/distance to traverse + * @returns {Object} Object containing connected nodes and links + */ + getConnectedSubgraph(startNodeId, maxDepth = 1) { + const connectedNodes = new Map(); + const connectedLinks = new Set(); + const queue = [{id: startNodeId, depth: 0}]; + const visited = new Set([startNodeId]); + + // First make sure we have the start node + const startNode = this.graphData.nodes.find(n => n.id === startNodeId); + if (!startNode) return {nodes: [], links: []}; + + connectedNodes.set(startNodeId, startNode); + + // BFS to find connected nodes up to maxDepth + while (queue.length > 0) { + const {id, depth} = queue.shift(); + + if (depth >= maxDepth) continue; + + // Find all links connected to this node + this.graphData.links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + if (sourceId === id || targetId === id) { + const linkId = `${sourceId}-${targetId}`; + + // If we've already processed this link, skip it + if (connectedLinks.has(linkId)) return; + + connectedLinks.add(linkId); + + // Get the ID of the node on the other end of the link + const otherId = sourceId === id ? targetId : sourceId; + + // If we haven't visited this node yet, add it to the queue + if (!visited.has(otherId)) { + visited.add(otherId); + const otherNode = this.graphData.nodes.find(n => n.id === otherId); + if (otherNode) { + connectedNodes.set(otherId, otherNode); + queue.push({id: otherId, depth: depth + 1}); + } + } + } + }); + } + + return { + nodes: Array.from(connectedNodes.values()), + links: this.graphData.links.filter(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + return connectedNodes.has(sourceId) && connectedNodes.has(targetId); + }) + }; + } + + /** + * Store a node object reference + * @param {string} nodeId - The node ID + * @param {Object} nodeData - The node data + */ + storeNodeObject(nodeId, nodeData) { + this.graphObjects.nodes.set(nodeId, nodeData); + } + + /** + * Store a link object reference + * @param {string} linkId - The link ID (format: "sourceId-targetId") + * @param {Object} linkData - The link data + */ + storeLinkObject(linkId, linkData) { + this.graphObjects.links.set(linkId, linkData); + } + + /** + * Get links connected to a node + * @param {string} nodeId - The node ID + * @returns {Array} Array of connected links + */ + getConnectedLinks(nodeId) { + return this.graphData.links.filter(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + return sourceId === nodeId || targetId === nodeId; + }); + } + + /** + * Track selected node + * @param {string} nodeId - The selected node ID + */ + setSelectedNode(nodeId) { + this.selectedNode = nodeId; + + // Find and track connected nodes and links + if (nodeId) { + // Clear previous + this.neighborNodes.clear(); + this.activeLinks.clear(); + + // Find connected links + this.graphData.links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + if (sourceId === nodeId || targetId === nodeId) { + // This is a connected link + const otherNodeId = sourceId === nodeId ? targetId : sourceId; + this.neighborNodes.add(otherNodeId); + + // Track active link + const linkKey = `${sourceId}-${targetId}`; + this.activeLinks.add(linkKey); + } + }); + } + } + + /** + * Clear selected node + */ + clearSelectedNode() { + this.selectedNode = null; + this.neighborNodes.clear(); + this.activeLinks.clear(); + } + + /** + * Track hovered node + * @param {string} nodeId - The hovered node ID + */ + setHoveredNode(nodeId) { + this.hoveredNode = nodeId; + } + + /** + * Clear hovered node + */ + clearHoveredNode() { + this.hoveredNode = null; + } +} + +/** + * GraphObjectManager - Creates and manages visual objects for nodes and links + */ +class GraphObjectManager { + constructor(sceneManager, dataManager, themeManager) { + this.sceneManager = sceneManager; + this.dataManager = dataManager; + this.themeManager = themeManager; + } + + /** + * Create a visual node object + * @param {Object} node - The node data + * @returns {Object} The created node object with visual elements + */ + createNodeObject(node) { + // Determine node type if not set + if (!node.type) { + node.type = this.dataManager.determineNodeType(node.id); + } + + // Add to NodeCloud for efficient rendering + if (this.graphController && this.graphController.nodeCloud) { + // Add to node cloud + const nodeIndex = this.graphController.nodeCloud.addNode(node); + + // Create a virtual object for compatibility + // This is needed because other code expects a THREE.Object3D + const virtualObject = { + position: new THREE.Vector3(node.x || 0, node.y || 0, this.themeManager.config.zPos.node), + visible: true, + userData: { id: node.id, type: node.type, label: node.label } + }; + + // Store virtual object reference + node.object = virtualObject; + node.nodeCloudIndex = nodeIndex; + } + + // Create label if enabled + let labelObject = null; + if (this.themeManager.config.showLabels) { + labelObject = this.createLabel(node); + } + + // Store label reference + node.labelObject = labelObject; + + // Store in dataManager + this.dataManager.storeNodeObject(node.id, node); + + // Add to spatial grid if simulation manager is available + if (this.graphController && this.graphController.simulationManager) { + this.graphController.simulationManager.addNodeToSpatialGrid(node); + } + + return node; + } + + /** + * Create a visual link object + * @param {Object} link - The link data + * @returns {Object} The created link object with visual elements + */ + createLinkObject(link) { + // Get node IDs + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + // Get node objects + const sourceNode = this.dataManager.graphObjects.nodes.get(sourceId); + const targetNode = this.dataManager.graphObjects.nodes.get(targetId); + + if (!sourceNode || !targetNode) { + return null; + } + + // Create line geometry + const points = [ + new THREE.Vector3(sourceNode.x, sourceNode.y, this.themeManager.config.zPos.line), + new THREE.Vector3(targetNode.x, targetNode.y, this.themeManager.config.zPos.line) + ]; + + // Create material and line + const material = new THREE.LineDashedMaterial({ + color: this.themeManager.getLinkColor(), + dashSize: 2.5, + gapSize: 1.5, + transparent: true, + opacity: 0.6, + linewidth: 1, + depthWrite: false + }); + + const geometry = new THREE.BufferGeometry().setFromPoints(points); + const line = new THREE.Line(geometry, material); + + // Important: Disable frustum culling to ensure lines are always visible + line.frustumCulled = false; + line.renderOrder = 0; // Ensure lines render before nodes + + this.sceneManager.addToScene(line); + + // Store connection info + link.sourceId = sourceId; + link.targetId = targetId; + link.line = line; + + // Create label if needed + let labelObject = null; + if (this.themeManager.config.showLabels && link.label) { + labelObject = this.createLinkLabel(link, sourceNode, targetNode); + } + + link.labelObject = labelObject; + + // Store reference + const linkId = `${sourceId}-${targetId}`; + this.dataManager.storeLinkObject(linkId, link); + + return link; + } + + /** + * Truncate text and add ellipsis in the middle + * @param {string} text - The text to truncate + * @returns {string} Truncated text with ellipsis + */ + truncateWithEllipsis(text) { + // Show only if longer than 15 characters (6 + 3 + 6) + if (!text || text.length <= 15) { + return text; + } + + // Take exactly 6 chars from start and 6 from end + return text.substring(0, 6) + '...' + text.substring(text.length - 6); + } + + /** + * Create a label for a node + * @param {Object} node - The node data + * @returns {Object} The created label object + */ + createLabel(node) { + // Get display text and truncate it if needed + const displayText = this.truncateWithEllipsis(node.label || node.id); + const label = new SpriteText(displayText); + + // Improved text rendering settings + label.fontFace = 'Arial, Helvetica, sans-serif'; + label.fontSize = 32; + label.fontWeight = '600'; + label.strokeWidth = 0; // No stroke for sharper text + label.color = '#000000'; + label.backgroundColor = 'rgba(255,255,255,0.95)'; + label.padding = 3; + label.textHeight = 5; // Increased for better resolution with larger text + label.borderWidth = 0; // No border for sharper edges + + // Position above node with pixel-perfect positioning + const offset_val = 100; + const isSimple = node.type === 'simple'; + const offset = isSimple ? + this.themeManager.config.nodeSize.simple + offset_val : + this.themeManager.config.nodeSize.composite + offset_val; + + // Round to whole pixels to avoid subpixel rendering + const x = Math.round(node.x || 0); + const y = Math.round((node.y || 0) + offset); + const z = this.themeManager.config.zPos.label; + + label.position.set(x, y, z); + label.renderOrder = 20; + + this.sceneManager.addToScene(label); + + return label; + } + + /** + * Create a label for a link + * @param {Object} link - The link data + * @param {Object} sourceNode - The source node + * @param {Object} targetNode - The target node + * @returns {Object} The created label object + */ + createLinkLabel(link, sourceNode, targetNode) { + const midPoint = new THREE.Vector3( + (sourceNode.x + targetNode.x) / 2, + (sourceNode.y + targetNode.y) / 2, + this.themeManager.config.zPos.label + ); + + let label_text = link.label; + if (link.label.length == 43) { + label_text = this.truncateWithEllipsis(link.label); + } + + const label = new SpriteText(label_text); + + // Improved text rendering settings + label.fontFace = 'Arial, Helvetica, sans-serif'; + label.fontSize = 32; + label.fontWeight = '600'; + label.strokeWidth = 0; // No stroke for sharper text + label.color = '#000000'; + label.backgroundColor = 'rgba(255,255,255,0.95)'; + label.padding = 3; + label.textHeight = 4; // Better resolution for link labels + label.borderWidth = 0; // No border for sharper edges + + // Round to whole pixels to avoid subpixel rendering + midPoint.x = Math.round(midPoint.x); + midPoint.y = Math.round(midPoint.y); + + label.position.copy(midPoint); + label.renderOrder = 20; + + // Hide link labels by default - only show when node is selected + label.visible = false; + + this.sceneManager.addToScene(label); + + return label; + } + + /** + * Update the position of a node object + * @param {Object} node - The node object to update + */ + updateNodePosition(node) { + if (!node) return; + + if (this.graphController && this.graphController.nodeCloud) { + // Update position in the NodeCloud + this.graphController.nodeCloud.updateNodePosition(node.id, node.x || 0, node.y || 0); + + // Also update the virtual object for compatibility + if (node.object && node.object.position) { + node.object.position.x = node.x || 0; + node.object.position.y = node.y || 0; + } + } + + // Update label position if it exists + if (node.labelObject) { + const offset_val = 6; // Same value as in createLabel + const isSimple = node.type === 'simple'; + const offset = isSimple ? + this.themeManager.config.nodeSize.simple + offset_val : + this.themeManager.config.nodeSize.composite + offset_val; + + node.labelObject.position.x = node.x || 0; + node.labelObject.position.y = (node.y || 0) + offset; + } + } + + /** + * Update the position of a link object + * @param {Object} link - The link object to update + */ + updateLinkPosition(link) { + if (!link.line) return; + + const sourceId = link.sourceId || (typeof link.source === 'object' ? link.source.id : link.source); + const targetId = link.targetId || (typeof link.target === 'object' ? link.target.id : link.target); + + const sourceNode = this.dataManager.graphObjects.nodes.get(sourceId); + const targetNode = this.dataManager.graphObjects.nodes.get(targetId); + + if (sourceNode && targetNode) { + // Update the link line geometry + const points = [ + new THREE.Vector3(sourceNode.x || 0, sourceNode.y || 0, this.themeManager.config.zPos.line), + new THREE.Vector3(targetNode.x || 0, targetNode.y || 0, this.themeManager.config.zPos.line) + ]; + + // Update the line geometry + link.line.geometry.setFromPoints(points); + + // Ensure frustum culling remains disabled after updates + link.line.frustumCulled = false; + + // Update the link label position if it exists + if (link.labelObject) { + const midPoint = new THREE.Vector3( + (sourceNode.x + targetNode.x) / 2, + (sourceNode.y + targetNode.y) / 2, + this.themeManager.config.zPos.label + ); + link.labelObject.position.copy(midPoint); + } + } + } + + /** + * Update node colors based on selection and hover state + * @param {string} nodeId - The ID of the node to update + */ + updateNodeColors(nodeId) { + const node = this.dataManager.graphObjects.nodes.get(nodeId); + if (!node) return; + + let state = 'default'; + + // Determine state based on selection and hover + if (nodeId === this.dataManager.selectedNode) { + state = 'selected'; + } else if (this.dataManager.neighborNodes.has(nodeId)) { + state = 'neighbor'; + } else if (nodeId === this.dataManager.hoveredNode) { + state = 'hover'; + } + + if (this.graphController && this.graphController.nodeCloud) { + // Update the color in the node cloud + this.graphController.nodeCloud.updateNodeColor(nodeId, node.type, state); + } + } + + /** + * Update link colors based on selection state + * @param {string} linkId - The ID of the link to update (format: "sourceId-targetId") + */ + updateLinkColors(linkId) { + const link = this.dataManager.graphObjects.links.get(linkId); + if (!link || !link.line) return; + + // Determine if this is an active link + const isActive = this.dataManager.activeLinks.has(linkId); + + // Set color and opacity based on active state + link.line.material.color.set(this.themeManager.getLinkColor(isActive ? 'active' : 'default')); + link.line.material.opacity = isActive ? 1.0 : 0.6; + } + + /** + * Remove a node and its visual elements from the scene + * @param {string} nodeId - The ID of the node to remove + */ + removeNode(nodeId) { + const node = this.dataManager.graphObjects.nodes.get(nodeId); + if (!node) return; + + // Remove from NodeCloud if available + if (this.graphController && this.graphController.nodeCloud) { + this.graphController.nodeCloud.removeNode(nodeId); + } + + // Remove label from scene + if (node.labelObject) { + this.sceneManager.removeFromScene(node.labelObject); + } + + // Remove from data manager + this.dataManager.graphObjects.nodes.delete(nodeId); + } + + /** + * Remove a link and its visual elements from the scene + * @param {string} linkId - The ID of the link to remove + */ + removeLink(linkId) { + const link = this.dataManager.graphObjects.links.get(linkId); + if (!link) return; + + // Remove line from scene + if (link.line) { + this.sceneManager.removeFromScene(link.line); + } + + // Remove label from scene + if (link.labelObject) { + this.sceneManager.removeFromScene(link.labelObject); + } + + // Remove from data manager + this.dataManager.graphObjects.links.delete(linkId); + } + + /** + * Clear all visible nodes and links from the scene + */ + clearVisibleObjects() { + // Clear NodeCloud if available + if (this.graphController && this.graphController.nodeCloud) { + this.graphController.nodeCloud.clear(); + } + + // Remove label objects from the scene + this.dataManager.graphObjects.nodes.forEach((node, id) => { + if (node.labelObject) { + this.sceneManager.removeFromScene(node.labelObject); + } + }); + + this.dataManager.graphObjects.links.forEach((link, id) => { + if (link.line) this.sceneManager.removeFromScene(link.line); + if (link.labelObject) this.sceneManager.removeFromScene(link.labelObject); + }); + + // Clear references in data manager + this.dataManager.graphObjects.nodes.clear(); + this.dataManager.graphObjects.links.clear(); + } + + /** + * Toggle visibility of all labels + * @returns {boolean} The new label visibility state + */ + toggleLabels() { + const showLabels = this.themeManager.toggleLabels(); + + // Update node labels + this.dataManager.graphObjects.nodes.forEach((node, id) => { + if (node.labelObject) { + node.labelObject.visible = showLabels; + } else if (showLabels) { + // Create label if it doesn't exist yet + node.labelObject = this.createLabel(node); + } + }); + + // Update link labels - only show for active links connected to selected node + this.dataManager.graphObjects.links.forEach((link, id) => { + if (link.labelObject) { + // Only show if labels are enabled AND this is an active link + const isActive = this.dataManager.activeLinks.has(id); + link.labelObject.visible = showLabels && isActive; + } else if (showLabels && link.label) { + // Create label if it doesn't exist yet + const sourceNode = this.dataManager.graphObjects.nodes.get(link.sourceId); + const targetNode = this.dataManager.graphObjects.nodes.get(link.targetId); + + if (sourceNode && targetNode) { + link.labelObject = this.createLinkLabel(link, sourceNode, targetNode); + // Only show if this is an active link + link.labelObject.visible = this.dataManager.activeLinks.has(id); + } + } + }); + + return showLabels; + } +} + +/** + * SpatialGrid - Simple spatial partitioning for efficient queries + */ +class SpatialGrid { + constructor(cellSize = 200) { + this.cellSize = cellSize; + this.grid = new Map(); + this.objects = new Set(); + } + + /** + * Get the cell key for a position + * @param {THREE.Vector3} position - The position to get the cell for + * @returns {string} The cell key + */ + getCellKey(position) { + const x = Math.floor(position.x / this.cellSize); + const y = Math.floor(position.y / this.cellSize); + const z = Math.floor(position.z / this.cellSize); + return `${x},${y},${z}`; + } + + /** + * Add an object to the grid + * @param {Object} object - The object to add + */ + addObject(object) { + if (!object.object || !object.object.position) return; + + const position = object.object.position; + const cellKey = this.getCellKey(position); + + // Create cell if it doesn't exist + if (!this.grid.has(cellKey)) { + this.grid.set(cellKey, new Set()); + } + + // Add to cell + this.grid.get(cellKey).add(object); + this.objects.add(object); + } + + /** + * Remove an object from the grid + * @param {Object} object - The object to remove + */ + removeObject(object) { + if (!object.object || !object.object.position) return; + + // Remove from all cells (in case it moved) + this.grid.forEach(cell => { + cell.delete(object); + }); + + this.objects.delete(object); + + // Clean up empty cells + this.grid.forEach((cell, key) => { + if (cell.size === 0) { + this.grid.delete(key); + } + }); + } + + /** + * Find objects within a radius of a position + * @param {THREE.Vector3} position - The center position + * @param {number} radius - The radius to search within + * @returns {Array} Array of objects within the radius + */ + findNearbyObjects(position, radius) { + // Calculate the cell range to check + const cellRadius = Math.ceil(radius / this.cellSize); + const centerX = Math.floor(position.x / this.cellSize); + const centerY = Math.floor(position.y / this.cellSize); + const centerZ = Math.floor(position.z / this.cellSize); + + const result = []; + const radiusSquared = radius * radius; + + // Check each cell in the range + for (let x = centerX - cellRadius; x <= centerX + cellRadius; x++) { + for (let y = centerY - cellRadius; y <= centerY + cellRadius; y++) { + for (let z = centerZ - cellRadius; z <= centerZ + cellRadius; z++) { + const cellKey = `${x},${y},${z}`; + const cell = this.grid.get(cellKey); + + if (cell) { + // Check each object in the cell + cell.forEach(object => { + if (object.object && object.object.position) { + const distSquared = position.distanceToSquared(object.object.position); + if (distSquared <= radiusSquared) { + result.push(object); + } + } + }); + } + } + } + } + + return result; + } + + /** + * Clear all objects from the grid + */ + clear() { + this.grid.clear(); + this.objects.clear(); + } + + /** + * Get the total number of objects in the grid + * @returns {number} The number of objects + */ + size() { + return this.objects.size; + } +} + +/** + * NodeCloud - Manages efficient point cloud rendering for graph nodes + */ +class NodeCloud { + constructor(scene, themeManager) { + this.scene = scene; + this.sceneManager = null; // Will be set by the EventManager + this.themeManager = themeManager; + + // Capacity tracking + this.maxNodes = 1000; // Initial capacity + this.nodeCount = 0; + + // Node tracking + this.nodeIndices = new Map(); // Maps node IDs to their index in the arrays + this.nodeTypes = new Map(); // Maps node IDs to their types + this.positions = null; + this.colors = null; + this.sizes = null; + + // Selection tracking + this.selectedIndices = new Set(); + this.neighborIndices = new Set(); + this.hoverIndex = -1; + + // Create base texture for all nodes + this.baseTexture = createCircleTexture(64, 0xffffff); + + // Initialize geometry and point cloud + this.initialize(); + } + + /** + * Initialize buffers and point cloud with initial capacity + */ + initialize() { + // Create buffer attributes with initial capacity + this.positions = new Float32Array(this.maxNodes * 3); + this.colors = new Float32Array(this.maxNodes * 3); + this.sizes = new Float32Array(this.maxNodes); + + // Create buffer geometry + this.geometry = new THREE.BufferGeometry(); + this.geometry.setAttribute('position', new THREE.BufferAttribute(this.positions, 3)); + this.geometry.setAttribute('color', new THREE.BufferAttribute(this.colors, 3)); + this.geometry.setAttribute('size', new THREE.BufferAttribute(this.sizes, 1)); + + // Set draw range to only render active nodes + this.geometry.setDrawRange(0, 0); + + // Create point material + this.material = new THREE.ShaderMaterial({ + uniforms: { + pointTexture: { value: this.baseTexture } + }, + vertexShader: ` + precision highp float; + attribute float size; + attribute vec3 color; + varying vec3 vColor; + + void main() { + vColor = color; + vec4 mvPosition = modelViewMatrix * vec4(position, 1.0); + gl_PointSize = size * (1200.0 / -mvPosition.z); + gl_Position = projectionMatrix * mvPosition; + } + `, + fragmentShader: ` + precision highp float; + uniform sampler2D pointTexture; + varying vec3 vColor; + + void main() { + vec4 texColor = texture2D(pointTexture, gl_PointCoord); + if (texColor.a < 0.5) discard; + gl_FragColor = vec4(vColor, 1.0) * texColor; + } + `, + transparent: true, + depthWrite: false, + blending: THREE.NormalBlending + }); + + // Create points + this.points = new THREE.Points(this.geometry, this.material); + this.points.frustumCulled = false; // Disable frustum culling + this.points.renderOrder = 10; + + // Add to scene + this.scene.add(this.points); + } + + /** + * Resize buffers if needed + */ + ensureCapacity(requiredNodes) { + if (requiredNodes <= this.maxNodes) return; + + // Calculate new capacity (1.5x required or 2x current, whichever is larger) + const newCapacity = Math.max(Math.ceil(requiredNodes * 1.5), this.maxNodes * 2); + + // Create new arrays + const newPositions = new Float32Array(newCapacity * 3); + const newColors = new Float32Array(newCapacity * 3); + const newSizes = new Float32Array(newCapacity); + + // Copy existing data + newPositions.set(this.positions); + newColors.set(this.colors); + newSizes.set(this.sizes); + + // Update references + this.positions = newPositions; + this.colors = newColors; + this.sizes = newSizes; + this.maxNodes = newCapacity; + + // Update buffer attributes + this.geometry.setAttribute('position', new THREE.BufferAttribute(this.positions, 3)); + this.geometry.setAttribute('color', new THREE.BufferAttribute(this.colors, 3)); + this.geometry.setAttribute('size', new THREE.BufferAttribute(this.sizes, 1)); + } + + /** + * Add a node to the point cloud + * @param {Object} node - Node data with position and type + * @returns {number} Index of the node in the point cloud + */ + addNode(node) { + // Get node ID + const nodeId = node.id; + + // Check if this node is already in the cloud + if (this.nodeIndices.has(nodeId)) { + return this.nodeIndices.get(nodeId); + } + + // Ensure we have enough capacity + this.ensureCapacity(this.nodeCount + 1); + + // Add to the end of the arrays + const index = this.nodeCount; + const i3 = index * 3; + + // Set position + this.positions[i3] = node.x || 0; + this.positions[i3 + 1] = node.y || 0; + this.positions[i3 + 2] = this.themeManager.config.zPos.node; + + // Store node type for later reference + this.nodeTypes.set(nodeId, node.type || 'simple'); + + // Set color based on node type + const color = new THREE.Color(this.themeManager.getNodeColor(node.type)); + this.colors[i3] = color.r; + this.colors[i3 + 1] = color.g; + this.colors[i3 + 2] = color.b; + + // Set size based on node type - use larger sizes to make selection easier + const baseSize = this.themeManager.getNodeSize(node.type); + this.sizes[index] = baseSize * 4; // Increase size to improve interaction + + // Track this node + this.nodeIndices.set(nodeId, index); + this.nodeCount++; + + // Update draw range + this.geometry.setDrawRange(0, this.nodeCount); + + // Mark attributes as needing update + this.geometry.attributes.position.needsUpdate = true; + this.geometry.attributes.color.needsUpdate = true; + this.geometry.attributes.size.needsUpdate = true; + + return index; + } + + /** + * Update a node's position + * @param {string} nodeId - ID of the node to update + * @param {number} x - New X position + * @param {number} y - New Y position + */ + updateNodePosition(nodeId, x, y) { + if (!this.nodeIndices.has(nodeId)) return; + + const index = this.nodeIndices.get(nodeId); + const i3 = index * 3; + + this.positions[i3] = x; + this.positions[i3 + 1] = y; + + // Mark position attribute as needing update + this.geometry.attributes.position.needsUpdate = true; + } + + /** + * Update a node's color based on state + * @param {string} nodeId - ID of the node to update + * @param {string} nodeType - Type of the node + * @param {string} state - State of the node (default, selected, neighbor, hover) + */ + updateNodeColor(nodeId, nodeType, state = 'default') { + if (!this.nodeIndices.has(nodeId)) return; + + // Store node type if provided + if (nodeType) { + this.nodeTypes.set(nodeId, nodeType); + } else { + // Use stored type if available + nodeType = this.nodeTypes.get(nodeId) || 'simple'; + } + + const index = this.nodeIndices.get(nodeId); + const i3 = index * 3; + + // Get color for this node state + const color = new THREE.Color(this.themeManager.getNodeColor(nodeType, state)); + + // Update color in buffer + this.colors[i3] = color.r; + this.colors[i3 + 1] = color.g; + this.colors[i3 + 2] = color.b; + + // Mark colors attribute as needing update + this.geometry.attributes.color.needsUpdate = true; + } + + /** + * Update colors for all nodes based on selection state + * @param {string} selectedId - ID of the selected node + * @param {Set} neighborIds - Set of neighbor node IDs + * @param {string} hoveredId - ID of the hovered node + */ + updateColors(selectedId, neighborIds, hoveredId) { + // Reset tracking sets + this.selectedIndices.clear(); + this.neighborIndices.clear(); + this.hoverIndex = -1; + + // Track indices for faster updates + if (selectedId && this.nodeIndices.has(selectedId)) { + this.selectedIndices.add(this.nodeIndices.get(selectedId)); + } + + neighborIds.forEach(id => { + if (this.nodeIndices.has(id)) { + this.neighborIndices.add(this.nodeIndices.get(id)); + } + }); + + if (hoveredId && this.nodeIndices.has(hoveredId)) { + this.hoverIndex = this.nodeIndices.get(hoveredId); + } + + // Update all node colors based on state + for (const [nodeId, index] of this.nodeIndices.entries()) { + const i3 = index * 3; + let state = 'default'; + + // Determine state based on selection and hover + if (index === this.hoverIndex) { + state = 'hover'; + } else if (this.selectedIndices.has(index)) { + state = 'selected'; + } else if (this.neighborIndices.has(index)) { + state = 'neighbor'; + } + + // Get node type from our stored map + const nodeType = this.nodeTypes.get(nodeId) || 'simple'; + + // Get color for this state + const color = new THREE.Color(this.themeManager.getNodeColor(nodeType, state)); + + // Update color + this.colors[i3] = color.r; + this.colors[i3 + 1] = color.g; + this.colors[i3 + 2] = color.b; + } + + // Mark attributes as needing update + this.geometry.attributes.color.needsUpdate = true; + } + + /** + * Find the closest node to a mouse position + * @param {THREE.Raycaster} raycaster - The raycaster + * @param {number} threshold - Maximum distance to consider a hit (in screen space) + * @returns {string|null} ID of the closest node or null if none found + */ + findClosestNode(raycaster, threshold = 0.05) { + if (this.nodeCount === 0 || !this.positions) return null; + + // Get the camera from the scene manager or from the global controller + let camera = null; + let mousePosition = null; + + if (this.sceneManager) { + camera = this.sceneManager.camera; + mousePosition = this.sceneManager.mouse; // This is the normalized mouse position + } else if (window.lastGraphController && window.lastGraphController.sceneManager) { + camera = window.lastGraphController.sceneManager.camera; + mousePosition = window.lastGraphController.sceneManager.mouse; + } + + // If we can't get a camera or mouse position, we can't continue + if (!camera || !mousePosition) return null; + + // Find the closest node based on screen space distance + let closestDistance = Infinity; + let closestNodeId = null; + + // Check each node + for (const [nodeId, index] of this.nodeIndices.entries()) { + const i3 = index * 3; + + // Get node position + const nodePos = new THREE.Vector3( + this.positions[i3], + this.positions[i3 + 1], + this.positions[i3 + 2] + ); + + // Project to screen space + const screenPos = nodePos.clone().project(camera); + + // Calculate 2D distance in screen space between mouse and node + const dx = screenPos.x - mousePosition.x; + const dy = screenPos.y - mousePosition.y; + const distance = Math.sqrt(dx * dx + dy * dy); + + // Get node size and adjust threshold based on size + const nodeSize = this.sizes[index]; + const adjustedThreshold = threshold * (1 + (nodeSize / 10)); + + // If this node is closer than the current closest and within threshold, update + if (distance < closestDistance && distance < adjustedThreshold) { + closestDistance = distance; + closestNodeId = nodeId; + } + } + + return closestNodeId; + } + + /** + * Remove a node from the point cloud + * @param {string} nodeId - ID of the node to remove + */ + removeNode(nodeId) { + if (!this.nodeIndices.has(nodeId)) return; + + const indexToRemove = this.nodeIndices.get(nodeId); + + // Only perform complex removal if not the last node + if (indexToRemove !== this.nodeCount - 1) { + // Move the last node to this position + const lastIndex = this.nodeCount - 1; + const lastI3 = lastIndex * 3; + const removeI3 = indexToRemove * 3; + + // Copy position + this.positions[removeI3] = this.positions[lastI3]; + this.positions[removeI3 + 1] = this.positions[lastI3 + 1]; + this.positions[removeI3 + 2] = this.positions[lastI3 + 2]; + + // Copy color + this.colors[removeI3] = this.colors[lastI3]; + this.colors[removeI3 + 1] = this.colors[lastI3 + 1]; + this.colors[removeI3 + 2] = this.colors[lastI3 + 2]; + + // Copy size + this.sizes[indexToRemove] = this.sizes[lastIndex]; + + // Find which node was at the last position + let lastNodeId = null; + for (const [id, index] of this.nodeIndices.entries()) { + if (index === lastIndex) { + lastNodeId = id; + break; + } + } + + // Update the moved node's index + if (lastNodeId) { + this.nodeIndices.set(lastNodeId, indexToRemove); + } + } + + // Remove the node from tracking + this.nodeIndices.delete(nodeId); + this.nodeCount--; + + // Update draw range + this.geometry.setDrawRange(0, this.nodeCount); + + // Mark attributes as needing update + this.geometry.attributes.position.needsUpdate = true; + this.geometry.attributes.color.needsUpdate = true; + this.geometry.attributes.size.needsUpdate = true; + } + + /** + * Clear all nodes from the point cloud + */ + clear() { + this.nodeIndices.clear(); + this.nodeTypes.clear(); + this.nodeCount = 0; + this.geometry.setDrawRange(0, 0); + + // Reset state tracking + this.selectedIndices.clear(); + this.neighborIndices.clear(); + this.hoverIndex = -1; + } + + /** + * Update all positions from the node objects + * @param {Map} nodes - Map of nodes with current positions + */ + updateAllPositions(nodes) { + // Update all node positions from data + for (const [nodeId, node] of nodes.entries()) { + if (this.nodeIndices.has(nodeId) && node.x !== undefined && node.y !== undefined) { + const index = this.nodeIndices.get(nodeId); + const i3 = index * 3; + + this.positions[i3] = node.x; + this.positions[i3 + 1] = node.y; + } + } + + // Mark position attribute as needing update + this.geometry.attributes.position.needsUpdate = true; + } +} + +/** + * SimulationManager - Manages the D3 force-directed simulation + */ +class SimulationManager { + constructor(dataManager, graphObjectManager, themeManager) { + this.dataManager = dataManager; + this.graphObjectManager = graphObjectManager; + this.themeManager = themeManager; + + this.simulation = null; + this.isRunning = false; + + // Initialize spatial partitioning + this.spatialGrid = null; + this.useSpatialIndex = true; + this.gridUpdateInterval = 30; // Update grid every 30 frames + this.gridUpdateCounter = 0; + this.gridCellSize = 200; // Size of each grid cell + + // Initialize the simulation + this.initSimulation(); + this.initSpatialIndex(); + } + + /** + * Initialize D3 force simulation + */ + initSimulation() { + // Calculate connection counts for better distribution + const connectionCounts = this.calculateConnectionCounts(); + + // Link distance based on connection count + const linkDistance = link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + const sourceConnections = connectionCounts.get(sourceId) || 0; + const targetConnections = connectionCounts.get(targetId) || 0; + + // Scale distance based on connection count + const baseDistance = this.themeManager.config.defaultDistance; + const connectionFactor = Math.max(sourceConnections, targetConnections); + + return baseDistance * (1 + Math.log(1 + connectionFactor * 0.2)); + }; + + // Collision radius based on connection count + const collisionRadius = node => { + const connections = connectionCounts.get(node.id) || 0; + const baseRadius = 15; + + // Increase collision radius for highly connected nodes + if (connections > this.themeManager.config.highConnectionThreshold) { + return baseRadius * (1 + Math.log(connections) * 0.1); + } + return baseRadius; + }; + + // For monitoring simulation progress + this.tickCounter = 0; + this.lastLogTime = 0; + + // Create the simulation with all forces + this.simulation = d3.forceSimulation() + .force('link', d3.forceLink().id(d => d.id).distance(linkDistance)) + .force('charge', d3.forceManyBody().strength(-15)) + .force('center', d3.forceCenter(0, 0)) + .force('collision', d3.forceCollide().radius(collisionRadius)) + .force('x', d3.forceX().strength(0.001)) + .force('y', d3.forceY().strength(0.005)) + .on('tick', () => { + this.tickCounter++; + this.onSimulationTick(); + this.monitorSimulationProgress(); + }) + .on('end', () => { + console.log('Simulation reached equilibrium!'); + console.log('Final alpha:', this.simulation.alpha()); + console.log('Alpha min:', this.simulation.alphaMin()); + console.log('Alpha decay:', this.simulation.alphaDecay()); + console.log('Node count:', this.simulation.nodes().length); + console.log('Total ticks:', this.tickCounter); + this.isRunning = false; + }); + + // Adjust alpha settings for longer simulation time + // Reduce decay rate (default is ~0.0228 which is 1% cooling per tick) + this.simulation.alphaDecay(0.0228); // Slower decay (about 0.5% cooling per tick) + + // Lower minimum alpha threshold (default is 0.001) + this.simulation.alphaMin(0.001); // Lower threshold for stopping + + // Reduce velocity decay for more momentum (default is 0.4) + this.simulation.velocityDecay(0.1); + } + + /** + * Calculate connection counts for each node + * @returns {Map} Map of node IDs to connection counts + */ + calculateConnectionCounts() { + const connectionCounts = new Map(); + + // Count connections for each node + this.dataManager.graphData.links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + connectionCounts.set(sourceId, (connectionCounts.get(sourceId) || 0) + 1); + connectionCounts.set(targetId, (connectionCounts.get(targetId) || 0) + 1); + }); + + return connectionCounts; + } + + /** + * Update the simulation with current nodes and links + * @param {boolean} restart - Whether to restart the simulation + */ + updateSimulation(restart = true) { + if (!this.simulation) return; + + // Get visible nodes and links from data manager + const visibleNodes = Array.from(this.dataManager.graphObjects.nodes.values()); + const visibleLinks = Array.from(this.dataManager.graphObjects.links.values()); + + console.log(`Updating simulation with ${visibleNodes.length} nodes and ${visibleLinks.length} links`); + + // Update nodes and links in the simulation + this.simulation.nodes(visibleNodes); + this.simulation.force('link').links(visibleLinks); + + // Restart simulation if needed + if (restart && this.themeManager.config.physicsEnabled) { + // Reset tick counter and timing + this.tickCounter = 0; + this.lastLogTime = Date.now(); + + // Set a higher alpha to ensure thorough exploration of layout space + const startingAlpha = 1.0; + console.log(`Starting simulation with alpha=${startingAlpha}, alphaMin=${this.simulation.alphaMin()}, alphaDecay=${this.simulation.alphaDecay()}`); + this.simulation.alpha(startingAlpha).restart(); + this.isRunning = true; + } else { + console.log("Simulation not restarted (either restart=false or physics is disabled)"); + this.simulation.alpha(0); + this.isRunning = false; + } + } + + /** + * Toggle physics simulation on/off + * @returns {boolean} The new physics state + */ + togglePhysics() { + const physicsEnabled = this.themeManager.togglePhysics(); + + console.log(`Physics simulation ${physicsEnabled ? 'enabled' : 'disabled'}`); + + // Use updateSimulation to properly handle the physics state + this.updateSimulation(physicsEnabled); + + return physicsEnabled; + } + + /** + * Handle force simulation tick events + * Updates the positions of nodes and links in the visualization + */ + onSimulationTick() { + this.updatePositions(); + + // Update spatial grid periodically + if (this.useSpatialIndex) { + this.updateSpatialGrid(); + } + } + + /** + * Update positions of all nodes and links + */ + updatePositions() { + // Check if we have a NodeCloud available + if (this.graphController && this.graphController.nodeCloud) { + // Bulk update the NodeCloud for better performance + this.graphController.nodeCloud.updateAllPositions(this.dataManager.graphObjects.nodes); + + // Update virtual objects for compatibility with other systems + this.dataManager.graphObjects.nodes.forEach(node => { + if (node.object && node.object.position) { + node.object.position.x = node.x || 0; + node.object.position.y = node.y || 0; + } + + // Update labels separately + if (node.labelObject) { + this.graphObjectManager.updateNodePosition(node); + } + }); + } + + // Update the position of links in the scene + this.dataManager.graphObjects.links.forEach((link) => { + this.graphObjectManager.updateLinkPosition(link); + }); + } + + /** + * Monitor simulation progress with periodic logging + */ + monitorSimulationProgress() { + // Only log every 100 ticks to avoid spamming the console + if (this.tickCounter % 100 === 0) { + const now = Date.now(); + const timeSinceLastLog = now - this.lastLogTime; + this.lastLogTime = now; + + // Only print if we're still running + if (this.isRunning) { + const currentAlpha = this.simulation.alpha(); + console.log(`Simulation progress: tick=${this.tickCounter}, alpha=${currentAlpha.toFixed(6)}, ticks/second=${(100 / (timeSinceLastLog / 1000)).toFixed(1)}`); + } + } + } + + /** + * Initialize spatial index using a simple grid system + */ + initSpatialIndex() { + this.spatialGrid = new SpatialGrid(this.gridCellSize); + } + + /** + * Add a node to the spatial grid + * @param {Object} node - The node to add + */ + addNodeToSpatialGrid(node) { + if (!this.useSpatialIndex || !this.spatialGrid || !node.object) return; + + // Add node to the grid + this.spatialGrid.addObject(node); + } + + /** + * Update the spatial grid with current node positions + */ + updateSpatialGrid() { + if (!this.useSpatialIndex || !this.spatialGrid) return; + + // Only update periodically for performance + this.gridUpdateCounter++; + if (this.gridUpdateCounter < this.gridUpdateInterval) return; + this.gridUpdateCounter = 0; + + // Rebuild the grid + this.spatialGrid.clear(); + + // Add all current nodes to the grid + this.dataManager.graphObjects.nodes.forEach(node => { + if (node.object) { + this.spatialGrid.addObject(node); + } + }); + } + + /** + * Get nodes within a specific radius of a position + * @param {THREE.Vector3} position - The center position + * @param {number} radius - The radius to search within + * @returns {Array} Array of nodes within the radius + */ + getNodesInRadius(position, radius) { + // Use spatial grid for more efficient spatial query + return this.spatialGrid.findNearbyObjects(position, radius); + } +} + +/** + * UIManager - Handles UI elements and user interaction + */ +class UIManager { + constructor(container, dataManager, graphController) { + this.container = container; + this.dataManager = dataManager; + this.graphController = graphController; + + // DOM elements + this.nodeCountEl = document.getElementById('node-count'); + this.linkCountEl = document.getElementById('link-count'); + this.loadingEl = document.getElementById('loading'); + this.searchInput = document.getElementById('search-input'); + this.initialMessageEl = null; + this.nodeInfoPanel = null; + + // Search state + this.previousSearchValue = ''; + this.isUpdatingAutocomplete = false; + + // Autocomplete state + this.autocompleteList = null; + this.autocompleteSuggestions = []; + this.autocompleteSelectedIndex = -1; + + // Initialize UI elements + this.createAutocompleteUI(); + this.createNodeInfoPanel(); + this.setupEventListeners(); + } + + /** + * Set up event listeners for UI controls + */ + setupEventListeners() { + // Button event listeners + const labelsBtn = document.getElementById('toggle-labels-btn'); + if (labelsBtn) { + // Set initial state based on ThemeManager config + // The initial state should be true by default + labelsBtn.classList.add('active'); + + // Add click handler + labelsBtn.addEventListener('click', () => { + const showLabels = this.graphController.toggleLabels(); + // Toggle active class based on the returned state + labelsBtn.classList.toggle('active', showLabels); + }); + } + + document.getElementById('reset-btn')?.addEventListener('click', () => { + this.graphController.resetView(); + this.hideAutocomplete(); + this.previousSearchValue = ''; + this.searchInput.value = ''; + }); + + // Add click handler for the Load All Nodes button + document.getElementById('load-all-btn')?.addEventListener('click', () => { + this.graphController.loadAllNodes(); + }); + + // Set up search input with debounced handling + if (this.searchInput) { + let searchTimeout = null; + + // Input event for search text changes + this.searchInput.addEventListener('input', (e) => { + if (this.isUpdatingAutocomplete) return; + + const currentValue = e.target.value.trim(); + if (currentValue === this.previousSearchValue) return; + + this.previousSearchValue = currentValue; + clearTimeout(searchTimeout); + + if (currentValue.length > 0) { + this.isUpdatingAutocomplete = true; + searchTimeout = setTimeout(() => { + this.updateAutocompleteSuggestions(currentValue); + this.isUpdatingAutocomplete = false; + }, 250); + } else { + this.hideAutocomplete(); + } + }); + + // Keyboard navigation in autocomplete dropdown + this.searchInput.addEventListener('keydown', (e) => { + if (['ArrowUp', 'ArrowDown', 'Enter', 'Escape'].includes(e.key)) { + this.handleAutocompleteKeydown(e); + } + }); + + // Focus handler to show autocomplete + this.searchInput.addEventListener('focus', () => { + if (this.isUpdatingAutocomplete) return; + + const currentValue = this.searchInput.value.trim(); + if (currentValue.length > 0) { + this.previousSearchValue = currentValue; + this.isUpdatingAutocomplete = true; + this.updateAutocompleteSuggestions(currentValue); + this.isUpdatingAutocomplete = false; + } + }); + } + + // Hide autocomplete when clicking outside + document.addEventListener('click', (e) => { + if (this.autocompleteList && e.target !== this.searchInput && !this.autocompleteList.contains(e.target)) { + this.hideAutocomplete(); + } + }); + } + + /** + * Create the autocomplete UI elements + */ + createAutocompleteUI() { + if (!this.searchInput) return; + + // Create autocomplete container if it doesn't exist + if (!this.autocompleteList) { + this.autocompleteList = document.createElement('div'); + this.autocompleteList.className = 'autocomplete-items'; + this.autocompleteList.style.display = 'none'; + this.autocompleteList.style.position = 'absolute'; + this.autocompleteList.style.zIndex = '999'; + this.autocompleteList.style.maxHeight = '300px'; + this.autocompleteList.style.overflowY = 'auto'; + this.autocompleteList.style.width = '100%'; + this.autocompleteList.style.background = '#fff'; + this.autocompleteList.style.border = '1px solid #ddd'; + this.autocompleteList.style.borderRadius = '0 0 4px 4px'; + this.autocompleteList.style.boxShadow = '0 2px 4px rgba(0,0,0,0.2)'; + + // Append to parent container + const searchContainer = this.searchInput.parentNode; + searchContainer.appendChild(this.autocompleteList); + } + } + + /** + * Update autocomplete suggestions based on search term + * @param {string} searchTerm - The current search term + */ + updateAutocompleteSuggestions(searchTerm) { + if (!this.autocompleteList || !searchTerm) { + this.hideAutocomplete(); + return; + } + + // Clear previous suggestions + this.autocompleteList.innerHTML = ''; + this.autocompleteSuggestions = []; + this.autocompleteSelectedIndex = -1; + + const maxSuggestions = 10; + const searchLower = searchTerm.toLowerCase(); + + // Get all nodes that match the search term + const matchingNodes = this.dataManager.graphData.nodes + .filter(node => ( + (node.id && node.id.toLowerCase().includes(searchLower)) || + (node.label && node.label.toLowerCase().includes(searchLower)) + )) + .sort((a, b) => { + // Prioritize exact matches and matches at the beginning + const aId = a.id.toLowerCase(); + const bId = b.id.toLowerCase(); + const aLabel = (a.label || '').toLowerCase(); + const bLabel = (b.label || '').toLowerCase(); + + // Check for exact matches first + if (aId === searchLower || aLabel === searchLower) return -1; + if (bId === searchLower || bLabel === searchLower) return 1; + + // Then check for starting with search term + if (aId.startsWith(searchLower) || aLabel.startsWith(searchLower)) return -1; + if (bId.startsWith(searchLower) || bLabel.startsWith(searchLower)) return 1; + + // Fallback to alphabetical + return aId.localeCompare(bId); + }) + .slice(0, maxSuggestions); + + if (matchingNodes.length === 0) { + this.hideAutocomplete(); + return; + } + + // Save suggestions for keyboard navigation + this.autocompleteSuggestions = matchingNodes; + + // Create suggestion items + matchingNodes.forEach((node, index) => { + const item = document.createElement('div'); + item.className = 'autocomplete-item'; + item.style.padding = '8px 12px'; + item.style.cursor = 'pointer'; + item.style.borderBottom = '1px solid #f4f4f4'; + + // Highlight matching parts + const displayText = node.label || node.id; + const parts = displayText.split(new RegExp(`(${searchTerm})`, 'i')); + + parts.forEach(part => { + const span = document.createElement('span'); + span.textContent = part; + if (part.toLowerCase() === searchTerm.toLowerCase()) { + span.style.fontWeight = 'bold'; + span.style.backgroundColor = 'rgba(66, 133, 244, 0.1)'; + } + item.appendChild(span); + }); + + // Add node type indicator + const typeIndicator = document.createElement('span'); + typeIndicator.style.marginLeft = '8px'; + typeIndicator.style.padding = '2px 6px'; + typeIndicator.style.borderRadius = '10px'; + typeIndicator.style.fontSize = '0.8em'; + + // Different styling for different node types + if (node.type === 'simple') { + typeIndicator.textContent = 'item'; + typeIndicator.style.backgroundColor = 'rgba(100, 149, 237, 0.2)'; + typeIndicator.style.color = 'rgb(50, 90, 160)'; + } else { + typeIndicator.textContent = 'collection'; + typeIndicator.style.backgroundColor = 'rgba(240, 128, 128, 0.2)'; + typeIndicator.style.color = 'rgb(180, 70, 70)'; + } + + item.appendChild(typeIndicator); + + // Add hover effect + item.addEventListener('mouseover', () => { + this.autocompleteSelectedIndex = index; + this.highlightSelectedSuggestion(); + }); + + // Add click handler + item.addEventListener('click', () => { + this.searchInput.value = node.id; + this.hideAutocomplete(); + this.graphController.searchNodes(node.id); + }); + + this.autocompleteList.appendChild(item); + }); + + // Show the autocomplete list + this.autocompleteList.style.display = 'block'; + } + + /** + * Handle keyboard navigation in autocomplete list + * @param {KeyboardEvent} event - The keyboard event + */ + handleAutocompleteKeydown(event) { + // If no suggestions or hidden, do nothing special except for Enter + if (this.autocompleteSuggestions.length === 0 || + this.autocompleteList.style.display === 'none') { + if (event.key === 'Enter') { + const searchTerm = this.searchInput.value.trim(); + if (searchTerm) { + this.graphController.searchNodes(searchTerm); + this.hideAutocomplete(); + } + } + return; + } + + switch (event.key) { + case 'ArrowDown': + // Move selection down + event.preventDefault(); + this.autocompleteSelectedIndex = Math.min( + this.autocompleteSelectedIndex + 1, + this.autocompleteSuggestions.length - 1 + ); + this.highlightSelectedSuggestion(); + break; + + case 'ArrowUp': + // Move selection up + event.preventDefault(); + this.autocompleteSelectedIndex = Math.max(this.autocompleteSelectedIndex - 1, -1); + this.highlightSelectedSuggestion(); + break; + + case 'Enter': + // Select current suggestion or search with current text + event.preventDefault(); + if (this.autocompleteSelectedIndex >= 0) { + const selectedNode = this.autocompleteSuggestions[this.autocompleteSelectedIndex]; + this.searchInput.value = selectedNode.id; + this.graphController.searchNodes(selectedNode.id); + } else { + const searchTerm = this.searchInput.value.trim(); + if (searchTerm) { + this.graphController.searchNodes(searchTerm); + } + } + this.hideAutocomplete(); + break; + + case 'Escape': + // Hide autocomplete + event.preventDefault(); + this.hideAutocomplete(); + break; + } + } + + /** + * Highlight the currently selected suggestion item + */ + highlightSelectedSuggestion() { + // Remove highlight from all items + const items = this.autocompleteList.querySelectorAll('.autocomplete-item'); + items.forEach(item => { + item.style.backgroundColor = ''; + }); + + // Highlight selected item if any + if (this.autocompleteSelectedIndex >= 0 && this.autocompleteSelectedIndex < items.length) { + const selectedItem = items[this.autocompleteSelectedIndex]; + selectedItem.style.backgroundColor = 'rgba(66, 133, 244, 0.1)'; + + // Scroll into view if needed + if (selectedItem.offsetTop < this.autocompleteList.scrollTop) { + this.autocompleteList.scrollTop = selectedItem.offsetTop; + } else if (selectedItem.offsetTop + selectedItem.offsetHeight > + this.autocompleteList.scrollTop + this.autocompleteList.offsetHeight) { + this.autocompleteList.scrollTop = + selectedItem.offsetTop + selectedItem.offsetHeight - this.autocompleteList.offsetHeight; + } + } + } + + /** + * Hide the autocomplete list + */ + hideAutocomplete() { + if (this.autocompleteList) { + this.autocompleteList.style.display = 'none'; + this.autocompleteSelectedIndex = -1; + } + } + + /** + * Show or hide the loading indicator + * @param {boolean} show - Whether to show or hide the loading indicator + */ + showLoading(show) { + if (this.loadingEl) { + this.loadingEl.style.display = show ? 'block' : 'none'; + } + } + + /** + * Show an initial message in the graph area + * @param {string} message - The message to display + */ + showInitialMessage(message) { + // Create or update the message element + if (!this.initialMessageEl) { + this.initialMessageEl = document.createElement('div'); + this.initialMessageEl.style.position = 'absolute'; + this.initialMessageEl.style.top = '50%'; + this.initialMessageEl.style.left = '50%'; + this.initialMessageEl.style.transform = 'translate(-50%, -50%)'; + this.initialMessageEl.style.background = 'rgba(0, 0, 0, 0.7)'; + this.initialMessageEl.style.color = '#ffffff'; + this.initialMessageEl.style.padding = '20px'; + this.initialMessageEl.style.borderRadius = '8px'; + this.initialMessageEl.style.maxWidth = '80%'; + this.initialMessageEl.style.textAlign = 'center'; + this.initialMessageEl.style.fontSize = '18px'; + this.container.appendChild(this.initialMessageEl); + } + + this.initialMessageEl.textContent = message; + this.initialMessageEl.style.display = 'block'; + } + + /** + * Hide the initial message + */ + hideInitialMessage() { + if (this.initialMessageEl) { + this.initialMessageEl.style.display = 'none'; + } + } + + /** + * Show error message + * @param {string} message - The error message to display + */ + showError(message) { + console.error(message); + this.showInitialMessage(message); + } + + /** + * Update statistics display + */ + updateStats() { + if (this.nodeCountEl) { + this.nodeCountEl.textContent = this.dataManager.graphData.nodes.length; + } + if (this.linkCountEl) { + this.linkCountEl.textContent = this.dataManager.graphData.links.length; + } + } + + /** + * Create the node info panel + */ + createNodeInfoPanel() { + if (!this.nodeInfoPanel) { + this.nodeInfoPanel = document.createElement('div'); + this.nodeInfoPanel.className = 'node-info-panel'; + this.nodeInfoPanel.style.display = 'none'; + this.container.appendChild(this.nodeInfoPanel); + } + } + + /** + * Show node information in the info panel + * @param {string} nodeId - The ID of the node to display info for + */ + showNodeInfo(nodeId) { + if (!this.nodeInfoPanel) this.createNodeInfoPanel(); + + const node = this.dataManager.graphObjects.nodes.get(nodeId); + if (!node) return; + + // Store the current node ID for the Get Data button + this.currentNodeId = nodeId; + + // Get the node's connections + const connectedLinks = this.dataManager.getConnectedLinks(nodeId); + const connectionCount = connectedLinks.length; + + // Get any additional properties + const nodeType = node.type || 'Unknown'; + const nodeLabel = node.label || nodeId; + + // Build HTML content + let html = ` +

${nodeLabel}

+

ID: ${this.truncateWithEllipsis(nodeId)}

+

Type: ${nodeType}

+

Connections: ${connectionCount}

+ `; + + // Only show the Get Data button for composite nodes where the ID doesn't start with "data/" + if (nodeType === 'composite' && !nodeId.startsWith('data/')) { + html += ` + + + `; + } + + // Add any other properties that exist + if (node.data) { + html += `

Data: ${JSON.stringify(node.data)}

`; + } + + // Add information about connected nodes + if (connectionCount > 0) { + html += `

Connected Nodes:

`; + html += `
`; + + // Get info about connected nodes + const connectedNodes = new Map(); // Use Map to avoid duplicates + + connectedLinks.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + + // Get the ID of the connected node (not the current node) + const connectedNodeId = sourceId === nodeId ? targetId : sourceId; + + // Store relationship type if available + const relationship = link.label || ''; + + // Get the connected node + const connectedNode = this.dataManager.graphObjects.nodes.get(connectedNodeId); + if (connectedNode && !connectedNodes.has(connectedNodeId)) { + connectedNodes.set(connectedNodeId, { + node: connectedNode, + relationship: relationship + }); + } + }); + + // Display connected nodes (limited to avoid overwhelming the panel) + const maxNodesToShow = 50; + let nodeCount = 0; + + connectedNodes.forEach((data, connectedNodeId) => { + if (nodeCount < maxNodesToShow) { + const connectedNode = data.node; + const relationship = data.relationship; + + const truncatedId = this.truncateWithEllipsis(connectedNodeId); + const nodeLabel = connectedNode.label || truncatedId; + const nodeType = connectedNode.type || 'Unknown'; + + html += `
`; + html += `${nodeLabel}`; + html += `
${nodeType}
`; + + if (relationship) { + html += `
+ Relationship: ${relationship}
`; + } + + html += `
`; + + nodeCount++; + } + }); + + // If there are more nodes than we're showing + if (connectedNodes.size > maxNodesToShow) { + html += `
...and ${connectedNodes.size - maxNodesToShow} more
`; + } + + html += `
`; + } + + // Set content and show panel + this.nodeInfoPanel.innerHTML = html; + this.nodeInfoPanel.style.display = 'block'; + + // Add event listener for the Get Data button + const getDataBtn = document.getElementById('get-node-data'); + if (getDataBtn) { + getDataBtn.addEventListener('click', () => this.fetchNodeData(nodeId)); + } + + // Add event listeners to connected node items + const nodeItems = this.nodeInfoPanel.querySelectorAll('.connected-node-item'); + nodeItems.forEach(item => { + // Hover effect + item.addEventListener('mouseover', () => { + item.style.backgroundColor = '#f5f5f5'; + item.style.borderLeftColor = '#4D90FE'; + }); + + item.addEventListener('mouseout', () => { + item.style.backgroundColor = ''; + item.style.borderLeftColor = '#eee'; + }); + + // Click to select the node + item.addEventListener('click', () => { + const clickedNodeId = item.getAttribute('data-node-id'); + if (clickedNodeId && this.graphController.eventManager) { + this.graphController.eventManager.selectNode(clickedNodeId); + this.graphController.eventManager.focusOnNode(clickedNodeId); + } + }); + }); + } + + /** + * Fetch data for a specific node from the server + * @param {string} nodeId - The ID of the node to fetch data for + */ + fetchNodeData(nodeId) { + // Get the result container + const resultContainer = document.getElementById('node-data-result'); + if (!resultContainer) return; + + // Show loading indicator + resultContainer.style.display = 'block'; + resultContainer.innerHTML = ` +
+
+
Loading data...
+
+ + `; + + // Construct the URL for the data endpoint + const url = `/${nodeId}`; + + // Fetch the data + fetch(url) + .then(response => { + if (!response.ok) { + throw new Error(`HTTP error ${response.status}`); + } + return response.text(); // Using text() instead of json() to handle any type of response + }) + .then(data => { + // Try to parse as JSON if possible + try { + const jsonData = JSON.parse(data); + this.displayNodeData(jsonData, resultContainer); + } catch (e) { + // If not JSON, display as text + this.displayNodeData(data, resultContainer, false); + } + }) + .catch(error => { + // Show error message + resultContainer.innerHTML = ` +
+ Error loading data: ${error.message} +
+ `; + }); + } + + /** + * Display node data in the result container + * @param {Object|string} data - The data to display + * @param {HTMLElement} container - The container to display the data in + * @param {boolean} isJson - Whether the data is JSON + */ + displayNodeData(data, container, isJson = true) { + if (isJson) { + // Format JSON for display + const formattedJson = JSON.stringify(data, null, 2); + container.innerHTML = ` +
+ ${formattedJson.replace(//g, '>')} +
+ `; + } else { + // Display as text + container.innerHTML = ` +
+ ${data.toString().replace(//g, '>')} +
+ `; + } + } + + /** + * Hide the node info panel + */ + hideNodeInfo() { + if (this.nodeInfoPanel) { + this.nodeInfoPanel.style.display = 'none'; + } + } + + /** + * Truncate text and add ellipsis in the middle + * @param {string} text - The text to truncate + * @returns {string} Truncated text with ellipsis + */ + truncateWithEllipsis(text) { + // Show only if longer than 15 characters (6 + 3 + 6) + if (!text || text.length <= 15) { + return text; + } + + // Take exactly 6 chars from start and 6 from end + return text.substring(0, 6) + '...' + text.substring(text.length - 6); + } +} + +/** + * EventManager - Handles user interaction events + */ +class EventManager { + constructor(sceneManager, dataManager, graphObjectManager) { + this.sceneManager = sceneManager; + this.dataManager = dataManager; + this.graphObjectManager = graphObjectManager; + + // Set up event listeners + this.setupEventListeners(); + } + + /** + * Set up graph-specific interaction handlers + */ + setupEventListeners() { + const renderer = this.sceneManager.renderer; + if (!renderer || !renderer.domElement) return; + + // Add click event listener for node selection + renderer.domElement.addEventListener('click', this.onMouseClick.bind(this)); + + // Add double-click event listener for camera focus + renderer.domElement.addEventListener('dblclick', this.onDoubleClick.bind(this)); + + // Add hover event listeners + renderer.domElement.addEventListener('mousemove', this.onMouseMove.bind(this)); + } + + /** + * Handle mouse click events for node selection + * @param {MouseEvent} event - The mouse event + */ + onMouseClick(event) { + // Calculate mouse position and find intersections + this.sceneManager.updateMousePosition(event); + + // Set the scene manager on the NodeCloud for camera access + this.graphController.nodeCloud.sceneManager = this.sceneManager; + + const nodeId = this.graphController.nodeCloud.findClosestNode( + this.sceneManager.raycaster, + 0.08 // Screen space threshold for clicks (0-1 normalized coordinates) + ); + + if (nodeId) { + // We clicked on a node + this.selectNode(nodeId); + } else { + // Clicked on empty space - deselect + this.deselectNode(); + } + } + + /** + * Handle double-click events for focusing on nodes + * @param {MouseEvent} event - The mouse event + */ + onDoubleClick(event) { + // Calculate mouse position and find intersections + this.sceneManager.updateMousePosition(event); + + // Set the scene manager on the NodeCloud for camera access + this.graphController.nodeCloud.sceneManager = this.sceneManager; + + const nodeId = this.graphController.nodeCloud.findClosestNode( + this.sceneManager.raycaster, + 0.08 // Screen space threshold for double-clicks (0-1 normalized coordinates) + ); + + if (nodeId) { + // Double-clicked on a node - focus camera on this node + this.focusOnNode(nodeId); + } + } + + /** + * Handle mouse movement for hover effects + * @param {MouseEvent} event - The mouse event + */ + onMouseMove(event) { + // Calculate mouse position + this.sceneManager.updateMousePosition(event); + + // Set the scene manager on the NodeCloud for camera access + this.graphController.nodeCloud.sceneManager = this.sceneManager; + + const nodeId = this.graphController.nodeCloud.findClosestNode( + this.sceneManager.raycaster, + 0.04 // Screen space threshold for hover (0-1 normalized coordinates) + ); + + if (nodeId) { + // Hovering over a node + this.hoverNode(nodeId); + } else { + // Not hovering over any node + this.unhoverNode(); + } + } + + /** + * Select a node and highlight it and its connections + * @param {string} nodeId - The ID of the node to select + */ + selectNode(nodeId) { + if (this.dataManager.selectedNode === nodeId) return; + + // Set selection in data manager + this.dataManager.setSelectedNode(nodeId); + + // Update colors in bulk + this.graphController.nodeCloud.updateColors( + nodeId, + this.dataManager.neighborNodes, + this.dataManager.hoveredNode + ); + + // Update link colors + this.updateLinkColors(); + + // First, hide all link labels + this.dataManager.graphObjects.links.forEach((link, id) => { + if (link.labelObject) { + link.labelObject.visible = false; + } + }); + + // Then show labels for active links + this.dataManager.activeLinks.forEach(linkId => { + const link = this.dataManager.graphObjects.links.get(linkId); + if (link && link.labelObject) { + link.labelObject.visible = true; + } + }); + + // Show node info panel + const graphController = this.sceneManager.graphController || + (this.graphObjectManager && this.graphObjectManager.graphController); + + if (graphController && graphController.uiManager) { + graphController.uiManager.showNodeInfo(nodeId); + } + } + + /** + * Deselect the currently selected node + */ + deselectNode() { + if (!this.dataManager.selectedNode) return; + + // Hide all link labels before clearing selection + this.dataManager.activeLinks.forEach(linkId => { + const link = this.dataManager.graphObjects.links.get(linkId); + if (link && link.labelObject) { + link.labelObject.visible = false; + } + }); + + // Clear selection in data manager + this.dataManager.clearSelectedNode(); + + // Update colors in bulk + this.graphController.nodeCloud.updateColors( + null, // No selected node + new Set(), // No neighbor nodes + this.dataManager.hoveredNode // Keep hover state + ); + + // Update link colors + this.updateLinkColors(); + + // Hide node info panel + const graphController = this.sceneManager.graphController || + (this.graphObjectManager && this.graphObjectManager.graphController); + + if (graphController && graphController.uiManager) { + graphController.uiManager.hideNodeInfo(); + } + } + + /** + * Focus the camera on a specific node + * @param {string} nodeId - The ID of the node to focus on + */ + focusOnNode(nodeId) { + const node = this.dataManager.graphObjects.nodes.get(nodeId); + if (!node || !node.object) return; + + const position = node.object.position.clone(); + this.sceneManager.focusCamera(position); + } + + /** + * Apply hover effect to a node + * @param {string} nodeId - The ID of the node to hover + */ + hoverNode(nodeId) { + // If already hovering over this node, do nothing + if (this.dataManager.hoveredNode === nodeId) return; + + // Set hover in data manager + this.dataManager.setHoveredNode(nodeId); + + // Update colors in bulk + this.graphController.nodeCloud.updateColors( + this.dataManager.selectedNode, + this.dataManager.neighborNodes, + nodeId + ); + + // Change cursor to pointer + this.sceneManager.renderer.domElement.style.cursor = 'pointer'; + } + + /** + * Remove hover effect from the currently hovered node + */ + unhoverNode() { + if (!this.dataManager.hoveredNode) return; + + // Get the node ID before clearing + const nodeId = this.dataManager.hoveredNode; + + // Clear hover in data manager + this.dataManager.clearHoveredNode(); + + // Update colors in bulk + this.graphController.nodeCloud.updateColors( + this.dataManager.selectedNode, + this.dataManager.neighborNodes, + null // No hover + ); + + // Reset cursor + this.sceneManager.renderer.domElement.style.cursor = 'auto'; + } + + /** + * Update the colors of all visible nodes based on selection state + */ + updateNodeColors() { + this.dataManager.graphObjects.nodes.forEach((node, id) => { + this.graphObjectManager.updateNodeColors(id); + }); + } + + /** + * Update the colors of all visible links based on selection state + */ + updateLinkColors() { + this.dataManager.graphObjects.links.forEach((link, id) => { + this.graphObjectManager.updateLinkColors(id); + }); + } +} + +/** + * DebugVisualizer - Generic visualization for debugging graph components + */ +class DebugVisualizer { + constructor(sceneManager, graphController) { + this.sceneManager = sceneManager; + this.graphController = graphController; + this.debugObjects = []; + this.enabled = false; + this.lastUpdateTime = 0; + this.updateInterval = 1000; // Update debug visuals every second + this.activeVisualizations = { + grid: true, + performance: true, + nodes: true + }; + this.stats = {}; + } + + /** + * Toggle debug visualization + * @param {boolean} enabled - Whether to enable or disable visualization + */ + toggle(enabled) { + this.enabled = enabled; + + // Show or hide debug UI elements + const debugPanel = document.getElementById('debug-info-panel'); + const frameGraph = document.getElementById('debug-frame-graph'); + + if (debugPanel) { + debugPanel.style.display = enabled ? 'block' : 'none'; + } + + if (frameGraph) { + frameGraph.style.display = enabled ? 'block' : 'none'; + } + + if (enabled) { + // Initialize frame history if needed + if (!this.frameHistory) { + const canvas = document.getElementById('debug-frame-canvas'); + if (canvas) { + this.frameHistory = new Array(canvas.width).fill(0); + } + } + + this.createDebugVisualization(); + } else { + this.clearDebugVisualization(); + } + } + + /** + * Toggle specific visualization types + * @param {string} type - Visualization type to toggle + */ + toggleVisualization(type) { + if (this.activeVisualizations.hasOwnProperty(type)) { + this.activeVisualizations[type] = !this.activeVisualizations[type]; + if (this.enabled) { + this.createDebugVisualization(); + } + } + } + + /** + * Create visual representation of debug data + */ + createDebugVisualization() { + this.clearDebugVisualization(); + + // Collect debug stats + this.collectStats(); + + // Create visualizations based on active settings + if (this.activeVisualizations.grid) { + this.createSpatialGridVisualization(); + } + + if (this.activeVisualizations.nodes) { + this.createNodeStatsVisualization(); + } + + if (this.activeVisualizations.performance) { + this.createPerformanceVisualization(); + } + + // Create debug panel with statistics + this.createDebugPanel(); + + this.lastUpdateTime = performance.now(); + } + + /** + * Collect statistics for debug display + */ + collectStats() { + // Clear previous stats + this.stats = { + fps: this.graphController.fps || 0, + nodeCount: 0, + visibleNodeCount: 0, + linkCount: 0, + gridStats: { + cells: 0, + objects: 0, + avgPerCell: 0 + }, + cameraPosition: { + x: 0, + y: 0, + z: 0 + }, + performanceMode: this.graphController.performanceMode + }; + + // Collect node and link stats + if (this.graphController.dataManager) { + const dataManager = this.graphController.dataManager; + + this.stats.nodeCount = dataManager.graphData.nodes.length; + this.stats.visibleNodeCount = dataManager.graphObjects.nodes.size; + this.stats.linkCount = dataManager.graphObjects.links.size; + } + + // Collect grid stats + if (this.graphController.simulationManager && + this.graphController.simulationManager.spatialGrid) { + + const grid = this.graphController.simulationManager.spatialGrid; + this.stats.gridStats.cells = grid.grid.size; + this.stats.gridStats.objects = grid.objects.size; + + if (grid.grid.size > 0 && grid.objects.size > 0) { + this.stats.gridStats.avgPerCell = + (grid.objects.size / grid.grid.size).toFixed(1); + } + } + + // Collect camera position + if (this.graphController.sceneManager && this.graphController.sceneManager.camera) { + const camera = this.graphController.sceneManager.camera; + this.stats.cameraPosition.x = camera.position.x.toFixed(1); + this.stats.cameraPosition.y = camera.position.y.toFixed(1); + this.stats.cameraPosition.z = camera.position.z.toFixed(1); + } + } + + /** + * Create spatial grid visualization + */ + createSpatialGridVisualization() { + const spatialGrid = this.graphController.simulationManager?.spatialGrid; + if (!spatialGrid) return; + + // Create wireframe boxes for each cell in the grid + spatialGrid.grid.forEach((cell, key) => { + const [x, y, z] = key.split(',').map(Number); + const cellSize = spatialGrid.cellSize; + + // Create box geometry for the cell + const geometry = new THREE.BoxGeometry(cellSize, cellSize, cellSize); + const material = new THREE.MeshBasicMaterial({ + color: 0x00ff00, + wireframe: true, + transparent: true, + opacity: 0.05 + (0.05 * Math.min(cell.size, 10)) // Brighter for more populated cells + }); + + const box = new THREE.Mesh(geometry, material); + box.position.set( + (x + 0.5) * cellSize, + (y + 0.5) * cellSize, + (z + 0.5) * cellSize + ); + + this.sceneManager.addToScene(box); + this.debugObjects.push(box); + + // Add text label showing object count in cell + if (cell.size > 0) { + const text = new SpriteText(`${cell.size}`, 12); + text.color = '#ffff00'; + text.backgroundColor = 'rgba(0,0,0,0.5)'; + text.padding = 2; + text.position.copy(box.position); + this.sceneManager.addToScene(text); + this.debugObjects.push(text); + } + }); + } + + /** + * Create node statistics visualization + */ + createNodeStatsVisualization() { + // Highlight nodes with different colors based on properties + const nodeManager = this.graphController.dataManager; + const nodeCloud = this.graphController.nodeCloud; + + if (!nodeManager || !nodeCloud || !nodeCloud.colors) return; + + // Store original colors to restore later + this.originalColors = new Float32Array(nodeCloud.colors.length); + this.originalColors.set(nodeCloud.colors); // Make a copy of all colors + + // Iterate through nodes and update colors in the buffer + nodeManager.graphObjects.nodes.forEach((node, id) => { + if (nodeCloud.nodeIndices.has(id)) { + // Get the node's index in the color buffer + const index = nodeCloud.nodeIndices.get(id); + const i3 = index * 3; + + // Get connection count + const connectedLinks = nodeManager.getConnectedLinks(id); + const connectionCount = connectedLinks.length; + + // Set color based on connection count + let color; + if (connectionCount > 10) { + color = new THREE.Color(0xff0000); // Red for highly connected + } else if (connectionCount > 5) { + color = new THREE.Color(0xff8800); // Orange for medium + } else if (connectionCount > 2) { + color = new THREE.Color(0xffff00); // Yellow for low + } else { + color = new THREE.Color(0x00ffff); // Cyan for minimal + } + + // Update the color buffer directly + nodeCloud.colors[i3] = color.r; + nodeCloud.colors[i3 + 1] = color.g; + nodeCloud.colors[i3 + 2] = color.b; + } + }); + + // Mark the color buffer as needing update + if (nodeCloud.geometry && nodeCloud.geometry.attributes.color) { + nodeCloud.geometry.attributes.color.needsUpdate = true; + } + } + + /** + * Create performance metrics visualization + */ + createPerformanceVisualization() { + // Update frame history and redraw + this.updateFrameGraph(); + } + + /** + * Update the frame rate graph + */ + updateFrameGraph() { + const canvas = document.getElementById('debug-frame-canvas'); + const fpsLabel = document.getElementById('debug-fps-label'); + + if (!canvas || !fpsLabel) return; + + // Update FPS label + fpsLabel.textContent = `${this.stats.fps} FPS`; + + // Add current FPS to history + if (!this.frameHistory) { + this.frameHistory = new Array(canvas.width).fill(0); + } + + this.frameHistory.push(this.stats.fps); + this.frameHistory.shift(); + + // Draw frame history + const ctx = canvas.getContext('2d'); + const width = canvas.width; + const height = canvas.height; + + // Clear canvas + ctx.clearRect(0, 0, width, height); + + // Calculate scale - find max FPS in history for scaling + const maxFPS = Math.max(60, ...this.frameHistory); + const scale = height / maxFPS; + + // Draw background grid + ctx.strokeStyle = '#333'; + ctx.lineWidth = 0.5; + + // Draw horizontal grid lines at 15, 30, 45, 60 FPS + [15, 30, 45, 60].forEach(fps => { + const y = height - (fps * scale); + if (y >= 0 && y <= height) { + ctx.beginPath(); + ctx.moveTo(0, y); + ctx.lineTo(width, y); + ctx.stroke(); + } + }); + + // Draw FPS graph + ctx.strokeStyle = '#4CAF50'; + ctx.lineWidth = 1.5; + ctx.beginPath(); + + // Start at bottom-left corner with 0 FPS + ctx.moveTo(0, height); + + // Draw lines for each frame sample + this.frameHistory.forEach((fps, x) => { + const y = height - (fps * scale); + ctx.lineTo(x, y); + }); + + // Finish at bottom-right corner + ctx.lineTo(width - 1, height); + ctx.closePath(); + + // Fill gradient + const gradient = ctx.createLinearGradient(0, 0, 0, height); + gradient.addColorStop(0, 'rgba(76, 175, 80, 0.7)'); + gradient.addColorStop(1, 'rgba(76, 175, 80, 0.1)'); + ctx.fillStyle = gradient; + ctx.fill(); + + // Stroke the line on top of the fill + ctx.strokeStyle = '#4CAF50'; + ctx.lineWidth = 1.5; + ctx.beginPath(); + + this.frameHistory.forEach((fps, x) => { + const y = height - (fps * scale); + if (x === 0) { + ctx.moveTo(x, y); + } else { + ctx.lineTo(x, y); + } + }); + + ctx.stroke(); + } + + /** + * Create debug panel with statistics + */ + createDebugPanel() { + // Update debug panel content using the existing HTML element + document.getElementById('debug-nodes').textContent = `${this.stats.visibleNodeCount}/${this.stats.nodeCount}`; + document.getElementById('debug-links').textContent = `${this.stats.linkCount}`; + document.getElementById('debug-cells').textContent = `${this.stats.gridStats.cells}`; + document.getElementById('debug-objects').textContent = `${this.stats.gridStats.objects}`; + document.getElementById('debug-avg-per-cell').textContent = `${this.stats.gridStats.avgPerCell}`; + + // Update camera position + document.getElementById('debug-camera-x').textContent = `${this.stats.cameraPosition.x}`; + document.getElementById('debug-camera-y').textContent = `${this.stats.cameraPosition.y}`; + document.getElementById('debug-camera-z').textContent = `${this.stats.cameraPosition.z}`; + } + + /** + * Remove all debug visualization objects + */ + clearDebugVisualization() { + // Remove all debug visualization objects from scene + this.debugObjects.forEach(obj => { + this.sceneManager.removeFromScene(obj); + }); + this.debugObjects = []; + + // Restore original node colors + if (this.originalColors && this.graphController.nodeCloud) { + const nodeCloud = this.graphController.nodeCloud; + + // Copy the original colors back to the nodeCloud color buffer + if (nodeCloud.colors && this.originalColors.length === nodeCloud.colors.length) { + nodeCloud.colors.set(this.originalColors); + + // Mark the color buffer as needing update + if (nodeCloud.geometry && nodeCloud.geometry.attributes.color) { + nodeCloud.geometry.attributes.color.needsUpdate = true; + } + } + + this.originalColors = null; + } + } + + /** + * Update debug visualization + */ + update() { + if (!this.enabled) return; + + // Update stats more frequently than full visualization refresh + this.collectStats(); + + // Update FPS graph and info panel more frequently + if (this.activeVisualizations.performance && document.getElementById('debug-frame-canvas')) { + this.updateFrameGraph(); + } + + // Update info panel if it exists + if (document.getElementById('debug-info-panel')) { + this.createDebugPanel(); // Updates panel content + } + + // Only update full visualization periodically to avoid performance impact + const now = performance.now(); + if (now - this.lastUpdateTime < this.updateInterval) return; + + // Recreate visualization + this.createDebugVisualization(); + } + + /** + * Handle keyboard shortcuts for debugging + * @param {KeyboardEvent} event - Keyboard event + */ + handleKeyPress(event) { + if (!this.enabled) return; + + switch (event.key) { + case '1': + this.toggleVisualization('grid'); + break; + case '2': + this.toggleVisualization('nodes'); + break; + case '3': + this.toggleVisualization('performance'); + break; + } + } +} + +/** + * Main controller class that coordinates all graph components + */ +class GraphController { + constructor(containerId) { + // DOM container reference + this.container = document.getElementById(containerId); + + // Initialize component managers + this.themeManager = new ThemeManager(); + this.sceneManager = new SceneManager(this.container, this.themeManager); + this.sceneManager.graphController = this; // Add reference to this controller + + this.dataManager = new DataManager(); + this.graphObjectManager = new GraphObjectManager(this.sceneManager, this.dataManager, this.themeManager); + this.graphObjectManager.graphController = this; // Add reference to this controller + + // Create the node cloud for efficient node rendering + this.nodeCloud = new NodeCloud(this.sceneManager.scene, this.themeManager); + + this.simulationManager = new SimulationManager(this.dataManager, this.graphObjectManager, this.themeManager); + this.simulationManager.graphController = this; // Add reference to this controller + + this.uiManager = new UIManager(this.container, this.dataManager, this); + this.eventManager = new EventManager(this.sceneManager, this.dataManager, this.graphObjectManager); + this.eventManager.graphController = this; // Add reference to this controller + + // Performance and debug settings + this.performanceMode = true; // Always on + this.debugMode = false; + + // Initialize generic debugger + this.debugger = new DebugVisualizer(this.sceneManager, this); + + // Initialize FPS counter + this.fpsCounter = document.getElementById('fps-counter'); + this.frameCount = 0; + this.lastTime = performance.now(); + this.fps = 0; + this.fpsUpdateInterval = 500; // Update FPS display every 500ms + + // Set up UI button handlers + this.setupButtonHandlers(); + + // Setup keyboard listeners for debug controls + document.addEventListener('keydown', this.handleKeyPress.bind(this)); + + // Store a global reference for convenience (used by NodeCloud) + window.lastGraphController = this; + + // Load data + this.loadGraphData(); + + // Always enable performance optimizations + this.enablePerformanceMode(); + + // Start animation loop + this.animate(); + } + + /** + * Set up handlers for UI buttons + */ + setupButtonHandlers() { + // Debug mode button + const debugBtn = document.getElementById('toggle-debug-btn'); + if (debugBtn) { + debugBtn.addEventListener('click', () => { + this.toggleDebugMode(); + debugBtn.classList.toggle('active', this.debugMode); + }); + } + } + + /** + * Enable performance optimizations + */ + enablePerformanceMode() { + // Enable spatial grid and frustum culling + if (this.simulationManager) { + this.simulationManager.useSpatialIndex = true; + } + if (this.sceneManager) { + this.sceneManager.enableFrustumCulling = true; + } + } + + /** + * Handle keyboard shortcuts + * @param {KeyboardEvent} event - Keyboard event + */ + handleKeyPress(event) { + // Pass to debugger if debug mode is on + if (this.debugMode && this.debugger) { + this.debugger.handleKeyPress(event); + } + } + + /** + * Toggle debug visualization mode + */ + toggleDebugMode() { + this.debugMode = !this.debugMode; + + if (this.debugger) { + this.debugger.toggle(this.debugMode); + } + + return this.debugMode; + } + + /** + * Animation loop + */ + animate() { + requestAnimationFrame(() => this.animate()); + + // Update FPS calculation + this.frameCount++; + const currentTime = performance.now(); + const elapsed = currentTime - this.lastTime; + + // Update FPS counter every interval + if (elapsed > this.fpsUpdateInterval) { + this.fps = Math.round((this.frameCount * 1000) / elapsed); + this.fpsCounter.textContent = `FPS: ${this.fps}`; + + // Reset counters + this.frameCount = 0; + this.lastTime = currentTime; + } + + // Update debug visualization if enabled + if (this.debugMode && this.debugger) { + this.debugger.update(); + } + + // Update scene + this.sceneManager.update(); + } + + /** + * Load graph data from the server + */ + loadGraphData() { + this.clearDisplay(); + // Show loading indicator + this.uiManager.showLoading(true); + + // Load data via the data manager + this.dataManager.loadData() + .then(data => { + // Initialize the force simulation with loaded data + this.simulationManager.updateSimulation(false); + + // Update statistics + this.uiManager.updateStats(); + + // Show initial message + this.uiManager.showInitialMessage("Enter a search term to display nodes"); + + // Hide loading indicator + this.uiManager.showLoading(false); + }) + .catch(error => { + // Show error message + this.uiManager.showError('Failed to load graph data: ' + error.message); + this.uiManager.showLoading(false); + }); + } + + /** + * Search for nodes by term and display them + * @param {string} searchTerm - The term to search for + */ + searchNodes(searchTerm) { + // Clear current display + this.clearDisplay(); + + // Hide the initial message + this.uiManager.hideInitialMessage(); + + if (!searchTerm) return; + + // Find matching nodes + const matchingNodeIds = this.dataManager.searchNodes(searchTerm); + + // If no nodes found, show a message + if (matchingNodeIds.length === 0) { + console.log(`No nodes found matching "${searchTerm}"`); + this.uiManager.showInitialMessage(`No nodes found matching "${searchTerm}"`); + return; + } + + console.log(`Found ${matchingNodeIds.length} nodes matching "${searchTerm}"`); + + // Show loading indicator during simulation + this.uiManager.showLoading(true); + + // Add each matching node and its connections with depth of 10 + const addedNodes = new Set(); + + matchingNodeIds.forEach(nodeId => { + // Use getConnectedSubgraph to get nodes and links up to depth 10 + const { nodes, links } = this.dataManager.getConnectedSubgraph(nodeId, 10); + + // Add all nodes to the scene + nodes.forEach(node => { + if (!this.dataManager.graphObjects.nodes.has(node.id)) { + this.graphObjectManager.createNodeObject(node); + addedNodes.add(node.id); + } + }); + + // Add all links to the scene + links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + const linkId = `${sourceId}-${targetId}`; + + if (!this.dataManager.graphObjects.links.has(linkId)) { + this.graphObjectManager.createLinkObject(link); + } + }); + }); + + // Update simulation and restart it properly + this.simulationManager.updateSimulation(true); + + // Center the view on the found nodes + this.centerOnNodes(Array.from(addedNodes)); + + // Hide loading indicator when view is centered + this.uiManager.showLoading(false); + } + + /** + * Clear the current display + */ + clearDisplay() { + this.graphObjectManager.clearVisibleObjects(); + } + + /** + * Center the view on a set of nodes + * @param {Array} nodeIds - Array of node IDs to center on + */ + centerOnNodes(nodeIds) { + if (!nodeIds || nodeIds.length === 0) return; + + // Calculate the center position of the specified nodes + let center = { x: 0, y: 0, z: 0 }; + let count = 0; + + nodeIds.forEach(nodeId => { + const nodeData = this.dataManager.graphObjects.nodes.get(nodeId); + if (nodeData && nodeData.object) { + center.x += nodeData.object.position.x; + center.y += nodeData.object.position.y; + center.z += nodeData.object.position.z; + count++; + } + }); + + if (count === 0) return; + + center.x /= count; + center.y /= count; + center.z /= count; + + // Set the camera target for perspective camera + this.sceneManager.controls.target.set(center.x, center.y, 0); + + // Position the perspective camera + const distance = 1000; + this.sceneManager.camera.position.set( + center.x, + center.y, + distance + ); + + // Update the camera and controls + this.sceneManager.camera.updateProjectionMatrix(); + this.sceneManager.controls.update(); + } + + /** + * Toggle label visibility + */ + toggleLabels() { + const showLabels = this.graphObjectManager.toggleLabels(); + return showLabels; + } + + /** + * Toggle physics simulation + */ + togglePhysics() { + this.simulationManager.togglePhysics(); + } + + /** + * Reset the view + */ + resetView() { + // Clear current display + this.clearDisplay(); + + // Reset camera + this.sceneManager.resetView(); + + // Show initial message + this.uiManager.showInitialMessage("Enter a search term to display nodes"); + } + + /** + * Load all nodes in the graph + */ + loadAllNodes() { + // Clear current display + this.clearDisplay(); + + // Hide the initial message + this.uiManager.hideInitialMessage(); + + // Show loading indicator + this.uiManager.showLoading(true); + + console.log(`Loading all ${this.dataManager.graphData.nodes.length} nodes`); + + // Store all added node IDs + const addedNodes = new Set(); + + // Add all nodes to the scene + this.dataManager.graphData.nodes.forEach(node => { + if (!this.dataManager.graphObjects.nodes.has(node.id)) { + this.graphObjectManager.createNodeObject(node); + addedNodes.add(node.id); + } + }); + + // Add all links between the visible nodes + this.dataManager.graphData.links.forEach(link => { + const sourceId = typeof link.source === 'object' ? link.source.id : link.source; + const targetId = typeof link.target === 'object' ? link.target.id : link.target; + const linkId = `${sourceId}-${targetId}`; + + // Only add links between nodes that are visible + if (addedNodes.has(sourceId) && addedNodes.has(targetId) && + !this.dataManager.graphObjects.links.has(linkId)) { + this.graphObjectManager.createLinkObject(link); + } + }); + + // Update simulation and restart it + this.simulationManager.updateSimulation(true); + + // Center the view on all nodes + this.centerOnNodes(Array.from(addedNodes)); + + // Hide loading indicator + this.uiManager.showLoading(false); + } +} + +// Initialize the application when the page loads +document.addEventListener('DOMContentLoaded', () => { + new GraphController('graph-container'); +}); \ No newline at end of file diff --git a/src/html/hyperbuddy@1.0/index.html b/src/html/hyperbuddy@1.0/index.html new file mode 100644 index 000000000..8afc56419 --- /dev/null +++ b/src/html/hyperbuddy@1.0/index.html @@ -0,0 +1,575 @@ + + + + + + + + + + HyperBEAM + + +
+
+ + + +
+

Operator:

+ +
+
+
+
+
+
+ +
+
+
+
+
+ Signature +

...

+
+
+ Signer +

...

+
+
+
+
+ + + +

Response

+
+
+
+ +
+
+
+
+ + + +

Links

+
+ +
+ +
+
+
+
+
+
+
+ + +
+
+
+
+
+
+
+
+
+
+

Status:

+
+
+

Live

+
+
+
+
+
+ + + + + + diff --git a/src/html/hyperbuddy@1.0/metrics.js b/src/html/hyperbuddy@1.0/metrics.js new file mode 100644 index 000000000..55dfa886f --- /dev/null +++ b/src/html/hyperbuddy@1.0/metrics.js @@ -0,0 +1,590 @@ +// Import the required functions from utils.js +import { get, formatDisplayAmount } from "/~hyperbuddy@1.0/utils.js"; + +/** + * Parse metrics data from Prometheus format + * @param {string} text - The raw metrics text in Prometheus format + * @returns {Object} - Organized metrics data + */ +function parseMetrics(text) { + const lines = text.split("\n"); + const groups = {}; + let currentMetric = null; + + lines.forEach((line) => { + line = line.trim(); + if (!line) return; + + if (line.startsWith("# TYPE")) { + const parts = line.split(/\s+/); + const metricName = parts[2]; + const metricType = parts[3]; + + if (!groups[metricType]) { + groups[metricType] = []; + } + + currentMetric = { + name: metricName, + help: "", + values: [], + }; + + groups[metricType].push(currentMetric); + } else if (line.startsWith("# HELP")) { + const parts = line.split(/\s+/); + const metricName = parts[2]; + const helpText = parts.slice(3).join(" "); + if (currentMetric && currentMetric.name === metricName) { + currentMetric.help = helpText; + } + } else if (line.startsWith("#")) { + // Skip other comments + } else { + if (currentMetric) { + const match = line.match(/(-?\d+(\.\d+)?)(\s*)$/); + if (match) { + let label = currentMetric.name; + const data = parseFloat(match[1]); + + const inputMatch = line.match(/\{([^}]*)\}/); + if (inputMatch) { + label = inputMatch[1]; + } + + if (label === `process_uptime_seconds`) { + document.getElementById("uptime-value").innerHTML = + formatDisplayAmount(data); + startUpdatingUptime(); + } else if (label === `system_load`) { + document.getElementById("system-value").innerHTML = + formatDisplayAmount(data); + } else if ( + currentMetric.name === "event" && + label.includes('topic="ao_result",event="ao_result"') + ) { + document.getElementById("executions-value").innerHTML = + formatDisplayAmount(data); + } + + currentMetric.values.push({ label, data }); + } + } + } + }); + + if (groups.counter) { + const spawnedProcesse = groups.counter.find( + (item) => item.name === "cowboy_spawned_processes_total" + ); + + if (spawnedProcesse?.values) { + const readsHandled = + spawnedProcesse.values.find((value) => + value.label.includes( + 'method="GET",reason="normal",status_class="success"' + ) + )?.data || 0; + + const writesHandled = + spawnedProcesse.values.find((value) => + value.label.includes( + 'method="POST",reason="normal",status_class="success"' + ) + )?.data || 0; + + document.getElementById("read-value").innerHTML = + formatDisplayAmount(readsHandled); + document.getElementById("write-value").innerHTML = + formatDisplayAmount(writesHandled); + } + } + + return groups; +} + +/** + * Renders the metrics data in the UI + * @param {Object} groups - The organized metrics data + */ +function renderMetricGroups(groups) { + // Save the currently active category or default to "AO Events" + let activeCategory = + localStorage.getItem("activeMetricsCategory") || "AO Events"; + + const container = document.getElementById("metrics-section"); + container.innerHTML = ""; + + // Create category sections to organize metrics + const categories = { + "AO Events": ["event"], + "System Stats": [ + "process_uptime_seconds", + "system_load", + "outbound_connections", + ], + "HTTP & Requests": [ + "cowboy_requests_total", + "cowboy_protocol_upgrades_total", + "cowboy_spawned_processes_total", + "cowboy_errors_total", + "cowboy_early_errors_total", + "cowboy_request_duration_seconds", + "http_request_duration_seconds", + "cowboy_receive_body_duration_seconds", + ], + Memory: [ + "erlang_vm_memory_atom_bytes_total", + "erlang_vm_memory_bytes_total", + "erlang_vm_memory_dets_tables", + "erlang_vm_memory_ets_tables", + "erlang_vm_memory_processes_bytes_total", + "erlang_vm_memory_system_bytes_total", + ], + "Network Stats": [ + "http_client_uploaded_bytes_total", + "http_client_downloaded_bytes_total", + "gun_requests_total", + ], + Telemetry: [ + "telemetry_scrape_size_bytes", + "telemetry_scrape_duration_seconds", + "telemetry_scrape_encoded_size_bytes", + ], + "VM Stats": [ + "erlang_vm_msacc_aux_seconds_total", + "erlang_vm_msacc_check_io_seconds_total", + "erlang_vm_msacc_emulator_seconds_total", + "erlang_vm_msacc_gc_seconds_total", + "erlang_vm_msacc_other_seconds_total", + ], + }; + + // Build an index to quickly find metrics + const metricsIndex = {}; + for (const type in groups) { + groups[type].forEach((metric) => { + metricsIndex[metric.name] = { metric, type }; + }); + } + + // Create the metrics navbar + const navbar = document.createElement("div"); + navbar.classList.add("metrics-navbar"); + + // Create content container to hold the metrics content + const metricsContent = document.createElement("div"); + metricsContent.classList.add("metrics-content"); + + // Create and add all category sections first (hidden initially) + const categoryContainers = {}; + + // Function to show the selected category + const showCategory = (categoryName) => { + // Hide all categories + Object.values(categoryContainers).forEach((container) => { + container.style.display = "none"; + }); + + // Show selected category + if (categoryContainers[categoryName]) { + categoryContainers[categoryName].style.display = "block"; + } + + // Update active tab in navbar + document.querySelectorAll(".metrics-nav-item").forEach((item) => { + if (item.textContent === categoryName) { + item.classList.add("active"); + } else { + item.classList.remove("active"); + } + }); + + // Save active category to localStorage + localStorage.setItem("activeMetricsCategory", categoryName); + activeCategory = categoryName; + }; + + // Create "Other Metrics" category for uncategorized metrics + const otherMetrics = []; + for (const type in groups) { + groups[type].forEach((metric) => { + let isCategorized = false; + for (const metricNames of Object.values(categories)) { + if (metricNames.includes(metric.name)) { + isCategorized = true; + break; + } + } + if (!isCategorized) { + otherMetrics.push({ metric, type }); + } + }); + } + + if (otherMetrics.length > 0) { + categories["Other Metrics"] = []; + } + + // Create navbar items + for (const categoryName in categories) { + // Create navbar item + const navItem = document.createElement("div"); + navItem.classList.add("metrics-nav-item"); + navItem.textContent = categoryName; + + if (categoryName === activeCategory) { + navItem.classList.add("active"); + } + + navItem.addEventListener("click", () => { + showCategory(categoryName); + }); + + navbar.appendChild(navItem); + + // Create section container for this category + const categoryContainer = document.createElement("div"); + categoryContainer.classList.add("metrics-category"); + categoryContainer.style.display = + categoryName === activeCategory ? "block" : "none"; + categoryContainers[categoryName] = categoryContainer; + + // Build the category's content + if (categoryName === "Other Metrics") { + otherMetrics.forEach(({ metric, type }) => { + const metricContainer = createMetricDisplay(metric, type, categoryName); + categoryContainer.appendChild(metricContainer); + }); + } else { + let hasMetrics = false; + for (const metricName of categories[categoryName]) { + if (!metricsIndex[metricName]) continue; + + hasMetrics = true; + const { metric, type } = metricsIndex[metricName]; + + const metricContainer = createMetricDisplay(metric, type, categoryName); + categoryContainer.appendChild(metricContainer); + } + + if (!hasMetrics) { + const noDataMsg = document.createElement("div"); + noDataMsg.textContent = "No data available for this category"; + noDataMsg.style.padding = "20px"; + noDataMsg.style.textAlign = "center"; + categoryContainer.appendChild(noDataMsg); + } + } + + metricsContent.appendChild(categoryContainer); + } + + // Add navbar and content to container + container.appendChild(navbar); + container.appendChild(metricsContent); + + // Helper function to create metric display + function createMetricDisplay(metric, type, categoryName) { + const metricContainer = document.createElement("div"); + metricContainer.classList.add("metric-container"); + + // Skip the metric header for the AO Events section + if (!(categoryName === "AO Events" && metric.name === "event")) { + const subheader = document.createElement("div"); + subheader.classList.add("metrics-section-lines-header"); + subheader.classList.add("section-lines-header"); + + const metricTitle = document.createElement("p"); + metricTitle.textContent = metric.name + " (" + type + ")"; + subheader.appendChild(metricTitle); + + metricContainer.appendChild(subheader); + } + + // Add help text if available + if ( + metric.help && + !(categoryName === "AO Events" && metric.name === "event") + ) { + const helpDiv = document.createElement("div"); + helpDiv.classList.add("section-line"); + helpDiv.style.fontStyle = "italic"; + helpDiv.style.fontSize = "12px"; + helpDiv.style.color = "#666"; + helpDiv.textContent = metric.help; + metricContainer.appendChild(helpDiv); + } + + const metricBody = document.createElement("div"); + metricBody.classList.add("section-lines"); + + // Special handling for event metrics - group by topic + if (metric.name === "event") { + const eventsByTopic = {}; + + metric.values.forEach((value) => { + const match = value.label.match(/topic="([^"]+)"/); + if (match) { + const topic = match[1]; + if (!eventsByTopic[topic]) { + eventsByTopic[topic] = []; + } + eventsByTopic[topic].push(value); + } + }); + + for (const [topic, events] of Object.entries(eventsByTopic)) { + const topicHeader = document.createElement("div"); + topicHeader.classList.add("section-line"); + topicHeader.style.fontWeight = "bold"; + topicHeader.textContent = `Topic: ${topic}`; + metricBody.appendChild(topicHeader); + + events.forEach((event) => { + const eventMatch = event.label.match(/event="([^"]+)"/); + if (eventMatch) { + const eventLine = document.createElement("div"); + eventLine.classList.add("section-line"); + + const eventName = document.createElement("p"); + eventName.textContent = eventMatch[1]; + eventName.style.paddingLeft = "20px"; + + const eventValue = document.createElement("p"); + eventValue.textContent = formatDisplayAmount(event.data); + + eventLine.appendChild(eventName); + eventLine.appendChild(eventValue); + metricBody.appendChild(eventLine); + } + }); + } + } else if (metric.values.length > 0) { + // Sort values for better readability + const sortedValues = [...metric.values].sort((a, b) => { + return a.label.localeCompare(b.label); + }); + + // Handle histogram metrics - group by method + if ( + metric.name.includes("duration_seconds") && + metric.name.includes("bucket") + ) { + const valuesByMethod = {}; + + sortedValues.forEach((value) => { + const methodMatch = value.label.match(/method="([^"]+)"/); + const method = methodMatch ? methodMatch[1] : "unknown"; + + if (!valuesByMethod[method]) { + valuesByMethod[method] = []; + } + valuesByMethod[method].push(value); + }); + + for (const method in valuesByMethod) { + const methodHeader = document.createElement("div"); + methodHeader.classList.add("section-line"); + methodHeader.style.fontWeight = "bold"; + methodHeader.textContent = `Method: ${method}`; + metricBody.appendChild(methodHeader); + + // Only show summary stats for histograms to avoid clutter + const count = + valuesByMethod[method].find((v) => v.label.includes("count")) + ?.data || 0; + const sum = + valuesByMethod[method].find((v) => v.label.includes("sum"))?.data || + 0; + const avg = count > 0 ? sum / count : 0; + + const statsLine = document.createElement("div"); + statsLine.classList.add("section-line"); + statsLine.style.paddingLeft = "20px"; + + statsLine.innerHTML = ` +

Count: ${formatDisplayAmount(count)}

+

Sum: ${formatDisplayAmount(sum)}

+

Avg: ${formatDisplayAmount(avg.toFixed(4))}

+ `; + metricBody.appendChild(statsLine); + } + } else { + // Group HTTP metrics by method if they contain method in their labels + if ( + categoryName === "HTTP & Requests" && + (metric.name.includes("cowboy") || metric.name.includes("http")) && + sortedValues.some((v) => v.label.includes("method=")) + ) { + // Group values by HTTP method + const valuesByMethod = {}; + + sortedValues.forEach((value) => { + const methodMatch = value.label.match(/method="([^"]+)"/); + const method = methodMatch ? methodMatch[1] : "unknown"; + + if (!valuesByMethod[method]) { + valuesByMethod[method] = []; + } + valuesByMethod[method].push(value); + }); + + // Display metrics grouped by method + for (const [method, values] of Object.entries(valuesByMethod)) { + const methodHeader = document.createElement("div"); + methodHeader.classList.add("section-line"); + methodHeader.style.fontWeight = "bold"; + methodHeader.style.backgroundColor = "#f0f0f0"; + methodHeader.textContent = `Method: ${method}`; + metricBody.appendChild(methodHeader); + + values.forEach((valueMap) => { + const metricLine = document.createElement("div"); + metricLine.classList.add("section-line"); + metricLine.style.paddingLeft = "20px"; + + const labelEl = document.createElement("p"); + // Extract just the reason and status for cleaner display + let displayLabel = "(default)"; + + const reasonMatch = valueMap.label.match(/reason="([^"]+)"/); + const statusMatch = valueMap.label.match( + /status_class="([^"]+)"/ + ); + const errorMatch = valueMap.label.match(/error="([^"]+)"/); + + if (reasonMatch || statusMatch) { + displayLabel = []; + if (reasonMatch) displayLabel.push(`reason: ${reasonMatch[1]}`); + if (statusMatch) displayLabel.push(`status: ${statusMatch[1]}`); + if (errorMatch) displayLabel.push(`error: ${errorMatch[1]}`); + displayLabel = displayLabel.join(", "); + } else { + displayLabel = valueMap.label.replace(/method="[^"]+",\s*/, ""); + displayLabel = + displayLabel === metric.name ? "(default)" : displayLabel; + } + + labelEl.textContent = displayLabel; + + const valueEl = document.createElement("p"); + valueEl.textContent = formatDisplayAmount(valueMap.data); + + metricLine.appendChild(labelEl); + metricLine.appendChild(valueEl); + metricBody.appendChild(metricLine); + }); + } + } else { + // Limit number of displayed values if too many + const maxValues = 15; + const valuesToShow = + sortedValues.length > maxValues + ? sortedValues.slice(0, maxValues) + : sortedValues; + + if (sortedValues.length > maxValues) { + const noticeDiv = document.createElement("div"); + noticeDiv.classList.add("section-line"); + noticeDiv.style.fontStyle = "italic"; + noticeDiv.textContent = `Showing ${maxValues} of ${sortedValues.length} values`; + metricBody.appendChild(noticeDiv); + } + + // Default rendering for non-HTTP metrics or metrics without method labels + valuesToShow.forEach((valueMap) => { + const metricLine = document.createElement("div"); + metricLine.classList.add("section-line"); + + const labelEl = document.createElement("p"); + // For better readability, format complex labels + const displayLabel = + valueMap.label === metric.name + ? "(default)" + : valueMap.label.replace(/,/g, ", "); + labelEl.textContent = displayLabel; + + const valueEl = document.createElement("p"); + valueEl.textContent = formatDisplayAmount(valueMap.data); + + metricLine.appendChild(labelEl); + metricLine.appendChild(valueEl); + metricBody.appendChild(metricLine); + }); + } + } + } else { + const metricLine = document.createElement("div"); + metricLine.classList.add("section-line"); + const metricValueLabel = document.createElement("p"); + metricValueLabel.textContent = "No data"; + metricLine.appendChild(metricValueLabel); + const metricValueData = document.createElement("p"); + metricValueData.textContent = "-"; + metricLine.appendChild(metricValueData); + metricBody.appendChild(metricLine); + } + + metricContainer.appendChild(metricBody); + return metricContainer; + } +} + +/** + * Fetches metrics data from the server + */ +async function fetchMetrics() { + try { + const metrics = await get("/~hyperbuddy@1.0/metrics"); + const metricGroups = parseMetrics(metrics); + renderMetricGroups(metricGroups); + } catch (error) { + console.error("Error fetching or parsing metrics:", error); + } +} + +/** + * Updates the uptime value on a regular basis + */ +function startUpdatingUptime() { + const uptimeElement = document.getElementById("uptime-value"); + if (!uptimeElement) return; + + const initialUptime = + parseFloat(uptimeElement.innerHTML.replaceAll(",", "")) || 0; + const startTime = Date.now(); + + function updateUptime() { + const elapsedSeconds = (Date.now() - startTime) / 1000; + const currentUptime = initialUptime + elapsedSeconds; + + uptimeElement.innerHTML = formatDisplayAmount(currentUptime.toFixed(2)); + requestAnimationFrame(updateUptime); + } + + updateUptime(); +} + +/** + * Start fetching metrics periodically + */ +async function startFetchingMetrics() { + await fetchMetrics(); + + // Initialize metrics display + if (!localStorage.getItem("activeMetricsCategory")) { + localStorage.setItem("activeMetricsCategory", "AO Events"); + } + + setInterval(fetchMetrics, 50000); +} + +// Export functions for use in other modules +export { + parseMetrics, + renderMetricGroups, + fetchMetrics, + startUpdatingUptime, + startFetchingMetrics, +}; diff --git a/src/html/hyperbuddy@1.0/styles.css b/src/html/hyperbuddy@1.0/styles.css new file mode 100644 index 000000000..737560607 --- /dev/null +++ b/src/html/hyperbuddy@1.0/styles.css @@ -0,0 +1,1021 @@ +:root { + --bg-color: #fcfcfc; + --section-bg-color-primary: #fff; + --section-bg-color-alt1: #ffffff; + --section-bg-color-alt2: #ffffffbf; + --section-bg-color-alt3: #ffffffee; + --section-bg-color-accent: #f8faf8f5; + --border-color: #eaeaea; + --active-action-bg: #EDEDED; + --text-color-primary: #000000; + --text-color-alt1: #646464; + --text-color-light: #FFFFFF; + --accent-color: #e9ffc5; + --link-color: #1A73E8; + --link-hover-color: #0C47A1; + --pulse-color: #17a923; + --pulse-color-active: #17a923; + --indicator-color: #f17100; + --indicator-color-active: #f17100; + --border-radius: 7.5px; +} + +@keyframes rotateInfinite { + from { + transform: rotate(0deg); + } + + to { + transform: rotate(360deg); + } +} + +@keyframes pulse { + 0%, + 100% { + background: var(--pulse-color); + transform: scale(1); + } + + 50% { + background: var(--pulse-color-active); + transform: scale(1.15); + } +} + +* { + box-sizing: border-box; +} + +body { + padding: 0; + margin: 0; + font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", + Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans", "Helvetica Neue", + sans-serif; + font-family: "Open Sans", sans-serif; + background: var(--bg-color); + color: var(--text-color-primary); +} + +h1, +h2, +h3, +h4, +h5, +h6, +p, +ul, +li, +span { + margin: 0; + color: var(--text-color-primary); +} + +button { + appearance: none; + outline: none; + border: none; + background: none; + padding: 0; + margin: 0; + transition: all 100ms; + font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", + Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans", "Helvetica Neue", + sans-serif; + font-family: "Open Sans", sans-serif; +} + +h1 { + font-size: clamp(24px, 2.75vw, 32px); + font-weight: 600; + letter-spacing: 0.5px; + color: var(--text-color-primary); +} + +ul { + list-style: none; + padding: 0; +} + +a { + text-decoration: none; + color: var(--link-color); + font-size: 14px; +} + +a:hover { + text-decoration: underline; + text-decoration-thickness: 1.5px; + color: var(--link-hover-color); +} + +.collapsible-header { + cursor: pointer; + display: flex; + align-items: center; + justify-content: space-between; +} + +.collapsible-header:hover { + background-color: var(--section-bg-color-alt1); +} + +.collapsible-header .header-text { + flex: 1; +} + +.collapse-icon { + font-size: 20px; + margin-right: 8px; + transition: transform 0.3s ease; +} + +.collapsed .collapse-icon { + transform: rotate(-90deg); +} + +.collapsible-content { + max-height: 2000px; + overflow: hidden; + transition: max-height 0.5s ease-in-out; +} + +.collapsed .collapsible-content { + max-height: 0; +} + +.video-container { + display: flex; + align-items: center; + height: 190px; +} + +header { + position: fixed; + top: 0; + background-color: var(--section-bg-color-alt1); + width: 100%; + border-bottom: 1px solid var(--border-color); + box-shadow: 0px 2px 2px 0px rgba(0, 0, 0, 0.03); + + z-index: 20; +} + +footer { + position: fixed; + bottom: 0; + background-color: var(--section-bg-color-alt1); + width: 100%; + border-top: 1px solid var(--border-color); + z-index: 20; +} + +.footer-inner { + width: 100%; + display: flex; + align-items: flex-end; + justify-content: space-between; + overflow: hidden; + gap: 10px; + padding: 8px 24px; +} + +.header-inner { + width: 100%; + max-width: 2000px; + padding: 15px 40px; + margin: 0 auto; + display: flex; + align-items: center; + justify-content: space-between; +} + +.logo { + display: flex; + align-items: center; + height: 25px; +} + +.logo p { + font-size: clamp(0.65rem, 1.75vw, 0.7rem); + color: var(--text-color-alt1) +} + +.logo img { + height: 100%; +} + +.subheader { + display: flex; + flex-direction: column; + gap: 15px; + align-items: flex-start; + z-index: 2; +} + +.subheader-value { + display: flex; + align-items: center; + gap: 7.5px; +} + +.subheader-value p { + font-size: clamp(0.65rem, 1.75vw, 0.75rem); + font-weight: 400; + color: var(--text-color-alt1); +} + +.subheader-indicator-wrapper { + display: flex; + align-items: center; + gap: 7.5px; +} + +.subheader-indicator-wrapper p { + color: var(--text-color-primary); + font-weight: 600; +} + +.subheader-indicator { + height: 8px; + width: 8px; + background: var(--pulse-color); + border-radius: 50%; + animation: pulse 1.075s infinite; +} + +.subheader-value button { + letter-spacing: 0.5px; + padding: 0; + font-size: clamp(0.7rem, 1.5vw, 0.75rem); + font-weight: 600; + color: var(--text-color-primary); + text-decoration: underline; + text-decoration-thickness: 1.5px; +} + +.subheader-value button:hover { + cursor: pointer; + color: var(--text-color-alt1); +} + +.subheader-value button:disabled { + cursor: default; + color: var(--text-color-alt1); +} + +.section-groups { + position: relative; + width: 100%; + display: flex; + flex-direction: column; + gap: 20px; +} + +.section-group, +.tabs-wrapper { + z-index: 10; +} + +.section-group { + width: 100%; + display: flex; + flex-wrap: wrap; + gap: 20px; +} + +.section { + height: fit-content; + flex: 1; + border-radius: var(--border-radius); + background: var(--section-bg-color-alt3); + border: 1px solid var(--border-color); + box-shadow: 0px 4px 4px 0px rgba(0, 0, 0, 0.02); + gap: 48px; + padding: 12px; +} + +.section-header { + font-size: clamp(0.7rem, 1.75vw, 0.85rem); + font-weight: 500; + padding: 15px; + border-bottom: 1px solid var(--border-color); + color: var(--text-color-primary); +} + +.key-metric-wrapper { + display: flex; + flex-direction: column; + gap: 7.5px; + padding: 5px 15px 15px 15px; +} + +.key-metric-header { + border-bottom: none; +} + +.key-metric-value { + font-size: clamp(1rem, 2.75vw, 1.8rem); + font-weight: 400; + +} + +.key-metric-label { + font-size: clamp(0.6rem, 1.5vw, 0.75rem); + font-weight: 400; + color: var(--text-color-alt1); + text-transform: uppercase; +} + +.metrics-section-header { + border-bottom: none; +} + +.section-lines-header { + height: 35px; + width: 100%; + display: flex; + align-items: center; + justify-content: space-between; + padding: 0 15px; + border-bottom: 1px solid var(--border-color); + background: var(--section-bg-color-alt1); +} + +.section-lines-header p { + font-size: clamp(13px, 1.75vw, 14px); + font-weight: 500; + color: var(--text-color-alt1); + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; +} + +.section-lines { + width: 100%; + display: flex; + flex-direction: column; +} + +.section-line { + height: 35px; + width: 100%; + display: flex; + align-items: center; + justify-content: space-between; + padding: 0 15px; + border-bottom: 1px solid var(--border-color); + font-size: 14px; +} + +.info-line:nth-child(even) { + background: var(--section-bg-color-primary); +} + +.info-line:nth-child(odd) { + background: var(--bg-color); +} + +.loading-info-line p { + font-size: 14px; + font-weight: 500; + padding: 0 10px 10px 5px; +} + +.section-line:last-child { + border-bottom: none; + border-bottom-left-radius: var(--border-radius); + border-bottom-right-radius: var(--border-radius); +} + +.section-line p { + font-size: clamp(13px, 1.75vw, 14px); + font-weight: 500; + max-width: 60%; + overflow: hidden; + text-overflow: ellipsis; +} + +#metrics-section { + width: 100%; + display: flex; + flex-direction: column; +} + +#console-section { + border-right: 1px solid var(--border-color); + border-left: 1px solid var(--border-color); + border-bottom: 1px solid var(--border-color); + border-bottom-right-radius: var(--border-radius); + border-bottom-left-radius: var(--border-radius); +} + +#ledger-section { + padding: 15px 30px; + background: var(--section-bg-color-primary); + border-right: 1px solid var(--border-color); + border-left: 1px solid var(--border-color); + border-bottom: 1px solid var(--border-color); + border-bottom-right-radius: var(--border-radius); + border-bottom-left-radius: var(--border-radius); +} + +.json-type-label { + font-size: clamp(13px, 1.75vw, 14px); + font-weight: 500; + color: var(--text-color-primary); + display: block; + margin: 0 0 5px 0; +} + +.json-key { + font-size: clamp(13px, 1.75vw, 14px); + font-weight: 500; + color: var(--text-color-alt1); +} + +.json-separator { + font-size: clamp(13px, 1.75vw, 14px); + font-weight: 500; + color: var(--text-color-alt1); +} + +.json-string { + font-size: clamp(13px, 1.75vw, 14px); + font-weight: 600; + color: var(--text-color-primary); +} + +.json-number { + font-size: clamp(13px, 1.75vw, 14px); + font-weight: 600; + color: var(--text-color-primary); +} + +.tabs-wrapper { + display: flex; + flex-direction: column; + border-radius: var(--border-radius); + box-shadow: 0px 4px 4px 0px rgba(0, 0, 0, 0.02); +} + +.tabs { + position: relative; + display: flex; + gap: 10px; + padding: 10px 10px 5px 10px; + background: var(--section-bg-color-alt1); + border: 1px solid var(--border-color); + border-top-right-radius: var(--border-radius); + border-top-left-radius: var(--border-radius); +} + +.tab-button { + font-size: clamp(0.75rem, 1.75vw, 0.9rem); + font-weight: 500; + color: var(--text-color-alt1); + cursor: pointer; + position: relative; + padding: 10px 20px 10px 20px; +} + +.tab-button.active { + color: var(--text-color-primary); +} + +.tab-button.active::after { + width: 100%; + content: ""; + background: var(--indicator-color); + height: 2.5px; + display: block; + position: absolute; + bottom: -6px; + left: 0; +} + +.hyperstate-tab-button.active::after { + background: var(--indicator-color); +} + +.dashboard-tab-button.active::after { + background: var(--indicator-color-active); +} + +.tab-button:hover { + color: var(--text-color-primary); +} + +.tab-content { + display: none; +} + +.tab-content.active { + display: block; +} + +@media (max-width: 1024px) { + .header-inner { + padding: 15px 10px !important; + } + + .view-wrapper { + padding: 55px 10px 80px 10px !important;; + } + + .section-group { + flex-direction: column; + } + + .tab-content .device-cards-container { + grid-template-columns: repeat(1, 1fr); + } + + .section-group { + flex-direction: column; + } + + .tab-content .device-cards-container { + grid-template-columns: repeat(1, 1fr); + + } +} + +/* Add styles for device cards */ +.device-cards-container { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 10px; + padding: 20px 10px 10px 10px; + border-right: 1px solid var(--border-color); + border-left: 1px solid var(--border-color); + border-bottom: 1px solid var(--border-color); + border-radius: 0px 0px var(--border-radius) var(--border-radius); +} + +.device-card { + background: var(--section-bg-color-alt1); + display: flex; + border-radius: var(--border-radius); + flex-direction: row; + justify-content: space-between; + align-items: center; + margin-bottom: 5px; + border: 1px solid var(--border-color); + padding: 10px; + width: 100%; +} + +.device-name { + font-size: clamp(0.6rem, 1.75vw, 0.9rem); + font-weight: 500; + color: var(--text-color-primary); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + margin-right: 10px; +} + +.device-variant { + font-size: clamp(0.8rem, 2.25vw, 0.9rem); + font-weight: 600; + white-space: nowrap; +} + +.device-variant-high { + font-size: clamp(10px, 1.75vw, 15px); + color: var(--indicator-color); +} + +.device-variant-medium { + color: #ff9800; +} + +.device-variant-low { + color: #f44336; +} + +.device-label { + font-size: clamp(12px, 1.5vw, 13px); + font-weight: 400; + color: var(--text-color-alt1); + text-transform: uppercase; + padding: 0 8px 8px 8px; +} + +#metrics-tab { + border: 1px solid (var(--border-color)); +} + +.metrics-navbar { + display: flex; + overflow-x: auto; +} + +.metrics-nav-item { + flex: 1; + padding: 10px 15px; + cursor: pointer; + background-color: var(--section-bg-color-alt2); + border-left: 1px solid var(--border-color); + font-size: 14px; + white-space: nowrap; +} + +.metrics-nav-item:last-child { + border-right: 1px solid var(--border-color); +} + +.metrics-nav-item:hover { + background: var(--active-action-bg); +} + +.metrics-nav-item.active { + background: var(--active-action-bg); + color: var(--text-color-primary); + position: relative; + font-weight: 600; +} + +.metrics-content { + background-color: var(--bg-color); +} + +.metrics-footer { + padding: 17.5px; + background-color: var(--section-bg-color-alt1); + border: 1px solid var(--border-color); + border-radius: 0 0 var(--border-radius) var(--border-radius); +} + +.metrics-footer span { + color: var(--text-color-alt1); + font-size: 13px; +} + +.metrics-category { + padding: 0; + background-color: var(--bg-color); +} + +.metric-container { + border-top: 1px solid var(--border-color); + border-left: 1px solid var(--border-color); + border-right: 1px solid var(--border-color); +} + +.section-lines-header { + border-bottom: 1px solid var(--border-color); +} + +.init-loading-wrapper { + padding: 0 10px 7.5px 10px; +} + +.init-loading-wrapper p { + color: var(--text-color-primary); + font-weight: 600; + font-size: 14px; +} + +.border-wrapper-primary { + background: var(--section-bg-color-primary); + border: 1px solid var(--border-color); + border-radius: var(--border-radius); +} + +.view-wrapper { + width: 100%; + max-width: 2000px; + padding: 55px 40px 80px 40px; + margin: 33.5px auto 0 auto; + display: flex; + flex-direction: column; + gap: 25px; +} + +.explorer-view { + width: 100%; + display: flex; + flex-direction: column; + gap: 25px; +} + +.explorer-view-flex { + position: relative; + width: 100%; + display: flex; + gap: 25px; +} + +.bg-video { + position: absolute; + top: -240px; + left: -240px; + + z-index: 5; +} + +.signature-wrapper, +.explorer-wrapper { + box-shadow: 0px 4px 4px 0px rgba(0, 0, 0, 0.04); + border-radius: var(--border-radius); + +} + +.signature-wrapper { + width: 100%; + display: flex; + flex-direction: column; + gap: 20px; + padding: 20px; +} + +.signature-line { + display: flex; + align-items: center; + justify-content: space-between; + gap: 7.5px; +} + +.signature-line p { + font-size: 14px; + font-weight: 600; + color: var(--text-color-primary); + white-space: nowrap; + max-width: 50%; + overflow: hidden; + text-overflow: ellipsis; +} + +.signature-line span { + font-size: 14px; + font-weight: 500; + color: var(--text-color-alt1); +} + +.message-id-wrapper { + width: 100%; + display: flex; + padding: 0px 20px; + justify-content: end; + gap: 7.5px; + align-items: center; + z-index: 10; +} + +.message-id-wrapper p { + font-size: 14px; + font-weight: 600; + color: var(--text-color-primary); + white-space: nowrap; + max-width: 50%; + overflow: hidden; + text-overflow: ellipsis; +} + +.message-id-wrapper span { + font-size: 14px; + font-weight: 500; + color: var(--text-color-alt1); +} + +.explorers-wrapper { + width: 45%; + display: flex; + flex-direction: column; + gap: 25px; + z-index: 10; +} + +.explorer-wrapper { + width: 100%; +} + +.visual-wrapper { + min-height: calc(600px + 56px); + width: 60%; + z-index: 10; +} + +.visual-body { + min-height: calc(600px + 56px); + width: 100%; + background: var(--bg-color); +} + +.visual-body iframe { + min-height: calc(600px + 56px); + width: 100%; + border: none; + border-bottom: 1px solid var(--border-color); + border-left: 1px solid var(--border-color); + border-right: 1px solid var(--border-color); + border-bottom-left-radius: var(--border-radius); + border-bottom-right-radius: var(--border-radius); +} + +.explorer-header { + display: flex; + align-items: center; + gap: 6px; + width: 100%; + overflow: auto; + padding: 20px; + background: var(--section-bg-color-alt1); + border-top: 1px solid var(--border-color); + border-left: 1px solid var(--border-color); + border-right: 1px solid var(--border-color); + border-top-left-radius: var(--border-radius); + border-top-right-radius: var(--border-radius); +} + +.explorer-footer { + width: 100%; + overflow: auto; + padding: 10px 15px; + background: var(--section-bg-color-alt1); + border-bottom: 1px solid var(--border-color); + border-left: 1px solid var(--border-color); + border-right: 1px solid var(--border-color); + border-bottom-left-radius: var(--border-radius); + border-bottom-right-radius: var(--border-radius); +} + +.explorer-header p { + font-weight: 600; + font-size: 15px; + color: var(--text-color-primary); +} + +.explorer-footer p { + font-weight: 500; + font-size: 13px; + color: var(--text-color-alt1); +} + +.explorer { + max-height: 450px; + width: 100%; + overflow: auto; + border-top: 1px solid var(--border-color); + border-left: 1px solid var(--border-color); + border-right: 1px solid var(--border-color); +} + +.explorer-body { + width: 100%; + display: flex; + flex-direction: column; +} + +.explorer-body-row-wrapper { + display: flex; + flex-direction: column; +} + +.explorer-body-row { + height: 55px; + display: flex; + align-items: center; + gap: 7.5px; + background: var(--bg-color); + border-bottom: 1px solid var(--border-color); +} + +.explorer-body-row-wrapper:nth-child(odd) .explorer-body-row { + background: var(--bg-color); +} + +.explorer-body-row-wrapper:nth-child(even) .explorer-body-row { + background: var(--section-bg-color-alt1); +} + +#explorer-body-row-loader { + padding: 0 15px; +} + +.explorer-body-row-open { + background: var(--active-action-bg) !important; +} + +.explorer-body-row-indicator { + transition: all 100ms; +} + +.explorer-body-row-indicator::before { + content: "›"; + font-size: 24px; + color: var(--indicator-color); + display: flex; + margin: -1.5px 2.5px 0 0; +} + +.explorer-body-row-indicator-open { + transform: rotate(90deg); + margin: 5px 2.5px 0 0; +} + +.explorer-body-row span { + font-size: 13px; + font-weight: 500; + color: var(--text-color-alt1); + white-space: nowrap; +} + +.explorer-body-row p { + max-width: 80%; + font-size: 13px; + font-weight: 600; + color: var(--text-color-primary); + white-space: nowrap; + text-overflow: ellipsis; + display: block; + overflow: hidden; +} + +.explorer-body-row-flag { + padding: 0.5px 7.5px; + background: var(--indicator-color); + border-radius: var(--border-radius); +} + +.status-indicator { + padding: 0.5px 7.5px !important; + color: var(--text-color-light) !important; + font-size: 12px !important; + font-weight: 500 !important; + background: var(--pulse-color) !important; + border-radius: var(--border-radius) !important; +} + +#metrics-tab { + border-bottom: 1px solid var(--border-color); + border-bottom-left-radius: var(--border-radius); + border-bottom-right-radius: var(--border-radius); + overflow: hidden; +} + +#cache-section { + border-bottom: 1px solid var(--border-color); + border-left: 1px solid var(--border-color); + border-right: 1px solid var(--border-color); + border-bottom-left-radius: var(--border-radius); + border-bottom-right-radius: var(--border-radius); +} + +.explorer-body-row-flag span { + color: var(--text-color-light); + font-size: 12px; + font-weight: 500; +} + +.explorer-body-link-value { + text-decoration: underline; + text-decoration-color: var(--indicator-color); + text-decoration-thickness: 2px; +} + +.tx-address { + text-decoration: underline !important; + text-decoration-color: var(--indicator-color) !important; + text-decoration-thickness: 1px !important; +} + +.explorer-action { + width: 100%; +} + +.explorer-action:hover { + cursor: pointer; + background: var(--active-action-bg) !important; +} + +/* Graph visualization styles */ +.graph-link { + font-size: clamp(0.65rem, 1.75vw, 0.75rem); + color: var(--indicator-color-active) +} + +.graph-link:hover { + opacity: 75%; + +} + +.copy-hover { + transition: opacity 0.1s ease; + cursor: pointer; + +} + +.copy-hover:hover { + opacity: 0.6; + +} + +.custom-tooltip { + transition: opacity 0.2s ease; + opacity: 0.95; + pointer-events: none; +} \ No newline at end of file diff --git a/src/html/hyperbuddy@1.0/utils.js b/src/html/hyperbuddy@1.0/utils.js new file mode 100644 index 000000000..af9e35859 --- /dev/null +++ b/src/html/hyperbuddy@1.0/utils.js @@ -0,0 +1,399 @@ +/** + * Base route for all API calls + */ +const BASE_ROUTE = window.location.origin; + +/** + * Format an address for display, shortening it if needed + * @param {string} address - The full address to format + * @param {boolean} wrap - Whether to wrap the address in parentheses + * @returns {string} - The formatted address + */ +function formatAddress(address, wrap) { + if (!address) return ""; + const formattedAddress = + address.substring(0, 6) + "..." + address.substring(36, address.length); + return wrap ? `(${formattedAddress})` : formattedAddress; +} + +/** + * Format a number for display, adding commas and handling decimals + * @param {number|string} amount - The amount to format + * @returns {string} - The formatted amount + */ +function formatDisplayAmount(amount) { + if (amount === null) return "-"; + if (amount.toString().includes(".")) { + let parts = amount.toString().split("."); + parts[0] = parts[0].replace(/\B(?=(\d{3})+(?!\d))/g, ","); + + let firstTwoDecimals = parts[1].substring(0, 2); + + if (firstTwoDecimals === "00") { + parts[1] = parts[1].substring(0, 6); + } else { + parts[1] = parts[1].substring(0, 4); + } + + return parts.join("."); + } else { + return amount.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); + } +} + +/** + * Make a GET request to the server + * @param {string} endpoint - The endpoint to request + * @returns {Promise} - The response text + */ +async function get(endpoint) { + return await (await fetch(`${BASE_ROUTE}${endpoint}`)).text(); +} + +/** + * Copy text to clipboard and update button text + * @param {HTMLButtonElement} button - The button that was clicked + */ +function copyToClipboard(button) { + const text = button.innerHTML; + + navigator.clipboard + .writeText(button.value) + .then(() => { + button.textContent = "Copied!"; + button.disabled = true; + + setTimeout(() => { + button.textContent = text; + button.disabled = false; + }, 2000); + }) + .catch((err) => console.error("Error copying text: ", err)); +} + +function enableTooltipAndCopy(selector) { + const el = document.querySelector(selector); + if (!el) return; + + // Create tooltip + const tooltip = document.createElement("div"); + tooltip.className = "custom-tooltip"; + tooltip.style.position = "absolute"; + tooltip.style.backgroundColor = "#333"; + tooltip.style.color = "#fff"; + tooltip.style.padding = "5px 10px"; + tooltip.style.borderRadius = "4px"; + tooltip.style.fontSize = "12px"; + tooltip.style.whiteSpace = "nowrap"; + tooltip.style.display = "none"; + tooltip.style.zIndex = "9999"; + tooltip.style.pointerEvents = "none"; + document.body.appendChild(tooltip); + + el.style.cursor = "pointer"; + + el.addEventListener("mouseenter", (e) => { + tooltip.textContent = "Click to copy"; + tooltip.style.display = "block"; + }); + + el.addEventListener("mousemove", (e) => { + const tooltipRect = tooltip.getBoundingClientRect(); + tooltip.style.left = `${e.clientX - tooltipRect.width / 2}px`; + tooltip.style.top = `${e.clientY - tooltipRect.height - 10}px`; + }); + + el.addEventListener("mouseleave", () => { + tooltip.style.display = "none"; + }); + + el.addEventListener("click", async () => { + try { + await navigator.clipboard.writeText(el.innerText); + tooltip.textContent = "Copied!"; + setTimeout(() => { + tooltip.style.display = "none"; + }, 1000); + } catch (err) { + console.error("Copy failed", err); + } + }); +} + +/** + * Show a specific tab content and update tab button styles + * @param {string} tabId - The ID of the tab to show + */ +function showTab(tabId) { + // Hide all tab content + document.querySelectorAll(".tab-content").forEach((content) => { + content.classList.remove("active"); + }); + + // Deactivate all tab buttons + document.querySelectorAll(".tab-button").forEach((button) => { + button.classList.remove("active"); + }); + + // Show the selected tab content + document.getElementById(tabId).classList.add("active"); + + // Activate the matching tab button + document + .querySelector(`.tab-button[data-tab="${tabId}"]`) + .classList.add("active"); +} + +/** + * Initialize tab event listeners + */ +function initTabListeners() { + document.querySelectorAll(".tab-button").forEach((button) => { + button.addEventListener("click", () => { + const tabId = button.getAttribute("data-tab"); + showTab(tabId); + }); + }); +} + +/** + * SimpleJsonViewer - A lightweight JSON visualization library + */ +class SimpleJsonViewer { + /** + * Create a new JSON viewer + * @param {Object} options - Configuration options + * @param {HTMLElement} options.container - DOM element to render in + * @param {string|Object} options.data - JSON string or object to display + * @param {string} [options.theme="light"] - Display theme (light/dark) + * @param {boolean} [options.expand=false] - Expand all nodes by default + */ + constructor(options) { + this.options = Object.assign( + { + theme: "light", + container: null, + data: "{}", + expand: false, + }, + options + ); + + if (!this.options.container) { + throw new Error("Container: DOM element is required"); + } + + this.render(); + } + + /** + * Render the JSON viewer + */ + render() { + const container = this.options.container; + container.innerHTML = ""; + container.className = `json-viewer ${this.options.theme}`; + + let data; + try { + if (typeof this.options.data === "string") { + data = JSON.parse(this.options.data); + } else { + data = this.options.data; + } + } catch (e) { + container.innerHTML = `
Invalid JSON: ${e.message}
`; + return; + } + + const rootElement = this.createNode(data, null, 0); + container.appendChild(rootElement); + } + + /** + * Create a DOM node for a JSON value + * @param {*} value - The value to display + * @param {string|null} key - The property key (or null for root/array elements) + * @param {number} level - Nesting level + * @returns {HTMLElement} The created DOM element + */ + createNode(value, key, level) { + const item = document.createElement("div"); + item.className = "json-item"; + item.style.marginLeft = level === 0 ? "0" : "20px"; + + const valueType = this.getType(value); + + // For objects and arrays, create expandable sections + if (valueType === "object" || valueType === "array") { + // Create the item label part + const itemHead = document.createElement("div"); + const toggle = document.createElement("span"); + toggle.className = `json-toggle ${this.options.expand ? "expanded" : ""}`; + itemHead.appendChild(toggle); + + // Add the key part if this isn't the root + if (key !== null) { + const keyEl = document.createElement("span"); + keyEl.className = "json-key"; + keyEl.textContent = key; + itemHead.appendChild(keyEl); + + const separator = document.createElement("span"); + separator.className = "json-separator"; + separator.textContent = " : "; + itemHead.appendChild(separator); + } + + // Add type label (object/array) with count + const typeLabel = document.createElement("span"); + typeLabel.className = "json-type-label"; + const count = + valueType === "object" ? Object.keys(value).length : value.length; + typeLabel.textContent = `${valueType} {${count}}`; + itemHead.appendChild(typeLabel); + + // Add the full item to the parent + item.appendChild(itemHead); + + // Create the container for child items + const container = document.createElement("div"); + container.className = `json-container ${ + this.options.expand ? "" : "collapsed" + }`; + + // For objects + if (valueType === "object") { + const keys = Object.keys(value); + keys.forEach((objKey) => { + const childNode = this.createNode(value[objKey], objKey, level + 1); + container.appendChild(childNode); + }); + } + // For arrays + else if (valueType === "array") { + value.forEach((arrItem, index) => { + const childNode = this.createNode( + arrItem, + index.toString(), + level + 1 + ); + container.appendChild(childNode); + }); + } + + // Add the container (no closing bracket/brace element) + item.appendChild(container); + + // Add click handler + toggle.addEventListener("click", () => { + toggle.classList.toggle("expanded"); + container.classList.toggle("collapsed"); + }); + } + // For primitive values, just show the key-value pair + else { + // Create the row with key and value + if (key !== null) { + const keyEl = document.createElement("span"); + keyEl.className = "json-key"; + keyEl.textContent = key; + item.appendChild(keyEl); + + const separator = document.createElement("span"); + separator.className = "json-separator"; + separator.textContent = " : "; + item.appendChild(separator); + } + + // Add the value with appropriate styling + const valueEl = document.createElement("span"); + valueEl.className = `json-${valueType}`; + + if (valueType === "string") { + valueEl.textContent = `"${this.escapeString(value)}"`; + } else { + valueEl.textContent = String(value); + } + + item.appendChild(valueEl); + } + + return item; + } + + /** + * Get the type of a value + * @param {*} value - The value to check + * @returns {string} The type name + */ + getType(value) { + if (value === null) return "null"; + if (Array.isArray(value)) return "array"; + if (typeof value === "object") return "object"; + if (typeof value === "string") return "string"; + if (typeof value === "number") return "number"; + if (typeof value === "boolean") return "boolean"; + return "unknown"; + } + + /** + * Escape a string for display + * @param {string} str - The string to escape + * @returns {string} The escaped string + */ + escapeString(str) { + return str + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/\n/g, "\\n") + .replace(/\r/g, "\\r") + .replace(/\t/g, "\\t"); + } + + /** + * Update the theme + * @param {string} theme - The theme to use ('light' or 'dark') + */ + setTheme(theme) { + this.options.container.className = `json-viewer ${theme}`; + this.options.theme = theme; + } + + /** + * Expand all nodes + */ + expandAll() { + const toggles = this.options.container.querySelectorAll(".json-toggle"); + const containers = + this.options.container.querySelectorAll(".json-container"); + + toggles.forEach((toggle) => toggle.classList.add("expanded")); + containers.forEach((container) => container.classList.remove("collapsed")); + } + + /** + * Collapse all nodes + */ + collapseAll() { + const toggles = this.options.container.querySelectorAll(".json-toggle"); + const containers = + this.options.container.querySelectorAll(".json-container"); + + toggles.forEach((toggle) => toggle.classList.remove("expanded")); + containers.forEach((container) => container.classList.add("collapsed")); + } +} + +// Export functions for use in other modules +export { + BASE_ROUTE, + formatAddress, + formatDisplayAmount, + get, + copyToClipboard, + showTab, + initTabListeners, + SimpleJsonViewer, + enableTooltipAndCopy, +}; diff --git a/src/include/ar.hrl b/src/include/ar.hrl index 0a6c29c77..655928791 100644 --- a/src/include/ar.hrl +++ b/src/include/ar.hrl @@ -2,6 +2,8 @@ -define(DEFAULT_ID, << 0:256 >>). -define(DEFAULT_OWNER, << 0:4096 >>). -define(DEFAULT_DATA, <<>>). +-define(DEFAULT_LAST_TX, <<>>). +-define(DEFAULT_TARGET, <<>>). -define(MAX_TAG_NAME_SIZE, 3072). -define(MAX_TAG_VALUE_SIZE, 3072). @@ -15,13 +17,13 @@ %% Either the identifier of the previous transaction from %% the same wallet or the identifier of one of the %% last ?MAX_TX_ANCHOR_DEPTH blocks. - last_tx = <<>>, + anchor = ?DEFAULT_LAST_TX, %% The public key the transaction is signed with. owner = ?DEFAULT_OWNER, %% A list of arbitrary key-value pairs. Keys and values are binaries. tags = [], %% The address of the recipient, if any. The SHA2-256 hash of the public key. - target = <<>>, + target = ?DEFAULT_TARGET, %% The amount of Winstons to send to the recipient, if any. quantity = 0, %% The data to upload, if any. For v2 transactions, the field is optional - a fee diff --git a/src/include/cargo.hrl b/src/include/cargo.hrl new file mode 100644 index 000000000..bcbb63656 --- /dev/null +++ b/src/include/cargo.hrl @@ -0,0 +1,8 @@ +-cargo_header_version 1. +-ifndef(CARGO_LOAD_APP). +-define(CARGO_LOAD_APP,hb). +-endif. +-ifndef(CARGO_HRL). +-define(CARGO_HRL, 1). +-define(load_nif_from_crate(__CRATE,__INIT),(fun()->__APP=?CARGO_LOAD_APP,__PATH=filename:join([code:priv_dir(__APP),"crates",__CRATE,__CRATE]),erlang:load_nif(__PATH,__INIT)end)()). +-endif. diff --git a/src/include/hb.hrl b/src/include/hb.hrl index 2043c9753..69ab91229 100644 --- a/src/include/hb.hrl +++ b/src/include/hb.hrl @@ -1,14 +1,18 @@ -include("include/ar.hrl"). +-define(HYPERBEAM_VERSION, 0.9). + %% @doc Macro for checking if a message is empty, ignoring its hashpath. --define(IS_EMPTY_MESSAGE(Msg), (map_size(Msg) == 0) orelse (map_size(Msg) == 1 andalso is_map_key(hashpath, Msg))). +-define(IS_EMPTY_MESSAGE(Msg), (map_size(Msg) == 0) orelse (map_size(Msg) == 1 andalso (is_map_key(priv, Msg) orelse is_map_key(<<"priv">>, Msg)))). %% @doc Macro usable in guards that validates whether a term is a %% human-readable ID encoding. --define(IS_ID(X), (is_binary(X) andalso (byte_size(X) == 43 orelse byte_size(X) == 32))). -%% @doc List of special keys that are used in the Converge Protocol. --define(CONVERGE_KEYS, [path, hashpath, priv]). +-define(IS_ID(X), (is_binary(X) andalso (byte_size(X) == 42 orelse byte_size(X) == 43 orelse byte_size(X) == 32))). +%% @doc Macro for checking a term is a link. +-define(IS_LINK(X), (is_tuple(X) andalso element(1, X) == link)). +%% @doc List of special keys that are used in the AO-Core protocol. +-define(AO_CORE_KEYS, [<<"path">>, <<"hashpath">>, <<"priv">>]). %% @doc Keys that can be regenerated losslessly. --define(REGEN_KEYS, [id, unsigned_id]). +-define(REGEN_KEYS, [<<"unsigned_id">>, <<"content-digest">>]). %% @doc Record used for parsing relevant components of a cursor-browsable %% response. @@ -22,20 +26,24 @@ %%% Functional macros that pass the current module and line number to the %%% underlying function. --define(event(X), hb:event(?MODULE, X, ?MODULE, ?FUNCTION_NAME, ?LINE)). --define(event(Topic, X), hb:event(Topic, X, ?MODULE, ?FUNCTION_NAME, ?LINE)). --define(event(Topic, X, Opts), hb:event(maps:get(topic, Opts, Topic), X, ?MODULE, ?FUNCTION_NAME, ?LINE), Opts). +-define(event(X), hb_event:log(global, X, ?MODULE, ?FUNCTION_NAME, ?LINE)). +-define(event(Topic, X), hb_event:log(Topic, X, ?MODULE, ?FUNCTION_NAME, ?LINE)). +-define(event(Topic, X, Opts), hb_event:log(maps:get(topic, Opts, Topic), X, ?MODULE, ?FUNCTION_NAME, ?LINE, Opts)). -define(debug_wait(T), hb:debug_wait(T, ?MODULE, ?FUNCTION_NAME, ?LINE)). +-define(debug_print(X), hb_format:print(X, ?MODULE, ?FUNCTION_NAME, ?LINE)). -define(no_prod(X), hb:no_prod(X, ?MODULE, ?LINE)). %%% Macro shortcuts for debugging. %% @doc A macro for marking that you got 'here'. --define(h(), hb:event("[Debug point reached.]", ?MODULE, ?FUNCTION_NAME, ?LINE)). +-define(h(), hb_event:log("[Debug point reached.]", ?MODULE, ?FUNCTION_NAME, ?LINE)). %% @doc Quickly print a value in the logs. Currently uses the event %% function, but should be moved to a debug-specific function once we %% build out better logging infrastructure. --define(p(X), hb:event(X, ?MODULE, ?FUNCTION_NAME, ?LINE)). +-define(p(X), hb_event:log(X, ?MODULE, ?FUNCTION_NAME, ?LINE)). %% @doc Print the trace of the current stack, up to the first non-hyperbeam %% module. --define(trace(), hb_util:trace_macro_helper(fun hb_util:print_trace/4, catch error(test), ?MODULE, ?FUNCTION_NAME, ?LINE)). --define(trace_short(), hb_util:trace_macro_helper(fun hb_util:print_trace_short/4, catch error(test), ?MODULE, ?FUNCTION_NAME, ?LINE)). \ No newline at end of file +-define(trace(), hb_format:trace_macro_helper(fun hb_format:print_trace/4, catch error(test), ?MODULE, ?FUNCTION_NAME, ?LINE)). +-define(trace_short(), hb_format:trace_macro_helper(fun hb_format:print_trace_short/4, catch error(test), ?MODULE, ?FUNCTION_NAME, ?LINE)). +%% @doc Draw a horizontal line in the logs. +-define(hr(), io:format(standard_error, "--------------------------------------------------------------------------------~n", [])). +-define(hr(Str), io:format(standard_error, iolist_to_binary(["---------------------------------------- ", Str, " ----------------------------------------~n"]), [])). diff --git a/src/sec.erl b/src/sec.erl deleted file mode 100644 index 291de1f32..000000000 --- a/src/sec.erl +++ /dev/null @@ -1,101 +0,0 @@ -%%%--------------------------------------------------------------------- -%%% Module: sec -%%%--------------------------------------------------------------------- -%%% Purpose: -%%% This module handles the generation and verification of attestation -%%% reports using both TPM and SEV-SNP. It combines attestation reports -%%% from both technologies into a single binary and provides functionality -%%% to verify the combined reports. -%%% -%%% It uses the `sec_tpm` and `sec_tee` modules to interact with the TPM -%%% hardware and SEV-SNP for generating and verifying attestation reports. -%%%--------------------------------------------------------------------- -%%% Exports -%%%--------------------------------------------------------------------- -%%% generate_attestation(Nonce) -%%% Generates a combined attestation report using the provided nonce. -%%% It generates attestation reports from both TPM and SEV-SNP, calculates -%%% their sizes, creates a header containing the sizes, and combines them -%%% into a single binary. -%%% -%%% verify_attestation(AttestationBinary) -%%% Verifies the provided attestation binary by extracting the TPM and -%%% SEV-SNP reports, verifying them using their respective verification -%%% methods, and then combining the results into a single binary. -%%%--------------------------------------------------------------------- - --module(sec). --export([generate_attestation/1, verify_attestation/1]). - --include("include/hb.hrl"). - --hb_debug(print). - -%% Generate attestation based on the provided nonce (both TPM and SEV-SNP) -generate_attestation(Nonce) -> - ?event({"Generating TPM attestation..."}), - - case sec_tpm:generate_attestation(Nonce) of - {ok, TPMAttestation} -> - ?event({"TPM attestation generated, size:", byte_size(TPMAttestation)}), - - ?event({"Generating SEV-SNP attestation..."}), - - case sec_tee:generate_attestation(Nonce) of - {ok, TEEAttestation} -> - ?event({"SEV-SNP attestation generated, size:", byte_size(TEEAttestation)}), - - %% Calculate sizes of the two attestation binaries - TPMSize = byte_size(TPMAttestation), - TEESize = byte_size(TEEAttestation), - - %% Create the header containing the sizes - Header = <>, - ?event({"Header created, TPMSize:", TPMSize, "TEESize:", TEESize}), - - %% Combine the header with the two attestation binaries - CombinedAttestation = <
>, - ?event({"Combined attestation binary created, total size:", byte_size(CombinedAttestation)}), - - {ok, CombinedAttestation}; - {error, Reason} -> - ?event({"Error generating SEV-SNP attestation:", Reason}), - {error, Reason} - end; - {error, Reason} -> - ?event({"Error generating TPM attestation:", Reason}), - {error, Reason} - end. - -%% Verify attestation report based on the provided binary (both TPM and SEV-SNP) -verify_attestation(AttestationBinary) -> - ?event("Verifying attestation..."), - - %% Extract the header (size info) and the attestation binaries - <> = AttestationBinary, - ?event({"Header extracted, TPMSize:", TPMSize, "TEESize:", TEESize}), - - %% Extract the TPM and SEV-SNP attestation binaries based on their sizes - <> = Rest, - ?event({"Extracted TPM and SEV-SNP attestation binaries"}), - - %% Verify TPM attestation - case sec_tpm:verify_attestation(TPMAttestation) of - {ok, _TPMVerification} -> - ?event({"TPM attestation verification completed"}), - - %% Verify SEV-SNP attestation - case sec_tee:verify_attestation(TEEAttestation) of - {ok, _TEEVerification} -> - ?event({"SEV-SNP attestation verification completed"}), - - %% Return success if both verifications succeeded - {ok, "Verified"}; - {error, Reason} -> - ?event({"Error verifying SEV-SNP attestation:", Reason}), - {error, Reason} - end; - {error, Reason} -> - ?event({"Error verifying TPM attestation:", Reason}), - {error, Reason} - end. \ No newline at end of file diff --git a/src/sec_helpers.erl b/src/sec_helpers.erl deleted file mode 100644 index e28b69c2b..000000000 --- a/src/sec_helpers.erl +++ /dev/null @@ -1,32 +0,0 @@ --module(sec_helpers). --export([write_to_file/2, read_file/1, run_command/1]). - --include("include/hb.hrl"). --hb_debug(print). - -%% Helper function to write data to a file -write_to_file(FilePath, Data) -> - case file:write_file(FilePath, Data) of - ok -> ?event({"Written data to file", FilePath}); - {error, Reason} -> ?event({"Failed to write to file", FilePath, Reason}) - end. - -%% Helper function to read a file -read_file(FilePath) -> - ?event({"Reading file", FilePath}), - case file:read_file(FilePath) of - {ok, Data} -> {FilePath, Data}; - {error, Reason} -> {error, Reason} - end. - -%% Generalized function to run a shell command and optionally apply a success function -%% When SuccessFun is provided, it is called upon successful execution -run_command(Command) -> - ?event({"Executing command", Command}), - Output = os:cmd(Command ++ " 2>&1"), - case Output of - % Empty output interpreted as success if no output is expected - "" -> {ok, []}; - % Return output for further inspection - _ -> {ok, Output} - end. \ No newline at end of file diff --git a/src/sec_tee.erl b/src/sec_tee.erl deleted file mode 100644 index 472880a68..000000000 --- a/src/sec_tee.erl +++ /dev/null @@ -1,207 +0,0 @@ -%%%--------------------------------------------------------------------- -%%% Module: sec_tee -%%%--------------------------------------------------------------------- -%%% Purpose: -%%% This module handles SEV-SNP attestation and verification processes. -%%% It generates attestation reports, retrieves necessary certificates, -%%% and verifies the attestation against AMD's root of trust using the -%%% snpguest and OpenSSL commands. -%%%--------------------------------------------------------------------- -%%% Exports -%%%--------------------------------------------------------------------- -%%% generate_attestation(Nonce) -%%% Generates an attestation report and retrieves certificates. -%%% Returns a binary with the attestation report and public key. -%%% -%%% verify_attestation(AttestationBinary) -%%% Verifies the attestation report against the VCEK certificate -%%% and AMD root of trust. -%%%--------------------------------------------------------------------- - --module(sec_tee). --export([ - generate_attestation/1, - verify_attestation/1 -]). --include("include/hb.hrl"). --hb_debug(print). - -%% Define the file paths --define(ROOT_DIR, "/tmp/tee"). --define(REQUEST_FILE, ?ROOT_DIR ++ "/request-file.txt"). --define(REPORT_FILE, ?ROOT_DIR ++ "/report.bin"). --define(CERT_CHAIN_FILE, ?ROOT_DIR ++ "/cert_chain.pem"). --define(VCEK_FILE, ?ROOT_DIR ++ "/vcek.pem"). - -%% Define the commands --define(SNP_GUEST_REPORT_CMD, "snpguest report " ++ ?REPORT_FILE ++ " " ++ ?REQUEST_FILE). --define(SNP_GUEST_CERTIFICATES_CMD, "snpguest certificates PEM " ++ ?ROOT_DIR). --define(VERIFY_VCEK_CMD, "openssl verify --CAfile " ++ ?CERT_CHAIN_FILE ++ " " ++ ?VCEK_FILE). --define(VERIFY_REPORT_CMD, "snpguest verify attestation " ++ ?ROOT_DIR ++ " " ++ ?REPORT_FILE). - -%% Temporarily hard-code the VCEK download command --define(DOWNLOAD_VCEK_CMD, - "curl --proto \'=https\' --tlsv1.2 -sSf https://kdsintf.amd.com/vcek/v1/Milan/cert_chain -o " ++ ?CERT_CHAIN_FILE -). - -%% Generate attestation, request certificates, download VCEK, and upload a transaction -generate_attestation(Nonce) -> - % Check if the root directory exists, and create it if not - case filelib:is_dir(?ROOT_DIR) of - true -> ok; - false -> file:make_dir(?ROOT_DIR) - end, - - % Debug: Print starting attestation generation - ?event("Starting attestation generation..."), - - % Generate request file and attestation report - ?event("Generating request file with nonce..."), - generate_request_file(Nonce), - ?event("Generating attestation report..."), - generate_attestation_report(), - - % Request certificates, download VCEK, and upload the attestation - ?event("Fetching certificates from host memory..."), - fetch_certificates(), - ?event("Downloading VCEK root of trust certificate..."), - download_vcek_cert(), - - % Debug: Print reading the attestation report and public key - ?event("Reading the attestation report and public key..."), - - % Ensure that read_file returns the binary data as expected - {_, ReportBin} = sec_helpers:read_file(?REPORT_FILE), - {_, PublicKeyBin} = sec_helpers:read_file(?VCEK_FILE), - - % Get sizes of the individual files (in binary) - ReportSize = byte_size(ReportBin), - PublicKeySize = byte_size(PublicKeyBin), - - % Debug: Print the sizes of the files - ?event({"Report size", ReportSize}), - ?event({"Public key size", PublicKeySize}), - - % Create a binary header with the sizes and offsets - Header = <>, - - % Create a binary with both the report and public key data concatenated after the header - AttestationBinary = <
>, - - % Debug: Print the final binary data size - ?event({"Generated attestation binary size", byte_size(AttestationBinary)}), - - % Return the binary containing the attestation data - {ok, AttestationBinary}. - -%% Helper to generate the request file with the padded address and nonce -generate_request_file(Nonce) -> - RequestFile = ?REQUEST_FILE, - NonceHex = binary_to_list(binary:encode_hex(Nonce)), - % Debug: Print the nonce - ?event({"Nonce in hex", NonceHex}), - case sec_helpers:write_to_file(RequestFile, NonceHex) of - {"Written data to file", FilePath} when FilePath == RequestFile -> - ?event({"Request file written successfully", RequestFile}), - ok; - {error, Reason} -> - ?event({"Failed to write request file", RequestFile, Reason}), - {error, failed_to_write_request_file} - end. - -% Helper to generate the attestation report -generate_attestation_report() -> - Command = ?SNP_GUEST_REPORT_CMD, - ?event({"Running command to generate attestation report", Command}), - case sec_helpers:run_command(Command) of - {ok, _} -> - ?event("SEV-SNP report generated successfully"), - ok; - {error, Reason} -> - ?event({"Failed to generate SEV-SNP report", Reason}), - {error, failed_to_generate_report} - end. - -%% Verify the attestation report using snpguest and VCEK certificate -verify_attestation(AttestationBinary) -> - % Extract the header (size info) - <> = AttestationBinary, - - % Extract the individual components using the sizes from the header - <> = Rest, - <> = Rest1, - - % Debug: Print the extracted components - ?event({"Extracted report data", ReportData}), - ?event({"Extracted public key data", PublicKeyData}), - - % Write the components to temporary files (if needed for verification) - sec_helpers:write_to_file(?REPORT_FILE, ReportData), - sec_helpers:write_to_file(?VCEK_FILE, PublicKeyData), - - % Verify the VCEK certificate - ?event("Verifying VCEK certificate..."), - case sec_helpers:run_command(?VERIFY_VCEK_CMD) of - {ok, CertOutput} -> - TrimmedOutput = string:trim(CertOutput), - ?event({"VCEK certificate verification output", TrimmedOutput}), - % Compute outside the guard - ExpectedOutput = ?VCEK_FILE ++ ": OK", - if - TrimmedOutput =:= ExpectedOutput -> - ?event("VCEK certificate signature verified successfully"), - verify_attestation_report(); - true -> - ?event({"VCEK signature verification failed", CertOutput}), - {error, invalid_signature} - end; - {error, Reason} -> - ?event({"Failed to verify VCEK signature", Reason}), - {error, verification_failed} - end. - -%% Verify the attestation report -verify_attestation_report() -> - Command = ?VERIFY_REPORT_CMD, - ?event({"Running command to verify attestation report", Command}), - case sec_helpers:run_command(Command) of - {ok, Output} -> - ?event({"Attestation verification result", Output}), - case string:find(Output, "VEK signed the Attestation Report!", leading) of - nomatch -> - ?event("Attestation verification failed"), - {error, verification_failed}; - _ -> - ?event("Attestation verified successfully"), - {ok, Output} - end; - {error, Reason} -> - ?event({"Failed to verify attestation", Reason}), - {error, verification_failed} - end. - -%% Fetch certificates from host memory and store as PEM files -fetch_certificates() -> - Command = ?SNP_GUEST_CERTIFICATES_CMD, - ?event({"Fetching SEV-SNP certificates from host memory", Command}), - case sec_helpers:run_command(Command) of - {ok, _} -> - ?event("Certificates fetched successfully"), - ok; - {error, Reason} -> - ?event({"Failed to fetch certificates", Reason}), - {error, failed_to_fetch_certificates} - end. - -%% Download VCEK root of trust certificate !!TEMPORARY!! -download_vcek_cert() -> - Command = ?DOWNLOAD_VCEK_CMD, - ?event({"Downloading VCEK root of trust certificate", Command}), - case sec_helpers:run_command(Command) of - {ok, _} -> - ?event("VCEK root of trust certificate downloaded successfully"), - ok; - {error, Reason} -> - ?event({"Failed to download VCEK certificate", Reason}), - {error, failed_to_download_cert} - end. \ No newline at end of file diff --git a/src/sec_tpm.erl b/src/sec_tpm.erl deleted file mode 100644 index c8dd9dc4c..000000000 --- a/src/sec_tpm.erl +++ /dev/null @@ -1,177 +0,0 @@ -%%%--------------------------------------------------------------------- -%%% Module: sec_tpm -%%%--------------------------------------------------------------------- -%%% Purpose: -%%% This module handles TPM-based attestation and key management processes. -%%% It generates attestation reports, creates and loads TPM keys, and -%%% verifies attestation reports using the TPM hardware. -%%%--------------------------------------------------------------------- -%%% Exports -%%%--------------------------------------------------------------------- -%%% setup_keys() -%%% Sets up the primary and attestation keys on the TPM. -%%% -%%% generate_attestation(Nonce) -%%% Generates an attestation report using a provided nonce and returns -%%% the attestation binary containing the quote, signature, and public key. -%%% -%%% verify_attestation(AttestationBinary) -%%% Verifies the attestation report by comparing the quote with the -%%% signature using TPM2's verification commands. -%%%--------------------------------------------------------------------- - --module(sec_tpm). --export([setup_keys/0, generate_attestation/1, verify_attestation/1]). - --include("include/hb.hrl"). --hb_debug(print). - -%% Define the file paths --define(ROOT_DIR, "/tmp/tpm"). --define(QUOTE_MSG_FILE, ?ROOT_DIR ++ "/quote.msg"). --define(QUOTE_SIG_FILE, ?ROOT_DIR ++ "/quote.sig"). --define(AK_PUB_FILE, ?ROOT_DIR ++ "/ak.pub"). --define(AK_PRIV_FILE, ?ROOT_DIR ++ "/ak.priv"). --define(AK_CTX_FILE, ?ROOT_DIR ++ "/ak.ctx"). --define(PRIMARY_CTX_FILE, ?ROOT_DIR ++ "/primary.ctx"). - -%% Define the TPM2 commands using the defined file paths --define(TPM2_CREATEPRIMARY_CMD, "tpm2_createprimary -C e -g sha256 -G rsa -c " ++ ?PRIMARY_CTX_FILE). --define(TPM2_CREATE_CMD, - "tpm2_create -C " ++ ?PRIMARY_CTX_FILE ++ " -G rsa -u " ++ ?AK_PUB_FILE ++ " -r " ++ ?AK_PRIV_FILE -). --define(TPM2_LOAD_CMD, - "tpm2_load -C " ++ ?PRIMARY_CTX_FILE ++ " -u " ++ ?AK_PUB_FILE ++ " -r " ++ ?AK_PRIV_FILE ++ " -c " ++ ?AK_CTX_FILE -). --define(TPM2_READPUBLIC_CMD, "tpm2_readpublic -c " ++ ?AK_CTX_FILE ++ " -o " ++ ?AK_PUB_FILE ++ " -f pem"). --define(TPM2_QUOTE_CMD, - "tpm2_quote -Q --key-context " ++ ?AK_CTX_FILE ++ " -l sha256:0,1 --message " ++ ?QUOTE_MSG_FILE ++ " --signature " ++ - ?QUOTE_SIG_FILE ++ " --qualification " -). --define(TPM2_VERIFY_CMD, - "tpm2_verifysignature -c " ++ ?AK_CTX_FILE ++ " -g sha256 -m " ++ ?QUOTE_MSG_FILE ++ " -s " ++ ?QUOTE_SIG_FILE -). - -%% Generate an attestation using the provided nonce (as a string) -generate_attestation(Nonce) -> - % Check if the root directory exists, and create it if not - case filelib:is_dir(?ROOT_DIR) of - true -> ok; - false -> file:make_dir(?ROOT_DIR) - end, - - % Setup the keys - ok = setup_keys(), - - % Use the address in hex format as the nonce - NonceHex = binary_to_list(binary:encode_hex(Nonce)), - Command = ?TPM2_QUOTE_CMD ++ NonceHex, - ?event({"Running command", Command}), - case sec_helpers:run_command(Command) of - {ok, _} -> - % Read the quote.msg, quote.sig, and ak.pub files - {_, QuoteBin} = sec_helpers:read_file(?QUOTE_MSG_FILE), - {_, SignatureBin} = sec_helpers:read_file(?QUOTE_SIG_FILE), - {_, AkPubBin} = sec_helpers:read_file(?AK_PUB_FILE), - - % Get sizes of the individual files (in binary) - QuoteSize = byte_size(QuoteBin), - SignatureSize = byte_size(SignatureBin), - AkPubSize = byte_size(AkPubBin), - - % Create a binary header with the sizes and offsets - Header = <>, - - % Create a binary with all three files' data concatenated after the header - AttestationBinary = <
>, - - % Log the data (if you need to debug) - ?event({"Attestation generated, binary data:", AttestationBinary}), - - % Return the binary containing the header and files - {ok, AttestationBinary}; - {error, Reason} -> - ?event({"Error generating attestation", Reason}), - {error, Reason} - end. - -%% Verify the attestation using the AttestationBinary -verify_attestation(AttestationBinary) -> - % Extract the header (size info) - <> = AttestationBinary, - - % Extract the individual components using the sizes from the header - <> = Rest, - <> = Rest1, - <> = Rest2, - - % Write the components to temporary files (if needed for verification) - sec_helpers:write_to_file(?QUOTE_MSG_FILE, QuoteData), - sec_helpers:write_to_file(?QUOTE_SIG_FILE, SignatureData), - sec_helpers:write_to_file(?AK_PUB_FILE, AkPubData), - - % Run the TPM verification command using the files - CommandVerify = ?TPM2_VERIFY_CMD, - ?event({"Running command", CommandVerify}), - case sec_helpers:run_command(CommandVerify) of - {ok, VerificationMessage} -> - {ok, VerificationMessage}; - {error, {failed, _}} = Error -> - ?event({"Verification failed", Error}), - Error - end. - -%% Set up primary and attestation keys -setup_keys() -> - ?event("Starting key setup..."), - create_primary_key(), - create_attestation_key(). - -create_primary_key() -> - CommandPrimary = ?TPM2_CREATEPRIMARY_CMD, - ?event({"Running command", CommandPrimary}), - case sec_helpers:run_command(CommandPrimary) of - {ok, _} -> - ?event("Primary key created successfully"), - create_attestation_key(); - {error, Reason} -> - ?event({"Error creating primary key", Reason}), - {error, Reason} - end. - -create_attestation_key() -> - CommandCreateAK = ?TPM2_CREATE_CMD, - ?event({"Running command", CommandCreateAK}), - case sec_helpers:run_command(CommandCreateAK) of - {ok, _} -> - ?event("Attestation key created successfully"), - load_attestation_key(); - {error, Reason} -> - ?event({"Error creating attestation key", Reason}), - {error, Reason} - end. - -load_attestation_key() -> - CommandLoadAK = ?TPM2_LOAD_CMD, - ?event({"Running command", CommandLoadAK}), - case sec_helpers:run_command(CommandLoadAK) of - {ok, _} -> - ?event("Attestation key loaded successfully"), - export_ak_public_key(); - {error, Reason} -> - ?event({"Error loading attestation key", Reason}), - {error, Reason} - end. - -%% Helper to export the AK public key -export_ak_public_key() -> - Command = ?TPM2_READPUBLIC_CMD, - ?event({"Running command", Command}), - case sec_helpers:run_command(Command) of - {ok, _} -> - ?event("AK public key exported successfully"), - ok; - {error, Reason} -> - ?event({"Error exporting AK public key", Reason}), - {error, Reason} - end. \ No newline at end of file diff --git a/test/OVMF-1.55.fd b/test/OVMF-1.55.fd new file mode 100644 index 000000000..6642e48d6 Binary files /dev/null and b/test/OVMF-1.55.fd differ diff --git a/test/admissible-report-wallet.json b/test/admissible-report-wallet.json new file mode 100644 index 000000000..52905a778 --- /dev/null +++ b/test/admissible-report-wallet.json @@ -0,0 +1 @@ +{"p":"v1JKa5JENrE5AOfbxQJKP5G_Q3miLbxobazHhAoznYVXtrklpNiYa5tbTT4yK1cH5dZpW2gNyYP6FuxfdwWN6EKXMlugH8BDCchLhcPvzuQd9WL7uyxO3CTTvJ3Rz_E87rAGfPTbVtM1kIXmK582HLZtbe1eDwvlMoEYOglobRO2YOc2_EL7P8v6spUvUW_y3UpwQnnE5zb-ZIb3W7A_3Xzb6sJB8KTVn8cjSX4eFAWA6WKZFvxyQ3yVnqLtwawon966lH-z2QOcV0fZa35K4GW8k2PX3dp4zJk4bremyG4bE0WY5xUSbqOhCOf8mjlGbdDYI1onPQQRBoi0ex-DPQ","q":"-xLVvoLT7YqbrFKg4BzxzD_MUJ-_EHcoj0K7z6S9Eb523nxoFMPFomd-kYCDCdDzGfN_d5lESI-3wv651OStG8phIBybCVWpUyGo1Kxp2d-qE8sI4JOhXX7p2oqjsgD-5iJATDwcKEXklkvCSv4PeRlsvhL55Y6fGblBzNec6fPrDrNOnk4O1mANIAzuSdO9MH1qVEKVCLOIfxyWoo3vQI6MXJdqxS-X5qjD_5PnhcyWH1rNMG9tb7RZJXd8tMRu5donJhYUdcgewI5_lEpem_9xB06wx7O6_Armd2z57yoq4xwLLVxfZDfsqJTTi6sRid2JlFpCXH4VYndCIybLfw","d":"BIOBc7rd_aah2JO7WgAa_bEO5bG2qES_-UdPe0MaECLakpm8YqnZKu-pR8yy5utGXwJiESR5XhhHWkI_tLY9WxNybAFKmqrL1LmfcITz63KySxxOSxQPIIicffOzOMQkc7vkeJ5Nyp1NveubKC_jcWSKH3F5Okf1GR-5kIWUUiEs3kRzK75VOierSLPp2PTc80Rl15oGQd6x7cfU_GEF34IQPHNekrtWY81gHuv7EyOMeWIKCxErecKJWfXwnAsIoDP9KEP_e2DynTynJQkbFs8DUTVrIrPtpkPG-lrnR9x-7tOx0DtTTZuxsCM1W5XkQQXNQH4RXRqGjAm8NoqdbEG0WKLRCH9uTXQxw7OD6v2miTTfgus2NBaH4D1yzvgpTTwtS__KMbjY2QefwWiZ-gEbEcb1XLFDCDwAIacwIeniQDb2E4JpXQqflAav6cpcB9VOMeQQxk2LUYo1mMbv5Ij-HpK2OAqAd38W8ImV_uD5rHaujyiGBSN5j8TAONl2D7_GJNHzKNy7yfEl0KSDca5wrP9CM2ri6aUe6LEBnoPkT9Xh7ZfEwPwOoMXYG0gD7WCfplVDm1hIheGi5TM5LDa8u7WqXG_jZVtYX-NnIsA_Sq-VbvaUF9CiYuii6OgUvr_5eDaJ7ck9v8lfg8yF1k0ZsBf5RxSOhXs6fc38kwU","n":"u6PCfoy-ExmoBXPZojc1oqL3wmSSywwjFGzr4ERqucFFp-ACA3KylqpgiBNGfobTokOQht4QKyMRDnf_LoO0is6idVuaVxp37L_3QRlSebn4VilPtlWcOr2rfSYbhM8TjBzwFoj1BpeQJzD4vYK9thyp2hlHVBB2ZFNAwB_22_Mr1atOYdZstv23eWo1Pb2Rwv_vwuvsK_-kffjRL6lTy7OjMwGL6UBnyWaInTSH74m4ihxS_vql8jMBySlB2ChhTpzsJi-JrrcBvCqbWhVK7ULXhVs8LvCMqCNG5w38ts8rfYtZYYjlcHOA_NER6PtYkS5nz9LVZtpZ4_QGNy7Xohqf9t0jrcYEmeSOV4EXhbLqxdwm_ITwoZbe6ZnXcSBUMOdMzaM7q0aIT8HOOI3MvLzl5uFRUdjfZ_irJOC0vb0IrsvRicC7jGQo0mBv8M2EE5l2otBO5aQFgpFqYphiCAMLhyTnsOUTvkA4yw40xXVstMzEJ2pxYOWiEvP1qPj-kjnq_7uHBa9TWzJ7bxiO_BzHMbKd7sb3r2jicsemmeMm4yeQzOT6Tf3jvNE17emV6K51SyWN6UNJUTJ9HpEhqZ0Y17nI3oOLOPsNMMXQ9_rmbP7Tv8dCbq9sRGnjmX1Svrs6SwNVpfoUT8NmG2tqM5VGKXo4mknLOr-1g-PBekM","e":"AQAB","ext":true,"dp":"HtDmY8U_b3_EKr0tzOG9i9ex8vBYiv1Z5LB7wmzSO4EKy8eupIquokZ3wk1OT2TJRN_wQGTWM6sqUR7pkYY3gT2YlOflNrgFFEJKx9Tzf2OG38t9uHw-h373C95vuQqmQdvgb6gQ3D9Q1WJ73HLciGtp3Nbq24mS9TuN52s0gr02Fw2m8aLoTTJRwwn8gSWC_NnMkyiB6qwU9aQ3m3EcGFTQJ1P6wwQJ1J6CtIe32Im6Zd0Xw3gN_4jFoLOlkBhmwrlhXCHlmgLW38gW4RWKgfJhGWxvjLBv3KShTlQObSIvAj-njTD7sw5wFbsoGL849N86sRcIUu-gvmiuiVZeEQ","dq":"6m952bu7O1Bzb4Jv6RPdy0O--YFQHIXG_43mZEqEqG7Z-4DahpkOj0hn7GC8-ot6kz7ERN593eskQRUsW9dytEJSUnOjaCHuS0tgo8ShyeiInJa2oUv4Hp8EqSVPGETJvgU5WHXALPKmMJhowTFdLUxKN2jsoiZ79L8A685gHCu_zigrPrHQNOfXGZg5YAIv43kXsbnCAy_wQhBlrz8sqXDxKvvPnHOGOMBY0uo-Arc3beuRMKq62tThcJSTgw7wJft_FpcDX78Ox-nGwqZ2lN79oT8e3jm6XOGotNaywVj0Vr-2yBI6mA-IERl2NjHz3HFZp4Zn9IleWmTVApGU7Q","kty":"RSA","qi":"VHkl0nQEL-BDIRm_o24lBpAKnWro3xpsh3bbWJD8KQy0ybi8txxPyNuMEeo8mMSYluxO2QBjO-9ATqx5jLNuWW07_e607UX6xDPY1UrG0irTgqcv_Iabel82qoA_RJ8dOw4ng_wBd9fdyRPliOWcmLpWMEZerkfhXNYRimw1z2kh3sOQSnibHQnBPzol0lNbp_Mi8MY378ssBIvCTN7p3aiQAiC5gyGyiP9y7e4ejl32G6zyco9X98-4n1s4ZPP1LZDkN9YoY_lrrDnUFJpQ8o6RuthiBxpN8gmutK4blU_6hE9ze5OYJkLCb-eyRVPqmbXX0WWAZJFQagFYgMX_PA"} \ No newline at end of file diff --git a/test/admissible-report.eterm b/test/admissible-report.eterm new file mode 100644 index 000000000..64ffbaedc --- /dev/null +++ b/test/admissible-report.eterm @@ -0,0 +1,41 @@ +#{ + <<"address">> => <<"-q_bZCJGupSiZvjNg-KoD0bjx53pqnay_5Ojvo0597s">>, + <<"local-hashes">> => + #{ + <<"append">> => + <<"95a34faced5e487991f9cc2253a41cbd26b708bf00328f98dddbbf6b3ea2892e">>, + <<"firmware">> => + <<"b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510">>, + <<"guest_features">> => 1, + <<"initrd">> => + <<"544045560322dbcd2c454bdc50f35edf0147829ec440e6cb487b4a1503f923c1">>, + <<"kernel">> => + <<"69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576">>, + <<"vcpu_type">> => 5, + <<"vcpus">> => 32, + <<"vmm_type">> => 1 + }, + <<"node-message">> => + #{ + <<"address">> => <<"-q_bZCJGupSiZvjNg-KoD0bjx53pqnay_5Ojvo0597s">>, + <<"snp_trusted">> => + [#{ + <<"append">> => + <<"95a34faced5e487991f9cc2253a41cbd26b708bf00328f98dddbbf6b3ea2892e">>, + <<"firmware">> => + <<"b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510">>, + <<"guest_features">> => 1, + <<"initrd">> => + <<"544045560322dbcd2c454bdc50f35edf0147829ec440e6cb487b4a1503f923c1">>, + <<"kernel">> => + <<"69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576">>, + <<"vcpu_type">> => 5, + <<"vcpus">> => 32, + <<"vmm_type">> => 1 + }] + }, + <<"nonce">> => + <<"-q_bZCJGupSiZvjNg-KoD0bjx53pqnay_5Ojvo0597uwXbjdkxgcnt621T0zh93SeA0lGDsu1JEZUfZLZPKycw">>, + <<"report">> => + <<"{\"version\":2,\"guest_svn\":0,\"policy\":196608,\"family_id\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"image_id\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"vmpl\":1,\"sig_algo\":1,\"current_tcb\":{\"bootloader\":4,\"tee\":0,\"_reserved\":[0,0,0,0],\"snp\":22,\"microcode\":213},\"plat_info\":3,\"_author_key_en\":0,\"_reserved_0\":0,\"report_data\":[250,175,219,100,34,70,186,148,162,102,248,205,131,226,168,15,70,227,199,157,233,170,118,178,255,147,163,190,141,57,247,187,181,160,210,21,56,188,91,240,50,131,40,188,234,18,130,134,180,231,217,163,72,45,87,74,139,105,40,115,207,229,115,46],\"measurement\":[135,166,103,101,166,120,21,18,52,110,203,71,81,17,101,194,107,109,163,231,41,151,61,151,16,160,197,103,199,74,166,87,130,58,240,193,98,213,22,248,67,0,84,255,163,46,194,73],\"host_data\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"id_key_digest\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"author_key_digest\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"report_id\":[241,10,197,48,71,124,10,164,25,84,217,143,57,33,170,252,188,35,183,1,18,3,169,47,254,196,204,111,197,155,181,36],\"report_id_ma\":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255],\"reported_tcb\":{\"bootloader\":4,\"tee\":0,\"_reserved\":[0,0,0,0],\"snp\":22,\"microcode\":213},\"_reserved_1\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"chip_id\":[6,154,118,21,12,115,53,47,251,19,195,100,155,12,239,102,116,131,103,189,106,202,153,3,114,175,16,182,24,166,229,214,231,126,164,15,129,52,233,142,196,43,43,8,89,238,118,246,21,144,209,16,165,197,134,105,214,250,155,148,50,78,87,203],\"committed_tcb\":{\"bootloader\":4,\"tee\":0,\"_reserved\":[0,0,0,0],\"snp\":22,\"microcode\":213},\"current_build\":20,\"current_minor\":55,\"current_major\":1,\"_reserved_2\":0,\"committed_build\":20,\"committed_minor\":55,\"committed_major\":1,\"_reserved_3\":0,\"launch_tcb\":{\"bootloader\":4,\"tee\":0,\"_reserved\":[0,0,0,0],\"snp\":22,\"microcode\":213},\"_reserved_4\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"signature\":{\"r\":[136,143,81,241,242,123,84,48,177,36,241,167,35,181,109,42,134,193,44,88,162,240,140,195,117,252,151,27,83,156,188,69,14,114,21,107,105,163,12,20,128,144,61,65,233,31,205,111,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"s\":[64,250,227,155,71,182,238,179,71,128,109,219,182,33,119,151,202,50,123,211,22,167,104,241,222,221,82,34,138,148,86,254,190,174,191,86,202,194,207,91,110,250,3,196,127,105,133,135,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"_reserved\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}}">> +}. \ No newline at end of file diff --git a/test/admissible-report.json b/test/admissible-report.json new file mode 100644 index 000000000..9ccc4a3d1 --- /dev/null +++ b/test/admissible-report.json @@ -0,0 +1 @@ +{"version":2,"guest_svn":0,"policy":196608,"family_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"image_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"vmpl":1,"sig_algo":1,"current_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"plat_info":3,"_author_key_en":0,"_reserved_0":0,"report_data":[250,175,219,100,34,70,186,148,162,102,248,205,131,226,168,15,70,227,199,157,233,170,118,178,255,147,163,190,141,57,247,187,181,160,210,21,56,188,91,240,50,131,40,188,234,18,130,134,180,231,217,163,72,45,87,74,139,105,40,115,207,229,115,46],"measurement":[135,166,103,101,166,120,21,18,52,110,203,71,81,17,101,194,107,109,163,231,41,151,61,151,16,160,197,103,199,74,166,87,130,58,240,193,98,213,22,248,67,0,84,255,163,46,194,73],"host_data":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"id_key_digest":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"author_key_digest":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"report_id":[241,10,197,48,71,124,10,164,25,84,217,143,57,33,170,252,188,35,183,1,18,3,169,47,254,196,204,111,197,155,181,36],"report_id_ma":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255],"reported_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"_reserved_1":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"chip_id":[6,154,118,21,12,115,53,47,251,19,195,100,155,12,239,102,116,131,103,189,106,202,153,3,114,175,16,182,24,166,229,214,231,126,164,15,129,52,233,142,196,43,43,8,89,238,118,246,21,144,209,16,165,197,134,105,214,250,155,148,50,78,87,203],"committed_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"current_build":20,"current_minor":55,"current_major":1,"_reserved_2":0,"committed_build":20,"committed_minor":55,"committed_major":1,"_reserved_3":0,"launch_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"_reserved_4":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"signature":{"r":[136,143,81,241,242,123,84,48,177,36,241,167,35,181,109,42,134,193,44,88,162,240,140,195,117,252,151,27,83,156,188,69,14,114,21,107,105,163,12,20,128,144,61,65,233,31,205,111,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s":[64,250,227,155,71,182,238,179,71,128,109,219,182,33,119,151,202,50,123,211,22,167,104,241,222,221,82,34,138,148,86,254,190,174,191,86,202,194,207,91,110,250,3,196,127,105,133,135,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"_reserved":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}} \ No newline at end of file diff --git a/test/config.flat b/test/config.flat new file mode 100644 index 000000000..be8c0d876 --- /dev/null +++ b/test/config.flat @@ -0,0 +1,3 @@ +port: 1234 +host: https://ao.computer +await-inprogress: false \ No newline at end of file diff --git a/test/config.json b/test/config.json new file mode 100644 index 000000000..0be064746 --- /dev/null +++ b/test/config.json @@ -0,0 +1,13 @@ +{ + "port": 1234, + "example": 9001, + "host": "https://ao.computer", + "await_inprogress": false, + "store": [ + { + "store-module": "hb_store_fs", + "name": "cache-TEST/json-test-store", + "ao-types": "store-module=\"atom\"" + } + ] +} \ No newline at end of file diff --git a/test/dev_dummy.erl b/test/dev_dummy.erl new file mode 100644 index 000000000..f36f2edca --- /dev/null +++ b/test/dev_dummy.erl @@ -0,0 +1,7 @@ +%%% @doc A dummy module that we use for testing load-device-from-cache behaviors. + +-module(dev_dummy). +-export([echo/3]). + +echo(_M1, M2, _Opts) -> + {ok, M2}. \ No newline at end of file diff --git a/test/hyper-aos.lua b/test/hyper-aos.lua new file mode 100644 index 000000000..9fee87e21 --- /dev/null +++ b/test/hyper-aos.lua @@ -0,0 +1,2558 @@ + +local function load_json() + local json = {_version = "0.2.0"} + +------------------------------------------------------------------------------- +-- Encode +------------------------------------------------------------------------------- + +local encode + +local escape_char_map = { + ["\\"] = "\\", + ["\""] = "\"", + ["\b"] = "b", + ["\f"] = "f", + ["\n"] = "n", + ["\r"] = "r", + ["\t"] = "t" +} + +local escape_char_map_inv = {["/"] = "/"} +for k, v in pairs(escape_char_map) do escape_char_map_inv[v] = k end + +local function escape_char(c) + return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte())) +end + +local function encode_nil(val) return "null" end + +local function encode_table(val, stack) + local res = {} + stack = stack or {} + + -- Circular reference? + if stack[val] then error("circular reference") end + + stack[val] = true + + if rawget(val, 1) ~= nil or next(val) == nil then + -- Treat as array -- check keys are valid and it is not sparse + local n = 0 + for k in pairs(val) do + if type(k) ~= "number" then + error("invalid table: mixed or invalid key types") + end + n = n + 1 + end + if n ~= #val then error("invalid table: sparse array") end + -- Encode + for i = 1, #val do res[i] = encode(val[i], stack) end + stack[val] = nil + return "[" .. table.concat(res, ",") .. "]" + else + -- Treat as an object + local i = 1 + for k, v in pairs(val) do + if type(k) ~= "string" then + error("invalid table: mixed or invalid key types") + end + if type(v) ~= "function" then + res[i] = encode(k, stack) .. ":" .. encode(v, stack) + i = i + 1 + end + end + stack[val] = nil + return "{" .. table.concat(res, ",") .. "}" + end +end + +local function encode_string(val) + return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"' +end + +local function encode_number(val) + -- Check for NaN, -inf and inf + if val ~= val or val <= -math.huge or val >= math.huge then + error("unexpected number value '" .. tostring(val) .. "'") + end + return string.format("%.14g", val) +end + +local type_func_map = { + ["nil"] = encode_nil, + ["table"] = encode_table, + ["string"] = encode_string, + ["number"] = encode_number, + ["boolean"] = tostring +} + +encode = function(val, stack) + local t = type(val) + local f = type_func_map[t] + if f then return f(val, stack) end + error("unexpected type '" .. t .. "'") +end + +function json.encode(val) return (encode(val)) end + +------------------------------------------------------------------------------- +-- Decode +------------------------------------------------------------------------------- + +local parse + +local function create_set(...) + local res = {} + for i = 1, select("#", ...) do res[select(i, ...)] = true end + return res +end + +local space_chars = create_set(" ", "\t", "\r", "\n") +local delim_chars = create_set(" ", "\t", "\r", "\n", "]", "}", ",") +local escape_chars = create_set("\\", "/", '"', "b", "f", "n", "r", "t", "u") +local literals = create_set("true", "false", "null") + +local literal_map = {["true"] = true, ["false"] = false, ["null"] = nil} + +local function next_char(str, idx, set, negate) + for i = idx, #str do if set[str:sub(i, i)] ~= negate then return i end end + return #str + 1 +end + +local function decode_error(str, idx, msg) + local line_count = 1 + local col_count = 1 + for i = 1, idx - 1 do + col_count = col_count + 1 + if str:sub(i, i) == "\n" then + line_count = line_count + 1 + col_count = 1 + end + end + error(string.format("%s at line %d col %d", msg, line_count, col_count)) +end + +local function codepoint_to_utf8(n) + local f = math.floor + if n <= 0x7f then + return string.char(n) + elseif n <= 0x7ff then + return string.char(f(n / 64) + 192, n % 64 + 128) + elseif n <= 0xffff then + return string.char(f(n / 4096) + 224, f(n % 4096 / 64) + 128, + n % 64 + 128) + elseif n <= 0x10ffff then + return string.char(f(n / 262144) + 240, f(n % 262144 / 4096) + 128, + f(n % 4096 / 64) + 128, n % 64 + 128) + end + error(string.format("invalid unicode codepoint '%x'", n)) +end + +local function parse_unicode_escape(s) + local n1 = tonumber(s:sub(1, 4), 16) + local n2 = tonumber(s:sub(7, 10), 16) + if n2 then + return + codepoint_to_utf8((n1 - 0xd800) * 0x400 + (n2 - 0xdc00) + 0x10000) + else + return codepoint_to_utf8(n1) + end +end + +local function parse_string(str, i) + local res = {} + local j = i + 1 + local k = j + + while j <= #str do + local x = str:byte(j) + + if x < 32 then + decode_error(str, j, "control character in string") + elseif x == 92 then -- `\`: Escape + res[#res + 1] = str:sub(k, j - 1) + j = j + 1 + local c = str:sub(j, j) + if c == "u" then + local hex = str:match("^[dD][89aAbB]%x%x\\u%x%x%x%x", j + 1) or + str:match("^%x%x%x%x", j + 1) or + decode_error(str, j - 1, + "invalid unicode escape in string") + res[#res + 1] = parse_unicode_escape(hex) + j = j + #hex + else + if not escape_chars[c] then + decode_error(str, j - 1, + "invalid escape char '" .. c .. "' in string") + end + res[#res + 1] = escape_char_map_inv[c] + end + k = j + 1 + elseif x == 34 then -- `"`: End of string + res[#res + 1] = str:sub(k, j - 1) + return table.concat(res), j + 1 + end + j = j + 1 + end + + decode_error(str, i, "expected closing quote for string") +end + +local function parse_number(str, i) + local x = next_char(str, i, delim_chars) + local s = str:sub(i, x - 1) + local n = tonumber(s) + if not n then decode_error(str, i, "invalid number '" .. s .. "'") end + return n, x +end + +local function parse_literal(str, i) + local x = next_char(str, i, delim_chars) + local word = str:sub(i, x - 1) + if not literals[word] then + decode_error(str, i, "invalid literal '" .. word .. "'") + end + return literal_map[word], x +end + +local function parse_array(str, i) + local res = {} + local n = 1 + i = i + 1 + while true do + local x + i = next_char(str, i, space_chars, true) + if str:sub(i, i) == "]" then + i = i + 1 + break + end + x, i = parse(str, i) + res[n] = x + n = n + 1 + i = next_char(str, i, space_chars, true) + local chr = str:sub(i, i) + i = i + 1 + if chr == "]" then break end + if chr ~= "," then decode_error(str, i, "expected ']' or ','") end + end + return res, i +end + +local function parse_object(str, i) + local res = {} + i = i + 1 + while true do + local key, val + i = next_char(str, i, space_chars, true) + if str:sub(i, i) == "}" then + i = i + 1 + break + end + if str:sub(i, i) ~= '"' then + decode_error(str, i, "expected string for key") + end + key, i = parse(str, i) + i = next_char(str, i, space_chars, true) + if str:sub(i, i) ~= ":" then + decode_error(str, i, "expected ':' after key") + end + i = next_char(str, i + 1, space_chars, true) + val, i = parse(str, i) + res[key] = val + i = next_char(str, i, space_chars, true) + local chr = str:sub(i, i) + i = i + 1 + if chr == "}" then break end + if chr ~= "," then decode_error(str, i, "expected '}' or ','") end + end + return res, i +end + +local char_func_map = { + ['"'] = parse_string, + ["0"] = parse_number, + ["1"] = parse_number, + ["2"] = parse_number, + ["3"] = parse_number, + ["4"] = parse_number, + ["5"] = parse_number, + ["6"] = parse_number, + ["7"] = parse_number, + ["8"] = parse_number, + ["9"] = parse_number, + ["-"] = parse_number, + ["t"] = parse_literal, + ["f"] = parse_literal, + ["n"] = parse_literal, + ["["] = parse_array, + ["{"] = parse_object +} + +parse = function(str, idx) + local chr = str:sub(idx, idx) + local f = char_func_map[chr] + if f then return f(str, idx) end + decode_error(str, idx, "unexpected character '" .. chr .. "'") +end + +function json.decode(str) + if type(str) ~= "string" then + error("expected argument of type string, got " .. type(str)) + end + local res, idx = parse(str, next_char(str, 1, space_chars, true)) + idx = next_char(str, idx, space_chars, true) + if idx <= #str then decode_error(str, idx, "trailing garbage") end + return res +end + +return json + +end +_G.package.loaded[".json"] = load_json() +print("loaded json") + + + +local function load_stringify() + --- The Stringify module provides utilities for formatting and displaying Lua tables in a more readable manner. Returns the stringify table. +-- @module stringify + +--- The stringify table +-- @table stringify +-- @field _version The version number of the stringify module +-- @field isSimpleArray The isSimpleArray function +-- @field format The format function +local stringify = { _version = "0.0.1" } + +-- ANSI color codes +local colors = { + red = "\27[31m", + green = "\27[32m", + blue = "\27[34m", + reset = "\27[0m" +} + +--- Checks if a table is a simple array (i.e., an array with consecutive numeric keys starting from 1). +-- @function isSimpleArray +-- @tparam {table} tbl The table to check +-- @treturn {boolean} Whether the table is a simple array +function stringify.isSimpleArray(tbl) + local arrayIndex = 1 + for k, v in pairs(tbl) do + if k ~= arrayIndex or (type(v) ~= "number" and type(v) ~= "string") then + return false + end + arrayIndex = arrayIndex + 1 + end + return true +end + +--- Formats a table for display, handling circular references and formatting strings and tables recursively. +-- @function format +-- @tparam {table} tbl The table to format +-- @tparam {number} indent The indentation level (default is 0) +-- @tparam {table} visited A table to track visited tables and detect circular references (optional) +-- @treturn {string} A string representation of the table +function stringify.format(tbl, indent, visited) + indent = indent or 0 + local toIndent = string.rep(" ", indent) + local toIndentChild = string.rep(" ", indent + 2) + + local result = {} + local isArray = true + local arrayIndex = 1 + + if stringify.isSimpleArray(tbl) then + for _, v in ipairs(tbl) do + if type(v) == "string" then + v = colors.green .. '"' .. v .. '"' .. colors.reset + else + v = colors.blue .. tostring(v) .. colors.reset + end + table.insert(result, v) + end + return "{ " .. table.concat(result, ", ") .. " }" + end + + for k, v in pairs(tbl) do + if isArray then + if k == arrayIndex then + arrayIndex = arrayIndex + 1 + if type(v) == "table" then + v = stringify.format(v, indent + 2) + elseif type(v) == "string" then + v = colors.green .. '"' .. v .. '"' .. colors.reset + else + v = colors.blue .. tostring(v) .. colors.reset + end + table.insert(result, toIndentChild .. v) + else + isArray = false + result = {} + end + end + if not isArray then + if type(v) == "table" then + visited = visited or {} + if visited[v] then + return "" + end + visited[v] = true + + v = stringify.format(v, indent + 2, visited) + elseif type(v) == "string" then + v = colors.green .. '"' .. v .. '"' .. colors.reset + else + v = colors.blue .. tostring(v) .. colors.reset + end + k = colors.red .. k .. colors.reset + table.insert(result, toIndentChild .. k .. " = " .. v) + end + end + + local prefix = isArray and "{\n" or "{\n " + local suffix = isArray and "\n" .. toIndent .. " }" or "\n" .. toIndent .. "}" + local separator = isArray and ",\n" or ",\n " + return prefix .. table.concat(result, separator) .. suffix +end + +return stringify + +end +_G.package.loaded[".stringify"] = load_stringify() +print("loaded stringify") + + + +local function load_eval() + --- The Eval module provides a handler for evaluating Lua expressions. Returns the eval function. +-- @module eval + +local stringify = require(".stringify") +local json = require('.json') +--- The eval function. +-- Handler for executing and evaluating Lua expressions. +-- After execution, the result is stringified and placed in ao.outbox.Output. +-- @function eval +-- @tparam {table} ao The ao environment object +-- @treturn {function} The handler function, which takes a message as an argument. +-- @see stringify +return function (ao) + return function (req) + local msg = req.body + -- exec expression + local expr = msg.body and msg.body.body or msg.data or "" + local func, err = load("return " .. expr, 'aos', 't', _G) + local output = "" + local e = nil + if err then + func, err = load(expr, 'aos', 't', _G) + end + if func then + output, e = func() + else + ao.outbox.Error = err + return + end + if e then + ao.outbox.Error = e + return + end + if HandlerPrintLogs and output then + table.insert(HandlerPrintLogs, + type(output) == "table" + and stringify.format(output) + or tostring(output) + ) + -- print(stringify.format(HandlerPrintLogs)) + -- else + -- -- set result in outbox.Output (Left for backwards compatibility) + -- ao.outbox.Output = { + -- data = type(output) == "table" + -- and stringify.format(output) or tostring(output), + -- prompt = Prompt() + -- } + -- + end + end +end + +end +_G.package.loaded[".eval"] = load_eval() +print("loaded eval") + + + +local function load_utils() + --- The Utils module provides a collection of utility functions for functional programming in Lua. It includes functions for array manipulation such as concatenation, mapping, reduction, filtering, and finding elements, as well as a property equality checker. +-- @module utils + +--- The utils table +-- @table utils +-- @field _version The version number of the utils module +-- @field matchesPattern The matchesPattern function +-- @field matchesSpec The matchesSpec function +-- @field curry The curry function +-- @field concat The concat function +-- @field reduce The reduce function +-- @field map The map function +-- @field filter The filter function +-- @field find The find function +-- @field propEq The propEq function +-- @field reverse The reverse function +-- @field compose The compose function +-- @field prop The prop function +-- @field includes The includes function +-- @field keys The keys function +-- @field values The values function +local utils = { _version = "0.0.5" } + +--- Given a pattern, a value, and a message, returns whether there is a pattern match. +-- @usage utils.matchesPattern(pattern, value, msg) +-- @param pattern The pattern to match +-- @param value The value to check for in the pattern +-- @param msg The message to check for the pattern +-- @treturn {boolean} Whether there is a pattern match +function utils.matchesPattern(pattern, value, msg) + -- If the key is not in the message, then it does not match + if (not pattern) then + return false + end + -- if the patternMatchSpec is a wildcard, then it always matches + if pattern == '_' then + return true + end + -- if the patternMatchSpec is a function, then it is executed on the tag value + if type(pattern) == "function" then + if pattern(value, msg) then + return true + else + return false + end + end + -- if the patternMatchSpec is a string, check it for special symbols (less `-` alone) + -- and exact string match mode + if (type(pattern) == 'string') then + if string.match(pattern, "[%^%$%(%)%%%.%[%]%*%+%?]") then + if string.match(value, pattern) then + return true + end + else + if value == pattern then + return true + end + end + end + + -- if the pattern is a table, recursively check if any of its sub-patterns match + if type(pattern) == 'table' then + for _, subPattern in pairs(pattern) do + if utils.matchesPattern(subPattern, value, msg) then + return true + end + end + end + + return false +end + +--- Given a message and a spec, returns whether there is a spec match. +-- @usage utils.matchesSpec(msg, spec) +-- @param msg The message to check for the spec +-- @param spec The spec to check for in the message +-- @treturn {boolean} Whether there is a spec match +function utils.matchesSpec(msg, spec) + if type(spec) == 'function' then + return spec(msg) + -- If the spec is a table, step through every key/value pair in the pattern and check if the msg matches + -- Supported pattern types: + -- - Exact string match + -- - Lua gmatch string + -- - '_' (wildcard: Message has tag, but can be any value) + -- - Function execution on the tag, optionally using the msg as the second argument + -- - Table of patterns, where ANY of the sub-patterns matching the tag will result in a match + end + if type(spec) == 'table' then + for key, pattern in pairs(spec) do + -- The key can either be in the top level of the 'msg' object + -- or in the body table of the msg + local msgValue = msg[key] or msg.body[key] + if not msgValue then + return false + end + local matchesMsgValue = utils.matchesPattern(pattern, msgValue, msg) + if not matchesMsgValue then + return false + end + + end + return true + end + + if type(spec) == 'string' and msg.action and msg.action == spec then + return true + end + if type(spec) == 'string' and msg.body.action and msg.body.action == spec then + return true + end + return false +end + +--- Given a table, returns whether it is an array. +-- An 'array' is defined as a table with integer keys starting from 1 and +-- having no gaps between the keys. +-- @lfunction isArray +-- @param table The table to check +-- @treturn {boolean} Whether the table is an array +local function isArray(table) + if type(table) == "table" then + local maxIndex = 0 + for k, v in pairs(table) do + if type(k) ~= "number" or k < 1 or math.floor(k) ~= k then + return false -- If there's a non-integer key, it's not an array + end + maxIndex = math.max(maxIndex, k) + end + -- If the highest numeric index is equal to the number of elements, it's an array + return maxIndex == #table + end + return false +end + +--- Curries a function. +-- @tparam {function} fn The function to curry +-- @tparam {number} arity The arity of the function +-- @treturn {function} The curried function +utils.curry = function (fn, arity) + assert(type(fn) == "function", "function is required as first argument") + arity = arity or debug.getinfo(fn, "u").nparams + if arity < 2 then return fn end + + return function (...) + local args = {...} + + if #args >= arity then + return fn(table.unpack(args)) + else + return utils.curry(function (...) + return fn(table.unpack(args), ...) + end, arity - #args) + end + end +end + +--- Concat two Array Tables +-- @function concat +-- @usage utils.concat(a)(b) +-- @usage utils.concat({1, 2})({3, 4}) --> {1, 2, 3, 4} +-- @tparam {table} a The first array +-- @tparam {table} b The second array +-- @treturn {table} The concatenated array +utils.concat = utils.curry(function (a, b) + assert(type(a) == "table", "first argument should be a table that is an array") + assert(type(b) == "table", "second argument should be a table that is an array") + assert(isArray(a), "first argument should be a table") + assert(isArray(b), "second argument should be a table") + + local result = {} + for i = 1, #a do + result[#result + 1] = a[i] + end + for i = 1, #b do + result[#result + 1] = b[i] + end + return result +end, 2) + +--- Applies a function to each element of a table, reducing it to a single value. +-- @function utils.reduce +-- @usage utils.reduce(fn)(initial)(t) +-- @usage utils.reduce(function(acc, x) return acc + x end)(0)({1, 2, 3}) --> 6 +-- @tparam {function} fn The function to apply +-- @param initial The initial value +-- @tparam {table} t The table to reduce +-- @return The reduced value +utils.reduce = utils.curry(function (fn, initial, t) + assert(type(fn) == "function", "first argument should be a function that accepts (result, value, key)") + assert(type(t) == "table" and isArray(t), "third argument should be a table that is an array") + local result = initial + for k, v in pairs(t) do + if result == nil then + result = v + else + result = fn(result, v, k) + end + end + return result +end, 3) + +--- Applies a function to each element of an array table, mapping it to a new value. +-- @function utils.map +-- @usage utils.map(fn)(t) +-- @usage utils.map(function(x) return x * 2 end)({1, 2, 3}) --> {2, 4, 6} +-- @tparam {function} fn The function to apply to each element +-- @tparam {table} data The table to map over +-- @treturn {table} The mapped table +utils.map = utils.curry(function (fn, data) + assert(type(fn) == "function", "first argument should be a unary function") + assert(type(data) == "table" and isArray(data), "second argument should be an Array") + + local function map (result, v, k) + result[k] = fn(v, k) + return result + end + + return utils.reduce(map, {}, data) +end, 2) + +--- Filters an array table based on a predicate function. +-- @function utils.filter +-- @usage utils.filter(fn)(t) +-- @usage utils.filter(function(x) return x > 1 end)({1, 2, 3}) --> {2,3} +-- @tparam {function} fn The predicate function to determine if an element should be included. +-- @tparam {table} data The array to filter +-- @treturn {table} The filtered table +utils.filter = utils.curry(function (fn, data) + assert(type(fn) == "function", "first argument should be a unary function") + assert(type(data) == "table" and isArray(data), "second argument should be an Array") + + local function filter (result, v, _k) + if fn(v) then + table.insert(result, v) + end + return result + end + + return utils.reduce(filter,{}, data) +end, 2) + +--- Finds the first element in an array table that satisfies a predicate function. +-- @function utils.find +-- @usage utils.find(fn)(t) +-- @usage utils.find(function(x) return x > 1 end)({1, 2, 3}) --> 2 +-- @tparam {function} fn The predicate function to determine if an element should be included. +-- @tparam {table} t The array table to search +-- @treturn The first element that satisfies the predicate function +utils.find = utils.curry(function (fn, t) + assert(type(fn) == "function", "first argument should be a unary function") + assert(type(t) == "table", "second argument should be a table that is an array") + for _, v in pairs(t) do + if fn(v) then + return v + end + end +end, 2) + +--- Checks if a property of an object is equal to a value. +-- @function utils.propEq +-- @usage utils.propEq(propName)(value)(object) +-- @usage utils.propEq("name")("Lua")({name = "Lua"}) --> true +-- @tparam {string} propName The property name to check +-- @tparam {string} value The value to check against +-- @tparam {table} object The object to check +-- @treturn {boolean} Whether the property is equal to the value +utils.propEq = utils.curry(function (propName, value, object) + assert(type(propName) == "string", "first argument should be a string") + assert(type(value) == "string", "second argument should be a string") + assert(type(object) == "table", "third argument should be a table") + + return object[propName] == value +end, 3) + +--- Reverses an array table. +-- @function utils.reverse +-- @usage utils.reverse(data) +-- @usage utils.reverse({1, 2, 3}) --> {3, 2, 1} +-- @tparam {table} data The array table to reverse +-- @treturn {table} The reversed array table +utils.reverse = function (data) + assert(type(data) == "table", "argument needs to be a table that is an array") + return utils.reduce( + function (result, v, i) + result[#data - i + 1] = v + return result + end, + {}, + data + ) +end + +--- Composes a series of functions into a single function. +-- @function utils.compose +-- @usage utils.compose(fn1)(fn2)(fn3)(v) +-- @usage utils.compose(function(x) return x + 1 end)(function(x) return x * 2 end)(3) --> 7 +-- @tparam {function} ... The functions to compose +-- @treturn {function} The composed function +utils.compose = utils.curry(function (...) + local mutations = utils.reverse({...}) + + return function (v) + local result = v + for _, fn in pairs(mutations) do + assert(type(fn) == "function", "each argument needs to be a function") + result = fn(result) + end + return result + end +end, 2) + +--- Returns the value of a property of an object. +-- @function utils.prop +-- @usage utils.prop(propName)(object) +-- @usage utils.prop("name")({name = "Lua"}) --> "Lua" +-- @tparam {string} propName The property name to get +-- @tparam {table} object The object to get the property from +-- @treturn The value of the property +utils.prop = utils.curry(function (propName, object) + return object[propName] +end, 2) + +--- Checks if an array table includes a value. +-- @function utils.includes +-- @usage utils.includes(val)(t) +-- @usage utils.includes(2)({1, 2, 3}) --> true +-- @param val The value to check for +-- @tparam {table} t The array table to check +-- @treturn {boolean} Whether the value is in the array table +utils.includes = utils.curry(function (val, t) + assert(type(t) == "table", "argument needs to be a table") + assert(isArray(t), "argument should be a table that is an array") + return utils.find(function (v) return v == val end, t) ~= nil +end, 2) + +--- Returns the keys of a table. +-- @usage utils.keys(t) +-- @usage utils.keys({name = "Lua", age = 25}) --> {"name", "age"} +-- @tparam {table} t The table to get the keys from +-- @treturn {table} The keys of the table +utils.keys = function (t) + assert(type(t) == "table", "argument needs to be a table") + local keys = {} + for key in pairs(t) do + table.insert(keys, key) + end + return keys +end + +--- Returns the values of a table. +-- @usage utils.values(t) +-- @usage utils.values({name = "Lua", age = 25}) --> {"Lua", 25} +-- @tparam {table} t The table to get the values from +-- @treturn {table} The values of the table +utils.values = function (t) + assert(type(t) == "table", "argument needs to be a table") + local values = {} + for _, value in pairs(t) do + table.insert(values, value) + end + return values +end + +--- Convert a message's tags to a table of key-value pairs +-- @function Tab +-- @tparam {table} msg The message containing tags +-- @treturn {table} A table with tag names as keys and their values +function utils.Tab(msg) + local inputs = {} + for _, o in ipairs(msg.Tags) do + if not inputs[o.name] then + inputs[o.name] = o.value + end + end + return inputs +end + + +return utils + +end +_G.package.loaded[".utils"] = load_utils() +print("loaded utils") + + + +local function load_handlers_utils() + --- The Handler Utils module is a lightweight Lua utility library designed to provide common functionalities for handling and processing messages within the AOS computer system. It offers a set of functions to check message attributes and send replies, simplifying the development of more complex scripts and modules. This document will guide you through the module's functionalities, installation, and usage. Returns the _utils table. +-- @module handlers-utils + +--- The _utils table +-- @table _utils +-- @field _version The version number of the _utils module +-- @field hasMatchingTag The hasMatchingTag function +-- @field hasMatchingTagOf The hasMatchingTagOf function +-- @field hasMatchingData The hasMatchingData function +-- @field reply The reply function +-- @field continue The continue function +local _utils = { _version = "0.0.2" } + +local _ = require('.utils') + +--- Checks if a given message has a tag that matches the specified name and value. +-- @function hasMatchingTag +-- @tparam {string} name The tag name to check +-- @tparam {string} value The value to match for in the tag +-- @treturn {function} A function that takes a message and returns whether there is a tag match (-1 if matches, 0 otherwise) +function _utils.hasMatchingTag(name, value) + assert(type(name) == 'string' and type(value) == 'string', 'invalid arguments: (name : string, value : string)') + + return function (msg) + return msg.Tags[name] == value + end +end + +--- Checks if a given message has a tag that matches the specified name and one of the specified values. +-- @function hasMatchingTagOf +-- @tparam {string} name The tag name to check +-- @tparam {string[]} values The list of values of which one should match +-- @treturn {function} A function that takes a message and returns whether there is a tag match (-1 if matches, 0 otherwise) +function _utils.hasMatchingTagOf(name, values) + assert(type(name) == 'string' and type(values) == 'table', 'invalid arguments: (name : string, values : string[])') + return function (msg) + for _, value in ipairs(values) do + local patternResult = Handlers.utils.hasMatchingTag(name, value)(msg) + + if patternResult ~= 0 and patternResult ~= false and patternResult ~= "skip" then + return patternResult + end + end + + return 0 + end +end + +--- Checks if a given message has data that matches the specified value. +-- @function hasMatchingData +-- @tparam {string} value The value to match against the message data +-- @treturn {function} A function that takes a message and returns whether the data matches the value (-1 if matches, 0 otherwise) +function _utils.hasMatchingData(value) + assert(type(value) == 'string', 'invalid arguments: (value : string)') + return function (msg) + return msg.Data == value + end +end + +--- Given an input, returns a function that takes a message and replies to it. +-- @function reply +-- @tparam {table | string} input The content to send back. If a string, it sends it as data. If a table, it assumes a structure with `Tags`. +-- @treturn {function} A function that takes a message and replies to it +function _utils.reply(input) + assert(type(input) == 'table' or type(input) == 'string', 'invalid arguments: (input : table or string)') + return function (msg) + if type(input) == 'string' then + msg.reply({ Data = input }) + return + end + msg.reply(input) + end +end + +--- Inverts the provided pattern's result if it matches, so that it continues execution with the next matching handler. +-- @function continue +-- @tparam {table | function} pattern The pattern to check for in the message +-- @treturn {function} Function that executes the pattern matching function and returns `1` (continue), so that the execution of handlers continues. +function _utils.continue(pattern) + return function (msg) + local match = _.matchesSpec(msg, pattern) + + if not match or match == 0 or match == "skip" then + return match + end + return 1 + end +end + +return _utils + +end +_G.package.loaded[".handlers-utils"] = load_handlers_utils() +print("loaded handlers-utils") + + + +local function load_handlers() + --- The Handlers library provides a flexible way to manage and execute a series of handlers based on patterns. Each handler consists of a pattern function, a handle function, and a name. This library is suitable for scenarios where different actions need to be taken based on varying input criteria. Returns the handlers table. +-- @module handlers + +--- The handlers table +-- @table handlers +-- @field _version The version number of the handlers module +-- @field list The list of handlers +-- @field onceNonce The nonce for the once handlers +-- @field utils The handlers-utils module +-- @field generateResolver The generateResolver function +-- @field receive The receive function +-- @field once The once function +-- @field add The add function +-- @field append The append function +-- @field prepend The prepend function +-- @field remove The remove function +-- @field evaluate The evaluate function +local handlers = { _version = "0.0.5" } +local utils = require('.utils') + +handlers.utils = require('.handlers-utils') +-- if update we need to keep defined handlers +if Handlers then + handlers.list = Handlers.list or {} +else + handlers.list = {} +end +handlers.onceNonce = 0 + +--- Given an array, a property name, and a value, returns the index of the object in the array that has the property with the value. +-- @lfunction findIndexByProp +-- @tparam {table[]} array The array to search through +-- @tparam {string} prop The property name to check +-- @tparam {any} value The value to check for in the property +-- @treturn {number | nil} The index of the object in the array that has the property with the value, or nil if no such object is found +local function findIndexByProp(array, prop, value) + for index, object in ipairs(array) do + if object[prop] == value then + return index + end + end + return nil +end + +--- Given a name, a pattern, and a handle, asserts that the arguments are valid. +-- @lfunction assertAddArgs +-- @tparam {string} name The name of the handler +-- @tparam {table | function | string} pattern The pattern to check for in the message +-- @tparam {function} handle The function to call if the pattern matches +-- @tparam {number | string | nil} maxRuns The maximum number of times the handler should run, or nil if there is no limit +local function assertAddArgs(name, pattern, handle, maxRuns) + assert( + type(name) == 'string' and + (type(pattern) == 'function' or type(pattern) == 'table' or type(pattern) == 'string'), + 'Invalid arguments given. Expected: \n' .. + '\tname : string, ' .. + '\tpattern : action : string | MsgMatch : table,\n' .. + '\t\tfunction(msg: Message) : {-1 = break, 0 = skip, 1 = continue},\n' .. + '\thandle(msg : Message) : void) | Resolver,\n' .. + '\tMaxRuns? : number | "inf" | nil') +end + +--- Given a resolver specification, returns a resolver function. +-- @function generateResolver +-- @tparam {table | function} resolveSpec The resolver specification +-- @treturn {function} A resolver function +function handlers.generateResolver(resolveSpec) + return function(msg) + -- If the resolver is a single function, call it. + -- Else, find the first matching pattern (by its matchSpec), and exec. + if type(resolveSpec) == "function" then + return resolveSpec(msg) + else + for matchSpec, func in pairs(resolveSpec) do + if utils.matchesSpec(msg, matchSpec) then + return func(msg) + end + end + end + end +end + +--- Given a pattern, returns the next message that matches the pattern. +-- This function uses Lua's coroutines under-the-hood to add a handler, pause, +-- and then resume the current coroutine. This allows us to effectively block +-- processing of one message until another is received that matches the pattern. +-- @function receive +-- @tparam {table | function} pattern The pattern to check for in the message +function handlers.receive(pattern) + return 'not implemented' +end + +--- Given a name, a pattern, and a handle, adds a handler to the list. +-- If name is not provided, "_once_" prefix plus onceNonce will be used as the name. +-- Adds handler with maxRuns of 1 such that it will only be called once then removed from the list. +-- @function once +-- @tparam {string} name The name of the handler +-- @tparam {table | function | string} pattern The pattern to check for in the message +-- @tparam {function} handle The function to call if the pattern matches +function handlers.once(...) + local name, pattern, handle + if select("#", ...) == 3 then + name = select(1, ...) + pattern = select(2, ...) + handle = select(3, ...) + else + name = "_once_" .. tostring(handlers.onceNonce) + handlers.onceNonce = handlers.onceNonce + 1 + pattern = select(1, ...) + handle = select(2, ...) + end + handlers.prepend(name, pattern, handle, 1) +end + +--- Given a name, a pattern, and a handle, adds a handler to the list. +-- @function add +-- @tparam {string} name The name of the handler +-- @tparam {table | function | string} pattern The pattern to check for in the message +-- @tparam {function} handle The function to call if the pattern matches +-- @tparam {number | string | nil} maxRuns The maximum number of times the handler should run, or nil if there is no limit +function handlers.add(...) + local name, pattern, handle, maxRuns + local args = select("#", ...) + if args == 2 then + name = select(1, ...) + pattern = select(1, ...) + handle = select(2, ...) + maxRuns = nil + elseif args == 3 then + name = select(1, ...) + pattern = select(2, ...) + handle = select(3, ...) + maxRuns = nil + else + name = select(1, ...) + pattern = select(2, ...) + handle = select(3, ...) + maxRuns = select(4, ...) + end + assertAddArgs(name, pattern, handle, maxRuns) + + handle = handlers.generateResolver(handle) + + -- update existing handler by name + local idx = findIndexByProp(handlers.list, "name", name) + if idx ~= nil and idx > 0 then + -- found update + handlers.list[idx].pattern = pattern + handlers.list[idx].handle = handle + handlers.list[idx].maxRuns = maxRuns + else + -- not found then add + table.insert(handlers.list, { pattern = pattern, handle = handle, name = name, maxRuns = maxRuns }) + + end + return #handlers.list +end + +--- Appends a new handler to the end of the handlers list. +-- @function append +-- @tparam {string} name The name of the handler +-- @tparam {table | function | string} pattern The pattern to check for in the message +-- @tparam {function} handle The function to call if the pattern matches +-- @tparam {number | string | nil} maxRuns The maximum number of times the handler should run, or nil if there is no limit +function handlers.append(...) + local name, pattern, handle, maxRuns + local args = select("#", ...) + if args == 2 then + name = select(1, ...) + pattern = select(1, ...) + handle = select(2, ...) + maxRuns = nil + elseif args == 3 then + name = select(1, ...) + pattern = select(2, ...) + handle = select(3, ...) + maxRuns = nil + else + name = select(1, ...) + pattern = select(2, ...) + handle = select(3, ...) + maxRuns = select(4, ...) + end + assertAddArgs(name, pattern, handle, maxRuns) + + handle = handlers.generateResolver(handle) + -- update existing handler by name + local idx = findIndexByProp(handlers.list, "name", name) + if idx ~= nil and idx > 0 then + -- found update + handlers.list[idx].pattern = pattern + handlers.list[idx].handle = handle + handlers.list[idx].maxRuns = maxRuns + else + table.insert(handlers.list, { pattern = pattern, handle = handle, name = name, maxRuns = maxRuns }) + end +end + +--- Prepends a new handler to the beginning of the handlers list. +-- @function prepend +-- @tparam {string} name The name of the handler +-- @tparam {table | function | string} pattern The pattern to check for in the message +-- @tparam {function} handle The function to call if the pattern matches +-- @tparam {number | string | nil} maxRuns The maximum number of times the handler should run, or nil if there is no limit +function handlers.prepend(...) + local name, pattern, handle, maxRuns + local args = select("#", ...) + if args == 2 then + name = select(1, ...) + pattern = select(1, ...) + handle = select(2, ...) + maxRuns = nil + elseif args == 3 then + name = select(1, ...) + pattern = select(2, ...) + handle = select(3, ...) + maxRuns = nil + else + name = select(1, ...) + pattern = select(2, ...) + handle = select(3, ...) + maxRuns = select(4, ...) + end + assertAddArgs(name, pattern, handle, maxRuns) + + handle = handlers.generateResolver(handle) + + -- update existing handler by name + local idx = findIndexByProp(handlers.list, "name", name) + if idx ~= nil and idx > 0 then + -- found update + handlers.list[idx].pattern = pattern + handlers.list[idx].handle = handle + handlers.list[idx].maxRuns = maxRuns + else + table.insert(handlers.list, 1, { pattern = pattern, handle = handle, name = name, maxRuns = maxRuns }) + end +end + +--- Returns an object that allows adding a new handler before a specified handler. +-- @function before +-- @tparam {string} handleName The name of the handler before which the new handler will be added +-- @treturn {table} An object with an `add` method to insert the new handler +function handlers.before(handleName) + assert(type(handleName) == 'string', 'Handler name MUST be a string') + + local idx = findIndexByProp(handlers.list, "name", handleName) + return { + add = function (name, pattern, handle, maxRuns) + assertAddArgs(name, pattern, handle, maxRuns) + handle = handlers.generateResolver(handle) + if idx then + table.insert(handlers.list, idx, { pattern = pattern, handle = handle, name = name, maxRuns = maxRuns }) + end + end + } +end + +--- Returns an object that allows adding a new handler after a specified handler. +-- @function after +-- @tparam {string} handleName The name of the handler after which the new handler will be added +-- @treturn {table} An object with an `add` method to insert the new handler +function handlers.after(handleName) + assert(type(handleName) == 'string', 'Handler name MUST be a string') + local idx = findIndexByProp(handlers.list, "name", handleName) + return { + add = function (name, pattern, handle, maxRuns) + assertAddArgs(name, pattern, handle, maxRuns) + handle = handlers.generateResolver(handle) + if idx then + table.insert(handlers.list, idx + 1, { pattern = pattern, handle = handle, name = name, maxRuns = maxRuns }) + end + end + } + +end + +--- Removes a handler from the handlers list by name. +-- @function remove +-- @tparam {string} name The name of the handler to be removed +function handlers.remove(name) + assert(type(name) == 'string', 'name MUST be string') + if #handlers.list == 1 and handlers.list[1].name == name then + handlers.list = {} + end + + local idx = findIndexByProp(handlers.list, "name", name) + if idx ~= nil and idx > 0 then + table.remove(handlers.list, idx) + end +end + +--- Evaluates each handler against a given message and environment. Handlers are called in the order they appear in the handlers list. +-- Return 0 to not call handler, -1 to break after handler is called, 1 to continue +-- @function evaluate +-- @tparam {table} msg The message to be processed by the handlers. +-- @tparam {table} env The environment in which the handlers are executed. +-- @treturn The response from the handler(s). Returns a default message if no handler matches. +function handlers.evaluate(msg, env) + local handled = false + assert(type(msg) == 'table', 'msg is not valid') + assert(type(env) == 'table', 'env is not valid') + for _, o in ipairs(handlers.list) do + if o.name ~= "_default" then + local match = utils.matchesSpec(msg, o.pattern) + if not (type(match) == 'number' or type(match) == 'string' or type(match) == 'boolean') then + error("Pattern result is not valid, it MUST be string, number, or boolean") + end + -- handle boolean returns + if type(match) == "boolean" and match == true then + match = -1 + elseif type(match) == "boolean" and match == false then + match = 0 + end + + -- handle string returns + if type(match) == "string" then + if match == "continue" then + match = 1 + elseif match == "break" then + match = -1 + else + match = 0 + end + end + + if match ~= 0 then + if match < 0 then + handled = true + end + -- each handle function can accept, the msg, env + local status, err = pcall(o.handle, msg, env) + if not status then + error(err) + end + -- remove handler if maxRuns is reached. maxRuns can be either a number or "inf" + if o.maxRuns ~= nil and o.maxRuns ~= "inf" then + o.maxRuns = o.maxRuns - 1 + if o.maxRuns == 0 then + handlers.remove(o.name) + end + end + end + if match < 0 then + return handled + end + end + end + -- do default + if not handled then + local idx = findIndexByProp(handlers.list, "name", "_default") + handlers.list[idx].handle(msg,env) + end +end + +return handlers + +end +_G.package.loaded[".handlers"] = load_handlers() +print("loaded handlers") + + + +local function load_dump() + -- +-- Copyright (C) 2018 Masatoshi Teruya +-- +-- Permission is hereby granted, free of charge, to any person obtaining a copy +-- of this software and associated documentation files (the "Software"), to deal +-- in the Software without restriction, including without limitation the rights +-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +-- copies of the Software, and to permit persons to whom the Software is +-- furnished to do so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in +-- all copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +-- THE SOFTWARE. +-- +-- dump.lua +-- lua-dump +-- Created by Masatoshi Teruya on 18/04/22. +-- +--- file-scope variables +local type = type +local floor = math.floor +local tostring = tostring +local tblsort = table.sort +local tblconcat = table.concat +local strmatch = string.match +local strformat = string.format +--- constants +local INFINITE_POS = math.huge +local LUA_FIELDNAME_PAT = '^[a-zA-Z_][a-zA-Z0-9_]*$' +local FOR_KEY = 'key' +local FOR_VAL = 'val' +local FOR_CIRCULAR = 'circular' +local RESERVED_WORD = { + -- primitive data + ['nil'] = true, + ['true'] = true, + ['false'] = true, + -- declaraton + ['local'] = true, + ['function'] = true, + -- boolean logic + ['and'] = true, + ['or'] = true, + ['not'] = true, + -- conditional statement + ['if'] = true, + ['elseif'] = true, + ['else'] = true, + -- iteration statement + ['for'] = true, + ['in'] = true, + ['while'] = true, + ['until'] = true, + ['repeat'] = true, + -- jump statement + ['break'] = true, + ['goto'] = true, + ['return'] = true, + -- block scope statement + ['then'] = true, + ['do'] = true, + ['end'] = true, +} +local DEFAULT_INDENT = 4 + +--- filter function for dump +--- @param val any +--- @param depth integer +--- @param vtype string +--- @param use string +--- @param key any +--- @param udata any +--- @return any val +--- @return boolean nodump +local function DEFAULT_FILTER(val) + return val +end + +--- sort_index +--- @param a table +--- @param b table +local function sort_index(a, b) + if a.typ == b.typ then + if a.typ == 'boolean' then + return b.key + end + + return a.key < b.key + end + + return a.typ == 'number' +end + +--- dumptbl +--- @param tbl table +--- @param depth integer +--- @param indent string +--- @param nestIndent string +--- @param ctx table +--- @return string +local function dumptbl(tbl, depth, indent, nestIndent, ctx) + local ref = tostring(tbl) + + -- circular reference + if ctx.circular[ref] then + local val, nodump = ctx.filter(tbl, depth, type(tbl), FOR_CIRCULAR, tbl, + ctx.udata) + + if val ~= nil and val ~= tbl then + local t = type(val) + + if t == 'table' then + -- dump table value + if not nodump then + return dumptbl(val, depth + 1, indent, nestIndent, ctx) + end + return tostring(val) + elseif t == 'string' then + return strformat('%q', val) + elseif t == 'number' or t == 'boolean' then + return tostring(val) + end + + return strformat('%q', tostring(val)) + end + + return '""' + end + + local res = {} + local arr = {} + local narr = 0 + local fieldIndent = indent .. nestIndent + + -- save reference + ctx.circular[ref] = true + + for k, v in pairs(tbl) do + -- check key + local key, nokdump = ctx.filter(k, depth, type(k), FOR_KEY, nil, + ctx.udata) + + if key ~= nil then + -- check val + local val, novdump = ctx.filter(v, depth, type(v), FOR_VAL, key, + ctx.udata) + local kv + + if val ~= nil then + local kt = type(key) + local vt = type(val) + + -- convert key to suitable to be safely read back + -- by the Lua interpreter + if kt == 'number' or kt == 'boolean' then + k = key + key = '[' .. tostring(key) .. ']' + -- dump table value + elseif kt == 'table' and not nokdump then + key = '[' .. + dumptbl(key, depth + 1, fieldIndent, nestIndent, + ctx) .. ']' + k = key + kt = 'string' + elseif kt ~= 'string' or RESERVED_WORD[key] or + not strmatch(key, LUA_FIELDNAME_PAT) then + key = strformat("[%q]", tostring(key), v) + k = key + kt = 'string' + end + + -- convert key-val pair to suitable to be safely read back + -- by the Lua interpreter + if vt == 'number' or vt == 'boolean' then + kv = strformat('%s%s = %s', fieldIndent, key, tostring(val)) + elseif vt == 'string' then + -- dump a string-value + if not novdump then + kv = strformat('%s%s = %q', fieldIndent, key, val) + else + kv = strformat('%s%s = %s', fieldIndent, key, val) + end + elseif vt == 'table' and not novdump then + kv = strformat('%s%s = %s', fieldIndent, key, dumptbl(val, + depth + + 1, + fieldIndent, + nestIndent, + ctx)) + else + kv = strformat('%s%s = %q', fieldIndent, key, tostring(val)) + end + + -- add to array + narr = narr + 1 + arr[narr] = { + typ = kt, + key = k, + val = kv, + } + end + end + end + + -- remove reference + ctx.circular[ref] = nil + -- concat result + if narr > 0 then + tblsort(arr, sort_index) + + for i = 1, narr do + res[i] = arr[i].val + end + res[1] = '{' .. ctx.LF .. res[1] + res = tblconcat(res, ',' .. ctx.LF) .. ctx.LF .. indent .. '}' + else + res = '{}' + end + + return res +end + +--- is_uint +--- @param v any +--- @return boolean ok +local function is_uint(v) + return type(v) == 'number' and v < INFINITE_POS and v >= 0 and floor(v) == v +end + +--- dump +--- @param val any +--- @param indent integer +--- @param padding integer +--- @param filter function +--- @param udata +--- @return string +local function dump(val, indent, padding, filter, udata) + local t = type(val) + + -- check indent + if indent == nil then + indent = DEFAULT_INDENT + elseif not is_uint(indent) then + error('indent must be unsigned integer', 2) + end + + -- check padding + if padding == nil then + padding = 0 + elseif not is_uint(padding) then + error('padding must be unsigned integer', 2) + end + + -- check filter + if filter == nil then + filter = DEFAULT_FILTER + elseif type(filter) ~= 'function' then + error('filter must be function', 2) + end + + -- dump table + if t == 'table' then + local ispace = '' + local pspace = '' + + if indent > 0 then + ispace = strformat('%' .. tostring(indent) .. 's', '') + end + + if padding > 0 then + pspace = strformat('%' .. tostring(padding) .. 's', '') + end + + return dumptbl(val, 1, pspace, ispace, { + LF = ispace == '' and ' ' or '\n', + circular = {}, + filter = filter, + udata = udata, + }) + end + + -- dump value + local v, nodump = filter(val, 0, t, FOR_VAL, nil, udata) + if nodump == true then + return tostring(v) + end + return strformat('%q', tostring(v)) +end + +return dump +end +_G.package.loaded[".dump"] = load_dump() +print("loaded dump") + + + +local function load_pretty() + local pretty = { _version = "0.0.1" } + +pretty.tprint = function (tbl, indent) + if not indent then indent = 0 end + local output = "" + for k, v in pairs(tbl) do + local formatting = string.rep(" ", indent) .. k .. ": " + if type(v) == "table" then + output = output .. formatting .. "\n" + output = output .. pretty.tprint(v, indent+1) + elseif type(v) == 'boolean' then + output = output .. formatting .. tostring(v) .. "\n" + else + output = output .. formatting .. v .. "\n" + end + end + return output +end + +return pretty + +end +_G.package.loaded[".pretty"] = load_pretty() +print("loaded pretty") + + + +local function load_chance() + --- The Chance module provides utilities for generating random numbers and values. Returns the chance table. +-- @module chance + +local N = 624 +local M = 397 +local MATRIX_A = 0x9908b0df +local UPPER_MASK = 0x80000000 +local LOWER_MASK = 0x7fffffff + +--- Initializes mt[N] with a seed +-- @lfunction init_genrand +-- @tparam {table} o The table to initialize +-- @tparam {number} s The seed +local function init_genrand(o, s) + o.mt[0] = s & 0xffffffff + for i = 1, N - 1 do + o.mt[i] = (1812433253 * (o.mt[i - 1] ~ (o.mt[i - 1] >> 30))) + i + -- See Knuth TAOCP Vol2. 3rd Ed. P.106 for multiplier. + -- In the previous versions, MSBs of the seed affect + -- only MSBs of the array mt[]. + -- 2002/01/09 modified by Makoto Matsumoto + o.mt[i] = o.mt[i] & 0xffffffff + -- for >32 bit machines + end + o.mti = N +end + +--- Generates a random number on [0,0xffffffff]-interval +-- @lfunction genrand_int32 +-- @tparam {table} o The table to generate the random number from +-- @treturn {number} The random number +local function genrand_int32(o) + local y + local mag01 = {} -- mag01[x] = x * MATRIX_A for x=0,1 + mag01[0] = 0x0 + mag01[1] = MATRIX_A + if o.mti >= N then -- generate N words at one time + if o.mti == N + 1 then -- if init_genrand() has not been called, + init_genrand(o, 5489) -- a default initial seed is used + end + for kk = 0, N - M - 1 do + y = (o.mt[kk] & UPPER_MASK) | (o.mt[kk + 1] & LOWER_MASK) + o.mt[kk] = o.mt[kk + M] ~ (y >> 1) ~ mag01[y & 0x1] + end + for kk = N - M, N - 2 do + y = (o.mt[kk] & UPPER_MASK) | (o.mt[kk + 1] & LOWER_MASK) + o.mt[kk] = o.mt[kk + (M - N)] ~ (y >> 1) ~ mag01[y & 0x1] + end + y = (o.mt[N - 1] & UPPER_MASK) | (o.mt[0] & LOWER_MASK) + o.mt[N - 1] = o.mt[M - 1] ~ (y >> 1) ~ mag01[y & 0x1] + + o.mti = 0 + end + + y = o.mt[o.mti] + o.mti = o.mti + 1 + + -- Tempering + y = y ~ (y >> 11) + y = y ~ ((y << 7) & 0x9d2c5680) + y = y ~ ((y << 15) & 0xefc60000) + y = y ~ (y >> 18) + + return y +end + +local MersenneTwister = {} +MersenneTwister.mt = {} +MersenneTwister.mti = N + 1 + + +--- The Random table +-- @table Random +-- @field seed The seed function +-- @field random The random function +-- @field integer The integer function +local Random = {} + +--- Sets a new random table given a seed. +-- @function seed +-- @tparam {number} seed The seed +function Random.seed(seed) + init_genrand(MersenneTwister, seed) +end + +--- Generates a random number on [0,1)-real-interval. +-- @function random +-- @treturn {number} The random number +function Random.random() + return genrand_int32(MersenneTwister) * (1.0 / 4294967296.0) +end + +--- Returns a random integer. The min and max are INCLUDED in the range. +-- The max integer in lua is math.maxinteger +-- The min is math.mininteger +-- @function Random.integer +-- @tparam {number} min The minimum value +-- @tparam {number} max The maximum value +-- @treturn {number} The random integer +function Random.integer(min, max) + assert(max >= min, "max must bigger than min") + return math.floor(Random.random() * (max - min + 1) + min) +end + +return Random +end +_G.package.loaded[".chance"] = load_chance() +print("loaded chance") + + + +local function load_boot() + --- The Boot module provides functionality for booting the process. Returns the boot function. +-- @module boot + +-- This is for aop6 Boot Loader +-- See: https://github.com/permaweb/aos/issues/342 +-- For the Process as the first Message, if On-Boot +-- has the value 'data' then evaluate the data +-- if it is a tx id, then download and evaluate the tx + +local drive = { _version = "0.0.1" } + +function drive.getData(txId) + local file = io.open('/data/' .. txId) + if not file then + return nil, "File not found!" + end + local contents = file:read( + file:seek('end') + ) + file:close() + return contents +end + +--- The boot function. +-- If the message has no On-Boot tag, do nothing. +-- If the message has an On-Boot tag with the value 'Data', then evaluate the message. +-- If the message has an On-Boot tag with a tx id, then download and evaluate the tx data. +-- @function boot +-- @param ao The ao environment object +-- @see eval +return function (ao) + local eval = require(".eval")(ao) + return function (msg) + if #Inbox == 0 then + table.insert(Inbox, msg) + end + if msg.Tags['On-Boot'] == nil then + return + end + if msg.Tags['On-Boot'] == 'Data' then + eval(msg) + else + local loadedVal = drive.getData(msg.Tags['On-Boot']) + eval({ Data = loadedVal }) + end + end +end +end +_G.package.loaded[".boot"] = load_boot() +print("loaded boot") + + + +local function load_default() + local json = require('.json') +-- default handler for aos +return function (insertInbox) + return function (msg) + -- Add Message to Inbox + insertInbox(msg) + + -- local txt = Colors.gray .. "New Message From " .. Colors.green .. + -- (msg.From and (msg.From:sub(1,3) .. "..." .. msg.From:sub(-3)) or "unknown") .. Colors.gray .. ": " + -- if msg.Action then + -- txt = txt .. Colors.gray .. (msg.Action and ("Action = " .. Colors.blue .. msg.Action:sub(1,20)) or "") .. Colors.reset + -- else + -- local data = msg.Data + -- if type(data) == 'table' then + -- data = json.encode(data) + -- end + -- txt = txt .. Colors.gray .. "Data = " .. Colors.blue .. (data and data:sub(1,20) or "") .. Colors.reset + -- end + -- Print to Output + -- print(txt) + print("New Message") + end + +end + +end +_G.package.loaded[".default"] = load_default() +print("loaded default") + + + +local function load_ao() + Handlers = Handlers or require('.handlers') + +local oldao = ao or {} + +local utils = require('.utils') + +local ao = { + _version = "0.0.6", + id = oldao.id or "", + _module = oldao._module or "", + authorities = oldao.authorities or {}, + reference = oldao.reference or 0, + outbox = oldao.outbox or + {Output = {}, Messages = {}, Spawns = {}, Assignments = {}}, + nonExtractableTags = { + 'data-protocol', 'variant', 'from-process', 'from-module', 'type', + 'from', 'owner', 'anchor', 'target', 'data', 'tags', 'read-only' + }, + nonForwardableTags = { + 'data-protocol', 'variant', 'from-process', 'from-module', 'type', + 'from', 'owner', 'anchor', 'target', 'tags', 'tagArray', 'hash-chain', + 'timestamp', 'nonce', 'slot', 'epoch', 'signature', 'forwarded-by', + 'pushed-for', 'read-only', 'cron', 'block-height', 'reference', 'id', + 'reply-to' + }, + Nonce = nil +} + +function ao.clearOutbox() + ao.outbox = { Output = {}, Messages = {}, Spawns = {}, Assignments = {}} +end + +local function getId(m) + local id = "" + utils.map(function (k) + local c = m.commitments[k] + if c.type == "rsa-pss-sha512" then + id = k + elseif c.type == "signed" and c['commitment-device'] == "ans104" then + id = k + end + end, utils.keys(m.commitments) + ) + return id +end + +function ao.init(env) + if _G.ao.id == "" then _G.ao.id = getId(env.process) end + + -- if ao._module == "" then + -- ao._module = env.Module.Id + -- end + -- TODO: need to deal with assignables + if # _G.ao.authorities < 1 then + if type(env.process.authority) == 'string' then + _G.ao.authorities = { env.process.authority } + else + _G.ao.authorities = env.process.authority + end + end + _G.ao.outbox = {Output = {}, Messages = {}, Spawns = {}, Assignments = {}} + _G.ao.env = env + +end + +function ao.send(msg) + assert(type(msg) == 'table', 'msg should be a table') + + ao.reference = ao.reference + 1 + local referenceString = tostring(ao.reference) + -- set kv + msg.reference = referenceString + + -- clone message info and add to outbox + table.insert(ao.outbox.Messages, utils.reduce( + function (acc, key) + acc[key] = msg[key] + return acc + end, + {}, + utils.keys(msg) + )) + + if msg.target then + msg.onReply = function(...) + local from, resolver + if select("#", ...) == 2 then + from = select(1, ...) + resolver = select(2, ...) + else + from = msg.target + resolver = select(1, ...) + end + Handlers.once({ + from = from, + ["x-reference"] = referenceString + }, resolver) + end + end + return msg +end + +function ao.spawn(module, msg) + assert(type(module) == "string", "Module source id is required!") + assert(type(msg) == "table", "Message must be a table.") + + ao.reference = ao.reference + 1 + + local spawnRef = tostring(ao.reference) + + msg["reference"] = spawnRef + + -- clone message info and add to outbox + table.insert(ao.outbox.Spawns, utils.reduce( + function (acc, key) + acc[key] = msg[key] + return acc + end, + {}, + utils.keys(msg) + )) + + msg.onReply = function(cb) + Handlers.once({ + action = "Spawned", + from = ao.id, + ["x-reference"] = spawnRef + }, cb) + end + + return msg + +end + +function ao.result(result) + if ao.outbox.Error or result.Error then + return { Error = result.Error or ao.outbox.Error } + end + return { + Output = result.Output or ao.output.Output, + Messages = ao.outbox.Messages, + Spawns = ao.outbox.Spawns, + Assignments = ao.outbox.Assignments + } +end + +-- set global Send and Spawn +Send = Send or ao.send +Spawn = Spawn or ao.spawn + +return ao + +end +_G.package.loaded[".ao"] = load_ao() +print("loaded ao") + + + +local function load_base64() + --[[ + + base64 -- v1.5.3 public domain Lua base64 encoder/decoder + no warranty implied; use at your own risk + + Needs bit32.extract function. If not present it's implemented using BitOp + or Lua 5.3 native bit operators. For Lua 5.1 fallbacks to pure Lua + implementation inspired by Rici Lake's post: + http://ricilake.blogspot.co.uk/2007/10/iterating-bits-in-lua.html + + author: Ilya Kolbin (iskolbin@gmail.com) + url: github.com/iskolbin/lbase64 + + COMPATIBILITY + + Lua 5.1+, LuaJIT + + LICENSE + + See end of file for license information. + +--]] + + +local base64 = {} + +local extract = _G.bit32 and _G.bit32.extract -- Lua 5.2/Lua 5.3 in compatibility mode +if not extract then + if _G.bit then -- LuaJIT + local shl, shr, band = _G.bit.lshift, _G.bit.rshift, _G.bit.band + extract = function( v, from, width ) + return band( shr( v, from ), shl( 1, width ) - 1 ) + end + elseif _G._VERSION == "Lua 5.1" then + extract = function( v, from, width ) + local w = 0 + local flag = 2^from + for i = 0, width-1 do + local flag2 = flag + flag + if v % flag2 >= flag then + w = w + 2^i + end + flag = flag2 + end + return w + end + else -- Lua 5.3+ + extract = load[[return function( v, from, width ) + return ( v >> from ) & ((1 << width) - 1) + end]]() + end +end + + +function base64.makeencoder( s62, s63, spad ) + local encoder = {} + for b64code, char in pairs{[0]='A','B','C','D','E','F','G','H','I','J', + 'K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y', + 'Z','a','b','c','d','e','f','g','h','i','j','k','l','m','n', + 'o','p','q','r','s','t','u','v','w','x','y','z','0','1','2', + '3','4','5','6','7','8','9',s62 or '+',s63 or'/',spad or'='} do + encoder[b64code] = char:byte() + end + return encoder +end + +function base64.makedecoder( s62, s63, spad ) + local decoder = {} + for b64code, charcode in pairs( base64.makeencoder( s62, s63, spad )) do + decoder[charcode] = b64code + end + return decoder +end + +local DEFAULT_ENCODER = base64.makeencoder() +local DEFAULT_DECODER = base64.makedecoder() + +local char, concat = string.char, table.concat + +function base64.encode( str, encoder, usecaching ) + encoder = encoder or DEFAULT_ENCODER + local t, k, n = {}, 1, #str + local lastn = n % 3 + local cache = {} + for i = 1, n-lastn, 3 do + local a, b, c = str:byte( i, i+2 ) + local v = a*0x10000 + b*0x100 + c + local s + if usecaching then + s = cache[v] + if not s then + s = char(encoder[extract(v,18,6)], encoder[extract(v,12,6)], encoder[extract(v,6,6)], encoder[extract(v,0,6)]) + cache[v] = s + end + else + s = char(encoder[extract(v,18,6)], encoder[extract(v,12,6)], encoder[extract(v,6,6)], encoder[extract(v,0,6)]) + end + t[k] = s + k = k + 1 + end + if lastn == 2 then + local a, b = str:byte( n-1, n ) + local v = a*0x10000 + b*0x100 + t[k] = char(encoder[extract(v,18,6)], encoder[extract(v,12,6)], encoder[extract(v,6,6)], encoder[64]) + elseif lastn == 1 then + local v = str:byte( n )*0x10000 + t[k] = char(encoder[extract(v,18,6)], encoder[extract(v,12,6)], encoder[64], encoder[64]) + end + return concat( t ) +end + +function base64.decode( b64, decoder, usecaching ) + decoder = decoder or DEFAULT_DECODER + local pattern = '[^%w%+%/%=]' + if decoder then + local s62, s63 + for charcode, b64code in pairs( decoder ) do + if b64code == 62 then s62 = charcode + elseif b64code == 63 then s63 = charcode + end + end + pattern = ('[^%%w%%%s%%%s%%=]'):format( char(s62), char(s63) ) + end + b64 = b64:gsub( pattern, '' ) + local cache = usecaching and {} + local t, k = {}, 1 + local n = #b64 + local padding = b64:sub(-2) == '==' and 2 or b64:sub(-1) == '=' and 1 or 0 + for i = 1, padding > 0 and n-4 or n, 4 do + local a, b, c, d = b64:byte( i, i+3 ) + local s + if usecaching then + local v0 = a*0x1000000 + b*0x10000 + c*0x100 + d + s = cache[v0] + if not s then + local v = decoder[a]*0x40000 + decoder[b]*0x1000 + decoder[c]*0x40 + decoder[d] + s = char( extract(v,16,8), extract(v,8,8), extract(v,0,8)) + cache[v0] = s + end + else + local v = decoder[a]*0x40000 + decoder[b]*0x1000 + decoder[c]*0x40 + decoder[d] + s = char( extract(v,16,8), extract(v,8,8), extract(v,0,8)) + end + t[k] = s + k = k + 1 + end + if padding == 1 then + local a, b, c = b64:byte( n-3, n-1 ) + local v = decoder[a]*0x40000 + decoder[b]*0x1000 + decoder[c]*0x40 + t[k] = char( extract(v,16,8), extract(v,8,8)) + elseif padding == 2 then + local a, b = b64:byte( n-3, n-2 ) + local v = decoder[a]*0x40000 + decoder[b]*0x1000 + t[k] = char( extract(v,16,8)) + end + return concat( t ) +end + +return base64 + +--[[ +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2018 Ilya Kolbin +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +--]] +end +_G.package.loaded[".base64"] = load_base64() +print("loaded base64") + + + +local function load_state() + ao = ao or require('.ao') +local state = {} +local stringify = require('.stringify') +local utils = require('.utils') + +Colors = { red = "\27[31m", green = "\27[32m", + blue = "\27[34m", reset = "\27[0m", gray = "\27[90m" +} +Bell = "\x07" + +Initialized = Initialized or false +Name = Name or "aos" + +Owner = Owner or "" +Inbox = Inbox or {} + +-- global prompt function +function Prompt() + return "aos> " +end + +local maxInboxCount = 10000 + +function state.insertInbox(msg) + table.insert(Inbox, msg) + local overflow = #Inbox - maxInboxCount + for i = 1,overflow do + table.remove(Inbox,1) + end +end +local function getOwnerAddress(m) + local _owner = nil + utils.map(function (k) + local c = m.commitments[k] + if c.alg == "rsa-pss-sha512" then + _owner = c.committer + elseif c.alg == "signed" and c['commitment-device'] == "ans104" then + _owner = c.commiter + end + end, utils.keys(m.commitments)) + return _owner +end + +local function isFromOwner(m) + local _owner = getOwnerAddress(m) + local _fromProcess = m['from-process'] or _owner + return _owner ~= nil and _fromProcess == _owner +end + +local function getOwner(m) + local id = "" + if m['from-process'] then + return m['from-process'] + end + + utils.map(function (k) + local c = m.commitments[k] + if c.type == "rsa-pss-sha512" then + id = c.committer + elseif c.alg == "signed" and c['commitment-device'] == "ans104" then + id = c.committer + end +end, utils.keys(m.commitments) +) + return id +end + +function state.init(req, base) + if not Initialized then + Owner = getOwner(base.process) + -- if process id is equal to message id then set Owner + -- TODO: need additional check, like msg.Slot == 1 + -- if env.Process.Id == msg.Id and Owner ~= msg.Id then + -- Owner = env.Process['From-Process'] or msg.From + -- end + -- if env.Process.Name then + -- Name = Name == "aos" and env.Process.Name + -- end + -- global print function + function print(a) + if type(a) == "table" then + a = stringify.format(a) + end + + if type(a) == "boolean" then + a = Colors.blue .. tostring(a) .. Colors.reset + end + if type(a) == "nil" then + a = Colors.red .. tostring(a) .. Colors.reset + end + if type(a) == "number" then + a = Colors.green .. tostring(a) .. Colors.reset + end + + if HandlerPrintLogs then + table.insert(HandlerPrintLogs, a) + return nil + end + + return tostring(a) + end + + Initialized = true + end +end + +function state.getFrom(req) + return getOwner(req.body) +end + +function state.isTrusted(req) + if isFromOwner(req.body) then + return true + end + local _trusted = false + + if req.body['from-process'] then + _trusted = utils.includes( + req.body['from-process'], + ao.authorities + ) +end + +if not _trusted then + _trusted = utils.includes( + getOwner(req.body), ao.authorities + ) +end + + return _trusted +end + +function state.checkSlot(req, ao) + -- slot check + if not ao.slot then + ao.slot = tonumber(req.slot) + else + if tonumber(req.slot) ~= (ao.slot + 1) then + print(table.concat({ + Colors.red, + "WARNING: Slot did not match, may be due to an error generated by process", + Colors.reset + })) + print("") + end + end +end + +function state.reset(tbl) + tbl = nil + collectgarbage() + return {} +end + +return state + +end +_G.package.loaded[".state"] = load_state() +print("loaded state") + + + +local function load_process() + ao = ao or require('.ao') +Handlers = require('.handlers') +Utils = require('.utils') +Dump = require('.dump') + +local process = { _version = "2.0.7" } +local state = require('.state') +local eval = require('.eval') +local default = require('.default') +local json = require('.json') + +function Prompt() + return "aos> " +end + +function process.handle(req, base) + HandlerPrintLogs = state.reset(HandlerPrintLogs) + os.time = function () return tonumber(req['block-timestamp']) end + + ao.init(base) + -- initialize state + state.init(req, base) + + + -- magic table + req.body.data = req.body['Content-Type'] == 'application/json' + and json.decode(req.body.data or "{}") + or req.body.data + + Errors = Errors or {} + -- clear outbox + ao.clearOutbox() + + if not state.isTrusted(req) then + return ao.result({ + Output = { + data = "Message is not trusted." + } + }) + end + req.reply = function (_reply) + local _from = state.getFrom(req) + _reply.target = _reply.target and _reply.target or _from + _reply['x-reference'] = req.body.reference or nil + _reply['x-origin'] = req.body['x-origin'] or nil + return ao.send(_reply) + end + + + -- state.checkSlot(msg, ao) + Handlers.add("_eval", function (_req) + local function getOwnerFrom(m) + local from = "" + Utils.map( + function (k) + local c = m.commitments[k] + if c.type == "rsa-pss-sha512" then + from = c.committer + end + end, + Utils.keys(m.commitments) + ) + return from + end + return _req.body.action == "Eval" and Owner == getOwnerFrom(_req.body) + end, eval(ao)) + + Handlers.add("_default", + function () return true end, + default(state.insertInbox) + ) + + local status, error = pcall(Handlers.evaluate, req, base) + + -- cleanup handlers so that they are always at the end of the pipeline + Handlers.remove("_eval") + Handlers.remove("_default") + + local printData = table.concat(HandlerPrintLogs, "\n") + if not status then + if req.body.action == "Eval" then + return { + Error = table.concat({ + printData, + "\n", + Colors.red, + "error: " .. error, + Colors.reset, + }) + } + end + print(Colors.red .. "Error" .. Colors.gray .. " handling message " .. Colors.reset) + print(Colors.green .. error .. Colors.reset) + -- print("\n" .. Colors.gray .. debug.traceback() .. Colors.reset) + return ao.result({ + Output = { + data = printData .. '\n\n' .. Colors.red .. 'error:\n' .. Colors.reset .. error + }, + Messages = {}, + Spawns = {}, + Assignments = {} + }) + end + + local response = {} + + if req.body.action == "Eval" then + response = ao.result({ + Output = { + data = printData, + prompt = Prompt() + } + }) + else + response = ao.result({ + Output = { + data = printData, + prompt = Prompt(), + print = true + } + }) + end + + HandlerPrintLogs = state.reset(HandlerPrintLogs) -- clear logs + -- ao.Slot = msg.Slot + return response +end + +function Version() + print("version: " .. process._version) +end + +return process + +end +_G.package.loaded[".process"] = load_process() +print("loaded process") + + +ao = require('.ao') +local _process = require('.process') + +function compute(base, req, opts) + local _results = _process.handle(req, base) + base.results = { + outbox = {}, + output = _results.Output + } + for i=1,#_results.Messages do + base.results.outbox[tostring(i)] = _results.Messages[i] + end + return base +end + + +print [[ _ ___ ____ + / \ / _ \/ ___| + / _ \| | | \___ \ + / ___ \ |_| |___) | + /_/ \_\___/|____/ + ]] diff --git a/test/large-message.eterm b/test/large-message.eterm new file mode 100644 index 000000000..a13b94c01 --- /dev/null +++ b/test/large-message.eterm @@ -0,0 +1,194 @@ +#{<<"address">> => <<"XgN1kN-ZyAWtYvdUlPEM3EIIi-budUx81mjcHQ1mSNU">>, + <<"append">> => + <<"aaf13c9ed2e821ea8c82fcc7981c73a14dc2d01c855f09262d42090fa0424422">>, + <<"commitments">> => + #{<<"6nWXloEFP1kqslsOCKCLxWvzr5GVJvvWlQAO0Uwv7MM">> => + #{<<"commitment-device">> => <<"httpsig@1.0">>, + <<"committed">> => + #{<<"1">> => <<"address">>,<<"10">> => <<"public-key">>, + <<"11">> => <<"report">>,<<"12">> => <<"vcpu_type">>, + <<"13">> => <<"vcpus">>,<<"14">> => <<"vmm_type">>, + <<"2">> => <<"ao-types">>,<<"3">> => <<"append">>, + <<"4">> => <<"firmware">>,<<"5">> => <<"guest_features">>, + <<"6">> => <<"initrd">>,<<"7">> => <<"kernel">>, + <<"8">> => <<"node-message">>,<<"9">> => <<"nonce">>}, + <<"keyid">> => <<"YW8">>, + <<"signature">> => + <<"6nWXloEFP1kqslsOCKCLxWvzr5GVJvvWlQAO0Uwv7MM">>, + <<"type">> => <<"hmac-sha256">>}, + <<"uHPsVJP_v7B46__xiMPluCZf_6IXjsMfsUTHHC46uz4">> => + #{<<"commitment-device">> => <<"httpsig@1.0">>, + <<"committed">> => + #{<<"1">> => <<"address">>,<<"10">> => <<"public-key">>, + <<"11">> => <<"report">>,<<"12">> => <<"vcpu_type">>, + <<"13">> => <<"vcpus">>,<<"14">> => <<"vmm_type">>, + <<"2">> => <<"ao-types">>,<<"3">> => <<"append">>, + <<"4">> => <<"firmware">>,<<"5">> => <<"guest_features">>, + <<"6">> => <<"initrd">>,<<"7">> => <<"kernel">>, + <<"8">> => <<"node-message">>,<<"9">> => <<"nonce">>}, + <<"committer">> => + <<"ggltHF0Cnv9ylH3vM1p7amR2vXLMoPLQIUQmAEwLP-k">>, + <<"keyid">> => + <<"rh3EGHNtQyto38UnoKwbnEkw3TqlR5b18n_uVySFBdI-t0O7lcrRM0MrL8l09YHb9ZAG5A-cB3ExZpdDCnnCGcvxFesdnapzWhYkPpXZKvgVk7NyZdRFWUJXoSaw3n5fajGShjAiOkiEzDEwHKozZnqJl-DRuuHVGcw4Q1VMk1CtzJJBCrZeNTG1crWynHa-3AfCEQ6Ou8AMs8a7ibHP57Jcmhe8xfVO4MtEvTtuMejAAjLaLIPWtzhct3rJIZmpSvwG7fAfWafqPuAx_hVjLWFv41wwhH1yA_bCU_x61n41Lqg06prUCNPBf8Ypz1EUIqKipV0O69UGHylOvA3cbpLFV6QyF1w8icae3vcKHPmQ-Fni0xD3PXZCCaiBHng4FianlIfms6IB7jXLr4dZ4utq-32wRR2anksxz6tyZ_RFWN8Tb7-qPXb5X0ESsAxAshh4vfYWRYdaMeO79I4IN2vB6EFhRHO_Jo5U-E3KDm2vyABly5x9Aw2gBUEhmEdM0T-fUZx_30UoqcqRYlAKCzpIQ4_irckldfZMGRoIlWxQP2qRTUCrOxR8rtSLWUKQTyOW9GcI0pV04q7lxRRCpZK-B76GKU8ZQnDqQerdvsFro66LXGXOESk1HN2isCV_Pwik9G1SFKftHWzDzrRlRrDWcoyoqzEMjE0wj7ttjJE">>, + <<"signature">> => + <<"YG9aPmui8jvXL_SZz97oyj1HV596j4dKe9gaHQYkqEqwJhgG2_apue9V7_adypNeFlHiQTTU-BgNV1SNSAsSqYXXBPAQCh1Yw8BQ3NhPPhFr-as7tiwrJBwsZUe7QoHZWyBdNOLIY-E7w4QWOszRNZRnmKNWEPutyvP9a_5fxz7c8Z7zIkYvfwZUa9OZwOw4uAoUjSZ8fmv4F6bFdElrkdWpADwsATFFZiaFy4v-qdRpCKwegWAKdu2ClMEY973YoR47zoAcPqX3vX0nlzvZ9EKLVlytPanFM29yTgnDy1LokUq7hxC4I0PEz3dhubVFWhdisKConKbFCJqDX7YoaS3hu6EpeknP9Ru1TqiWSFMKe9IMQppzP7FtQOb6JKE8mvPwPbWp1WPTr9V4_qooUvUcNKPXapIJc5gWaF59aiyN9jNRjKL1XLBhCFf59BDf072oOdI52UVjCRcE58X-5HL1u7m79BWzIeGXCE6KIlqYWuGV3VIt7hE8E4lWiTEBKy8FAoScx7n5JDE1K0x8AYBff7ipzKAHZMo8vOzZYNRl5f9tNJRurKnK0CU-85TNsORCVuDgOuMAWi9MmsduicGYvINOnXSXhht1y4L-AiQExIHVVBf5kGpJi0FV-FNKhwH3Vihgx24lZ2f9HuPZUt8vmjQwoEgCZT0PYTvHi3w">>, + <<"type">> => <<"rsa-pss-sha512">>}}, + <<"firmware">> => + <<"b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510">>, + <<"guest_features">> => 1, + <<"initrd">> => + <<"da6dffff50373e1d393bf92cb9b552198b1930068176a046dda4e23bb725b3bb">>, + <<"kernel">> => + <<"69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576">>, + <<"node-message">> => + #{<<"debug_metadata">> => true, + <<"gateway">> => <<"https://arweave.net">>, + <<"process_now_from_cache">> => false, + <<"debug_print_map_line_threshold">> => 30,<<"http_client">> => gun, + <<"http_keepalive">> => 120000,<<"short_trace_len">> => 5, + <<"debug_print_trace">> => short, + <<"date">> => <<"Fri, 18 Apr 2025 19:01:31 GMT">>, + <<"scheduler_location_ttl">> => 604800000, + <<"access_control_allow_origin">> => <<"*">>, + <<"compute_mode">> => lazy,<<"preload_devices">> => [], + <<"hb_config_location">> => <<"config.flat">>, + <<"process_workers">> => false, + <<"access_control_allow_methods">> => + <<"GET, POST, PUT, DELETE, OPTIONS">>, + <<"initialized">> => permanent,<<"only">> => local, + <<"trusted">> => + #{<<"append">> => + <<"aaf13c9ed2e821ea8c82fcc7981c73a14dc2d01c855f09262d42090fa0424422">>, + <<"firmware">> => + <<"b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510">>, + <<"guest_features">> => 1, + <<"initrd">> => + <<"da6dffff50373e1d393bf92cb9b552198b1930068176a046dda4e23bb725b3bb">>, + <<"kernel">> => + <<"69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576">>, + <<"vcpu_type">> => 5,<<"vcpus">> => 1,<<"vmm_type">> => 1}, + <<"snp_hashes">> => + #{<<"append">> => + <<"aaf13c9ed2e821ea8c82fcc7981c73a14dc2d01c855f09262d42090fa0424422">>, + <<"firmware">> => + <<"b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510">>, + <<"guest_features">> => 1, + <<"initrd">> => + <<"da6dffff50373e1d393bf92cb9b552198b1930068176a046dda4e23bb725b3bb">>, + <<"kernel">> => + <<"69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576">>, + <<"vcpu_type">> => 5,<<"vcpus">> => 1,<<"vmm_type">> => 1}, + <<"cache_lookup_hueristics">> => false, + <<"address">> => <<"XgN1kN-ZyAWtYvdUlPEM3EIIi-budUx81mjcHQ1mSNU">>, + <<"store">> => + [#{<<"prefix">> => <<"cache-mainnet">>, + <<"store-module">> => hb_store_fs}, + #{<<"store">> => + [#{<<"prefix">> => <<"cache-mainnet">>, + <<"store-module">> => hb_store_fs}], + <<"store-module">> => hb_store_gateway}], + <<"postprocessor">> => undefined,<<"debug_show_priv">> => false, + <<"routes">> => + [#{<<"node">> => #{<<"prefix">> => <<"http://localhost:6363">>}, + <<"template">> => <<"/result/.*">>}, + #{<<"nodes">> => + [#{<<"opts">> => + #{<<"http_client">> => httpc, + <<"protocol">> => http2}, + <<"prefix">> => + <<"https://arweave-search.goldsky.com">>}, + #{<<"opts">> => + #{<<"http_client">> => gun,<<"protocol">> => http2}, + <<"prefix">> => <<"https://arweave.net">>}], + <<"template">> => <<"/graphql">>}, + #{<<"node">> => + #{<<"opts">> => + #{<<"http_client">> => gun,<<"protocol">> => http2}, + <<"prefix">> => <<"https://arweave.net">>}, + <<"template">> => <<"/raw">>}], + <<"debug_print">> => false, + <<"bundler_ans104">> => <<"https://up.arweave.net:443">>, + <<"client_error_strategy">> => throw, + <<"http_request_send_timeout">> => 60000, + <<"http_extra_opts">> => + #{<<"cache_control">> => [<<"always">>], + <<"force_message">> => true}, + <<"force_signed">> => true, + <<"server">> => <<"nginx/1.18.0 (Ubuntu)">>, + <<"access_remote_cache_for_client">> => false, + <<"await_inprogress">> => named, + <<"content_type">> => + <<"multipart/form-data; boundary=\"5zd5yKhUfgSnCzBVlt29-Vs4rQQeUBGuVqNJubt07jI\"">>, + <<"http_connect_timeout">> => 5000,<<"status">> => 200, + <<"body_keys">> => + <<"\"routes\", \"routes\", \"routes\", \"routes\", \"routes\", \"routes\", \"routes\", \"routes\", \"routes\", \"routes\"">>, + <<"stack_print_prefixes">> => ["hb","dev","ar"], + <<"scheduling_mode">> => disabled, + <<"http_server">> => <<"XgN1kN-ZyAWtYvdUlPEM3EIIi-budUx81mjcHQ1mSNU">>, + <<"debug_print_binary_max">> => 60,<<"debug_ids">> => false, + <<"relay_http_client">> => httpc,<<"debug_stack_depth">> => 40, + <<"preloaded_devices">> => + [#{<<"module">> => dev_codec_ans104, + <<"name">> => <<"ans104@1.0">>}, + #{<<"module">> => dev_cu,<<"name">> => <<"compute@1.0">>}, + #{<<"module">> => dev_cache,<<"name">> => <<"cache@1.0">>}, + #{<<"module">> => dev_cacheviz,<<"name">> => <<"cacheviz@1.0">>}, + #{<<"module">> => dev_cron,<<"name">> => <<"cron@1.0">>}, + #{<<"module">> => dev_dedup,<<"name">> => <<"dedup@1.0">>}, + #{<<"module">> => dev_delegated_compute, + <<"name">> => <<"delegated-compute@1.0">>}, + #{<<"module">> => dev_faff,<<"name">> => <<"faff@1.0">>}, + #{<<"module">> => dev_codec_flat,<<"name">> => <<"flat@1.0">>}, + #{<<"module">> => dev_genesis_wasm, + <<"name">> => <<"genesis-wasm@1.0">>}, + #{<<"module">> => dev_green_zone, + <<"name">> => <<"greenzone@1.0">>}, + #{<<"module">> => dev_codec_httpsig, + <<"name">> => <<"httpsig@1.0">>}, + #{<<"module">> => dev_hyperbuddy, + <<"name">> => <<"hyperbuddy@1.0">>}, + #{<<"module">> => dev_codec_json,<<"name">> => <<"json@1.0">>}, + #{<<"module">> => dev_json_iface, + <<"name">> => <<"json-iface@1.0">>}, + #{<<"module">> => dev_lookup,<<"name">> => <<"lookup@1.0">>}, + #{<<"module">> => dev_lua,<<"name">> => <<"lua@5.3a">>}, + #{<<"module">> => dev_message,<<"name">> => <<"message@1.0">>}, + #{<<"module">> => dev_meta,<<"name">> => <<"meta@1.0">>}, + #{<<"module">> => dev_monitor,<<"name">> => <<"monitor@1.0">>}, + #{<<"module">> => dev_multipass, + <<"name">> => <<"multipass@1.0">>}, + #{<<"module">> => dev_p4,<<"name">> => <<"p4@1.0">>}, + #{<<"module">> => dev_patch,<<"name">> => <<"patch@1.0">>}, + #{<<"module">> => dev_poda,<<"name">> => <<"poda@1.0">>}, + #{<<"module">> => dev_process,<<"name">> => <<"process@1.0">>}, + #{<<"module">> => dev_push,<<"name">> => <<"push@1.0">>}, + #{<<"module">> => dev_relay,<<"name">> => <<"relay@1.0">>}, + #{<<"module">> => dev_router,<<"name">> => <<"router@1.0">>}, + #{<<"module">> => dev_scheduler, + <<"name">> => <<"scheduler@1.0">>}, + #{<<"module">> => dev_simple_pay, + <<"name">> => <<"simple-pay@1.0">>}, + #{<<"module">> => dev_snp,<<"name">> => <<"snp@1.0">>}, + #{<<"module">> => dev_stack,<<"name">> => <<"stack@1.0">>}, + #{<<"module">> => dev_codec_structured, + <<"name">> => <<"structured@1.0">>}, + #{<<"module">> => dev_test,<<"name">> => <<"test-device@1.0">>}, + #{<<"module">> => dev_wasi,<<"name">> => <<"wasi@1.0">>}, + #{<<"module">> => dev_wasm,<<"name">> => <<"wasm-64@1.0">>}], + <<"cache_writers">> => + [<<"XgN1kN-ZyAWtYvdUlPEM3EIIi-budUx81mjcHQ1mSNU">>], + <<"wasm_allow_aot">> => false,<<"store_all_signed">> => true, + <<"ans104_trust_gql">> => true, + <<"commitment_device">> => <<"httpsig@1.0">>, + <<"preprocessor">> => undefined,<<"debug_committers">> => false, + <<"port">> => 10000,<<"load_remote_devices">> => false, + <<"node_history">> => [],<<"mode">> => debug, + <<"trusted_device_signers">> => [],<<"debug_print_indent">> => 2, + <<"host">> => <<"localhost">>}, + <<"nonce">> => + <<"XgN1kN-ZyAWtYvdUlPEM3EIIi-budUx81mjcHQ1mSNU4bfGIgeSRG0fgGtFEdPTXAcI6szcsMPvbZmwMWraKxA">>, + <<"public-key">> => + <<"g2gCaAJ3A3JzYWIAAQABbQAAAgCj3ztG+laFiAMD2u171YmZWfCzbPphf+SGEGtA4EI7LFAbmWPeLtokhd8TBX5fcY9ZFIxD/uKciLgXDuhxQMhengRaGIJJTglPLDHNvByBHMfjZtOTwCwSbu5CshaoJhuKdD3ZCyfeqdbiOFylrXA8BJGW/e9gBAi7p5cq6WhekiPHysU4tPNON700VVaaPKk/2hKG2MjxjtV0SOOXkgsyq7ll/CUiyyxvH0nqMO832cwKPz6PVjIqH5Srdokp5jF7K1JJi8ZTiYesbePYdiihy30tc7BOHz2xexzuAK9Jp+kjyPtkUZGs/bwWFGEKPNiGIJPzx5+xbECOS/ULUlh/mMku0sV8SX5jf2OkY9echqh94wn1fZf9d/Hzoo4pKqg5T4OYUp8923gmOppp8jX8Cy/GnrR6dLUF2+B8uvfgryCrpjKizjsWvOGoJb8FY4Xvu+Cn7Dk9X61yza47WgqlSCHLV5pN5410oeOVCP2G46O/uQ4wEZ85a1kNbWw70KFgHcMNNPcADCjptWOPSKKLBYZvNZ5zBP048t61ELKZU8QW9NBZcrr7VC7dY2hioW0L+kWoej6jDFodLfvJJN8HUDTgLlrqbdMQ/2EtZ17nYLl+sx19yFi7Lmmqg2oxvezOcZFCE7CZxbFve6er23K0Bg2YSlCHNn6wdqlOzVjTzQ==">>, + <<"report">> => + <<"{\"version\":2,\"guest_svn\":0,\"policy\":196608,\"family_id\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"image_id\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"vmpl\":1,\"sig_algo\":1,\"current_tcb\":{\"bootloader\":4,\"tee\":0,\"_reserved\":[0,0,0,0],\"snp\":22,\"microcode\":213},\"plat_info\":3,\"_author_key_en\":0,\"_reserved_0\":0,\"report_data\":[94,3,117,144,223,153,200,5,173,98,247,84,148,241,12,220,66,8,139,230,238,117,76,124,214,104,220,29,13,102,72,213,56,109,241,136,129,228,145,27,71,224,26,209,68,116,244,215,1,194,58,179,55,44,48,251,219,102,108,12,90,182,138,196],\"measurement\":[57,145,156,83,216,77,75,73,242,17,131,95,71,138,47,87,91,44,72,35,26,227,241,105,23,212,36,168,251,224,172,31,33,175,179,197,217,106,227,52,65,65,10,56,83,147,108,11],\"host_data\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"id_key_digest\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"author_key_digest\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"report_id\":[76,118,205,58,46,223,75,32,141,9,113,137,69,229,230,102,49,239,64,145,130,106,21,147,154,141,222,210,40,40,121,150],\"report_id_ma\":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255],\"reported_tcb\":{\"bootloader\":4,\"tee\":0,\"_reserved\":[0,0,0,0],\"snp\":22,\"microcode\":213},\"_reserved_1\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"chip_id\":[6,154,118,21,12,115,53,47,251,19,195,100,155,12,239,102,116,131,103,189,106,202,153,3,114,175,16,182,24,166,229,214,231,126,164,15,129,52,233,142,196,43,43,8,89,238,118,246,21,144,209,16,165,197,134,105,214,250,155,148,50,78,87,203],\"committed_tcb\":{\"bootloader\":4,\"tee\":0,\"_reserved\":[0,0,0,0],\"snp\":22,\"microcode\":213},\"current_build\":20,\"current_minor\":55,\"current_major\":1,\"_reserved_2\":0,\"committed_build\":20,\"committed_minor\":55,\"committed_major\":1,\"_reserved_3\":0,\"launch_tcb\":{\"bootloader\":4,\"tee\":0,\"_reserved\":[0,0,0,0],\"snp\":22,\"microcode\":213},\"_reserved_4\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"signature\":{\"r\":[208,7,136,60,224,138,74,102,17,217,164,154,114,117,150,30,151,247,93,90,52,122,13,58,58,169,124,13,168,74,187,144,221,165,218,93,104,175,212,214,248,23,142,13,132,232,208,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"s\":[67,238,106,198,241,37,137,212,124,29,212,20,95,170,10,247,160,213,57,13,223,123,32,246,242,28,76,121,72,170,113,199,14,128,130,162,198,59,138,105,227,27,136,207,243,245,201,196,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"_reserved\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}}">>, + <<"vcpu_type">> => 5,<<"vcpus">> => 1,<<"vmm_type">> => 1}. \ No newline at end of file diff --git a/test/snp-attestation.json b/test/snp-attestation.json new file mode 100644 index 000000000..7fd0ed89f --- /dev/null +++ b/test/snp-attestation.json @@ -0,0 +1 @@ +{"version":2,"guest_svn":0,"policy":196608,"family_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"image_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"vmpl":1,"sig_algo":1,"current_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"plat_info":3,"_author_key_en":0,"_reserved_0":0,"report_data":[208,120,131,217,114,39,198,164,204,217,195,57,124,86,164,109,32,202,124,88,10,192,67,24,189,205,77,113,76,47,190,120,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"measurement":[68,96,162,147,247,146,208,74,212,11,55,215,117,33,42,122,17,21,124,118,203,181,169,138,54,218,98,162,214,19,16,56,35,142,64,229,240,143,249,82,158,125,61,16,25,217,49,96],"host_data":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"id_key_digest":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"author_key_digest":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"report_id":[175,98,224,32,248,224,81,83,49,238,39,53,221,235,57,116,75,140,222,254,255,16,138,8,11,2,130,31,195,151,86,101],"report_id_ma":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255],"reported_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"_reserved_1":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"chip_id":[140,186,24,26,13,93,198,89,109,169,156,117,74,171,119,218,227,248,161,29,156,249,196,253,0,133,213,176,104,236,220,229,27,64,240,249,213,207,232,136,152,246,240,221,96,1,178,159,177,108,253,113,102,214,196,175,132,105,188,140,137,98,86,52],"committed_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"current_build":20,"current_minor":55,"current_major":1,"_reserved_2":0,"committed_build":20,"committed_minor":55,"committed_major":1,"_reserved_3":0,"launch_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"_reserved_4":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"signature":{"r":[1,222,131,193,36,228,32,83,188,153,86,140,116,70,132,2,34,160,82,83,59,33,181,220,114,102,1,163,249,80,112,43,215,219,62,226,132,183,232,186,53,22,179,16,70,221,120,238,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s":[48,27,250,204,171,131,243,100,96,161,7,68,223,70,61,221,227,88,83,65,120,34,36,175,197,225,5,193,238,92,134,27,21,105,35,201,66,183,244,232,203,117,190,44,2,180,13,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"_reserved":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}} \ No newline at end of file diff --git a/test/snp-measurement.json b/test/snp-measurement.json new file mode 100644 index 000000000..319605154 --- /dev/null +++ b/test/snp-measurement.json @@ -0,0 +1 @@ +{"version":2,"guest_svn":0,"policy":196608,"family_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"image_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"vmpl":1,"sig_algo":1,"current_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"plat_info":3,"_author_key_en":0,"_reserved_0":0,"report_data":[107,177,15,108,76,181,154,193,86,58,37,11,6,178,145,224,233,19,70,184,151,201,166,72,200,158,145,236,87,146,8,219,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"measurement":[94,87,4,197,20,11,255,129,179,197,146,104,8,212,152,248,110,11,60,246,82,254,24,55,201,47,157,229,163,82,108,66,191,138,241,229,40,144,133,170,116,109,17,62,20,241,144,119],"host_data":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"id_key_digest":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"author_key_digest":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"report_id":[246,37,173,24,24,99,21,145,60,28,73,1,217,65,121,45,114,58,91,219,210,122,81,63,152,72,238,19,167,185,155,173],"report_id_ma":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255],"reported_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"_reserved_1":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"chip_id":[140,186,24,26,13,93,198,89,109,169,156,117,74,171,119,218,227,248,161,29,156,249,196,253,0,133,213,176,104,236,220,229,27,64,240,249,213,207,232,136,152,246,240,221,96,1,178,159,177,108,253,113,102,214,196,175,132,105,188,140,137,98,86,52],"committed_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"current_build":20,"current_minor":55,"current_major":1,"_reserved_2":0,"committed_build":20,"committed_minor":55,"committed_major":1,"_reserved_3":0,"launch_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213}} \ No newline at end of file diff --git a/test/snp_attestation b/test/snp_attestation new file mode 100644 index 000000000..4b20ccd73 --- /dev/null +++ b/test/snp_attestation @@ -0,0 +1,169 @@ +node-message/http_default_remote_port: 8734 +node-message/scheduler_location_ttl: 2592000 +node-message/load_remote_devices: "false" +node-message/ao-type-trusted_device_signers: empty-list +node-message/trusted/guest_features: 1 +node-message/preloaded_devices/meta@1.0: "dev_meta" +commitments/hmac-sha256/signature-input: http-sig-a2e132bfc4af5fc9=("address" "append" "body" "content-type" "ao-type-guest_features" "ao-type-vcpu_type" "ao-type-vcpus" "ao-type-vmm_type" "firmware" "guest_features" "initrd" "kernel" "nonce" "report" "vcpu_type" "vcpus" "vmm_type");alg="rsa-pss-sha512";keyid="3E7eC9SS-hxAYZ5PZ5eUyYXLPVoHDvkKZ_Fcj7-p9U5R4kl_-Ty1yLT1eCQrTFQk3ZXOB9w7JgTLgmfGS65Tief_4Dd0SsbKxbddRAgNhYtbSIwL2UEyv3BWJ5vlqbUq1NEILBnrCfP4DPljRvCvTq2mHhn4bKlA2Rb-uixKh2kal6nSugAWlkNZwRowxatvTD57NGDccEWVsloCpWbAUBePQBw7akUnV4WuxDWjibljW8DZ1FI3EqPgZngSuYmc9pfeh3a1TbsW40fETgcBIioXSNCvuhiSXcb89136zDM581RDNuwgxqWgJRyqmnzzZFDn4WceKOiUfINFTT_ggiYOtShWui6pP3En80qZp2iZwbQIwXxZrkWnzK3OkRZxChxaVVdTUK6nkmRvrRHHYtnEnapqrXrp1QKcpwcqDInxfn0A9OQpy34JfPuMgEDLgFuVi68-Itt3-T4XLTc7_MAsWAOdNkMfOMTUdzwMyLugQlyx50_HzKuyN3luNW8kMxuntRkKOZ09NVFhlyxdAKE7gKPo0I2jZ86-KNecqUDQGnu49dGkx_pSNZ9NTvKfuJ4hXf2gDdafP8IYIzJXOLwhCQoEcCIKnH2jkCS8-5OLVBTpJUePxZS0a6_D8kH7XqlAUJW2sfIFyC90xVCm1CPdoumWp1SFL0zbR70Ss3U";nonce="s6LhMr_Er1_JrDdHBWB8IejgO4H021ZR1fnZ919hDNW16J8u8IqYHDJog2bE5GZ2zB4Ir6f32Khq2C738NGk0A" +node-message/preloaded_devices/test-device@1.0: "dev_test" +node-message/preloaded_devices/ao-type-flat@1.0: atom +node-message/snp_hashes/vmm_type: 1 +node-message/preloaded_devices/ao-type-wasi@1.0: atom +node-message/preloaded_devices/router@1.0: "dev_router" +node-message/snp_hashes/ao-type-vcpu_type: integer +node-message/preloaded_devices/ao-type-stack@1.0: atom +node-message/preloaded_devices/ao-type-dedup@1.0: atom +node-message/trusted/append: 6cb8a0082b483849054f93b203aa7d98439736e44163d614f79380ca368cc77e +node-message/trusted/ao-type-vcpu_type: integer +node-message/protocol: "http2" +node-message/http_host: localhost +node-message/trusted/initrd: 853ebf56bc6ba5f08bd5583055a457898ffa3545897bee00103d3066b8766f5c +node-message/trusted/ao-type-vmm_type: integer +firmware: b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510 +ao-type-vcpus: integer +node-message/preloaded_devices/ao-type-relay@1.0: atom +node-message/ao-type-debug_print_binary_max: integer +node-message/port: 8734 +node-message/ao-type-load_remote_devices: atom +node-message/debug_print_binary_max: 60 +node-message/http_server: s6LhMr_Er1_JrDdHBWB8IejgO4H021ZR1fnZ919hDNU +node-message/preloaded_devices/ao-type-httpsig@1.0: atom +node-message/preloaded_devices/snp@1.0: "dev_snp" +node-message/preloaded_devices/ao-type-monitor@1.0: atom +node-message/preloaded_devices/message@1.0: "dev_message" +node-message/preloaded_devices/ao-type-compute@1.0: atom +node-message/ao-type-http_keepalive: integer +commitments/s6LhMr_Er1_JrDdHBWB8IejgO4H021ZR1fnZ919hDNU/signature: http-sig-a2e132bfc4af5fc9=:sR4pqe/ynt0q4aTSkBP8u9zz+Ob8MhqXcU3pIvcwcbZQW6HAkEONMbTt3+GiNxPeELU8Lg62FSlnOBOGhtlGicfeDCPidALf9Y+SD3GjTLI8o1Wsi7B7Q502nDxIkgP5+sVHVJcyO3RKqTIEJ9cdmr2fuv5f9Z6KIVg2YukqgoIxv/P9L03/E38N7oimlRSI4PWcTPXkPhcHr4EVrX1gkbfxZM+xfzt/3i95WL15Mz/kt6p5HBV75Jt5lJdTdRNgbH8SODio0Z6YTWMZgeHZ+Pw6WCPb08gpkWxQ81FtmkFsvb4XG6DSj087PGtnAbVuvkMl+hYQKHhGYVbQqRnS11jFOuZMOpcTU/3FbxJ+wKfUYzQeKiunLXaC+M1ao77LRNfe1QBJU7lVGcg6P7pKTbDKlNWNm5z5EDItd5e3+XB7zwQP5/gRxLHt75jaPcBviGWgTAERm3qfGD5DJw/+cM+miFBOQLXuscYdJ2U0D3t51ezN6yhEz7iqCQMv11DfgK1x4Av5f2pR0Q8mW11MUhAuCR7RfaZi7+hxVSK9m9JodrLaK/wmFtAPKmlBgtodpuMXTMSXpssZB9VYuFBtjWJt2ZrCMkl8aOU6sFKR40N15vzQ+c53dPTQLhjdLHsgfub3TB3P2LU1dc05xR+BvrhNL2jC1z7fTCyADqKmcwY=: +node-message/ao-type-debug_print_trace: atom +node-message/wasm_allow_aot: "false" +node-message/preloaded_devices/multipass@1.0: "dev_multipass" +node-message/ao-type-debug_print: atom +ao-type-vmm_type: integer +node-message/ao-type-mode: atom +node-message/http_response_timeout: 30000 +node-message/snp_hashes/firmware: b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510 +node-message/http_keepalive: 120000 +node-message/preloaded_devices/ao-type-multipass@1.0: atom +node-message/ao-type-protocol: atom +node-message/trusted/vmm_type: 1 +node-message/debug_hide_metadata: "false" +node-message/preloaded_devices/simple-pay@1.0: "dev_simple_pay" +commitments/s6LhMr_Er1_JrDdHBWB8IejgO4H021ZR1fnZ919hDNU/signature-input: http-sig-a2e132bfc4af5fc9=("address" "append" "body" "content-type" "ao-type-guest_features" "ao-type-vcpu_type" "ao-type-vcpus" "ao-type-vmm_type" "firmware" "guest_features" "initrd" "kernel" "nonce" "report" "vcpu_type" "vcpus" "vmm_type");alg="rsa-pss-sha512";keyid="3E7eC9SS-hxAYZ5PZ5eUyYXLPVoHDvkKZ_Fcj7-p9U5R4kl_-Ty1yLT1eCQrTFQk3ZXOB9w7JgTLgmfGS65Tief_4Dd0SsbKxbddRAgNhYtbSIwL2UEyv3BWJ5vlqbUq1NEILBnrCfP4DPljRvCvTq2mHhn4bKlA2Rb-uixKh2kal6nSugAWlkNZwRowxatvTD57NGDccEWVsloCpWbAUBePQBw7akUnV4WuxDWjibljW8DZ1FI3EqPgZngSuYmc9pfeh3a1TbsW40fETgcBIioXSNCvuhiSXcb89136zDM581RDNuwgxqWgJRyqmnzzZFDn4WceKOiUfINFTT_ggiYOtShWui6pP3En80qZp2iZwbQIwXxZrkWnzK3OkRZxChxaVVdTUK6nkmRvrRHHYtnEnapqrXrp1QKcpwcqDInxfn0A9OQpy34JfPuMgEDLgFuVi68-Itt3-T4XLTc7_MAsWAOdNkMfOMTUdzwMyLugQlyx50_HzKuyN3luNW8kMxuntRkKOZ09NVFhlyxdAKE7gKPo0I2jZ86-KNecqUDQGnu49dGkx_pSNZ9NTvKfuJ4hXf2gDdafP8IYIzJXOLwhCQoEcCIKnH2jkCS8-5OLVBTpJUePxZS0a6_D8kH7XqlAUJW2sfIFyC90xVCm1CPdoumWp1SFL0zbR70Ss3U";nonce="s6LhMr_Er1_JrDdHBWB8IejgO4H021ZR1fnZ919hDNW16J8u8IqYHDJog2bE5GZ2zB4Ir6f32Khq2C738NGk0A" +append: 6cb8a0082b483849054f93b203aa7d98439736e44163d614f79380ca368cc77e +guest_features: 1 +node-message/preloaded_devices/ao-type-poda@1.0: atom +commitments/s6LhMr_Er1_JrDdHBWB8IejgO4H021ZR1fnZ919hDNU/commitment-device: httpsig@1.0 +initrd: 853ebf56bc6ba5f08bd5583055a457898ffa3545897bee00103d3066b8766f5c +node-message/ao-type-stack_print_prefixes: list +node-message/preloaded_devices/poda@1.0: "dev_poda" +node-message/preloaded_devices/p4@1.0: "dev_p4" +node-message/ao-type-port: integer +node-message/preloaded_devices/monitor@1.0: "dev_monitor" +node-message/preloaded_devices/structured@1.0: "dev_codec_structured" +node-message/ao-type-http_connect_timeout: integer +node-message/debug_print: "false" +node-message/ao-type-http_default_remote_port: integer +node-message/preloaded_devices/ao-type-process@1.0: atom +node-message/preloaded_devices/compute@1.0: "dev_cu" +node-message/stack_print_prefixes: "(ao-type-list) \"(ao-type-integer) 104\", \"(ao-type-integer) 98\"", "(ao-type-list) \"(ao-type-integer) 100\", \"(ao-type-integer) 101\", \"(ao-type-integer) 118\"", "(ao-type-list) \"(ao-type-integer) 97\", \"(ao-type-integer) 114\"" +vmm_type: 1 +node-message/preloaded_devices/wasm-64@1.0: "dev_wasm" +node-message/ao-type-debug_hide_metadata: atom +node-message/ao-type-cache_results: atom +node-message/mode: "debug" +ao-type-status: integer +node-message/compute_mode: "lazy" +node-message/http_request_send_timeout: 60000 +node-message/only: "local" +node-message/preloaded_devices/relay@1.0: "dev_relay" +node-message/preloaded_devices/ans104@1.0: "dev_codec_ans104" +node-message/ao-type-wasm_allow_aot: atom +status: 200 +commitments/hmac-sha256/commitment-device: httpsig@1.0 +vcpus: 1 +node-message/debug_ids: "false" +node-message/preloaded_devices/faff@1.0: "dev_faff" +node-message/http_connect_timeout: 5000 +node-message/ao-type-scheduler_location_ttl: integer +commitments/hmac-sha256/signature: http-sig-a2e132bfc4af5fc9=:sR4pqe/ynt0q4aTSkBP8u9zz+Ob8MhqXcU3pIvcwcbZQW6HAkEONMbTt3+GiNxPeELU8Lg62FSlnOBOGhtlGicfeDCPidALf9Y+SD3GjTLI8o1Wsi7B7Q502nDxIkgP5+sVHVJcyO3RKqTIEJ9cdmr2fuv5f9Z6KIVg2YukqgoIxv/P9L03/E38N7oimlRSI4PWcTPXkPhcHr4EVrX1gkbfxZM+xfzt/3i95WL15Mz/kt6p5HBV75Jt5lJdTdRNgbH8SODio0Z6YTWMZgeHZ+Pw6WCPb08gpkWxQ81FtmkFsvb4XG6DSj087PGtnAbVuvkMl+hYQKHhGYVbQqRnS11jFOuZMOpcTU/3FbxJ+wKfUYzQeKiunLXaC+M1ao77LRNfe1QBJU7lVGcg6P7pKTbDKlNWNm5z5EDItd5e3+XB7zwQP5/gRxLHt75jaPcBviGWgTAERm3qfGD5DJw/+cM+miFBOQLXuscYdJ2U0D3t51ezN6yhEz7iqCQMv11DfgK1x4Av5f2pR0Q8mW11MUhAuCR7RfaZi7+hxVSK9m9JodrLaK/wmFtAPKmlBgtodpuMXTMSXpssZB9VYuFBtjWJt2ZrCMkl8aOU6sFKR40N15vzQ+c53dPTQLhjdLHsgfub3TB3P2LU1dc05xR+BvrhNL2jC1z7fTCyADqKmcwY=: +node-message/gateway: https://arweave.net +node-message/scheduling_mode: "local_confirmation" +node-message/debug_print_indent: 2 +node-message/ao-type-access_remote_cache_for_client: atom +node-message/preloaded_devices/stack@1.0: "dev_stack" +node-message/preloaded_devices/process@1.0: "dev_process" +node-message/cache_control: "no-cache", "no-store" +node-message/short_trace_len: 5 +node-message/client_error_strategy: "throw" +node-message/ao-type-compute_mode: atom +node-message/ao-type-scheduling_mode: atom +server: Cowboy +node-message/preloaded_devices/ao-type-wasm-64@1.0: atom +report: {"version":2,"guest_svn":0,"policy":196608,"family_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"image_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"vmpl":1,"sig_algo":1,"current_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"plat_info":3,"_author_key_en":0,"_reserved_0":0,"report_data":[179,162,225,50,191,196,175,95,201,172,55,71,5,96,124,33,232,224,59,129,244,219,86,81,213,249,217,247,95,97,12,213,181,232,159,46,240,138,152,28,50,104,131,102,196,228,102,118,204,30,8,175,167,247,216,168,106,216,46,247,240,209,164,208],"measurement":[225,76,136,16,138,45,59,152,100,179,55,144,221,98,117,227,70,176,119,39,27,231,20,246,56,124,198,164,250,249,141,169,126,16,27,52,241,154,89,228,151,7,82,57,144,195,21,108],"host_data":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"id_key_digest":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"author_key_digest":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"report_id":[164,76,236,34,82,53,152,194,35,16,54,201,87,9,27,106,226,224,16,90,93,28,16,222,79,216,101,51,133,143,248,143],"report_id_ma":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255],"reported_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"_reserved_1":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"chip_id":[140,186,24,26,13,93,198,89,109,169,156,117,74,171,119,218,227,248,161,29,156,249,196,253,0,133,213,176,104,236,220,229,27,64,240,249,213,207,232,136,152,246,240,221,96,1,178,159,177,108,253,113,102,214,196,175,132,105,188,140,137,98,86,52],"committed_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"current_build":20,"current_minor":55,"current_major":1,"_reserved_2":0,"committed_build":20,"committed_minor":55,"committed_major":1,"_reserved_3":0,"launch_tcb":{"bootloader":4,"tee":0,"_reserved":[0,0,0,0],"snp":22,"microcode":213},"_reserved_4":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"signature":{"r":[250,82,21,86,200,230,71,75,225,197,179,100,13,251,172,213,9,48,28,69,101,121,120,41,14,205,103,233,99,56,64,248,27,136,215,180,42,132,18,80,173,111,144,102,96,174,55,225,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"s":[123,231,116,141,20,78,52,24,154,106,51,180,112,0,132,79,133,255,41,158,221,224,176,14,79,215,20,73,199,43,115,170,234,203,63,126,183,153,37,135,203,201,162,178,32,109,171,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"_reserved":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}} +node-message/snp_hashes/kernel: 69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576 +node-message/preloaded_devices/push@1.0: "dev_mu" +ao-type-guest_features: integer +node-message/preloaded_devices/ao-type-test-device@1.0: atom +kernel: 69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576 +node-message/ao-type-debug_stack_depth: integer +node-message/preloaded_devices/ao-type-scheduler@1.0: atom +node-message/ao-type-debug_ids: atom +node-message/key_location: hyperbeam-key.json +node-message/preloaded_devices/ao-type-meta@1.0: atom +node-message/preloaded_devices/ao-type-simple-pay@1.0: atom +node-message/ao-type-http_request_send_timeout: integer +node-message/preloaded_devices/cron@1.0: "dev_cron" +node-message/snp_hashes/ao-type-guest_features: integer +node-message/trusted/vcpu_type: 5 +node-message/preloaded_devices/ao-type-message@1.0: atom +ao-type-vcpu_type: integer +node-message/ao-type-http_response_timeout: integer +node-message/preloaded_devices/ao-type-ans104@1.0: atom +node-message/snp_hashes/append: 6cb8a0082b483849054f93b203aa7d98439736e44163d614f79380ca368cc77e +node-message/default_page_limit: 5 +commitments/s6LhMr_Er1_JrDdHBWB8IejgO4H021ZR1fnZ919hDNU/id: #�M;+ +U!q�� +�1��ʀ��Ӫ��yDt�J� +node-message/debug_stack_depth: 40 +node-message/preloaded_devices/json-iface@1.0: "dev_json_iface" +address: s6LhMr_Er1_JrDdHBWB8IejgO4H021ZR1fnZ919hDNU +node-message/trusted/vcpus: 1 +node-message/snp_hashes/ao-type-vcpus: integer +node-message/ao-type-only: atom +node-message/ao-type-debug_print_indent: integer +node-message/snp_hashes/ao-type-vmm_type: integer +node-message/preloaded_devices/flat@1.0: "dev_codec_flat" +node-message/ao-type-cache_control: list +node-message/access_remote_cache_for_client: "false" +node-message/trusted/ao-type-guest_features: integer +node-message/snp_hashes/vcpu_type: 5 +nonce: s6LhMr_Er1_JrDdHBWB8IejgO4H021ZR1fnZ919hDNW16J8u8IqYHDJog2bE5GZ2zB4Ir6f32Khq2C738NGk0A +node-message/preloaded_devices/ao-type-push@1.0: atom +node-message/preloaded_devices/ao-type-router@1.0: atom +node-message/snp_hashes/vcpus: 1 +commitments/hmac-sha256/id: NvxJno3CqMV1yRXueeaOnHHy-U8oHTPmDdr-ye2prE8 +node-message/trusted/firmware: b8c5d4082d5738db6b0fb0294174992738645df70c44cdecf7fad3a62244b788e7e408c582ee48a74b289f3acec78510 +node-message/preloaded_devices/ao-type-json-iface@1.0: atom +node-message/ao-type-client_error_strategy: atom +node-message/snp_hashes/guest_features: 1 +node-message/ao-type-default_page_limit: integer +node-message/trusted/kernel: 69d0cd7d13858e4fcef6bc7797aebd258730f215bc5642c4ad8e4b893cc67576 +node-message/preloaded_devices/scheduler@1.0: "dev_scheduler" +node-message/ao-type-debug_print_map_line_threshold: integer +date: Thu, 30 Jan 2025 20:08:28 GMT +node-message/preloaded_devices/ao-type-cron@1.0: atom +node-message/debug_print_map_line_threshold: 30 +node-message/preloaded_devices/ao-type-structured@1.0: atom +node-message/preloaded_devices/wasi@1.0: "dev_wasi" +node-message/preloaded_devices/ao-type-p4@1.0: atom +node-message/preloaded_devices/ao-type-snp@1.0: atom +vcpu_type: 5 +node-message/trusted/ao-type-vcpus: integer +node-message/ao-type-short_trace_len: integer +node-message/preloaded_devices/httpsig@1.0: "dev_codec_httpsig" +node-message/preloaded_devices/ao-type-faff@1.0: atom +node-message/snp_hashes/initrd: 853ebf56bc6ba5f08bd5583055a457898ffa3545897bee00103d3066b8766f5c +node-message/cache_results: "false" +node-message/debug_print_trace: "short" +node-message/bundler: https://up.arweave.net +node-message/preloaded_devices/dedup@1.0: "dev_dedup" diff --git a/test/test-aos-2-pure-xs.aot b/test/test-aos-2-pure-xs.aot new file mode 100644 index 000000000..4a76a770f Binary files /dev/null and b/test/test-aos-2-pure-xs.aot differ diff --git a/test/test.lua b/test/test.lua new file mode 100644 index 000000000..01c29e43f --- /dev/null +++ b/test/test.lua @@ -0,0 +1,158 @@ +--- @module test +--- Contains a collection of test functions for the Lua device. + +--- @function AssocTable +--- @treturn table +--- @return a table with three key-value pairs. In Erlang, this will be +--- represented as `#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}`. +function assoctable() + return { + a = 1, + b = 2, + c = 3 + } +end + +function error_response() + return "error", "Very bad, but Lua caught it." +end + +--- @function ListTable +--- @treturn table +--- @return a table with three elements. In Erlang, this will be +--- represented as `[1, 2, 3]`. +function ListTable() + return {1, 2, 3} +end + +function ao_resolve() + local status, res = + ao.resolve({ + path = "/hello", + hello = "Hello, AO world!" + }) + return res +end + +function ao_relay() + local status, res = + ao.resolve({ + path = "/~relay@1.0/call?relay-path=http://localhost:10000/hello" + }) + return res +end + +--- @function compute +--- @tparam stringified AO process +--- @tparam stringified AO message +--- @return table An AO Process response, with the `ok` field set to `true`, +--- the `response` field set to a table with the `Output` field set to a string, +--- and the `messages` field set to an empty table. +function compute(process, message, opts) + process.results = { + output = { + body = 42 + } + } + return process +end + +--- @function json_result +--- @tparam table base +--- @tparam table request +--- @return table request with the `ok` field set to `true`, the `response` +--- field set to a table with the `Output` field set to `42`, and +--- the `messages` field set to an empty table. +function json_result(base, req, opts) + return [[ + { + "ok": true, + "response": {"Output": {"data": 42}, "Messages": []} + } + ]] +end + +--- @function hello +--- @tparam table base +--- @tparam table request +--- @return table request with the `hello` field set to `"world"`. +function hello(base, req, opts) + base.hello = req.name or "world" + return base +end + +--- @function preprocess +--- @tparam table base +--- @tparam table request +--- @return table an answer to every HTTP request with the words "i like turtles" +function request(base, req, opts) + return "ok", { body = { { body = "i like turtles" } } } +end + +--- @function sandboxed_fail +--- @tparam table base +--- @tparam table request +--- @error fails when inside the sandbox +function sandboxed_fail() + -- Do something that is not dangerous, but is sandboxed nonetheless. + return os.getenv("PWD") +end + +--- @function provider +--- @tparam table base +--- @tparam table request +--- @return table a static set of routes for testing purposes. +function provider(base, req, opts) + return { + { + node = base.node + } + } +end + +BaseRoutes = { + { + template = "test1", + host = "http://localhost:10000", + weight = 50 + }, + { + template = "test2", + strategy = "By-Weight", + choose = 1, + nodes = { + { + prefix = "http://localhost:10001/", + weight = 50 + }, + { + prefix = "http://localhost:10002/", + weight = 50 + } + } + } +} + +--- @function compute_routes +--- @tparam table base +--- @tparam table request +--- @return table the state of a process after adding a route. +function compute_routes(base, req, opts) + base["known-routes"] = base["known-routes"] or BaseRoutes + if req.body.path == "add-route" then + table.insert(base["known-routes"], req.body) + base.results = base.results or {} + base.results.output = { + status = 200, + ["content-type"] = "text/plain", + body = "Route added." + } + end + return base +end + +function inc(base, req, opts) + base.count = base.count or 0 + base.count = base.count + 1 + return base +end \ No newline at end of file