From 69cd8d2b8481af7a12463f794fa0dadfe2ae8b93 Mon Sep 17 00:00:00 2001 From: Connor Snitker Date: Mon, 21 Sep 2026 10:43:34 -0500 Subject: [PATCH 1/3] docs(deploy): add native Ubuntu RA and TL setup Signed-off-by: Connor Snitker --- README.md | 3 + deploy/ubuntu/Caddyfile | 32 ++++ deploy/ubuntu/README.md | 264 ++++++++++++++++++++++++++++++ deploy/ubuntu/ans-ra.service | 29 ++++ deploy/ubuntu/ans-tl.service | 29 ++++ deploy/ubuntu/install-packages.sh | 63 +++++++ deploy/ubuntu/ra.yaml | 52 ++++++ deploy/ubuntu/tl.yaml | 30 ++++ 8 files changed, 502 insertions(+) create mode 100644 deploy/ubuntu/Caddyfile create mode 100644 deploy/ubuntu/README.md create mode 100644 deploy/ubuntu/ans-ra.service create mode 100644 deploy/ubuntu/ans-tl.service create mode 100644 deploy/ubuntu/install-packages.sh create mode 100644 deploy/ubuntu/ra.yaml create mode 100644 deploy/ubuntu/tl.yaml diff --git a/README.md b/README.md index 1cc03e0..2ddf0fe 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,9 @@ The RA writes signed events; the TL verifies, ingests, and publishes them. An operator can run just the TL (for read-only verification) or both together (full registry). +For a native Ubuntu host with systemd, Caddy, OIDC, and Let’s Encrypt, see +the [RA/TL deployment guide](deploy/ubuntu/README.md). + ## Quickstart (60 seconds) ```bash diff --git a/deploy/ubuntu/Caddyfile b/deploy/ubuntu/Caddyfile new file mode 100644 index 0000000..784fe85 --- /dev/null +++ b/deploy/ubuntu/Caddyfile @@ -0,0 +1,32 @@ +{ + # Use Let's Encrypt explicitly for RA/TL service HTTPS. + acme_ca https://acme-v02.api.letsencrypt.org/directory +} + +ra.ans.example.com { + redir /docs/ /docs 308 + # Keep authenticated API responses out of shared proxy caches. + header Cache-Control "no-store" + # Keep operational endpoints accessible only through localhost. + @private path /v2/admin /v2/admin/* + respond @private 404 + reverse_proxy 127.0.0.1:18080 +} + +tl.ans.example.com { + redir /docs/ /docs 308 + # Fresh status/revocation evidence must not be served from an edge cache. + header Cache-Control "no-store" + # Public TL access is read-only. Ingestion and producer-key admin + # stay on the private backend, including both API versions. + @public { + method GET HEAD + path /docs /docs/* /root-keys /checkpoint /tile/* /v1/agents/* /v1/identities/* /v1/log/* + } + handle @public { + reverse_proxy 127.0.0.1:18081 + } + handle { + respond 404 + } +} diff --git a/deploy/ubuntu/README.md b/deploy/ubuntu/README.md new file mode 100644 index 0000000..ea43216 --- /dev/null +++ b/deploy/ubuntu/README.md @@ -0,0 +1,264 @@ +# Native Ubuntu deployment + +Run RA, TL, and Caddy directly on Ubuntu under systemd. No Docker, external +SQL server, or Node.js runtime is required for RA/TL. These instructions assume +a recent systemd-based Ubuntu server on amd64 or arm64, sudo access, and the +reviewed source release containing these deployment files. Build on a separate +Ubuntu machine of the same architecture if you do not want build tools on +the service host. + +Caddy terminates RA/TL HTTPS with Let's Encrypt production certificates. The +RA independently uses Let's Encrypt staging for agent server certificates. +The identity CA remains the local persistent issuer for initial staging. + +## Before starting + +Replace the example values in these templates before installing them: + +| File | Values to replace | +| --- | --- | +| `Caddyfile` | `ra.ans.example.com` and `tl.ans.example.com` with your service hostnames | +| `ra.yaml` | OIDC issuer and audience; `tl-client.public-base-url` with the same public TL URL | +| `tl.yaml` | `merkle.origin` with the same public TL hostname | + +Choose stable signer key IDs and RA IDs before first startup. Preserve them +and their keys on upgrades. The `__TL_SERVICE_KEY__` marker is replaced by the +installation command below; do not substitute a real secret into the repository. + +For Clerk, create a JWT template named `ans-ra` with an `aud` claim matching +`auth.oidc.audience` (the example uses `ans-ra`). Send the resulting template +JWT as `Authorization: Bearer ` to the RA. Use your own issuer; a Clerk +development instance is not a production identity deployment. Other OIDC +providers work when their discovery/JWKS and token claims match the config. +OIDC protects RA operations, independently of agents' ANS identity certificates. + +The example uses the persistent local identity CA. A managed private CA +requires its own adapter; selecting ACME does not replace the identity CA. + +## 1. Install packages + +From the repository root: + +```sh +sudo bash deploy/ubuntu/install-packages.sh +``` + +The script installs `ca-certificates`, `curl`, `gnupg`, `debian-keyring`, +`debian-archive-keyring`, `apt-transport-https`, `git`, `build-essential`, `jq`, +`openssl`, `dnsutils`, `python3`, and `python3-yaml` from Ubuntu; Caddy from +its official stable APT repository; and the latest Go 1.26 patch release from +`go.dev`, checking its published SHA-256. Stay on this Go release line until +the pinned linter supports newer compiler export formats; Go 1.27 failed the +current linter's type checks during the first server build. +The package installer requires outbound network access and sudo; it changes +APT sources and `/usr/local/bin/go` and `/usr/local/bin/gofmt`. +Go is installed in `/opt/ans-toolchains/` with `go` and `gofmt` +symlinks under `/usr/local/bin`. Existing toolchains are retained. +Caddy's package may start its default site; our hostnames are enabled below. + +Sources: [Caddy packages](https://caddyserver.com/docs/install#debian-ubuntu-raspbian), +[Go downloads](https://go.dev/dl/), +[Let's Encrypt staging roots](https://letsencrypt.org/docs/staging-environment/). + +## 2. Build the reviewed source + +As your ordinary user, from the repository root: + +```sh +export PATH="/usr/local/bin:$PATH" +go version +make check +make test-race +make build +sudo install -o root -g root -m 0755 bin/ans-ra bin/ans-tl bin/ans-verify /usr/local/bin/ +``` + +`make check` installs the repository's pinned golangci-lint when needed. +Do not continue past failed checks. These are build-time tools; Go and the +compiler are not needed by the installed running binaries. + +## 3. Create accounts and install configuration + +```sh +id ans-ra >/dev/null 2>&1 || sudo useradd --system --user-group --home-dir /var/lib/ans-ra --no-create-home --shell /usr/sbin/nologin ans-ra +id ans-tl >/dev/null 2>&1 || sudo useradd --system --user-group --home-dir /var/lib/ans-tl --no-create-home --shell /usr/sbin/nologin ans-tl +sudo install -d -o root -g root -m 0755 /etc/ans +``` + +Install first-time configurations with one shared random TL service secret. +This refuses to overwrite existing configurations. Secrets remain in files +readable by root and the corresponding service group, not in shell history: + +```sh +sudo python3 - <<'PYCONFIG' +import grp, os, secrets +from pathlib import Path +pairs = [('ra', 'ans-ra'), ('tl', 'ans-tl')] +for name, group in pairs: + if Path(f'/etc/ans/{name}.yaml').exists(): + raise SystemExit(f'/etc/ans/{name}.yaml already exists; preserve its secret and edit deliberately') +key = secrets.token_hex(32) +for name, group in pairs: + text = Path(f'deploy/ubuntu/{name}.yaml').read_text() + if text.count('__TL_SERVICE_KEY__') != 1: + raise SystemExit('Expected exactly one service secret marker') + text = text.replace('__TL_SERVICE_KEY__', key) + fd = os.open(f'/etc/ans/{name}.yaml', os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640) + with os.fdopen(fd, 'w') as f: + os.fchown(f.fileno(), 0, grp.getgrnam(group).gr_gid) + os.fchmod(f.fileno(), 0o640) + f.write(text) +print('Installed RA/TL configuration with a generated service secret.') +PYCONFIG +``` + +The templates bind both services to `127.0.0.1`, configure Clerk for RA users, +use DNS lookup and real did:web resolution, and disable vLEI (`off`) until a +real verifier is deployed. The TL accepts its service secret only on localhost; +Caddy publishes only read/verification routes. Do not use the checked-in secret +markers directly. Add your ACME contact email under `ca.server.acme.email` if +desired. ACME account creation accepts the issuer's terms as documented by the +adapter. + +Install staging roots **only into the RA-specific trust file**, not Ubuntu's +system/browser trust store: + +```sh +( + set -eu + ans_roots_work=$(mktemp -d) + trap 'rm -rf "$ans_roots_work"' EXIT + curl -fsSL https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem -o "$ans_roots_work/x1.pem" + curl -fsSL https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x2.pem -o "$ans_roots_work/x2.pem" + openssl x509 -in "$ans_roots_work/x1.pem" -noout -subject + openssl x509 -in "$ans_roots_work/x2.pem" -noout -subject + cat "$ans_roots_work/x1.pem" "$ans_roots_work/x2.pem" > "$ans_roots_work/roots.pem" + sudo install -o root -g ans-ra -m 0640 "$ans_roots_work/roots.pem" /etc/ans/le-staging-roots.pem +) +``` + +## 4. Start RA/TL and bootstrap signing trust + +```sh +sudo install -o root -g root -m 0644 deploy/ubuntu/ans-ra.service deploy/ubuntu/ans-tl.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now ans-tl ans-ra +sudo journalctl -u ans-tl -u ans-ra -n 80 --no-pager +curl --fail http://127.0.0.1:18080/v2/admin/ready +curl --fail http://127.0.0.1:18081/v2/admin/ready +``` + +Wait for both readiness checks to pass. Then seed the RA public key into the +TL configuration and restart TL. No private signing key is copied. This +initial seed uses the TL's existing ten-year bootstrap validity policy; manage +subsequent rotations through its private producer-key admin API. + +```sh +sudo python3 - <<'PYTRUST' +from pathlib import Path +import yaml +ra = yaml.safe_load(Path('/etc/ans/ra.yaml').read_text()) +p = Path('/etc/ans/tl.yaml') +tl = yaml.safe_load(p.read_text()) +kid = ra['signer']['keyId'] +entry = { + 'raId': ra['signer']['raId'], + 'keyId': kid, + 'algorithm': 'ES256', + 'publicKeyPem': (Path(ra['keys']['file']['path']) / (kid + '.pub')).read_text(), +} +existing = tl.setdefault('producerKeys', []) +matching = [e for e in existing if e['keyId'] == kid] +if matching and matching != [entry]: + raise SystemExit('Existing producer key differs; investigate before changing trust') +if not matching: + existing.append(entry) + p.write_text(yaml.safe_dump(tl, sort_keys=False)) +print('RA public key configured for TL bootstrap.') +PYTRUST +sudo systemctl restart ans-tl +curl --fail http://127.0.0.1:18081/v2/admin/ready +``` + +If the last readiness request races startup, retry it after checking the +journal. Confirm the TL journal reports successful producer-key bootstrap. +Health alone does not prove end-to-end event delivery. + +## 5. DNS, firewall, and HTTPS + +Point `ra.ans.example.com` and `tl.ans.example.com` A records at the server. +Add AAAA only with working public IPv6. Any CAA records must permit +`letsencrypt.org`. Allow inbound TCP 80/443 in the host/cloud firewall while +preserving the existing SSH rule. Never expose 18080/18081/18082. No DNS API +credentials are required for Caddy's normal HTTP/TLS challenges. Agent +registration still requires publishing its challenge and ANS DNS records. + +Use DNS-only records during initial setup. If you later enable a proxy, its +edge certificate must cover the exact hostnames; a wildcard for example.com +does not cover ra.ans.example.com. Agent certificate/TLSA checks must see the +certificate authorized by the agent registration. + +The example RA queries `1.1.1.1:53` for DNS verification. Choose a reachable +recursive resolver for your environment, or omit `dns.server` to use the +system resolver. An old answer from a recursive cache is not evidence that +an authoritative update failed; compare authoritative answers and TTLs. + +After DNS is ready, from the repository root: + +```sh +sudo install -o root -g caddy -m 0640 deploy/ubuntu/Caddyfile /etc/caddy/Caddyfile +sudo caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile +sudo systemctl enable --now caddy +sudo systemctl reload caddy +sudo journalctl -u caddy -n 80 --no-pager +curl --fail https://ra.ans.example.com/docs +curl --fail https://tl.ans.example.com/root-keys +curl -o /dev/null -w '%{http_code}\n' https://tl.ans.example.com/internal/v1/producer-keys +``` + +Also open `https://tl.ans.example.com/docs`; +the proxy explicitly allows both `/docs` and its static assets. Swagger uses +the public service origin for requests. + +The last request must return 404. Caddy obtains/renews RA/TL certificates and +redirects HTTP to HTTPS; no Certbot is needed. Its administration API remains +loopback-only. Check backend ports are unreachable from another machine. + +Back up `/etc/ans`, `/etc/caddy`, `/var/lib/ans-ra`, `/var/lib/ans-tl`, and +`/var/lib/caddy`, protecting keys/secrets. For a simple consistent backup, +stop RA and TL while copying their complete state directories, then restart +them. Run only one TL writer per tile directory. On upgrades stop a service +before replacing its binary, then start it again. + +When moving agent issuance to Let's Encrypt production, change the RA ACME +directory URL, use a separate production ACME data directory, and remove the +staging-only `ca.validation.roots-file` setting. Reissue/register suitable +trusted agent certificates. Caddy's RA/TL certificates already use production. + +## Verify a registration and maintain the deployment + +Use the RA Swagger UI with an OIDC bearer token to register an agent, publish +one returned ACME challenge, and call `verify-acme`. Retrieve the issued +certificates, publish the returned permanent DNS records, and call `verify-dns`. +Wait for ACTIVE and verify the TL receipt/status before serving the agent with +its registered certificate. Registration version and metadata hashes must +match the exact metadata documents the agent serves. RA/TL readiness alone +does not prove this full lifecycle. + +Run ordinary setup validation with a registration owned by your account. A +controlled renewal/revocation exercise and backup restoration are separate +acceptance steps; do not revoke a live user's agent merely to check setup. + +Caddy automatically renews the RA/TL service certificates. It does not rotate +agent certificates issued through the RA or install them into an agent. Plan +agent renewal, TL publication, DNS updates, and certificate installation +before expiry. Post-renewal verification and sealing of changed DNS evidence +is not implemented: `verify-dns` on an ACTIVE registration currently returns +without refreshing its sealed DNS snapshot. Do not treat it as a DNS-update API. + +Configure off-host encrypted backups and monitoring for service readiness, +outbox delivery failures, disk space, and certificate expiry. The units do not +install a backup schedule or certificate-installation automation. + +This guide covers RA/TL only. An A2A/MCP agent, its ANS authentication, +metadata, and any application frontend are separate deployments. diff --git a/deploy/ubuntu/ans-ra.service b/deploy/ubuntu/ans-ra.service new file mode 100644 index 0000000..6e836b7 --- /dev/null +++ b/deploy/ubuntu/ans-ra.service @@ -0,0 +1,29 @@ +[Unit] +Description=ANS Registration Authority +Wants=network-online.target ans-tl.service +After=network-online.target ans-tl.service + +[Service] +Type=simple +User=ans-ra +Group=ans-ra +ExecStart=/usr/local/bin/ans-ra --config /etc/ans/ra.yaml +WorkingDirectory=/var/lib/ans-ra +StateDirectory=ans-ra +StateDirectoryMode=0700 +UMask=0077 +Restart=on-failure +RestartSec=5s +TimeoutStopSec=30s +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictSUIDSGID=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/ubuntu/ans-tl.service b/deploy/ubuntu/ans-tl.service new file mode 100644 index 0000000..591c690 --- /dev/null +++ b/deploy/ubuntu/ans-tl.service @@ -0,0 +1,29 @@ +[Unit] +Description=ANS Transparency Log +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +User=ans-tl +Group=ans-tl +ExecStart=/usr/local/bin/ans-tl --config /etc/ans/tl.yaml +WorkingDirectory=/var/lib/ans-tl +StateDirectory=ans-tl +StateDirectoryMode=0700 +UMask=0077 +Restart=on-failure +RestartSec=5s +TimeoutStopSec=30s +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictSUIDSGID=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/ubuntu/install-packages.sh b/deploy/ubuntu/install-packages.sh new file mode 100644 index 0000000..aa57191 --- /dev/null +++ b/deploy/ubuntu/install-packages.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Install native-host build/runtime dependencies. Run with sudo on Ubuntu. +set -euo pipefail + +if [[ ${EUID} -ne 0 ]]; then + echo "Run with sudo bash deploy/ubuntu/install-packages.sh" >&2 + exit 1 +fi +. /etc/os-release +if [[ ${ID} != ubuntu ]]; then + echo "This installer targets Ubuntu." >&2 + exit 1 +fi + +apt-get update +apt-get install -y \ + ca-certificates curl gnupg debian-keyring debian-archive-keyring \ + apt-transport-https git build-essential jq openssl dnsutils \ + python3 python3-yaml + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# Official Caddy stable APT repository. +curl -fsSL --retry 3 https://dl.cloudsmith.io/public/caddy/stable/gpg.key -o "$work/caddy.asc" +gpg --batch --yes --dearmor -o "$work/caddy.gpg" "$work/caddy.asc" +install -m 0644 "$work/caddy.gpg" /usr/share/keyrings/caddy-stable-archive-keyring.gpg +curl -fsSL --retry 3 https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt -o "$work/caddy.list" +install -m 0644 "$work/caddy.list" /etc/apt/sources.list.d/caddy-stable.list +apt-get update +apt-get install -y caddy + +# Ubuntu's golang-go may be older than the project's Go 1.26 minimum. +# Stay on the project's Go 1.26 line: the pinned linter must understand +# the compiler's export format. Verify the selected patch release SHA-256. +case "$(dpkg --print-architecture)" in + amd64) ans_arch=amd64 ;; + arm64) ans_arch=arm64 ;; + *) echo "Supported server architectures: amd64 and arm64" >&2; exit 1 ;; +esac +curl -fsSL --retry 3 'https://go.dev/dl/?mode=json' -o "$work/releases.json" +ans_version=$(jq -er '[.[] | select(.stable == true and (.version | startswith("go1.26.")))][0].version' "$work/releases.json") +if [[ ! $ans_version =~ ^go1\.26\.[0-9]+$ ]]; then + echo "No supported Go 1.26 patch release found in the upstream release index." >&2 + exit 1 +fi +ans_archive=$(jq -er --arg v "$ans_version" --arg a "$ans_arch" \ + '.[] | select(.version == $v) | .files[] | select(.os == "linux" and .arch == $a and .kind == "archive") | .filename' "$work/releases.json") +ans_sha=$(jq -er --arg v "$ans_version" --arg f "$ans_archive" \ + '.[] | select(.version == $v) | .files[] | select(.filename == $f) | .sha256' "$work/releases.json") +curl -fsSL --retry 3 "https://go.dev/dl/$ans_archive" -o "$work/$ans_archive" +(cd "$work" && printf '%s %s\n' "$ans_sha" "$ans_archive" | sha256sum --check --status) +tar -C "$work" -xzf "$work/$ans_archive" +install -d -m 0755 /opt/ans-toolchains +if [[ ! -e /opt/ans-toolchains/$ans_version ]]; then + mv "$work/go" "/opt/ans-toolchains/$ans_version" +fi +ln -sfn "/opt/ans-toolchains/$ans_version/bin/go" /usr/local/bin/go +ln -sfn "/opt/ans-toolchains/$ans_version/bin/gofmt" /usr/local/bin/gofmt +/usr/local/bin/go version +caddy version + +echo "Packages installed. Follow deploy/ubuntu/README.md to build and configure RA/TL." diff --git a/deploy/ubuntu/ra.yaml b/deploy/ubuntu/ra.yaml new file mode 100644 index 0000000..7652363 --- /dev/null +++ b/deploy/ubuntu/ra.yaml @@ -0,0 +1,52 @@ +# Template: follow README.md to install with a generated service secret. +server: + host: "127.0.0.1" + port: 18080 +auth: + type: oidc + oidc: + issuer-url: "https://YOUR_INSTANCE.clerk.accounts.dev" + audience: "ans-ra" + client-id: "" + admin-groups: [] +ca: + type: self + self: + org: "ANS Staging Identity CA" + validity-days: 365 + data-dir: "/var/lib/ans-ra/ca" + server: + type: acme + acme: + directory-url: "https://acme-staging-v02.api.letsencrypt.org/directory" + data-dir: "/var/lib/ans-ra/acme-staging" + validation: + roots-file: "/etc/ans/le-staging-roots.pem" +dns: + type: lookup + server: "1.1.1.1:53" +identity: + resolver: + type: web + challenge-ttl: 5m +vlei: + type: "off" +keys: + type: file + file: + path: "/var/lib/ans-ra/keys" +store: + type: sqlite + sqlite: + path: "/var/lib/ans-ra/ans.db" +tl-client: + base-url: "http://127.0.0.1:18081" + public-base-url: "https://tl.ans.example.com" + api-key: "__TL_SERVICE_KEY__" + timeout: 10s +signer: + keyId: "ans-ra-example-signer" + raId: "ans-ra-example" +log: + level: info + format: json diff --git a/deploy/ubuntu/tl.yaml b/deploy/ubuntu/tl.yaml new file mode 100644 index 0000000..8f77db4 --- /dev/null +++ b/deploy/ubuntu/tl.yaml @@ -0,0 +1,30 @@ +# Template: follow README.md to install with a generated service secret. +server: + host: "127.0.0.1" + port: 18081 +auth: + type: static + static: + api-key: "__TL_SERVICE_KEY__" + public-read: true +keys: + type: file + file: + path: "/var/lib/ans-tl/keys" +store: + type: sqlite + sqlite: + path: "/var/lib/ans-tl/tl.db" +merkle: + origin: "tl.ans.example.com" + tile-storage: + type: filesystem + filesystem: + path: "/var/lib/ans-tl/tiles" + checkpoint-interval: 10s +attestation: + keyId: "ans-tl-example-attestation" +producerKeys: [] +log: + level: info + format: json From e0dadf036fe5d5713a7c21db6a49c295ec49a71f Mon Sep 17 00:00:00 2001 From: Connor Snitker Date: Mon, 21 Sep 2026 16:16:40 -0500 Subject: [PATCH 2/3] docs(deploy): link lifecycle and recovery upgrade guidance Signed-off-by: Connor Snitker --- deploy/ubuntu/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/ubuntu/README.md b/deploy/ubuntu/README.md index ea43216..0413d7c 100644 --- a/deploy/ubuntu/README.md +++ b/deploy/ubuntu/README.md @@ -262,3 +262,5 @@ install a backup schedule or certificate-installation automation. This guide covers RA/TL only. An A2A/MCP agent, its ANS authentication, metadata, and any application frontend are separate deployments. + +For existing installations, follow the [certificate lifecycle and TL recovery upgrade notes](../../docs/operations/deployment-fix-upgrade.md) before replacing binaries. From 3de914f7b4b26948da3440221286c13c82c0a5a9 Mon Sep 17 00:00:00 2001 From: Connor Snitker Date: Fri, 2 Oct 2026 13:13:08 -0500 Subject: [PATCH 3/3] docs(deploy): harden and validate native Ubuntu setup Signed-off-by: Connor Snitker --- deploy/ubuntu/Caddyfile | 8 +- deploy/ubuntu/README.md | 120 +++++++++++++++--------------- deploy/ubuntu/ans-ra.service | 10 +++ deploy/ubuntu/ans-tl.service | 10 +++ deploy/ubuntu/bootstrap-trust.sh | 70 +++++++++++++++++ deploy/ubuntu/install-config.sh | 42 +++++++++++ deploy/ubuntu/install-packages.sh | 18 +++-- deploy/ubuntu/ra.yaml | 3 +- 8 files changed, 210 insertions(+), 71 deletions(-) create mode 100644 deploy/ubuntu/bootstrap-trust.sh create mode 100644 deploy/ubuntu/install-config.sh diff --git a/deploy/ubuntu/Caddyfile b/deploy/ubuntu/Caddyfile index 784fe85..61845a9 100644 --- a/deploy/ubuntu/Caddyfile +++ b/deploy/ubuntu/Caddyfile @@ -5,6 +5,7 @@ ra.ans.example.com { redir /docs/ /docs 308 + header Strict-Transport-Security "max-age=31536000" # Keep authenticated API responses out of shared proxy caches. header Cache-Control "no-store" # Keep operational endpoints accessible only through localhost. @@ -15,8 +16,11 @@ ra.ans.example.com { tl.ans.example.com { redir /docs/ /docs 308 - # Fresh status/revocation evidence must not be served from an edge cache. - header Cache-Control "no-store" + header Strict-Transport-Security "max-age=31536000" + # Mutable status and revocation evidence stays fresh. Leave tile and + # checkpoint cache policy to the TL instead of overriding every route. + @current path /v1/agents/* /v1/identities/* /v1/log/* + header @current Cache-Control "no-store" # Public TL access is read-only. Ingestion and producer-key admin # stay on the private backend, including both API versions. @public { diff --git a/deploy/ubuntu/README.md b/deploy/ubuntu/README.md index 0413d7c..60a9c33 100644 --- a/deploy/ubuntu/README.md +++ b/deploy/ubuntu/README.md @@ -1,5 +1,12 @@ # Native Ubuntu deployment +Install and startup verified on Ubuntu **24.04.5 LTS arm64**, with Go **1.26.8** +and Caddy **2.11.6**, on 2026-10-02. Verification covered package installation, +binary builds, config permissions/overwrite refusal, systemd readiness, API +producer-key bootstrap, and proxy routes. Proxy HTTPS used a temporary local +CA; public DNS, Let's Encrypt issuance and authenticated registration remain +deployment-specific acceptance steps. + Run RA, TL, and Caddy directly on Ubuntu under systemd. No Docker, external SQL server, or Node.js runtime is required for RA/TL. These instructions assume a recent systemd-based Ubuntu server on amd64 or arm64, sudo access, and the @@ -25,7 +32,8 @@ Choose stable signer key IDs and RA IDs before first startup. Preserve them and their keys on upgrades. The `__TL_SERVICE_KEY__` marker is replaced by the installation command below; do not substitute a real secret into the repository. -For Clerk, create a JWT template named `ans-ra` with an `aud` claim matching +Configure an OIDC provider with a discoverable issuer, JWKS, and tokens whose +`aud` matches `auth.oidc.audience`. As one example, with Clerk create a JWT template named `ans-ra` with an `aud` claim matching `auth.oidc.audience` (the example uses `ans-ra`). Send the resulting template JWT as `Authorization: Bearer ` to the RA. Use your own issuer; a Clerk development instance is not a production identity deployment. Other OIDC @@ -46,10 +54,10 @@ sudo bash deploy/ubuntu/install-packages.sh The script installs `ca-certificates`, `curl`, `gnupg`, `debian-keyring`, `debian-archive-keyring`, `apt-transport-https`, `git`, `build-essential`, `jq`, `openssl`, `dnsutils`, `python3`, and `python3-yaml` from Ubuntu; Caddy from -its official stable APT repository; and the latest Go 1.26 patch release from -`go.dev`, checking its published SHA-256. Stay on this Go release line until -the pinned linter supports newer compiler export formats; Go 1.27 failed the -current linter's type checks during the first server build. +its official stable APT repository; and the latest stable Go patch on the +release line declared by the `go` directive in this checkout's `go.mod`, +checking its published SHA-256. The complete upstream release index is used +so the installer still finds that line after newer Go releases appear. The package installer requires outbound network access and sudo; it changes APT sources and `/usr/local/bin/go` and `/usr/local/bin/gofmt`. Go is installed in `/opt/ans-toolchains/` with `go` and `gofmt` @@ -90,32 +98,23 @@ This refuses to overwrite existing configurations. Secrets remain in files readable by root and the corresponding service group, not in shell history: ```sh -sudo python3 - <<'PYCONFIG' -import grp, os, secrets -from pathlib import Path -pairs = [('ra', 'ans-ra'), ('tl', 'ans-tl')] -for name, group in pairs: - if Path(f'/etc/ans/{name}.yaml').exists(): - raise SystemExit(f'/etc/ans/{name}.yaml already exists; preserve its secret and edit deliberately') -key = secrets.token_hex(32) -for name, group in pairs: - text = Path(f'deploy/ubuntu/{name}.yaml').read_text() - if text.count('__TL_SERVICE_KEY__') != 1: - raise SystemExit('Expected exactly one service secret marker') - text = text.replace('__TL_SERVICE_KEY__', key) - fd = os.open(f'/etc/ans/{name}.yaml', os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640) - with os.fdopen(fd, 'w') as f: - os.fchown(f.fileno(), 0, grp.getgrnam(group).gr_gid) - os.fchmod(f.fileno(), 0o640) - f.write(text) -print('Installed RA/TL configuration with a generated service secret.') -PYCONFIG +sudo bash deploy/ubuntu/install-config.sh ``` -The templates bind both services to `127.0.0.1`, configure Clerk for RA users, +The templates bind both services to `127.0.0.1`, configure OIDC for RA users, use DNS lookup and real did:web resolution, and disable vLEI (`off`) until a real verifier is deployed. The TL accepts its service secret only on localhost; -Caddy publishes only read/verification routes. Do not use the checked-in secret +Caddy publishes only read/verification routes. + +**Trust boundary:** the TL's static service key grants administrator privileges, +including producer-key changes, as well as ingestion. The RA holds that key. +Compromise of the RA process therefore permits changes to the TL producer trust +store over loopback. Separate Unix users and Caddy's public allowlist do not +remove this authority. This single-host profile assumes RA and TL share an +administrative trust boundary. Deployments requiring separation must provide a +least-privileged ingest authorization path before using this profile. + +Do not use the checked-in secret markers directly. Add your ACME contact email under `ca.server.acme.email` if desired. ACME account creation accepts the issuer's terms as documented by the adapter. @@ -149,40 +148,19 @@ curl --fail http://127.0.0.1:18081/v2/admin/ready ``` Wait for both readiness checks to pass. Then seed the RA public key into the -TL configuration and restart TL. No private signing key is copied. This -initial seed uses the TL's existing ten-year bootstrap validity policy; manage -subsequent rotations through its private producer-key admin API. +TL through its loopback producer-key admin API. No private signing key is +copied and no YAML rewrite or TL restart is needed. The initial key uses a +ten-year validity interval; plan rotations before expiry. Re-running the script +verifies an existing identical, active key and refuses conflicting/revoked keys. ```sh -sudo python3 - <<'PYTRUST' -from pathlib import Path -import yaml -ra = yaml.safe_load(Path('/etc/ans/ra.yaml').read_text()) -p = Path('/etc/ans/tl.yaml') -tl = yaml.safe_load(p.read_text()) -kid = ra['signer']['keyId'] -entry = { - 'raId': ra['signer']['raId'], - 'keyId': kid, - 'algorithm': 'ES256', - 'publicKeyPem': (Path(ra['keys']['file']['path']) / (kid + '.pub')).read_text(), -} -existing = tl.setdefault('producerKeys', []) -matching = [e for e in existing if e['keyId'] == kid] -if matching and matching != [entry]: - raise SystemExit('Existing producer key differs; investigate before changing trust') -if not matching: - existing.append(entry) - p.write_text(yaml.safe_dump(tl, sort_keys=False)) -print('RA public key configured for TL bootstrap.') -PYTRUST -sudo systemctl restart ans-tl -curl --fail http://127.0.0.1:18081/v2/admin/ready +sudo bash deploy/ubuntu/bootstrap-trust.sh ``` -If the last readiness request races startup, retry it after checking the -journal. Confirm the TL journal reports successful producer-key bootstrap. -Health alone does not prove end-to-end event delivery. +The script reads the administrator credential from root-controlled config; it +does not place it in command arguments, output, or shell history. A successful +bootstrap confirms trust-store configuration; health alone does not prove +end-to-end event delivery. ## 5. DNS, firewall, and HTTPS @@ -218,7 +196,9 @@ curl -o /dev/null -w '%{http_code}\n' https://tl.ans.example.com/internal/v1/pro Also open `https://tl.ans.example.com/docs`; the proxy explicitly allows both `/docs` and its static assets. Swagger uses -the public service origin for requests. +the public service origin for requests. Deploy with the Swagger fix in #135, +which pins the CDN JavaScript and CSS with Subresource Integrity. An SSH tunnel +alone does not protect bearer tokens from a tampered CDN script. The last request must return 404. Caddy obtains/renews RA/TL certificates and redirects HTTP to HTTPS; no Certbot is needed. Its administration API remains @@ -252,9 +232,17 @@ acceptance steps; do not revoke a live user's agent merely to check setup. Caddy automatically renews the RA/TL service certificates. It does not rotate agent certificates issued through the RA or install them into an agent. Plan agent renewal, TL publication, DNS updates, and certificate installation -before expiry. Post-renewal verification and sealing of changed DNS evidence -is not implemented: `verify-dns` on an ACTIVE registration currently returns -without refreshing its sealed DNS snapshot. Do not treat it as a DNS-update API. +before expiry. Post-renewal DNS resealing remains tracked in +[ans-registry#66](https://github.com/agentnameservice/ans-registry/issues/66): +`verify-dns` on an ACTIVE registration does not refresh its sealed snapshot. + +Use external monitoring for `GET https://tl.ans.example.com/root-keys` and +`GET https://ra.ans.example.com/docs`. These check public routing/availability; +run `/v2/admin/ready` checks locally because Caddy intentionally returns 404 +for them. Alert separately on private readiness and public endpoint failures. + +Inspect unit confinement after installation with +`sudo systemd-analyze security ans-ra.service ans-tl.service`. Configure off-host encrypted backups and monitoring for service readiness, outbox delivery failures, disk space, and certificate expiry. The units do not @@ -264,3 +252,13 @@ This guide covers RA/TL only. An A2A/MCP agent, its ANS authentication, metadata, and any application frontend are separate deployments. For existing installations, follow the [certificate lifecycle and TL recovery upgrade notes](../../docs/operations/deployment-fix-upgrade.md) before replacing binaries. + +## Source and merge sequence + +The source release must include #125, #126, #127, and #128 in that dependency +order, plus #135 for Swagger. After each parent merges, rebase its dependent +branch onto the merged `main` and retarget the PR before merging it. For a +squash merge, drop the old parent commits during that rebase; do not merge a +child whose diff still contains a second copy of the parent changes. Merge +this deployment PR onto `main` last so `Fixes #133` closes against the default +branch. Check the resulting diff before deleting the old stack branches. diff --git a/deploy/ubuntu/ans-ra.service b/deploy/ubuntu/ans-ra.service index 6e836b7..5f92268 100644 --- a/deploy/ubuntu/ans-ra.service +++ b/deploy/ubuntu/ans-ra.service @@ -17,6 +17,16 @@ RestartSec=5s TimeoutStopSec=30s NoNewPrivileges=true PrivateTmp=true +PrivateDevices=true +CapabilityBoundingSet= +ProtectClock=true +ProtectHostname=true +LockPersonality=true +RestrictRealtime=true +RestrictNamespaces=true +SystemCallFilter=@system-service +SystemCallArchitectures=native +ProtectProc=invisible ProtectSystem=strict ProtectHome=true ProtectKernelTunables=true diff --git a/deploy/ubuntu/ans-tl.service b/deploy/ubuntu/ans-tl.service index 591c690..12430e0 100644 --- a/deploy/ubuntu/ans-tl.service +++ b/deploy/ubuntu/ans-tl.service @@ -17,6 +17,16 @@ RestartSec=5s TimeoutStopSec=30s NoNewPrivileges=true PrivateTmp=true +PrivateDevices=true +CapabilityBoundingSet= +ProtectClock=true +ProtectHostname=true +LockPersonality=true +RestrictRealtime=true +RestrictNamespaces=true +SystemCallFilter=@system-service +SystemCallArchitectures=native +ProtectProc=invisible ProtectSystem=strict ProtectHome=true ProtectKernelTunables=true diff --git a/deploy/ubuntu/bootstrap-trust.sh b/deploy/ubuntu/bootstrap-trust.sh new file mode 100644 index 0000000..a0226ee --- /dev/null +++ b/deploy/ubuntu/bootstrap-trust.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Read only the RA public signer key; register it through the private TL admin API. +set -euo pipefail +if [[ ${EUID} -ne 0 ]]; then + echo "Run with sudo bash deploy/ubuntu/bootstrap-trust.sh" >&2 + exit 1 +fi +python3 - <<'PY' +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path +import urllib.error +import urllib.parse +import urllib.request +import yaml + +ra = yaml.safe_load(Path('/etc/ans/ra.yaml').read_text()) +tl = yaml.safe_load(Path('/etc/ans/tl.yaml').read_text()) +kid = ra['signer']['keyId'] +if not kid or Path(kid).name != kid or kid in ('.', '..'): + raise SystemExit('RA signer keyId is not a safe key filename') +entry = { + 'key_id': kid, + 'ra_id': ra['signer']['raId'], + 'algorithm': 'ES256', + 'public_key_pem': (Path(ra['keys']['file']['path']) / (kid + '.pub')).read_text(), +} +if tl['server']['host'] != '127.0.0.1': + raise SystemExit('Bootstrap requires the documented loopback TL listener') +base = f"http://127.0.0.1:{int(tl['server']['port'])}/internal/v1/producer-keys" +# Neither environment proxies nor redirects may receive the administrator key. +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None +opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) +def request(method, url, body=None): + req = urllib.request.Request(url, method=method, + data=None if body is None else json.dumps(body).encode(), + headers={'Authorization': 'Bearer ' + tl['auth']['static']['api-key'], + 'Content-Type': 'application/json'}) + try: + with opener.open(req, timeout=15) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + with error: + # Do not echo arbitrary server bodies or credential-bearing requests. + return error.code, None + except urllib.error.URLError: + raise SystemExit('Cannot reach the loopback TL; check readiness and its journal') from None +url = base + '/' + urllib.parse.quote(kid, safe='') +status, existing = request('GET', url) +if status == 404: + now = datetime.now(timezone.utc) + body = dict(entry, valid_from=now.isoformat(), expires_at=(now + timedelta(days=3650)).isoformat()) + status, _ = request('POST', base, body) + if status not in (200, 409): + raise SystemExit(f'TL producer-key creation failed with HTTP {status}') + status, existing = request('GET', url) +if status != 200: + raise SystemExit(f'TL producer-key lookup failed with HTTP {status}') +for field, expected in entry.items(): + if str(existing.get(field, '')).strip() != expected.strip(): + raise SystemExit(f'Existing producer key differs in {field}; investigate before changing trust') +now = datetime.now(timezone.utc) +if existing.get('status') != 'active' or not ( + datetime.fromisoformat(existing['valid_from'].replace('Z', '+00:00')) <= now < + datetime.fromisoformat(existing['expires_at'].replace('Z', '+00:00'))): + raise SystemExit('Existing producer key is revoked or outside its validity interval; rotate deliberately') +print('RA public signer key is active in the TL trust store. No TL restart required.') +PY diff --git a/deploy/ubuntu/install-config.sh b/deploy/ubuntu/install-config.sh new file mode 100644 index 0000000..f2a724c --- /dev/null +++ b/deploy/ubuntu/install-config.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Install first-time configuration; existing secrets/configuration are never replaced. +set -euo pipefail +if [[ ${EUID} -ne 0 ]]; then + echo "Run with sudo bash deploy/ubuntu/install-config.sh" >&2 + exit 1 +fi +ans_templates=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +python3 - "$ans_templates" <<'PY' +import grp +import os +from pathlib import Path +import secrets +import sys + +key = secrets.token_hex(32) +prepared = [] +for name, group in [('ra', 'ans-ra'), ('tl', 'ans-tl')]: + dest = Path('/etc/ans') / (name + '.yaml') + if dest.exists() or dest.is_symlink(): + raise SystemExit(f'{dest} exists; preserve its secret and edit deliberately') + text = (Path(sys.argv[1]) / (name + '.yaml')).read_text() + if text.count('__TL_SERVICE_KEY__') != 1: + raise SystemExit(f'{name}.yaml must contain exactly one service-secret marker') + prepared.append((dest, grp.getgrnam(group).gr_gid, text.replace('__TL_SERVICE_KEY__', key))) +created = [] +try: + for dest, gid, text in prepared: + fd = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640) + created.append(dest) + with os.fdopen(fd, 'w') as stream: + os.fchown(stream.fileno(), 0, gid) + os.fchmod(stream.fileno(), 0o640) + stream.write(text) + stream.flush() + os.fsync(stream.fileno()) +except BaseException: + for dest in created: + dest.unlink() + raise +print('Installed RA/TL configurations with one generated service secret.') +PY diff --git a/deploy/ubuntu/install-packages.sh b/deploy/ubuntu/install-packages.sh index aa57191..f1f44a9 100644 --- a/deploy/ubuntu/install-packages.sh +++ b/deploy/ubuntu/install-packages.sh @@ -30,18 +30,22 @@ install -m 0644 "$work/caddy.list" /etc/apt/sources.list.d/caddy-stable.list apt-get update apt-get install -y caddy -# Ubuntu's golang-go may be older than the project's Go 1.26 minimum. -# Stay on the project's Go 1.26 line: the pinned linter must understand -# the compiler's export format. Verify the selected patch release SHA-256. +# Build with the release line declared by this checkout; checksum the archive. +ans_repo=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) +ans_go_line=$(awk '$1 == "go" {split($2, v, "."); print v[1] "." v[2]; exit}' "$ans_repo/go.mod") +if [[ ! $ans_go_line =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "Cannot determine the Go release line from go.mod." >&2 + exit 1 +fi case "$(dpkg --print-architecture)" in amd64) ans_arch=amd64 ;; arm64) ans_arch=arm64 ;; *) echo "Supported server architectures: amd64 and arm64" >&2; exit 1 ;; esac -curl -fsSL --retry 3 'https://go.dev/dl/?mode=json' -o "$work/releases.json" -ans_version=$(jq -er '[.[] | select(.stable == true and (.version | startswith("go1.26.")))][0].version' "$work/releases.json") -if [[ ! $ans_version =~ ^go1\.26\.[0-9]+$ ]]; then - echo "No supported Go 1.26 patch release found in the upstream release index." >&2 +curl -fsSL --retry 3 'https://go.dev/dl/?mode=json&include=all' -o "$work/releases.json" +ans_version=$(jq -er --arg prefix "go${ans_go_line}." '[.[] | select(.stable == true and (.version | startswith($prefix)))][0].version' "$work/releases.json") +if [[ $ans_version != "go${ans_go_line}."* ]]; then + echo "No stable Go ${ans_go_line} patch release found in the upstream release index." >&2 exit 1 fi ans_archive=$(jq -er --arg v "$ans_version" --arg a "$ans_arch" \ diff --git a/deploy/ubuntu/ra.yaml b/deploy/ubuntu/ra.yaml index 7652363..e85c34c 100644 --- a/deploy/ubuntu/ra.yaml +++ b/deploy/ubuntu/ra.yaml @@ -5,7 +5,7 @@ server: auth: type: oidc oidc: - issuer-url: "https://YOUR_INSTANCE.clerk.accounts.dev" + issuer-url: "https://issuer.example.com" audience: "ans-ra" client-id: "" admin-groups: [] @@ -42,6 +42,7 @@ store: tl-client: base-url: "http://127.0.0.1:18081" public-base-url: "https://tl.ans.example.com" + # The static key grants TL administration as well as ingest. See README trust boundary. api-key: "__TL_SERVICE_KEY__" timeout: 10s signer: