diff --git a/.github/workflows/hosting-authority.yml b/.github/workflows/hosting-authority.yml new file mode 100644 index 00000000..d5756fc3 --- /dev/null +++ b/.github/workflows/hosting-authority.yml @@ -0,0 +1,68 @@ +name: Hosting authority regression +on: + push: + branches: ['fix/hosting-authority-*'] + pull_request: + paths: + - 'src/msg/hosting_runtime.py' + - 'src/msg/storage/read_only.py' + - 'tests/test_hosting_runtime_authority.py' + - '.github/workflows/hosting-authority.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: hosting-authority-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + authority: + runs-on: ubuntu-latest + timeout-minutes: 15 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest -d postgres" + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Bind evidence to source + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, pathlib, platform, subprocess + git=lambda *args: subprocess.check_output(['git',*args],text=True).strip() + pathlib.Path('artifacts/source.json').write_text(json.dumps({ + 'head':os.environ['HEAD_SHA'], 'checkout':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'), 'python':platform.python_version(), + 'run_id':os.environ['GITHUB_RUN_ID'], 'attempt':os.environ['GITHUB_RUN_ATTEMPT']},indent=2)+'\n') + PY + - name: Install disposable test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Exercise revoked keys and restorable ownership rights + run: python -m pytest tests/test_hosting_runtime_authority.py --junitxml=artifacts/authority.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: hosting-authority-evidence + path: artifacts/ + if-no-files-found: error diff --git a/.github/workflows/hosting-readonly.yml b/.github/workflows/hosting-readonly.yml new file mode 100644 index 00000000..6ed927d9 --- /dev/null +++ b/.github/workflows/hosting-readonly.yml @@ -0,0 +1,75 @@ +name: Hosting read-only boundary +on: + push: + branches: ['fix/hosting-readonly-20260929'] + pull_request: + paths: + - 'src/msg/**' + - 'tests/**' + - 'deploy/**' + - '.github/workflows/hosting-readonly.yml' + - 'pyproject.toml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: hosting-readonly-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + hosting: + runs-on: ubuntu-latest + timeout-minutes: 15 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest" + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install isolated test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Bind evidence to checkout and requested head + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, platform, shutil, subprocess + from pathlib import Path + git=lambda *args: subprocess.check_output(['git',*args],text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({ + 'head':os.environ['HEAD_SHA'], 'checkout':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'), 'python':platform.python_version(), + 'run_id':os.environ['GITHUB_RUN_ID'], 'attempt':os.environ['GITHUB_RUN_ATTEMPT']},indent=2)) + for source in Path('tests').glob('test_hosting*.py'): + shutil.copyfile(source,Path('artifacts')/source.name) + PY + - name: Exercise isolated hosting and retain established web regressions + run: >- + python -m pytest tests/test_hosting*.py tests/test_cli_hosting.py + tests/test_extensions.py::test_hosting_publish_is_explicit_versioned_on_service_origin + --junitxml=artifacts/hosting.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: hosting-readonly-evidence + path: artifacts/ + if-no-files-found: error diff --git a/.github/workflows/token-delivery-boundary.yml b/.github/workflows/token-delivery-boundary.yml new file mode 100644 index 00000000..5178b251 --- /dev/null +++ b/.github/workflows/token-delivery-boundary.yml @@ -0,0 +1,82 @@ +name: Credential delivery boundary +on: + push: + branches: [fix/token-delivery-boundary-20260929] + pull_request: + paths: + - 'src/msg/**' + - 'tests/test_token_delivery_boundary.py' + - '.github/workflows/token-delivery-boundary.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: token-delivery-${{ github.event.pull_request.head.sha || github.sha }} + cancel-in-progress: false +jobs: + focused: + runs-on: ubuntu-latest + timeout-minutes: 12 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest" + --health-interval 5s --health-timeout 5s --health-retries 10 + valkey: + image: valkey/valkey:9.0.6 + ports: ['6379:6379'] + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + MSG_TEST_VALKEY_URL: redis://127.0.0.1:6379/0 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install isolated test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Record source identity + env: + EVENT_HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, platform, subprocess + from pathlib import Path + def git(ref): + return subprocess.check_output(['git', 'rev-parse', ref], text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({ + 'head': os.environ['EVENT_HEAD'], 'checkout': git('HEAD'), 'tree': git('HEAD^{tree}'), + 'run': os.environ['GITHUB_RUN_ID'], 'attempt': os.environ['GITHUB_RUN_ATTEMPT'], + 'python': platform.python_version(), 'scope': 'isolated-regression-not-production' + }, indent=2) + '\n') + PY + - name: Exercise boundaries and unchanged security contracts + run: >- + python -m pytest tests/test_token_delivery_boundary.py tests/test_token_delivery.py + tests/test_credential_delivery_contract.py tests/test_credential_delivery_matrix.py + tests/test_credential_recovery_window.py tests/test_client_token_delivery.py + tests/test_batch_secret_release.py tests/test_security_patch_regressions.py + --junitxml=artifacts/token-delivery-boundary.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: token-delivery-boundary-evidence + path: artifacts/ + if-no-files-found: error diff --git a/deploy/hosting-role.sql b/deploy/hosting-role.sql new file mode 100644 index 00000000..3962fe74 --- /dev/null +++ b/deploy/hosting-role.sql @@ -0,0 +1,16 @@ +-- Review and run once as the installation database owner, in the chosen DB. +-- This file neither creates a database nor changes existing PUBLIC grants. +-- A password/peer map belongs in the operator's secret setup, never this file. +CREATE ROLE msgd_hosting LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE + NOINHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('GRANT CONNECT ON DATABASE %I TO msgd_hosting', current_database()) \gexec +GRANT USAGE ON SCHEMA public TO msgd_hosting; +GRANT SELECT ON public.schema_version, public.resources, public.revisions, + public.identities, public.memberships, public.credentials, public.certificates, + public.settings, public.share_grants, public.share_grants_v2, + public.dm_conversations, public.system_sources TO msgd_hosting; +ALTER ROLE msgd_hosting SET default_transaction_read_only = on; +ALTER ROLE msgd_hosting SET search_path = pg_catalog, public, pg_temp; +-- No role membership, ownership, CREATE, sequence use, other-table SELECT, +-- column/table writes or callable application SECURITY DEFINER functions. +-- If PUBLIC already grants those, startup refuses; review that DB separately. diff --git a/deploy/hosting.example.toml b/deploy/hosting.example.toml new file mode 100644 index 00000000..b8d6c54b --- /dev/null +++ b/deploy/hosting.example.toml @@ -0,0 +1,18 @@ +# Install as /etc/msgd-hosting/msgd.toml, root:msgd-hosting 0640. +# This is NOT a copy of the writer configuration or its PostgreSQL credentials. +[server] +service_url = "https://msg.example.org" +public_web_origin = "https://msg.example.org" + +[hosting] +# The same marker checked by the write service. Never select a private copy. +recovery_marker = "/etc/msgd/recovery-drill.json" + +[storage] +postgres_dsn = "service=msgd-hosting" +content = "/var/lib/msgd/git/content" +blobs = "/var/lib/msgd/blobs/sha256" +# These parser-compatible paths are deliberately inaccessible and never opened. +service_keys = "/var/lib/msgd/service" +staging = "/var/lib/msgd/transfers/staging" +repositories = "/var/lib/msgd/git/repos" diff --git a/deploy/msgd-hosting.service b/deploy/msgd-hosting.service index 16b12bd7..a7f26a0d 100644 --- a/deploy/msgd-hosting.service +++ b/deploy/msgd-hosting.service @@ -1,21 +1,28 @@ [Unit] -Description=msg.lmm.best separate-origin static hosting +Description=msg.lmm.best read-only hosted-content process After=network.target [Service] Type=simple -User=msgd -Group=msgd -WorkingDirectory=/var/lib/msgd -ExecStart=/opt/msgd/venv/bin/msgd hosting --listen 127.0.0.1 --port 8043 +User=msgd-hosting +Group=msgd-hosting +SupplementaryGroups=msgd-content-read +WorkingDirectory=/ +ExecStart=/opt/msgd/venv/bin/msgd --config-dir /etc/msgd-hosting hosting --listen 127.0.0.1 --port 8043 +Environment=PGSERVICEFILE=/etc/msgd-hosting/pg_service.conf +Environment=PGPASSFILE=/etc/msgd-hosting/pgpass Restart=on-failure UMask=0077 NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict ProtectHome=true -ReadWritePaths=/var/lib/msgd -InaccessiblePaths=/var/lib/msgd-root /etc/msgd/root +ReadOnlyPaths=/etc/msgd-hosting /var/lib/msgd/git/content /var/lib/msgd/blobs/sha256 +InaccessiblePaths=/var/lib/msgd-root -/etc/msgd/root /var/lib/msgd/service -/var/lib/msgd/service-keys /var/lib/msgd/git/repos /var/lib/msgd/transfers -/etc/msgd/msgd.toml -/etc/msgd/server.toml -/etc/msgd/mail.toml +RestrictSUIDSGID=true +CapabilityBoundingSet= +AmbientCapabilities= +LimitCORE=0 [Install] WantedBy=multi-user.target diff --git a/deploy/prepare-hosting-content.sh b/deploy/prepare-hosting-content.sh new file mode 100755 index 00000000..b4f910b8 --- /dev/null +++ b/deploy/prepare-hosting-content.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Explicit offline OS-administrator preparation, never called by hosting.load(). +# Stop writers first. This changes only OS identities and content permissions. +set -eu +[ "$(id -u)" = 0 ] || { echo 'OS administrator required' >&2; exit 1; } +if systemctl is-active --quiet msgd.service || systemctl is-active --quiet msgd-worker.service; then + echo 'Stop all writers, including any non-systemd instances, before preparation.' >&2 + exit 1 +fi +getent passwd msgd >/dev/null || { echo 'Prepare the write service first.' >&2; exit 1; } +for directory in /var/lib/msgd /var/lib/msgd/git /var/lib/msgd/blobs /var/lib/msgd/git/content /var/lib/msgd/blobs/sha256; do + [ -d "$directory" ] && [ ! -L "$directory" ] || { echo "Missing or symlinked directory: $directory" >&2; exit 1; } +done +for directory in /var/lib/msgd/git/content /var/lib/msgd/blobs/sha256; do + [ -z "$(find -P "$directory" -type l -print -quit)" ] || { echo 'Content symlinks require operator review.' >&2; exit 1; } +done +getent group msgd-content-read >/dev/null || groupadd --system msgd-content-read +if ! getent passwd msgd-hosting >/dev/null; then + useradd --system --user-group --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin msgd-hosting +fi +usermod -a -G msgd-content-read msgd +usermod -a -G msgd-content-read msgd-hosting +for directory in /var/lib/msgd /var/lib/msgd/git /var/lib/msgd/blobs; do + chgrp msgd-content-read "$directory" + chmod 0710 "$directory" +done +for directory in /var/lib/msgd/git/content /var/lib/msgd/blobs/sha256; do + chgrp -R msgd-content-read "$directory" + find "$directory" -type d -exec chmod 2750 {} + + find "$directory" -type f -exec chmod u+rw,g+r,g-w,o-rwx {} + +done +git -c safe.directory=/var/lib/msgd/git/content/private.git --git-dir=/var/lib/msgd/git/content/private.git config core.sharedRepository 0640 +install -d -o root -g msgd-hosting -m 0750 /etc/msgd-hosting +install -d -o root -g msgd-hosting -m 0750 /etc/msgd-hosting/trust +[ -e /etc/msgd/trust/root.json ] || { echo 'Public trust is missing.' >&2; exit 1; } +[ -e /etc/msgd-hosting/trust/root.json ] || ln -s /etc/msgd/trust/root.json /etc/msgd-hosting/trust/root.json +printf '%s\n' 'Now enable hosting.content_group_read=true in the WRITER config and review the separate reader config/DB role.' +printf '%s\n' 'No configuration secrets were copied, no service started, and no database permissions changed.' diff --git a/docs/HOSTING_AUTHORITY_PROGRESS.md b/docs/HOSTING_AUTHORITY_PROGRESS.md new file mode 100644 index 00000000..b6800247 --- /dev/null +++ b/docs/HOSTING_AUTHORITY_PROGRESS.md @@ -0,0 +1,61 @@ +# Hosting authority supplement (#161 / #173 / #174) + +## Single integration path + +#173 is the only product integration path. #174 is closed as superseded, not +merged or accepted as a completed #161. Its duplicate runtime/storage/publication +implementation is not stacked. Original #174 RED and failure-correction evidence +remain in that PR (comments5887238998,5887734281). Historical source +51af07f4835c37a00cf438055ca41a19f7cd6246 is retained, not deployed. + +#173 head6703f3de41e988977be09e509c91fc30a11738b1 retained #174's real UID, +ACL/cache, missing-schema and quarantine tests. The independent child branch +`fix/hosting-authority-174-to-173` adds only key-revocation/owner-authority tests +and nine lines of ownership checking; it never overwrites #173. Coordination: +#173 comments5887748079,5887771169,5887792936,5887839120; #83 comment5887825824. + +## Verified cloud evidence + +Test commit c314d1b2ff0c892e274d731d5c424154a44f2784 added six tests. Run36551691097 +failed container initialization because Actions did not group a single-quoted +Docker health command. No tests/artifacts: NOT a functional RED. Commit +99a0242bf9fdc089f61b7b127c21853e66b20a7d corrected only the workflow quoting. + +**RED**: head=checkout99a0242bf9fdc089f61b7b127c21853e66b20a7d, +tree70361477198cf11e08ec9d6876ac6ccdb08415c8, run36551843100/attempt1, +artifact11024911828, Python3.15.0rc2. JUnit **6 tests:3 passed/3 failed/0 errors/0 skipped**. +Downloaded ZIP SHA256 verified against provider: +5664e005eb1cd92ba8d7f1f4e97b980d823793639426a24fb94b1aed891a9146. + +The three failures are table/schema/sequence owners: the actual reader login +successfully restores its own permissions with GRANT, removes ACLs again, and +hosting.load incorrectly succeeds. Database ownership, real key revocation +(GET/HEAD/Range/ETag), and column-only UPDATE cases already passed. + +**GREEN**: head=checkoutd17a2875986b9f897c266f3632f3e8681ce58a96, +treeeef4fef2bf5a19908a4a8fb8be690c80bbd2dc01, run36552196870/attempt1, +artifact11024289895, Python3.15.0rc2. JUnit **the same 6 tests:6 passed/0 failures/0 errors/0 skipped**. +Downloaded ZIP SHA256 verified against provider: +47981a0025c74219454aef9f47f8b2cb396255f8e682f39e30a125f650c5b4db. +Source manifest and every test name/result were independently inspected. This +is a focused regression result, not a claim that canonical #173 or the full +project has passed. This later note-only commit is not the tested d17 head. + +## Repair and integration handoff + +`storage/read_only.py::require_readonly_role` rejects current-database, schema, +relation and routine ownership before checking effective ACLs. Source diff is +**9 added/0 removed lines**. Existing table-read allowlist, canonical transactions, +authentication, signatures, release validation and GitContentReader are unchanged. +No assertions were weakened, skipped or marked as expected failures. + +For integration into #173, retain the three source commits c314d1b (tests), +99a0242 (health quoting) and d17a287 (guard), plus this evidence note, or apply +their exact narrow diff with traceable attribution. Do not copy the old #174 +Application/Git implementation. Reconcile against the latest #173 head and run +its full combined four-shard/node-ID/conformance/build and isolation workflows. +The canonical branch owner retains schema/marker/real-UID fixes and serial merge. + +All project execution is in Actions. No production secrets, deployment, +permissions, real outbound delivery, Root/PIN, funds or recovery promotion. +#161 is not closed by this supplement; #80/#84 full acceptance remains separate. diff --git a/docs/HOSTING_RUNTIME.md b/docs/HOSTING_RUNTIME.md new file mode 100644 index 00000000..09227028 --- /dev/null +++ b/docs/HOSTING_RUNTIME.md @@ -0,0 +1,91 @@ +# Hosting runtime boundary (#161) + +## Runtime roles + +| Role | Composition | Runtime secrets | Metadata / content | Startup work | +|---|---|---|---|---| +| `msgd serve` | Application, full registered operations and executor | Online CA, receipt and token keys | Existing write authority | Verify installation and sync changed release rules | +| `msgd hosting` | HostingRuntime, callback-free Registry, existing authenticator/authorizer | Public trust only; separate read-only DB credential | Allowlisted SELECT, READ ONLY transactions; Git/CAS reads only | Verify trust, current authority, schema accessibility and exact installed release; never migrate/sync | +| `msgd worker` | Existing Application and registered effect worker | Existing channel/tool/issuer dependencies | Existing controlled write/effect authority | Existing lifecycle; not narrowed by this change | + +Hosting is a process/OS/DB capability boundary, not a sandbox for Python plugins. +The installed plugin order remains in `msg.plugins.install_registry`. Hosting uses +the identical schema, capability and operation metadata, discarding both handlers +and requirement closures. It holds no parent Application, issuer, signer, token +secret, cursor MAC, OperationExecutor, Valkey signal or outbound sender. + +`GitContentReader` binds the existing streamed Git/CAS read methods without a writer +instance or staging path. PostgreSQL uses the existing canonical transaction/session +implementation without initialization. The wrapper rejects write transactions; each +transaction also rejects a superuser, inherited role, owner/write/CREATE privileges, +unneeded table reads and callable application SECURITY DEFINER routines. This checks +configuration rather than repairing it. READ ONLY alone is not a role boundary. + +## Separate deployment configuration + +Review `deploy/msgd-hosting.service`, `deploy/hosting.example.toml` and +`deploy/hosting-role.sql`. Use a distinct `msgd-hosting` OS user and `msgd_hosting` +database login. The host config contains the same public service URL, plugin list +and content locations but **not** a copy of the writer DSN or mail/token keys. +A PostgreSQL service entry `msgd-hosting` plus a reader-owned 0600 pgpass/peer mapping +supplies the dedicated database credential. Protect those files and the config. + +The SQL template grants only SELECT on schema_version, resources, revisions, +identities, memberships, credentials, certificates, settings, share_grants, +share_grants_v2, dm_conversations and system_sources. It does not grant table +ownership, sequence use, CREATE, role membership or token delivery/vault/job/ledger +reads. Existing PUBLIC grants must be reviewed separately; the template does not +revoke other applications' permissions. The runtime refuses an overpowered login. +New schema/authorization dependencies require review plus regression coverage before +adding a table. Column/row data returned by public HTTP still uses current authority. + +Do not copy a possibly stale public trust file or recovery marker. Link the public +`trust/root.json` to the installed public trust, keep its directory traversable, and +set `hosting.recovery_marker` to the writer's real `recovery-drill.json` path. The +hosting account must be able to stat the marker's parent without reading writer +configuration or secrets. The sample marker is `/etc/msgd/recovery-drill.json`. +Missing/untraversable parents fail closed. An observed marker latches quarantine +until restart, even if someone subsequently removes it; the database gate is also +checked for every request. Root rotation requires restarting readers after the +approved trust update. The role cannot clear either recovery condition. + +## Existing and future content permissions + +Content sharing is **opt-in**: `[hosting] content_group_read=false` remains the +writer default. Do not simply add hosting to the writer's private group. + +After stopping every writer, review `deploy/prepare-hosting-content.sh` and run it +as an OS administrator only for the standard paths it names. This is an explicit +permission migration, not an automatic startup action. It grants the dedicated +`msgd-content-read` group traversal and read access only to internal content and +CAS; it does not copy secrets, start services or change database grants. It refuses +symlinked content. Review nonstandard layouts manually rather than widening paths. +The database, service keys, repositories and transfer staging remain out of scope. +All future writer processes need the new supplementary group after restart. + +Then explicitly set `content_group_read=true` in the **writer** configuration. New +content/index/CAS files are group-readable but not group-writable or world-readable; +setgid content directories retain the read group's ownership. New Git repositories +use `--shared=0640`; existing repositories must already have the reviewed mode. +Binary staging remains private; only verified publication adopts the destination's +read group. A private existing installation fails with `content_sharing_not_prepared` +instead of silently chmod'ing history. Backups/restores must preserve these modes +and restore quarantine; repeat the operator review before read service promotion. + +## Transport and acceptance + +This does not change the current **same-origin** contract. The edge proxy sends only +hosted-content paths to the loopback hosting listener; ordinary API/operation routes +remain on the main service. Response-level CSP sandbox, no same-origin allowance, +no-store/nosniff, signed private preview, current revocation, history, HEAD/Range/304 +and explicit publish/rollback semantics are preserved. Unknown host/path/methods do +not open a write route. Startup never generates a key, creates content directories, +repairs missing tables or publishes release revisions. + +`tests/test_hosting_runtime.py` exercises the actual SELECT-only PostgreSQL login, +an OS-unreadable service-key directory, unchanged business facts, denied writes, +missing/stale prerequisites and current authorization/quarantine. Permission tests +check new text/binary/index publication under umask 0077 and refusal to alter an +existing private installation. Existing transport/CLI checks and full four-shard +CI remain required. Evidence and outstanding checks: `HOSTING_RUNTIME_PROGRESS.md`. +Green isolated CI is not a production deployment or completion of #80/#84. diff --git a/docs/HOSTING_RUNTIME_PROGRESS.md b/docs/HOSTING_RUNTIME_PROGRESS.md new file mode 100644 index 00000000..da204daa --- /dev/null +++ b/docs/HOSTING_RUNTIME_PROGRESS.md @@ -0,0 +1,41 @@ +# Hosting read-only runtime — #161 + +Base `261c8816cd581deaf69fc0c810b8fa6dad5ead5e`, tree `6171f6041ffd0addaf72eadd35acf562f478281f` (#167). Branch `fix/hosting-readonly-20260929`, PR #173; coordination #83. + +## Scope and implementation + +Independent HostingRuntime, canonical Registry metadata without business callbacks, original authentication/authorization, real SELECT-only PostgreSQL login, no initialization/migration/write execution, read-only canonical Git/CAS streaming, public trust/recovery gates and dedicated OS/DB deployment. Explicit content-group sharing preserves future publication readability without exposing staging or changing default private permissions. Details: `HOSTING_RUNTIME.md`. + +Keep current same-origin/CSP, signing bytes, IDs, grants and operation versions. #80/#84 production acceptance is separate. Read the authoritative design/completed archive on 2026-09-29. Baseline artifact10999494535 independently verified SHA256 `cff2bbc5beb0575f79e1eaf069494a0cbdcfe5862b64c15401f974a1791340f0`; source tree matches base. + +## Independently verified hosting evidence + +| Head | Run / artifact | JUnit | ZIP SHA256 | +|---|---|---|---| +| `0942251` | 36547254507 / none | Docker setup failed; NOT RED | none | +| `6d24ee847c2e4c0349c71ce667d45b4ac04393f1` | 36547778158 / 11023785174 | 17: 11 failed, 6 passed | `c4f67c186d03cd57df2a85668dcd5487ac5f261a91f913dfb047432eb6b7f2c0` | +| `20399c0eacffece853d2d0e2035986e30b0636a4` | 36548533968 / 11023512357 | 20: 14 failed, 6 passed | `33ff93271408f4e2006891154bf4621aa660dce6c26324b9945fdc4f54faac60` | +| `246f0fd83f41185dde4e796d0f96f1cca8db7934` | 36549829088 / 11024435325 | 22: 8 failed, 14 passed | `f308a1c1dd112339cd6fb6912350c4f63f9964e6e4bd79acc37bd739c9789fe9` | +| `df7efdbaa637e7f316d0eab1e0ce3dc156f83954` | 36550574815 / 11024546594 | 22 passed | `6dd107a22546b392c6ce136d1eab61f9c7927c387731e73d73a4cbb9add2fd9f` | +| `6703f3de41e988977be09e509c91fc30a11738b1` | 36551192429 / 11025490714 | 35: 4 failed, 31 passed | `6543b5326faab8f1352328058279faf35e4a3bec8cee65324fc0a0b9f1852e3e` | +| `2b97ee9a3a3e7b01fbd60f7b19c84d3578ab47b7` | 36551916795 / 11024567848 | **35 passed** | `cdc5f7fc9c68de649c58098e251d9a2c84b0e29d3db1de6da30f13da0d4af281` | + +Every JUnit above has zero errors/skips. Source manifests were verified against exact heads/trees. Latest GREEN checkout `ff7f2d95c50522b58194b5a4aeaa7bba4227429a`, tree `75cf741527a4d62757b8a495ef34d0ce9175c009`, Python3.15.0rc2, attempt1. These are isolated source acceptance results, not deployment evidence. + +Initial implementation failures shared PostgreSQL predicate-evaluation ordering: sequence privilege checks received index OIDs. `df7efd` resolves sequences first. Extended RED found three real readiness gaps (missing version table, wrong version, missing required SELECT) and one CI sudoers launch failure before application code. `2b97ee9` verifies all required tables/grants and version1; the UID test uses trusted `setpriv` to drop to UID65534 and clear supplementary groups before Python. It does not alter sudoers or run the application as root. All assertions remain; the actual different-UID read/new-publication/private-key-denial/write-denial test passed in 3.914 seconds. + +## Parallel convergence + +#174 independently overlapped #161. Coordination: #83 comment5887561813 and #174 comments5887616225/5887713667. Only one runtime implementation should merge. Its independent-UID/failure-containment tests were retained from head `eafea15af0b8ff26c0208d5c0fcc039b4d32fb56` with attribution in `tests/test_hosting_runtime_isolation.py`, then extended with exact version/grant/sequence/secret-table/quarantine-latch checks. The 35-case GREEN above includes these tests. No old assertion was deleted; no default ACL or post-publication chmod hides new-file permissions. + +## Token-delivery integration (#162, #172, #175) + +#172 merged into the old executor feature branch, not main. Its merge commit is `b0b51205c8273c2d89bf99c7facf9e56767bc8d2`; comparison with main261c8816 is ahead4/behind1 with exactly its six changed files. PR #175 fixes the target using an independent promotion branch, without overwriting either original source branch. + +Promotion focused run `36551495692`, artifact `11024782079` independently verified: **122 passed, zero failures/errors/skips**, ZIP SHA256 `9079f195d8d49568fe502c140ec7e2f05412bf751852ef180d74e681610a3312`. Manifest headb0b51205, checkout `8c86062ba7c60f63d7a4824bce66db1db0f0001c`, tree `ebe1facca432803854d0c04aaa93519a589ff93f`, Python3.15.0rc2/attempt1. Read the complete token owner, authentication change and all seven added boundary tests. Original TDD evidence remains in `TOKEN_DELIVERY_BOUNDARY_PROGRESS.md`. + +This integration preserves both parents (hosting2b97 and tokenb0b512), reuses all five non-Application token blobs unchanged and combines only the Application wiring: shared Registry installer/group-read construction plus the single TokenDelivery owner. No business rule/operation version is rewritten. Both focused suites and full four-shard/node-ID/conformance/build/recovery gates must run on this combined head before merge. Focused GREEN on either separate parent is not substituted for that gate. + +Full runs36551916794 (hosting) and36551495771 (promotion) were still running at this note update. Final verification/merge evidence will be attached to #173/#175; do not close #161/#162 from this note alone. Merge serially, refresh actual main/tree, and retain #80/#84 production blockers. + +All project execution runs in GitHub Actions. Local work is source text/Git/AST inspection and artifact parsing only. No production probes, deployments, Root/PIN/private-key access, real outbound delivery, funds, destructive backup operations or restore promotion. diff --git a/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md b/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md new file mode 100644 index 00000000..d4855a63 --- /dev/null +++ b/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md @@ -0,0 +1,23 @@ +# Credential delivery boundary progress + +Issue #162; PR #172, stacked on #167 base2d1cc443c828dd2f735995cc6a5d20021fee6ac7 / tree6171f6041ffd0addaf72eadd35acf562f478281f. Branchfix/token-delivery-boundary-20260929. Coordination #83 comment5879008128. No other source branch is overwritten. + +## Single source and transaction timing + +security.token_delivery owns TokenDelivery and the one recovery_verifier function. Application constructs it, retains compatibility forwarding only, and binds HTTP/SSH executors directly to the same release method. Authentication consumes that verifier, not a second formula. Dependencies: metadata, the existing token derivation key, clock and current-window providers. No Root signer, vault, Registry, plugin or HTTP object. + +The original issued_token/release/verifier bodies are unchanged; record_token_delivery changes only current-window access. Live clock/settings changes are preserved. The handler still records recovery binding inside issuance. After commit, release independently commits the one-time claim before exposing the response token. Lost responses still require recovery. IDs, operation versions, purpose strings, deadlines, persistent results/events/receipts retain their existing meaning. No project code/test executed locally. + +## Independently verified cloud evidence + +RED b68526267a4aaea2df8418ad4d400f64a845fa63 / tree0f653206afd064f3e752cbb3442c202fbd72a4bb, run36486490339, artifact10999622677: **7 failed /115 passed /0 errors /0 skipped**. All seven new ownership checks failed; all existing suites passed. ZIP SHA2560fed854c2890019a5892fbabdde7a6fe3093b05d6805b64d701504f9f44de958; source.json confirms head/tree/attempt1/Python3.15.0rc2. + +First implementation f015c60f05469f0058255bd645afef084e87840d / tree46af80b065a8e4eaea319b87a70bb5ddb281f966, push run36486854337, artifact10998634353: **1 failed /121 passed /0 errors /0 skipped**. ZIP SHA2560186be18b2ed3393af3fe5eaa5ee11eb1caa315f941cd0a1ddd469fb39fdc189 and source.json independently verified. + +The sole failure is a new test's SQL query for events.request_id, which does not exist: events stores seq/id/body and canonical Event JSON includes request_id. All earlier assertions in that case (concurrent one-time release, receipt verification, secret-free persistent result, recovery binding) passed. The corrected test reads real body JSON, still requires exactly one matching event, and additionally checks its subject and absence of both token and recovery secret. No production schema/code or old assertion is changed to accommodate a fabricated column. + +The corrected head must rerun the full credential selection and full four-shard/node-ID/conformance/build/recovery gates. Neither RED nor first implementation is a passing merge candidate. Latest final status belongs to #172 comments/actual Actions, not historical evidence. + +## Boundaries + +#67 remains completed with every original token/credential/recovery-window/client/batch/security regression retained. This is ownership repair, not a new token protocol. No deployment, actual credentials, Root/PIN, funds, external delivery, backup destruction or promotion. #167 finishes its own stable combined-main gate separately. diff --git a/src/msg/application.py b/src/msg/application.py index 1862a579..90a8dec9 100644 --- a/src/msg/application.py +++ b/src/msg/application.py @@ -1,27 +1,25 @@ """Explicit composition root for network services. Contains no root signer.""" from __future__ import annotations -from dataclasses import replace -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from pathlib import Path -import hashlib import hmac -import importlib from msg.constants import ROOT_SPACE, ROOT_SUBJECT -from msg.core.codec import loads, decode, unb64, b64, canonical, wire +from msg.core.codec import loads, decode, unb64 from msg.core.cursors import CursorCodec -from msg.core.errors import Failure, require +from msg.core.errors import require from msg.core.executor import OperationExecutor from msg.core.models import Certificate, Scope -from msg.core.requests import SECRET_DELIVERY_MIN_VERSION from msg.core.registry import Registry from msg.security.authentication import AuthenticationService from msg.security.authorization import AuthorizationService -from msg.security.capabilities import install_capabilities, primary_ceiling, base_grants, temporary_ceiling +from msg.security.capabilities import primary_ceiling, base_grants, temporary_ceiling +from msg.plugins import install_registry from msg.security.certificates import CertificateValidator from msg.security.crypto import Ed25519Signer from msg.security.quarantine import active as quarantine_active +from msg.security.token_delivery import TokenDelivery, recovery_verifier from msg.storage.git import GitContentStore @@ -39,19 +37,9 @@ def __init__(self,settings, *, clock=None, selftest_run_id=None): self.metadata=None self.contents=None self.executor=None + self.token_delivery=None self._loaded=False - # Plugins are installed code, never resources, posts, or configuration expressions. - implemented=('identity','content','discussion','communication','discovery','achievements','recovery','sharing','money','offers','store','bounty','orders','delivery') - configured=set(settings.server.plugins) - require(configured<=set(implemented)|{'transfer','extensions','system','batch'},'unknown_plugin') - for plugin in implemented: - if plugin in configured: - importlib.import_module('msg.plugins.'+plugin).install(self) - for plugin in ('transfer','extensions','system','batch'): - if plugin in configured: - importlib.import_module('msg.plugins.'+plugin).install(self) - install_capabilities(self.registry) - self.registry.freeze() + install_registry(self, settings.server.plugins) def primary_ceiling(self): return primary_ceiling(self.registry,self.default_scope()) @@ -79,7 +67,8 @@ async def open_storage(self): if self.contents is None: self.contents=GitContentStore(self.settings.server.content_dir, binary_dir=self.settings.server.blob_dir, - staging_dir=self.settings.server.staging_dir) + staging_dir=self.settings.server.staging_dir, + group_read=self.settings.hosting_content_group_read) async def load(self): require(self.settings.trust_file.is_file(),'root_not_initialized') @@ -97,6 +86,9 @@ async def load(self): require(len(self._token_secret)==32,'invalid_service_key') self._vault_key=hmac.digest(self._token_secret,b'custodial-vault-aesgcm-v1','sha256') self.cursors=CursorCodec(hmac.digest(self._token_secret,b'cursor-key-v1','sha256')) + self.token_delivery=TokenDelivery(metadata=self.metadata,token_secret=self._token_secret, + clock=lambda: self.clock(), + recovery_window=lambda: self.settings.credential_delivery_recovery_window) self.certificates=CertificateValidator(self.registry,root_certificate,root_public,self.settings.service_url,self.clock) async with self.metadata.transaction(write=False) as tx: quarantined=quarantine_active(tx) @@ -133,7 +125,7 @@ def new_executor(self, authenticator=None): max_request_bytes=self.settings.server.limits.max_request_bytes, result_projection=self._result_projection, event_notifications=self._event_notifications) - executor.response_hook=self._secrets_for_caller + executor.response_hook=self.token_delivery.release executor.recovery_drill_marker=self.settings.config_dir/'recovery-drill.json' if self.executor is not None: executor.recovery_quarantined=self.executor.recovery_quarantined @@ -156,62 +148,15 @@ async def online_issuer(self,tx): return cert def issued_token(self,request,subject): - # Nonces are high-entropy claims, not random tokens persisted in results. - return hmac.digest(self._token_secret,b'issued-token-v1\0'+canonical({ - 'subject':subject,'request_id':request.request_id,'nonce':request.arguments['nonce']}),'sha256') + return self.token_delivery.issued_token(request,subject) - @staticmethod - def recovery_verifier(value): - secret=unb64(value,limit=64) - require(len(secret)>=32,'invalid_recovery_secret') - return hashlib.sha256(b'token-recovery-v1\0'+secret).hexdigest() + recovery_verifier=staticmethod(recovery_verifier) def record_token_delivery(self,tx,request,credential,now,*,recovery_deadline=None): - """Bind recovery before commit; neither token nor recovery secret is stored.""" - require(request.operation in SECRET_DELIVERY_MIN_VERSION and - request.contract_version>=SECRET_DELIVERY_MIN_VERSION[request.operation], - 'credential_delivery_upgrade_required') - field='new_recovery_secret' if request.operation=='identity.token_recover' else 'recovery_secret' - secret=unb64(request.arguments[field],limit=64) - require(secret!=unb64(request.arguments['nonce'],limit=64),'recovery_secret_not_independent') - if request.operation=='identity.token_recover': - require(secret!=unb64(request.arguments['recovery_secret'],limit=64), - 'recovery_secret_not_independent') - expires=min(credential.expires_at, - recovery_deadline if recovery_deadline is not None else - now+timedelta(seconds=self.settings.credential_delivery_recovery_window)) - require(expires>now,'recovery_unavailable') - tx.execute('''INSERT INTO token_deliveries - (credential_id,subject,request_id,request_digest,recovery_verifier,recovery_expires_at, - claimed_at,consumed_at) VALUES (?,?,?,?,?,?,NULL,NULL)''', - (credential.id,credential.subject_id,request.request_id,request.payload_digest, - self.recovery_verifier(request.arguments[field]),wire(expires)),write=True) + return self.token_delivery.record_token_delivery(tx,request,credential,now,recovery_deadline=recovery_deadline) async def _secrets_for_caller(self,request,result): - if result.status!='ok' or request.operation not in SECRET_DELIVERY_MIN_VERSION: - return result - require(request.contract_version>=SECRET_DELIVERY_MIN_VERSION[request.operation], - 'credential_delivery_upgrade_required') - token=self.issued_token(request,result.subject) - async with self.metadata.transaction(write=True) as tx: - credential=await tx.credential(result.data['credential_id']) - require(credential.revoked_at is None and credential.expires_at>self.clock(),'credential_expired') - require(hmac.compare_digest(credential.verifier,hashlib.sha256(token).digest()),'invalid_token_result') - row=tx.one('''SELECT subject,request_id,request_digest,claimed_at,consumed_at - FROM token_deliveries WHERE credential_id=?''',(credential.id,)) - require(row is not None and row[0]==result.subject and row[1]==request.request_id - and row[2]==request.payload_digest and row[4] is None,'token_delivery_unavailable', - details={'committed_result':wire(result,compact=True)}) - require(row[3] is None,'token_delivery_unavailable', - details={'committed_result':wire(result,compact=True)}) - updated=tx.execute('''UPDATE token_deliveries SET claimed_at=? WHERE credential_id=? - AND claimed_at IS NULL AND consumed_at IS NULL''', - (wire(self.clock()),credential.id),write=True) - require(updated.rowcount==1,'token_delivery_unavailable', - details={'committed_result':wire(result,compact=True)}) - # The claim commits before the adapter sees the token. A dropped response - # must use a separate, pre-bound recovery secret to rotate the credential. - return replace(result,data=dict(result.data,token=b64(token))) + return await self.token_delivery.release(request,result) async def close(self): if self.metadata is not None: diff --git a/src/msg/config.py b/src/msg/config.py index 21caaebd..54d8a974 100644 --- a/src/msg/config.py +++ b/src/msg/config.py @@ -70,6 +70,8 @@ class Settings: recovery_custodians: tuple[RecoveryCustodian,...]=() money: MoneyConfig=MoneyConfig() hosting_base_capacity_bytes: int=10*1024*1024 + hosting_recovery_marker: Path | None=None + hosting_content_group_read: bool=False @property def config_dir(self): @@ -108,11 +110,17 @@ def load_settings(config_dir=Path('/etc/msgd')): window_minutes=int(window[:-1]) require(1<=window_minutes<=60,'invalid_credential_delivery_recovery_window') hosting=data.get('hosting',{}) - require(isinstance(hosting,dict) and set(hosting)<={'base_capacity_bytes'}, + require(isinstance(hosting,dict) and set(hosting)<={'base_capacity_bytes','recovery_marker','content_group_read'}, 'unknown_hosting_configuration') hosting_base=hosting.get('base_capacity_bytes',10*1024*1024) require(type(hosting_base) is int and 0 < hosting_base <= 1024**4, 'invalid_hosting_capacity') + content_group_read=hosting.get('content_group_read',False) + require(type(content_group_read) is bool,'invalid_content_group_read') + recovery_marker=hosting.get('recovery_marker') + require(recovery_marker is None or (isinstance(recovery_marker,str) and + Path(recovery_marker).is_absolute() and '..' not in Path(recovery_marker).parts and + not any(ord(c)<32 for c in recovery_marker)), 'invalid_hosting_recovery_marker') money=data.get('money',{}) require(isinstance(money,dict) and set(money)<=set(MoneyConfig.__dataclass_fields__), 'unknown_money_configuration') @@ -255,7 +263,9 @@ def load_settings(config_dir=Path('/etc/msgd')): tool_methods=tuple(tools.get('methods',('GET','HEAD'))),tool_ports=tuple(tools.get('ports',(80,443))), recovery_custodians=tuple(custodians), money=MoneyConfig(display_name=display_name,code=code), - hosting_base_capacity_bytes=hosting_base) + hosting_base_capacity_bytes=hosting_base, + hosting_recovery_marker=Path(recovery_marker) if recovery_marker is not None else None, + hosting_content_group_read=content_group_read) def write_example(config_dir,data_dir,service_url='https://msg.lmm.best',*,postgres_dsn='service=msgd',valkey_url=None): @@ -283,6 +293,7 @@ def write_example(config_dir,data_dir,service_url='https://msg.lmm.best',*,postg [hosting] base_capacity_bytes = 10485760 +content_group_read = false [storage] postgres_dsn = {json.dumps(postgres_dsn)} diff --git a/src/msg/daemon.py b/src/msg/daemon.py index 637715da..5b0423c9 100644 --- a/src/msg/daemon.py +++ b/src/msg/daemon.py @@ -208,6 +208,26 @@ def main(argv=None): if args.command=='ssh-session': from msg.extensions.ssh import forced_session return asyncio.run(forced_session(args.config_dir,args.credential)) + if args.command=='hosting': + from contextlib import asynccontextmanager + from msg.config import load_settings + from msg.hosting_runtime import HostingRuntime + from msg.extensions.hosting import hosting_app + import uvicorn + settings=load_settings(args.config_dir) + network_runtime(settings) + app=HostingRuntime(settings) + @asynccontextmanager + async def lifespan(asgi): + try: + await app.load() + yield + finally: + await app.close() + asgi=hosting_app(app) + asgi.router.lifespan_context=lifespan + uvicorn.run(asgi,host=args.listen,port=args.port,access_log=False,ws='none') + return 0 app=load_application(args.config_dir) if args.command=='backup': from msg.admin.backups import backup @@ -222,21 +242,8 @@ async def save(): if result is not None:emit(result) return 0 import uvicorn - if args.command=='hosting': - require(app.settings.public_web_origin is not None,'hosting_origin_not_configured') - from msg.extensions.hosting import hosting_app - # The hosted origin exposes no operations, cookies, MCP or root routes. - from contextlib import asynccontextmanager - @asynccontextmanager - async def lifespan(asgi): - await app.load() - yield - await app.close() - asgi=hosting_app(app);asgi.router.lifespan_context=lifespan - uvicorn.run(asgi,host=args.listen,port=args.port,access_log=False,ws='none') - else: - from msg.transports.http import create_app - uvicorn.run(create_app(app),host=app.settings.listen,port=app.settings.port,access_log=False,ws='none') + from msg.transports.http import create_app + uvicorn.run(create_app(app),host=app.settings.listen,port=app.settings.port,access_log=False,ws='none') return 0 except (Failure,OSError,ValueError) as exc: code=exc.code if isinstance(exc,Failure) else 'local_operation_failed' diff --git a/src/msg/extensions/hosting.py b/src/msg/extensions/hosting.py index 380ddee4..47f9f7b9 100644 --- a/src/msg/extensions/hosting.py +++ b/src/msg/extensions/hosting.py @@ -161,6 +161,8 @@ async def serve_hosted(service,request): site_path='/'+parts[0]+'/'+parts[1] try: async with service.metadata.transaction(write=False) as tx: + from msg.security.quarantine import require_live_authority + require_live_authority(tx) try: rid=await tx.resolve(site_path) except Failure as exc: @@ -243,7 +245,12 @@ async def serve_hosted(service,request): def hosting_app(service): + require(getattr(service,'readonly_hosting',False),'hosting_runtime_required') async def dispatch(request): + try: + service.require_ready() + except Failure as exc: + return hosted_error(exc.code) expected=urlsplit(service.settings.service_url) supplied=urlsplit('//'+request.headers.get('host','')) if supplied.hostname!=expected.hostname: diff --git a/src/msg/hosting_runtime.py b/src/msg/hosting_runtime.py new file mode 100644 index 00000000..bcf9ef20 --- /dev/null +++ b/src/msg/hosting_runtime.py @@ -0,0 +1,138 @@ +"""Explicit read-only composition for the independent hosting HTTP process.""" +from dataclasses import replace +from datetime import UTC, datetime +from functools import partial +from types import SimpleNamespace +import os + +import psycopg + +from msg.constants import ROOT_SUBJECT +from msg.core.codec import decode, digest, loads, unb64 +from msg.core.errors import Failure, require +from msg.core.models import Certificate, ResourceRef +from msg.core.registry import Registry +from msg.plugins import install_registry +from msg.security.authentication import AuthenticationService +from msg.security.authorization import AuthorizationService +from msg.security.capabilities import primary_ceiling, temporary_ceiling +from msg.security.certificates import CertificateValidator +from msg.security.quarantine import require_live_authority +from msg.storage.read_only import GitContentReader, ReadOnlyPostgresStore + + +async def _no_execution(*args, **kwargs): + raise Failure('readonly_registry') + + +def _contract_only(spec): + if spec.handler is _no_execution and spec.requirements is _no_execution: + return spec + return replace(spec, handler=_no_execution, requirements=_no_execution) + + +class _ContractRegistry(Registry): + """Use the original contracts/validation but retain no business closures.""" + def add_operation(self, spec): + super().add_operation(_contract_only(spec)) + + def add(self, manifest): + super().add(replace(manifest, operations=tuple( + _contract_only(spec) for spec in manifest.operations))) + + +async def require_prepared_sources(tx): + """The installer owns publication; this role only compares its release inputs.""" + from msg.bootstrap import RULE_SPECS, SOURCE_PATH_OVERRIDES, SOURCE_HEADER, system_source_root + root = system_source_root() + expected = set() + for rule_id, rid, _, _, default_path, _ in RULE_SPECS: + path = SOURCE_PATH_OVERRIDES.get(rule_id, default_path) + raw = root.joinpath(path).read_bytes() + header = SOURCE_HEADER.match(raw.decode('utf-8')) + require(header is not None and header.group(1) == rule_id, + 'hosting_installation_stale') + row = tx.one('''SELECT source_path,rule_id,source_kind,source_version,source_digest,revision_id + FROM system_sources WHERE resource_id=?''', (rid,)) + require(row is not None and row[:5] == ( + 'docs/system/' + path, rule_id, 'release', int(header.group(2)), digest(raw)), + 'hosting_installation_stale') + resource = await tx.resource(rid) + require(resource.revision == row[5], 'hosting_installation_stale') + revision = await tx.revision(ResourceRef(id=rid, revision=row[5])) + require(revision.source_kind == 'release' and revision.source_digest == row[4] + and revision.source_version == row[3], 'hosting_installation_stale') + expected.add(rid) + require({row[0] for row in tx.rows('SELECT resource_id FROM system_sources')} == expected, + 'hosting_installation_stale') + + +class HostingRuntime: + """Current verification + existing bytes. No Application, issuer or executor.""" + readonly_hosting = True + + def __init__(self, settings, *, clock=None): + self.settings = settings + self.clock = clock or (lambda: datetime.now(UTC)) + self.registry = _ContractRegistry() + # Installers only register trusted code. The throw-away context has no + # storage, signer, policy service or parent Application to capture. + install_registry(SimpleNamespace(registry=self.registry), settings.server.plugins) + self.metadata = None + self.contents = None + self._loaded = False + self._quarantined = False + self._marker = settings.hosting_recovery_marker or settings.config_dir / 'recovery-drill.json' + + def require_ready(self, *, loading=False): + require(loading or self._loaded, 'hosting_not_ready') + try: + self._marker.lstat() # includes a dangling symlink: never a bypass + except FileNotFoundError: + require(self._marker.parent.is_dir() and os.access(self._marker.parent, os.X_OK), + 'hosting_recovery_state_unavailable') + except OSError as exc: + raise Failure('hosting_recovery_state_unavailable') from exc + else: + self._quarantined = True + require(not self._quarantined, 'recovery_quarantined') + + async def load(self): + require(not self._loaded, 'hosting_already_loaded') + require(self.settings.public_web_origin is not None, 'hosting_origin_not_configured') + self.require_ready(loading=True) + require(self.settings.trust_file.is_file(), 'root_not_initialized') + trust = loads(self.settings.trust_file.read_bytes()) + require(type(trust) is dict and set(trust) == {'version', 'public_key', 'certificate'} + and trust['version'] == 1, 'invalid_trust_anchor') + root = decode(Certificate, trust['certificate']) + public = unb64(trust['public_key'], limit=32) + require(len(public) == 32, 'invalid_trust_anchor') + self.contents = GitContentReader(self.settings.server.content_dir, + binary_dir=self.settings.server.blob_dir) + self.metadata = ReadOnlyPostgresStore(self.settings.server.postgres_dsn) + self.certificates = CertificateValidator(self.registry, root, public, + self.settings.service_url, self.clock) + try: + async with self.metadata.transaction(write=False) as tx: + require_live_authority(tx) + require(tx.setting('active_root_certificate', root.resource_id) == root.resource_id, + 'service_restart_required') + await self.certificates.validate(root.resource_id, tx) + require((await tx.subject(ROOT_SUBJECT)).local_only, 'root_policy_corrupt') + await require_prepared_sources(tx) + except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn, + psycopg.errors.InsufficientPrivilege) as exc: + raise Failure('hosting_installation_stale') from exc + self.authenticator = AuthenticationService(self.registry, self.certificates, + self.settings.service_url, self.clock, + partial(primary_ceiling, self.registry), partial(temporary_ceiling, self.registry)) + self.authorizer = AuthorizationService(self.registry, self.certificates) + self.require_ready(loading=True) + self._loaded = True + return self + + async def close(self): + self._loaded = False + if self.metadata is not None: + await self.metadata.close() diff --git a/src/msg/plugins/__init__.py b/src/msg/plugins/__init__.py index e69de29b..6f1c8086 100644 --- a/src/msg/plugins/__init__.py +++ b/src/msg/plugins/__init__.py @@ -0,0 +1,28 @@ +"""The installed plugin order has one owner for every runtime role.""" +import importlib + +from msg.core.errors import require + + +BUILTINS = ( + 'identity', 'content', 'discussion', 'communication', 'discovery', + 'achievements', 'recovery', 'sharing', 'money', 'offers', 'store', + 'bounty', 'orders', 'delivery', 'transfer', 'extensions', 'system', 'batch', +) + + +def install_registry(context, plugins): + """Register installed code only; the context determines available capabilities. + + The write Application supplies its own context. A read-only host supplies a + disposable registration context and a registry which discards callbacks. + No storage connection, runtime secret or business operation is needed here. + """ + from msg.security.capabilities import install_capabilities + configured = set(plugins) + require(configured <= set(BUILTINS), 'unknown_plugin') + for name in BUILTINS: + if name in configured: + importlib.import_module('msg.plugins.' + name).install(context) + install_capabilities(context.registry) + context.registry.freeze() diff --git a/src/msg/security/authentication.py b/src/msg/security/authentication.py index 3c2281ee..3a7291b1 100644 --- a/src/msg/security/authentication.py +++ b/src/msg/security/authentication.py @@ -11,6 +11,7 @@ from msg.core.requests import payload_fields,signing_bytes from msg.security.crypto import key_id,subject_id,verify from msg.security.quarantine import active as quarantine_active +from msg.security.token_delivery import recovery_verifier CUSTODIAL_SIGNED_WRITES=frozenset({ @@ -64,9 +65,7 @@ async def authenticate(self,request,session, *, entry): require(old.kind=='token' and old.subject_id==request.subject,'recovery_unavailable') actor=await session.subject(old.subject_id) require(not actor.local_only and actor.resource_id!=ROOT_SUBJECT,'local_only') - raw=unb64(request.arguments['recovery_secret'],limit=64) - require(len(raw)>=32,'invalid_recovery_secret') - verifier=hashlib.sha256(b'token-recovery-v1\0'+raw).hexdigest() + verifier=recovery_verifier(request.arguments['recovery_secret']) row=session.one('''SELECT recovery_verifier,recovery_expires_at,consumed_at,request_id FROM token_deliveries WHERE credential_id=? AND subject=?''', (old.id,actor.resource_id)) diff --git a/src/msg/security/token_delivery.py b/src/msg/security/token_delivery.py new file mode 100644 index 00000000..e60bc6b0 --- /dev/null +++ b/src/msg/security/token_delivery.py @@ -0,0 +1,93 @@ +"""One-time credential delivery, distinct from service composition. + +The handler records the verifier in its existing issuance transaction. After that +transaction commits, release claims the delivery in a separate write transaction; +only a committed claim can produce a response-only token. Lost responses must use +the pre-bound recovery operation, never replay a persistent secret. This module +owns no signer, vault, root capability, HTTP adapter or plugin registry. + +Clock and window providers preserve the configured current values without giving +the domain access to the whole settings/Application object. These are trusted +in-process dependencies, not a Python sandbox or an additional authorization path. +""" +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import replace +from datetime import datetime, timedelta +import hashlib +import hmac + +from msg.core.codec import b64, canonical, unb64, wire +from msg.core.contracts import MetadataStore +from msg.core.errors import require +from msg.core.requests import SECRET_DELIVERY_MIN_VERSION + + +def recovery_verifier(value): + secret=unb64(value,limit=64) + require(len(secret)>=32,'invalid_recovery_secret') + return hashlib.sha256(b'token-recovery-v1\0'+secret).hexdigest() + +class TokenDelivery: + def __init__(self, *, metadata: MetadataStore, token_secret: bytes, + clock: Callable[[], datetime], recovery_window: Callable[[], int]): + require(len(token_secret)==32,'invalid_service_key') + self.metadata=metadata + self._token_secret=token_secret + self.clock=clock + self.recovery_window=recovery_window + + def issued_token(self,request,subject): + # Nonces are high-entropy claims, not random tokens persisted in results. + return hmac.digest(self._token_secret,b'issued-token-v1\0'+canonical({ + 'subject':subject,'request_id':request.request_id,'nonce':request.arguments['nonce']}),'sha256') + + recovery_verifier=staticmethod(recovery_verifier) + + def record_token_delivery(self,tx,request,credential,now,*,recovery_deadline=None): + """Bind recovery before commit; neither token nor recovery secret is stored.""" + require(request.operation in SECRET_DELIVERY_MIN_VERSION and + request.contract_version>=SECRET_DELIVERY_MIN_VERSION[request.operation], + 'credential_delivery_upgrade_required') + field='new_recovery_secret' if request.operation=='identity.token_recover' else 'recovery_secret' + secret=unb64(request.arguments[field],limit=64) + require(secret!=unb64(request.arguments['nonce'],limit=64),'recovery_secret_not_independent') + if request.operation=='identity.token_recover': + require(secret!=unb64(request.arguments['recovery_secret'],limit=64), + 'recovery_secret_not_independent') + expires=min(credential.expires_at, + recovery_deadline if recovery_deadline is not None else + now+timedelta(seconds=self.recovery_window())) + require(expires>now,'recovery_unavailable') + tx.execute('''INSERT INTO token_deliveries + (credential_id,subject,request_id,request_digest,recovery_verifier,recovery_expires_at, + claimed_at,consumed_at) VALUES (?,?,?,?,?,?,NULL,NULL)''', + (credential.id,credential.subject_id,request.request_id,request.payload_digest, + self.recovery_verifier(request.arguments[field]),wire(expires)),write=True) + + async def release(self,request,result): + if result.status!='ok' or request.operation not in SECRET_DELIVERY_MIN_VERSION: + return result + require(request.contract_version>=SECRET_DELIVERY_MIN_VERSION[request.operation], + 'credential_delivery_upgrade_required') + token=self.issued_token(request,result.subject) + async with self.metadata.transaction(write=True) as tx: + credential=await tx.credential(result.data['credential_id']) + require(credential.revoked_at is None and credential.expires_at>self.clock(),'credential_expired') + require(hmac.compare_digest(credential.verifier,hashlib.sha256(token).digest()),'invalid_token_result') + row=tx.one('''SELECT subject,request_id,request_digest,claimed_at,consumed_at + FROM token_deliveries WHERE credential_id=?''',(credential.id,)) + require(row is not None and row[0]==result.subject and row[1]==request.request_id + and row[2]==request.payload_digest and row[4] is None,'token_delivery_unavailable', + details={'committed_result':wire(result,compact=True)}) + require(row[3] is None,'token_delivery_unavailable', + details={'committed_result':wire(result,compact=True)}) + updated=tx.execute('''UPDATE token_deliveries SET claimed_at=? WHERE credential_id=? + AND claimed_at IS NULL AND consumed_at IS NULL''', + (wire(self.clock()),credential.id),write=True) + require(updated.rowcount==1,'token_delivery_unavailable', + details={'committed_result':wire(result,compact=True)}) + # The claim commits before the adapter sees the token. A dropped response + # must use a separate, pre-bound recovery secret to rotate the credential. + return replace(result,data=dict(result.data,token=b64(token))) diff --git a/src/msg/storage/git.py b/src/msg/storage/git.py index 9846a21a..8c0223eb 100644 --- a/src/msg/storage/git.py +++ b/src/msg/storage/git.py @@ -218,20 +218,43 @@ def collect_unlinked(self, *, older_than: float, index: Path) -> tuple[int,int]: class GitContentStore: def __init__(self,path: Path, *, binary_dir: Path | None = None, - staging_dir: Path | None = None): + staging_dir: Path | None = None, group_read: bool = False): + require(type(group_read) is bool,'invalid_content_group_read') + self.group_read=group_read self.path=Path(path) self.repo=self.path/'private.git' self.index=self.path/'index' self.binary=Path(binary_dir) if binary_dir is not None else self.path/'binary' self.staging=Path(staging_dir) if staging_dir is not None else self.path/'staging' - for p in (self.path,self.index,self.binary,self.staging): - p.mkdir(parents=True,exist_ok=True) - if not self.repo.exists(): - subprocess.run(['git','init','--bare',str(self.repo)],check=True,capture_output=True) self.env={"PATH":os.environ.get('PATH','/usr/bin:/bin'),"LANG":"C.UTF-8", "HOME":str(self.path),"GIT_CONFIG_NOSYSTEM":"1","GIT_CONFIG_GLOBAL":"/dev/null", "GIT_AUTHOR_NAME":"msg","GIT_AUTHOR_EMAIL":"msg@localhost", "GIT_COMMITTER_NAME":"msg","GIT_COMMITTER_EMAIL":"msg@localhost"} + # An existing installation needs an explicit offline permission review. + # Validate before creating/chmod'ing anything, including staging paths. + if group_read: + for p in (self.path,self.index,self.binary): + if p.exists() or p.is_symlink(): + require(p.is_dir() and not p.is_symlink() and + p.stat().st_mode & 0o2077 == 0o2050, + 'content_sharing_not_prepared') + if self.repo.exists() or self.repo.is_symlink(): + require(self.repo.is_dir() and not self.repo.is_symlink(), + 'content_sharing_not_prepared') + config=subprocess.run(['git','--git-dir',str(self.repo),'config', + '--local','core.sharedRepository'],capture_output=True,env=self.env,timeout=30) + require(config.returncode==0 and config.stdout.strip()==b'0640', + 'content_sharing_not_prepared') + for p in (self.path,self.index,self.binary): + existed=p.exists() + p.mkdir(parents=True,exist_ok=True) + if group_read and not existed: + p.chmod(0o2750) + self.staging.mkdir(parents=True,exist_ok=True) + if not self.repo.exists(): + shared=['--shared=0640'] if group_read else [] + subprocess.run(['git','init','--bare',*shared,str(self.repo)], + check=True,capture_output=True,env=self.env,timeout=30) def _run(self,*args,input=None,stdin=None): result=subprocess.run(['git','--git-dir',str(self.repo),'-c','core.fsync=all', @@ -281,6 +304,13 @@ def save_git(): else: destination=self.binary/self._key(ref) if not destination.exists(): + if self.group_read: + # Rename retains the staging GID, so explicitly use the + # prepared destination group. Staging itself stays private. + os.chown(name,-1,self.binary.stat().st_gid) + os.chmod(name,0o640) + with open(name,'rb') as staged: + os.fsync(staged.fileno()) os.replace(name,destination) directory=os.open(destination.parent,os.O_RDONLY|os.O_DIRECTORY) try: @@ -288,7 +318,8 @@ def save_git(): finally: os.close(directory) entry={'kind':'binary','size':size} - durable_write(self.index/self._key(ref),canonical(entry)) + durable_write(self.index/self._key(ref),canonical(entry), + mode=0o640 if self.group_read else 0o600) return ref finally: if os.path.exists(name): diff --git a/src/msg/storage/read_only.py b/src/msg/storage/read_only.py new file mode 100644 index 00000000..deed3d7f --- /dev/null +++ b/src/msg/storage/read_only.py @@ -0,0 +1,126 @@ +"""Storage capabilities for a process which must never initialize or publish. + +PostgreSQL grants are an independent boundary, not replaced by READ ONLY or a +Python wrapper. Content reads share the exact existing Git/CAS implementation, +without constructing a writer or exposing its publication methods. +""" +from contextlib import asynccontextmanager +from pathlib import Path +import os + +from msg.core.errors import require +from msg.storage.git import GitContentStore +from msg.storage.postgres import PostgresMetadataStore + + +HOSTING_READ_TABLES = frozenset({ + 'schema_version', 'resources', 'revisions', 'identities', 'memberships', + 'credentials', 'certificates', 'settings', 'share_grants', 'share_grants_v2', + 'dm_conversations', 'system_sources', +}) + + +def require_readonly_role(tx): + """Refuse owner/writer/SET ROLE/secret-table authority; never repair grants.""" + role = tx.one('''SELECT current_user, session_user, rolsuper, rolcreaterole, + rolcreatedb, rolreplication, rolbypassrls FROM pg_catalog.pg_roles + WHERE rolname=current_user''') + require(role is not None and role[0] == role[1] and not any(role[2:]), + 'hosting_database_not_readonly') + # An owner can GRANT its own rights back after REVOKE ALL. Current ACL bits + # cannot establish read-only authority even when every write check is false. + require(tx.one('''SELECT 1 FROM ( + SELECT datdba AS owner_id FROM pg_catalog.pg_database WHERE datname=current_database() + UNION ALL SELECT nspowner FROM pg_catalog.pg_namespace + UNION ALL SELECT relowner FROM pg_catalog.pg_class + UNION ALL SELECT proowner FROM pg_catalog.pg_proc + ) AS owned WHERE owner_id=(SELECT oid FROM pg_catalog.pg_roles + WHERE rolname=current_user) LIMIT 1''') is None, 'hosting_database_not_readonly') + require(tx.one('''SELECT 1 FROM pg_catalog.pg_roles + WHERE rolname<>current_user AND pg_has_role(oid,'MEMBER') LIMIT 1''') is None, + 'hosting_database_not_readonly') + require(not tx.one("SELECT has_database_privilege(current_database(),'CREATE')")[0], + 'hosting_database_not_readonly') + require(tx.one('''SELECT 1 FROM pg_catalog.pg_namespace + WHERE left(nspname,3)<>'pg_' AND nspname<>'information_schema' + AND has_schema_privilege(oid,'CREATE') LIMIT 1''') is None, + 'hosting_database_not_readonly') + available = set() + for schema, name, readable, writable, full_read in tx.rows(''' + SELECT n.nspname,c.relname,has_any_column_privilege(c.oid,'SELECT'), + (has_table_privilege(c.oid,'INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER') + OR has_any_column_privilege(c.oid,'INSERT,UPDATE,REFERENCES')), + has_table_privilege(c.oid,'SELECT') + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE c.relkind IN ('r','p','v','m','f') + AND left(n.nspname,3)<>'pg_' AND n.nspname<>'information_schema' + '''): + require(not writable and (not readable or + (schema == 'public' and name in HOSTING_READ_TABLES)), + 'hosting_database_not_readonly') + if schema == 'public' and full_read: + available.add(name) + require(available == HOSTING_READ_TABLES, 'hosting_installation_stale') + # The installed PostgreSQL metadata format is version 1. A reader cannot + # initialize or migrate it, or infer readiness from a few working tables. + require(tx.rows('SELECT version FROM public.schema_version') == [(1,)], + 'hosting_installation_stale') + # Fetch sequence OIDs first: SQL predicates do not impose evaluation order, + # and has_sequence_privilege must never receive an index/table OID. + for (oid,) in tx.rows('''SELECT c.oid FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE c.relkind='S' AND left(n.nspname,3)<>'pg_' '''): + require(not tx.one("SELECT has_sequence_privilege(?::oid,'USAGE,UPDATE')", (oid,))[0], + 'hosting_database_not_readonly') + # A callable SECURITY DEFINER routine can reintroduce its owner's privileges. + require(tx.one('''SELECT 1 FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace + WHERE p.prosecdef AND left(n.nspname,3)<>'pg_' + AND n.nspname<>'information_schema' + AND has_function_privilege(p.oid,'EXECUTE') LIMIT 1''') is None, + 'hosting_database_not_readonly') + + +class ReadOnlyPostgresStore: + """Reuse canonical sessions/transactions, without migration or a write API.""" + def __init__(self, dsn): + self._store = PostgresMetadataStore(dsn, initialize=False) + + @asynccontextmanager + async def transaction(self, *, write=False): + require(not write, 'read_only_transaction') + async with self._store.transaction(write=False) as tx: + # Explicit ordering prevents writable/temp schemas from shadowing + # the installed public schema or PostgreSQL's privilege functions. + tx.execute('SET LOCAL search_path = pg_catalog, public, pg_temp') + require_readonly_role(tx) + yield tx + + async def close(self): + await self._store.close() + + +class GitContentReader: + """Read existing content; no mkdir, init, staging, pin, write or writer pointer.""" + def __init__(self, path, *, binary_dir=None): + self.path = Path(path) + self.repo = self.path / 'private.git' + self.index = self.path / 'index' + self.binary = Path(binary_dir) if binary_dir is not None else self.path / 'binary' + require(all(p.is_dir() for p in (self.path, self.repo, self.index, self.binary)), + 'content_store_not_initialized') + self.env = { + 'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'LANG': 'C.UTF-8', + 'HOME': str(self.path), 'GIT_CONFIG_NOSYSTEM': '1', + 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_OPTIONAL_LOCKS': '0', + # The approved content repository is owned by the writer, not this UID. + 'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'safe.directory', + 'GIT_CONFIG_VALUE_0': str(self.repo), + } + + # Unbound implementations, not a wrapped GitContentStore instance. Keep the + # original limits, references, streaming, range and truncation behavior. + _key = staticmethod(GitContentStore._key) + _entry = GitContentStore._entry + read = GitContentStore.read + read_bytes = GitContentStore.read_bytes diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 749c3b16..823f7895 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -6,6 +6,7 @@ from msg.extensions.hosting import hosting_app from msg.extensions.repositories import NativeGitStore from test_service import register,call +from test_hosting_runtime import reader_settings, runtime_class def test_public_key_encryption_recipient_and_tamper_checks(): @@ -44,7 +45,7 @@ async def test_keystore_stores_only_ciphertext_and_has_common_acl(installed): @pytest.mark.asyncio -async def test_hosting_publish_is_explicit_versioned_on_service_origin(installed): +async def test_hosting_publish_is_explicit_versioned_on_service_origin(installed,reader_settings): app,_=installed key,uid,_=await register(app,'site-owner') private_file=await call(app,'content.file_put',{'parent':'/@site-owner/files','name':'index.html', @@ -56,19 +57,24 @@ async def test_hosting_publish_is_explicit_versioned_on_service_origin(installed 'entries':[{'path':'index.html','source':wire(private_file.resources[0])}]},key=key,subject=uid, expected=((site.resources[0].id,site.data['generation']),)) assert published.status=='ok',wire(published) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(app)),base_url=app.settings.service_url) as http: - result=await http.get('/@site-owner/w/') - assert result.status_code==200 and result.content==b'

Published explicitly

' - assert 'set-cookie' not in result.headers - assert 'sandbox' in result.headers['content-security-policy'] - assert 'allow-same-origin' not in result.headers['content-security-policy'] - head=await http.head('/@site-owner/w/') - assert head.headers['content-security-policy']==result.headers['content-security-policy'] - cached=await http.get('/@site-owner/w/',headers={'If-None-Match':result.headers['etag']}) - assert cached.status_code==304 - assert cached.headers['content-security-policy']==result.headers['content-security-policy'] - denied=await http.get('/@site-owner/w/../../root',headers={'Host':'pages.example.test'}) - assert denied.status_code==403 + reader=runtime_class()(reader_settings,clock=app.clock) + await reader.load() + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(reader)),base_url=app.settings.service_url) as http: + result=await http.get('/@site-owner/w/') + assert result.status_code==200 and result.content==b'

Published explicitly

' + assert 'set-cookie' not in result.headers + assert 'sandbox' in result.headers['content-security-policy'] + assert 'allow-same-origin' not in result.headers['content-security-policy'] + head=await http.head('/@site-owner/w/') + assert head.headers['content-security-policy']==result.headers['content-security-policy'] + cached=await http.get('/@site-owner/w/',headers={'If-None-Match':result.headers['etag']}) + assert cached.status_code==304 + assert cached.headers['content-security-policy']==result.headers['content-security-policy'] + denied=await http.get('/@site-owner/w/../../root',headers={'Host':'pages.example.test'}) + assert denied.status_code==403 + finally: + await reader.close() source=await call(app,'discovery.get',{'id':private_file.resources[0].id}) assert source.error.code=='permission_denied' traversal=await call(app,'hosting.deploy',{'id':site.resources[0].id, diff --git a/tests/test_hosting_read_permissions.py b/tests/test_hosting_read_permissions.py new file mode 100644 index 00000000..dba21082 --- /dev/null +++ b/tests/test_hosting_read_permissions.py @@ -0,0 +1,52 @@ +"""Optional content-group sharing grants read, never staging or write authority.""" +import importlib +import os + +import pytest + +from msg.core.errors import Failure +from msg.storage.git import GitContentStore + + +@pytest.mark.parametrize('group_read', [False, True]) +async def test_new_publication_uses_exact_opt_in_read_group_mode(tmp_path, group_read): + previous = os.umask(0o077) + try: + store = GitContentStore(tmp_path / 'content', binary_dir=tmp_path / 'blobs', + staging_dir=tmp_path / 'staging', group_read=group_read) + text = await store.put_bytes(b'

group reader

', 'text/html') + binary = await store.put_bytes(b'\0\x01\x02\xff', 'application/octet-stream') + finally: + os.umask(previous) + expected = 0o040 if group_read else 0 + for path in (store.index / text.digest[7:], store.index / binary.digest[7:], + store.binary / binary.digest[7:]): + assert path.stat().st_mode & 0o077 == expected + assert store.staging.stat().st_mode & 0o077 == 0 + if group_read: + for path in (store.path, store.index, store.binary): + assert path.stat().st_mode & 0o2777 == 0o2750 + entry = store._entry(text) + obj = store.repo / 'objects' / entry['oid'][:2] / entry['oid'][2:] + assert obj.stat().st_mode & 0o077 == 0o040 + reader_class = importlib.import_module('msg.storage.read_only').GitContentReader + reader = reader_class(store.path, binary_dir=store.binary) + assert reader_class.read is GitContentStore.read + assert await reader.read_bytes(text) == b'

group reader

' + assert await reader.read_bytes(binary) == b'\0\x01\x02\xff' + assert not hasattr(reader, 'put_bytes') + assert not hasattr(reader, 'staging') + + +def test_enabling_group_reads_never_silently_changes_existing_permissions(tmp_path): + previous = os.umask(0o077) + try: + private = GitContentStore(tmp_path / 'content', binary_dir=tmp_path / 'blobs', + staging_dir=tmp_path / 'staging') + finally: + os.umask(previous) + before = {str(p): p.stat().st_mode for p in tmp_path.rglob('*')} + with pytest.raises(Failure, match='content_sharing_not_prepared'): + GitContentStore(private.path, binary_dir=private.binary, + staging_dir=private.staging, group_read=True) + assert before == {str(p): p.stat().st_mode for p in tmp_path.rglob('*')} diff --git a/tests/test_hosting_runtime.py b/tests/test_hosting_runtime.py new file mode 100644 index 00000000..2f432f05 --- /dev/null +++ b/tests/test_hosting_runtime.py @@ -0,0 +1,303 @@ +"""The hosting process reads current authority without owning a write service.""" +from dataclasses import replace +from datetime import timedelta +from pathlib import Path +import importlib +import uuid + +import httpx +import psycopg +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict, make_conninfo +import pytest + +from msg.core.codec import b64, canonical, wire +from msg.core.errors import Failure +from msg.core.requests import request_for +from test_service import NOW, call, register + +# Independent acceptance inventory: no ledger, token delivery, vault or job access. +READ_TABLES = ( + 'schema_version', 'resources', 'revisions', 'identities', 'memberships', + 'credentials', 'certificates', 'settings', 'share_grants', 'share_grants_v2', + 'dm_conversations', 'system_sources', +) + + +def runtime_class(): + # Import inside tests so the pre-implementation baseline fails tests, not collection. + return importlib.import_module('msg.hosting_runtime').HostingRuntime + + +@pytest.fixture +async def reader_settings(installed): + app, _ = installed + role = 'hosting_' + uuid.uuid4().hex + password = uuid.uuid4().hex + admin_dsn = app.settings.server.postgres_dsn + connection_info = conninfo_to_dict(admin_dsn) + with psycopg.connect(admin_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('CREATE ROLE {} LOGIN PASSWORD {} NOSUPERUSER NOCREATEDB ' + 'NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS').format( + sql.Identifier(role), sql.Literal(password))) + conn.execute(sql.SQL('GRANT CONNECT ON DATABASE {} TO {}').format( + sql.Identifier(connection_info['dbname']), sql.Identifier(role))) + conn.execute(sql.SQL('GRANT USAGE ON SCHEMA public TO {}').format(sql.Identifier(role))) + conn.execute(sql.SQL('GRANT SELECT ON {} TO {}').format( + sql.SQL(', ').join(sql.Identifier('public', name) for name in READ_TABLES), + sql.Identifier(role))) + dsn = make_conninfo(admin_dsn, user=role, password=password) + settings = replace(app.settings, server=replace(app.settings.server, + postgres_dsn=dsn, valkey_url=None, mail=None), + public_web_origin=app.settings.service_url) + try: + yield settings + finally: + with psycopg.connect(admin_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('DROP OWNED BY {}').format(sql.Identifier(role))) + conn.execute(sql.SQL('DROP ROLE {}').format(sql.Identifier(role))) + + +async def website(app): + key, subject, certificate = await register(app, 'readonly-host') + site = await call(app, 'hosting.create', {'parent': '/@readonly-host', 'name': 'web'}, + key=key, subject=subject) + source = await call(app, 'content.file_put', {'parent': '/@readonly-host/files', + 'name': 'index.html', 'data': b64(b'

published

'), 'media_type': 'text/html'}, + key=key, subject=subject) + assert site.status == source.status == 'ok' + entries = [{'path': 'index.html', 'source': wire(source.resources[0])}] + published = await call(app, 'hosting.deploy', {'id': site.resources[0].id, 'entries': entries}, + key=key, subject=subject, expected=((site.resources[0].id, site.data['generation']),)) + assert published.status == 'ok', wire(published) + preview = await call(app, 'hosting.preview', {'id': site.resources[0].id, 'entries': entries}, + key=key, subject=subject, expected=((site.resources[0].id, published.data['generation']),)) + assert preview.status == 'ok', wire(preview) + return key, subject, certificate, site, preview + + +def isolated(response): + assert 'sandbox' in response.headers['content-security-policy'] + assert 'allow-same-origin' not in response.headers['content-security-policy'] + assert response.headers['cache-control'] == 'no-store' + assert response.headers['x-content-type-options'] == 'nosniff' + assert 'set-cookie' not in response.headers + + +async def test_hosting_load_has_no_application_signers_or_startup_writes(installed, reader_settings, monkeypatch): + app, _ = installed + await website(app) + cls = runtime_class() + from msg.application import Application + import msg.bootstrap + from msg.security.crypto import Ed25519Signer + + def forbidden(*args, **kwargs): + pytest.fail('read-only hosting assembled a signer, full Application or installer') + + monkeypatch.setattr(Application, '__init__', forbidden) + monkeypatch.setattr(Ed25519Signer, 'from_bytes', forbidden) + monkeypatch.setattr(msg.bootstrap, 'sync_system_sources', forbidden) + before = sorted(str(p.relative_to(reader_settings.server.content_dir)) + for p in reader_settings.server.content_dir.rglob('*')) + keys = reader_settings.service_keys + original_mode = keys.stat().st_mode & 0o777 + keys.chmod(0) + reader = cls(reader_settings, clock=lambda: NOW) + try: + # This is an actual OS denial on the unprivileged CI runner, not a mock. + with pytest.raises(PermissionError): + (keys / 'online.key').read_bytes() + await reader.load() + for name in ('executor', 'online_signer', 'receipt_signer', '_token_secret', + '_vault_key', 'token_delivery', 'online_issuer', 'application', 'app'): + assert not hasattr(reader, name), name + assert not hasattr(reader.contents, 'put') + assert not hasattr(reader.contents, 'pin') + from msg.extensions.hosting import hosting_app + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(reader)), + base_url=reader_settings.service_url) as http: + response = await http.get('/@readonly-host/web/') + assert response.status_code == 200 and response.content == b'

published

' + isolated(response) + assert before == sorted(str(p.relative_to(reader_settings.server.content_dir)) + for p in reader_settings.server.content_dir.rglob('*')) + finally: + keys.chmod(original_mode) + await reader.close() + + +async def test_hosting_reuses_exact_registry_contracts_but_no_business_callbacks(installed, reader_settings): + app, _ = installed + reader = runtime_class()(reader_settings, clock=lambda: NOW) + await reader.load() + try: + assert reader.registry.catalog() == app.registry.catalog() + assert reader.registry.capabilities() == app.registry.capabilities() + assert reader.registry.resource_types() == app.registry.resource_types() + for operation in reader.registry.operations(): + assert reader.registry.schema(operation.input_schema) == app.registry.schema(operation.input_schema) + assert operation.handler.__closure__ is None + assert operation.requirements.__closure__ is None + with pytest.raises(Failure, match='readonly_registry'): + await operation.handler(None, None, None) + with pytest.raises(Failure, match='readonly_registry'): + await operation.requirements(None, None) + for manifest in reader.registry._plugins.values(): + assert all(spec is reader.registry.operation(spec.name, spec.version) + for spec in manifest.operations) + finally: + await reader.close() + + +async def test_hosting_database_has_both_transaction_and_role_write_denial(reader_settings): + reader = runtime_class()(reader_settings, clock=lambda: NOW) + await reader.load() + try: + with pytest.raises(Failure, match='read_only_transaction'): + async with reader.metadata.transaction(write=True): + pytest.fail('write transaction was entered') + async with reader.metadata.transaction(write=False) as tx: + assert tx.one('SHOW transaction_read_only') == ('on',) + with pytest.raises(Failure, match='read_only_transaction'): + tx.execute("UPDATE settings SET value=value", write=True) + # Even bypassing the wrapper and READ ONLY transaction, DB grants deny it. + with psycopg.connect(reader_settings.server.postgres_dsn) as conn: + with pytest.raises(psycopg.errors.InsufficientPrivilege): + conn.execute('UPDATE public.settings SET value=value') + with psycopg.connect(reader_settings.server.postgres_dsn) as conn: + with pytest.raises(psycopg.errors.InsufficientPrivilege): + conn.execute('SELECT * FROM public.token_deliveries') + finally: + await reader.close() + + +async def test_hosting_rejects_writer_database_account(installed): + app, _ = installed + reader = runtime_class()(replace(app.settings, public_web_origin=app.settings.service_url), + clock=lambda: NOW) + try: + with pytest.raises(Failure, match='hosting_database_not_readonly'): + await reader.load() + finally: + await reader.close() + + +@pytest.mark.parametrize('damage', ['trust', 'release', 'content']) +async def test_hosting_missing_or_stale_installation_fails_without_repair(installed, reader_settings, damage): + app, _ = installed + expected = {'trust': 'root_not_initialized', 'release': 'hosting_installation_stale', + 'content': 'content_store_not_initialized'}[damage] + settings = reader_settings + if damage == 'trust': + settings.trust_file.unlink() + elif damage == 'release': + async with app.metadata.transaction(write=True) as tx: + tx.execute("UPDATE system_sources SET source_digest='sha256:stale' WHERE resource_id='r_agents'", + write=True) + else: + settings = replace(settings, server=replace(settings.server, + content_dir=settings.server.content_dir.parent / 'missing-content')) + reader = runtime_class()(settings, clock=lambda: NOW) + try: + with pytest.raises(Failure, match=expected): + await reader.load() + if damage == 'trust': + assert not settings.trust_file.exists() + elif damage == 'content': + assert not settings.server.content_dir.exists() + else: + async with app.metadata.transaction(write=False) as tx: + assert tx.one("SELECT source_digest FROM system_sources WHERE resource_id='r_agents'") == ('sha256:stale',) + finally: + await reader.close() + + +async def test_hosting_private_preview_head_range_revocation_and_zero_effects(installed, reader_settings): + app, _ = installed + key, subject, certificate, site, preview = await website(app) + reader = runtime_class()(reader_settings, clock=lambda: NOW) + await reader.load() + from msg.extensions.hosting import hosting_app + candidate = preview.resources[0].id + path = f'/@readonly-host/web/_preview/{candidate}/index.html' + packet = request_for('discovery.raw', {'id': candidate}, app.settings.service_url, + signer=key, subject=subject, certificates=(certificate,), expires_at=NOW+timedelta(seconds=120)) + headers = {'X-Msg-Request': b64(canonical(wire(packet)))} + async with app.metadata.transaction(write=False) as tx: + before = {table: tuple(tx.rows(f'SELECT * FROM {table}')) for table in + ('resources', 'revisions', 'events', 'results', 'credentials', 'jobs')} + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(reader)), + base_url=app.settings.service_url) as http: + assert (await http.get(path)).status_code == 403 + page = await http.get(path, headers=headers) + assert page.status_code == 200 and page.content == b'

published

' + isolated(page) + assert (await http.head(path, headers=headers)).content == b'' + assert (await http.get(path, headers={**headers, 'Range': 'bytes=0-3'})).content == b'

' + assert (await http.get(path, headers={**headers, 'If-None-Match': page.headers['etag']})).status_code == 304 + assert (await http.get('/@readonly-host/web/missing.html')).status_code == 404 + assert (await http.post(path, headers=headers, content=b'write')).status_code == 405 + assert (await http.post('/-/p/content.post_create', content=b'{}')).status_code == 404 + assert (await http.get('/@readonly-host/web/', headers={'Host': 'other.invalid'})).status_code == 403 + async with app.metadata.transaction(write=False) as tx: + assert before == {table: tuple(tx.rows(f'SELECT * FROM {table}')) for table in before} + revoked = await call(app, 'content.chmod', {'id': candidate, 'mode': '0000'}, + key=key, subject=subject, expected=((candidate, preview.data['generation']),)) + assert revoked.status == 'ok' + assert (await http.get(path, headers=headers)).status_code == 403 + finally: + await reader.close() + + +@pytest.mark.parametrize('gate', ['database', 'marker']) +async def test_hosting_checks_quarantine_before_disclosing_bytes(installed, reader_settings, gate): + app, _ = installed + await website(app) + reader = runtime_class()(reader_settings, clock=lambda: NOW) + await reader.load() + from msg.extensions.hosting import hosting_app + try: + if gate == 'database': + async with app.metadata.transaction(write=True) as tx: + tx.execute("INSERT INTO settings(key,value) VALUES ('recovery_quarantine','{}')", write=True) + else: + (app.settings.config_dir / 'recovery-drill.json').write_text('{}') + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(reader)), + base_url=app.settings.service_url) as http: + response = await http.get('/@readonly-host/web/') + assert response.status_code != 200 + assert b'

published

' not in response.content + assert b'recovery_quarantined' in response.content + isolated(response) + finally: + await reader.close() + + +def test_hosting_unit_has_distinct_identity_and_no_data_write_grant(): + service = (Path(__file__).resolve().parents[1] / 'deploy/msgd-hosting.service').read_text() + assert 'User=msgd-hosting\n' in service + assert 'Group=msgd-hosting\n' in service + assert 'ReadWritePaths=' not in service + assert '--config-dir /etc/msgd-hosting' in service + assert 'ProtectSystem=strict' in service + assert '/var/lib/msgd/service' in service + + +def test_hosting_subcommand_never_assembles_main_application(reader_settings, monkeypatch): + from msg import daemon + import msg.config + import uvicorn + called = [] + + def forbidden(*args, **kwargs): + pytest.fail('hosting constructed the normal write Application') + + monkeypatch.setattr(daemon, 'load_application', forbidden) + monkeypatch.setattr(daemon, 'network_runtime', lambda settings: called.append(settings)) + monkeypatch.setattr(msg.config, 'load_settings', lambda directory: reader_settings) + monkeypatch.setattr(uvicorn, 'run', lambda asgi, **options: called.append((asgi, options))) + assert daemon.main(['--config-dir', str(reader_settings.config_dir), 'hosting']) == 0 + assert called[0] == reader_settings + assert len(called) == 2 and called[1][1]['access_log'] is False diff --git a/tests/test_hosting_runtime_authority.py b/tests/test_hosting_runtime_authority.py new file mode 100644 index 00000000..4b2fae0e --- /dev/null +++ b/tests/test_hosting_runtime_authority.py @@ -0,0 +1,106 @@ +"""Retained #174 authority cases on #173's single hosting implementation. + +These tests use disposable PostgreSQL objects and real signed calls. No second +runtime, shared installer or alternative publication policy is introduced. +""" +import httpx +import psycopg +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict +import pytest + +from msg.core.codec import b64, canonical, wire +from msg.core.errors import Failure +from msg.security.crypto import Ed25519Signer +from test_hosting_runtime import reader_settings, runtime_class +from test_hosting_runtime_isolation import preview +from test_service import call + + +async def test_revoked_signing_key_cannot_reuse_a_warm_private_preview(installed, reader_settings): + app, _ = installed + key, subject, _, payload, request = await preview(app, 'revocable-reader') + replacement = Ed25519Signer.generate() + public = b64(replacement.public_key) + proof = replacement.sign(canonical({'subject_id': subject, 'public_key': public}), purpose='key-add') + added = await call(app, 'identity.key_add', {'public_key': public, + 'possession_proof': wire(proof), 'ceiling': wire(app.primary_ceiling())}, key=key, subject=subject) + assert added.status == 'ok', wire(added) + service = runtime_class()(reader_settings, clock=app.clock) + await service.load() + try: + from msg.extensions.hosting import hosting_app + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(service)), + base_url=app.settings.service_url) as http: + headers = {'X-Msg-Request': request['header']} + accepted = await http.get(request['path'], headers=headers) + assert accepted.status_code == 200 and accepted.content == payload + revoked = await call(app, 'identity.key_revoke', {'key_id': key.key_id}, + key=replacement, subject=subject) + assert revoked.status == 'ok', wire(revoked) + for method, extra in (('GET', {}), ('HEAD', {}), ('GET', {'Range': 'bytes=0-3'}), + ('GET', {'If-None-Match': accepted.headers['etag']})): + denied = await http.request(method, request['path'], headers={**headers, **extra}) + assert denied.status_code == 403 and payload not in denied.content + finally: + await service.close() + + +async def test_column_only_update_is_not_a_readonly_database_role(installed, reader_settings): + app, _ = installed + reader = conninfo_to_dict(reader_settings.server.postgres_dsn)['user'] + with psycopg.connect(app.settings.server.postgres_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('GRANT UPDATE(value) ON settings TO {}').format(sql.Identifier(reader))) + service = runtime_class()(reader_settings, clock=app.clock) + try: + with pytest.raises(Failure, match='hosting_database_not_readonly'): + await service.load() + finally: + await service.close() + + +@pytest.mark.parametrize('kind', ['table', 'schema', 'sequence', 'database']) +async def test_owner_can_restore_revoked_rights_and_must_not_be_a_reader(installed, reader_settings, kind): + app, _ = installed + reader = conninfo_to_dict(reader_settings.server.postgres_dsn)['user'] + name = 'hosting_owned_probe' + owner_dsn = app.settings.server.postgres_dsn + service = runtime_class()(reader_settings, clock=app.clock) + with psycopg.connect(owner_dsn, autocommit=True) as conn: + original_owner, database = conn.info.user, conn.info.dbname + if kind == 'table': + conn.execute('CREATE TABLE public.hosting_owned_probe (value integer)') + conn.execute(sql.SQL('ALTER TABLE public.hosting_owned_probe OWNER TO {}').format(sql.Identifier(reader))) + grant, privilege = 'TABLE public.hosting_owned_probe', 'UPDATE' + elif kind == 'schema': + conn.execute(sql.SQL('CREATE SCHEMA hosting_owned_probe AUTHORIZATION {}').format(sql.Identifier(reader))) + grant, privilege = 'SCHEMA hosting_owned_probe', 'CREATE' + elif kind == 'sequence': + conn.execute('CREATE SEQUENCE public.hosting_owned_probe') + conn.execute(sql.SQL('ALTER SEQUENCE public.hosting_owned_probe OWNER TO {}').format(sql.Identifier(reader))) + grant, privilege = 'SEQUENCE public.hosting_owned_probe', 'USAGE' + else: + conn.execute(sql.SQL('ALTER DATABASE {} OWNER TO {}').format(sql.Identifier(database), sql.Identifier(reader))) + grant = sql.SQL('DATABASE {}').format(sql.Identifier(database)).as_string(conn) + privilege = 'CREATE, TEMPORARY' + # Keep CONNECT for a database owner; revoke every object ACL otherwise. + revoked = privilege if kind == 'database' else 'ALL' + query = sql.SQL('REVOKE {} ON {} FROM {}').format(sql.SQL(revoked), sql.SQL(grant), sql.Identifier(reader)) + conn.execute(query) + try: + # Demonstrate why current ACL bits alone cannot establish a read-only + # boundary: the owner can recover rights without an administrator. + with psycopg.connect(reader_settings.server.postgres_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('GRANT {} ON {} TO {}').format( + sql.SQL(privilege), sql.SQL(grant), sql.Identifier(reader))) + conn.execute(query) + with pytest.raises(Failure, match='hosting_database_not_readonly'): + await service.load() + finally: + await service.close() + if kind == 'database': + # DROP OWNED does not restore database ownership; restore it before + # the shared role fixture removes the disposable reader login. + with psycopg.connect(owner_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('ALTER DATABASE {} OWNER TO {}').format( + sql.Identifier(database), sql.Identifier(original_owner))) diff --git a/tests/test_hosting_runtime_isolation.py b/tests/test_hosting_runtime_isolation.py new file mode 100644 index 00000000..9d67c7e7 --- /dev/null +++ b/tests/test_hosting_runtime_isolation.py @@ -0,0 +1,284 @@ +"""Independent-UID and failure-containment cases retained from PR #174. + +Adapted from eafea15af0b8ff26c0208d5c0fcc039b4d32fb56 to the #173 reader API. +These are disposable CI installs, not production permission changes. +""" +from datetime import timedelta +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +import httpx +import psycopg +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict +import pytest + +from msg.core.codec import b64, canonical, wire +from msg.core.errors import Failure +from msg.core.requests import request_for +from test_hosting_runtime import reader_settings, runtime_class, isolated +from test_service import NOW, call, register + + +async def preview(app, name='readonly-preview', *, binary=False): + key, subject, _ = await register(app, name) + site = await call(app, 'hosting.create', {'parent': '/@'+name, 'name': 'web'}, + key=key, subject=subject) + assert site.status == 'ok', wire(site) + payload = b'\x00readonly-live-binary\xff' if binary else b'

private read-only preview

' + filename = 'asset.bin' if binary else 'index.html' + source = await call(app, 'content.file_put', {'parent': '/@'+name+'/files', + 'name': filename, 'data': b64(payload), + 'media_type': 'application/octet-stream' if binary else 'text/html'}, key=key, subject=subject) + assert source.status == 'ok', wire(source) + candidate = await call(app, 'hosting.preview', {'id': site.resources[0].id, + 'entries': [{'path': filename, 'source': wire(source.resources[0])}]}, + key=key, subject=subject, expected=((site.resources[0].id, site.data['generation']),)) + assert candidate.status == 'ok', wire(candidate) + packet = request_for('discovery.raw', {'id': candidate.resources[0].id}, app.settings.service_url, + signer=key, subject=subject, expires_at=NOW+timedelta(seconds=120)) + return key, subject, candidate, payload, { + 'path': f'/@{name}/web/_preview/{candidate.resources[0].id}/{filename}', + 'header': b64(canonical(wire(packet))), 'payload': b64(payload)} + + +async def test_preview_rechecks_acl_before_head_range_and_cached_response(installed, reader_settings): + app, _ = installed + key, subject, candidate, payload, request = await preview(app) + service = runtime_class()(reader_settings, clock=app.clock) + await service.load() + try: + from msg.extensions.hosting import hosting_app + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(service)), + base_url=app.settings.service_url) as http: + path, headers = request['path'], {'X-Msg-Request': request['header']} + assert (await http.get(path)).status_code == 403 + accepted = await http.get(path, headers=headers) + assert accepted.status_code == 200 and accepted.content == payload + isolated(accepted) + assert (await http.get('/-/schema')).status_code == 404 + assert (await http.get(path, headers={**headers, 'Host': 'other.example'})).status_code == 403 + denied = await call(app, 'content.chmod', {'id': candidate.resources[0].id, 'mode': '0000'}, + key=key, subject=subject, expected=((candidate.resources[0].id, candidate.data['generation']),)) + assert denied.status == 'ok', wire(denied) + for method, extra in (('GET', {}), ('HEAD', {}), ('GET', {'Range': 'bytes=0-7'}), + ('GET', {'If-None-Match': accepted.headers['etag']})): + response = await http.request(method, path, headers={**headers, **extra}) + assert response.status_code == 403 and payload not in response.content + finally: + await service.close() + + +@pytest.mark.parametrize('gate', ['database', 'marker', 'marker_symlink']) +async def test_quarantine_revokes_cached_reads_and_new_startup(installed, reader_settings, gate): + app, _ = installed + service = runtime_class()(reader_settings, clock=app.clock) + await service.load() + marker = reader_settings.config_dir/'recovery-drill.json' + try: + from msg.extensions.hosting import hosting_app + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(service)), + base_url=app.settings.service_url) as http: + original = await http.get('/@root/web/index.html') + assert original.status_code == 200 + if gate == 'database': + async with app.metadata.transaction(write=True) as tx: + tx.set_setting('recovery_quarantine', {'version': 1, 'test': True}) + elif gate == 'marker': + marker.write_text('{}') + else: + marker.symlink_to(marker.with_name('missing-marker-target')) + for method in ('GET', 'HEAD'): + response = await http.request(method, '/@root/web/index.html', + headers={'If-None-Match': original.headers['etag']}) + assert response.status_code not in (200, 206, 304) + assert original.content not in response.content + fresh = runtime_class()(reader_settings, clock=app.clock) + try: + with pytest.raises(Failure, match='recovery_quarantined'): + await fresh.load() + finally: + await fresh.close() + if gate != 'database': + marker.unlink() + assert (await http.get('/@root/web/index.html')).status_code not in (200, 206, 304) + finally: + marker.unlink(missing_ok=True) + await service.close() + + +@pytest.mark.parametrize('damage', ['missing', 'wrong_version', 'missing_read_grant']) +async def test_required_schema_is_verified_without_repair(installed, reader_settings, damage): + app, _ = installed + reader = conninfo_to_dict(reader_settings.server.postgres_dsn)['user'] + with psycopg.connect(app.settings.server.postgres_dsn, autocommit=True) as conn: + if damage == 'missing': + conn.execute('DROP TABLE schema_version') + elif damage == 'wrong_version': + conn.execute('UPDATE schema_version SET version=999') + else: + conn.execute(sql.SQL('REVOKE SELECT ON share_grants_v2 FROM {}').format(sql.Identifier(reader))) + service = runtime_class()(reader_settings, clock=app.clock) + try: + with pytest.raises(Failure, match='hosting_installation_stale'): + await service.load() + finally: + await service.close() + with psycopg.connect(app.settings.server.postgres_dsn) as conn: + if damage == 'missing': + assert conn.execute("SELECT to_regclass('public.schema_version')").fetchone() == (None,) + elif damage == 'wrong_version': + assert conn.execute('SELECT version FROM schema_version').fetchall() == [(999,)] + else: + assert not conn.execute('SELECT has_table_privilege(%s,%s,%s)', + (reader, 'public.share_grants_v2', 'SELECT')).fetchone()[0] + + +@pytest.mark.parametrize('extra', ['write', 'membership', 'definer', 'sequence', 'secret_read']) +async def test_effective_privilege_escalations_fail_startup(installed, reader_settings, extra): + app, _ = installed + reader = conninfo_to_dict(reader_settings.server.postgres_dsn)['user'] + with psycopg.connect(app.settings.server.postgres_dsn, autocommit=True) as conn: + if extra == 'write': + conn.execute(sql.SQL('GRANT UPDATE ON settings TO {}').format(sql.Identifier(reader))) + elif extra == 'membership': + conn.execute(sql.SQL('GRANT {} TO {}').format(sql.Identifier(conn.info.user), sql.Identifier(reader))) + elif extra == 'definer': + conn.execute('CREATE FUNCTION hosting_test_definer() RETURNS integer LANGUAGE sql ' + "SECURITY DEFINER AS 'SELECT 1'") + elif extra == 'sequence': + conn.execute('CREATE SEQUENCE hosting_test_sequence') + conn.execute(sql.SQL('GRANT USAGE ON SEQUENCE hosting_test_sequence TO {}').format(sql.Identifier(reader))) + else: + conn.execute(sql.SQL('GRANT SELECT ON token_deliveries TO {}').format(sql.Identifier(reader))) + service = runtime_class()(reader_settings, clock=app.clock) + try: + with pytest.raises(Failure, match='hosting_database_not_readonly'): + await service.load() + finally: + await service.close() + + +READ_PROCESS = r''' +import asyncio, json, os, sys +from datetime import UTC, datetime +from pathlib import Path +import httpx +from msg.config import load_settings +from msg.core.codec import unb64 +from msg.hosting_runtime import HostingRuntime +from msg.extensions.hosting import hosting_app + +async def main(): + assert os.geteuid() == 65534, os.geteuid() + settings = load_settings(Path(sys.argv[1])) + try: + Path(sys.argv[2]).read_bytes() + except PermissionError: + pass + else: + raise AssertionError('reader can read service signing material') + try: + (settings.server.content_dir/'forbidden-write').write_bytes(b'no') + except PermissionError: + pass + else: + raise AssertionError('reader can write business files') + service = HostingRuntime(settings, clock=lambda: datetime(2026, 9, 27, tzinfo=UTC)) + await service.load() + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(service)), + base_url=settings.service_url) as http: + assert (await http.get('/@root/web/index.html')).status_code == 200 + for request in json.loads(Path(sys.argv[3]).read_text()): + assert (await http.get(request['path'])).status_code == 403 + response = await http.get(request['path'], headers={'X-Msg-Request': request['header']}) + assert response.status_code == 200, response.status_code + assert response.content == unb64(request['payload']) + print(json.dumps({'uid': os.geteuid(), 'private_key_denied': True, + 'business_write_denied': True, 'live_previews': 2})) + finally: + await service.close() +asyncio.run(main()) +''' + + +async def test_real_other_uid_reads_new_text_and_binary_without_keys(installed, reader_settings): + assert shutil.which('sudo') and shutil.which('setpriv'), 'Linux isolation prerequisites missing' + app, _ = installed + base = Path(tempfile.mkdtemp(prefix='msg-hosting-reader-')) + changed_ancestors = {} + try: + shutil.copytree(Path(__file__).resolve().parents[1]/'src'/'msg', base/'src'/'msg') + from msg.config import write_example + config = write_example(base/'etc', base/'unused', app.settings.service_url, + postgres_dsn='service=hosting-test') + config_file = config.config_dir/'msgd.toml' + text = config_file.read_text().replace('[server]', '[server]\npublic_web_origin = '+json.dumps(app.settings.service_url)) + text = text.replace('[hosting]', '[hosting]\nrecovery_marker = '+json.dumps(str(app.settings.config_dir/'recovery-drill.json'))) + for previous, actual in ((config.server.content_dir, app.contents.path), + (config.server.blob_dir, app.contents.binary), + (config.server.staging_dir, app.contents.staging)): + text = text.replace(str(previous), str(actual)) + config_file.write_text(text) + config.trust_file.parent.mkdir(parents=True) + shutil.copyfile(reader_settings.trust_file, config.trust_file) + service_file = base/'pg_service.conf' + service_file.write_text('[hosting-test]\n'+''.join( + f'{key}={value}\n' for key, value in conninfo_to_dict(reader_settings.server.postgres_dsn).items())) + # Explicitly prepare existing content before opt-in. No post-publication + # chmod/default ACL can hide a mode bug in newly published objects. + for root in (app.contents.path, app.contents.binary): + for parent in root.parents: + if parent in (Path('/tmp'), Path('/')): + break + if parent not in changed_ancestors: + changed_ancestors[parent] = parent.stat().st_mode & 0o7777 + parent.chmod(changed_ancestors[parent] | 0o010) + for path in [root, *root.rglob('*')]: + path.chmod(0o2750 if path.is_dir() else 0o640) + subprocess.run(['git', '--git-dir', str(app.contents.repo), 'config', 'core.sharedRepository', '0640'], + check=True, capture_output=True) + from msg.storage.git import GitContentStore + old_umask = os.umask(0o077) + try: + app.contents = GitContentStore(app.contents.path, binary_dir=app.contents.binary, + staging_dir=app.contents.staging, group_read=True) + first = await preview(app, 'live-text-reader') + second = await preview(app, 'live-binary-reader', binary=True) + finally: + os.umask(old_umask) + requests = base/'requests.json' + requests.write_text(json.dumps([first[-1], second[-1]])) + script = base/'reader.py' + script.write_text(READ_PROCESS) + for path in [base, *base.rglob('*')]: + path.chmod(0o550 if path.is_dir() else 0o440) + def snapshots(): + return {str(path): hashlib.sha256(path.read_bytes()).hexdigest() + for root in (app.contents.path, app.contents.binary) + for path in root.rglob('*') if path.is_file()} + before = snapshots() + result = subprocess.run(['sudo', '-n', '--', 'setpriv', '--reuid=65534', + '--regid='+str(os.getgid()), '--clear-groups', '--no-new-privs', '--', 'env', 'PYTHONDONTWRITEBYTECODE=1', 'PYTHONPATH='+str(base/'src'), + 'PGSERVICEFILE='+str(service_file), sys.executable, str(script), str(config.config_dir), + str(app.settings.service_keys/'online.key'), str(requests)], + capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stderr[-6000:] + assert json.loads(result.stdout) == {'uid': 65534, 'private_key_denied': True, + 'business_write_denied': True, 'live_previews': 2} + assert before == snapshots() + assert not (app.contents.path/'forbidden-write').exists() + finally: + for path in [base, *base.rglob('*')]: + if path.is_dir(): + path.chmod(0o700) + shutil.rmtree(base) + for parent, mode in reversed(tuple(changed_ancestors.items())): + parent.chmod(mode) diff --git a/tests/test_token_delivery_boundary.py b/tests/test_token_delivery_boundary.py new file mode 100644 index 00000000..e8e4743f --- /dev/null +++ b/tests/test_token_delivery_boundary.py @@ -0,0 +1,153 @@ +"""Credential delivery has one owner, separate from service composition.""" +import ast +import asyncio +import hashlib +import hmac +import importlib +import importlib.util +import inspect +from pathlib import Path +import subprocess +import sys + +import pytest + +from msg.core.codec import b64, canonical, loads, unb64, wire +from msg.core.errors import Failure +from msg.core.requests import request_for, receipt_bytes +from msg.security.crypto import verify +from test_service import NOW, temporary_v3_args + + +def delivery_module(): + assert importlib.util.find_spec('msg.security.token_delivery') is not None + return importlib.import_module('msg.security.token_delivery') + + +def test_composition_only_delegates_the_existing_delivery_api(): + from msg.application import Application + module = delivery_module() + tree = ast.parse(inspect.getsource(Application)) + methods = {node.name: node for node in tree.body[0].body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))} + for name in ('issued_token', 'record_token_delivery', '_secrets_for_caller'): + method = methods[name] + assert len(method.body) == 1 and isinstance(method.body[0], ast.Return) + assert 'token_delivery' in ast.unparse(method) + assert Application.recovery_verifier is module.recovery_verifier + assert module.TokenDelivery.recovery_verifier is module.recovery_verifier + + +def test_domain_import_and_constructor_need_no_application_or_plugins(): + module = delivery_module() + assert set(inspect.signature(module.TokenDelivery).parameters) == { + 'metadata', 'token_secret', 'clock', 'recovery_window'} + code = """ +import importlib.abc +import sys +class NoApplication(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.startswith(('msg.application', 'msg.plugins', 'msg.transports', 'msg.storage')): + raise AssertionError('delivery imports composition/adapter: ' + fullname) +sys.meta_path.insert(0, NoApplication()) +from msg.security.token_delivery import TokenDelivery +instance = TokenDelivery(metadata=None, token_secret=b'x'*32, + clock=lambda: None, recovery_window=lambda: 900) +assert not hasattr(instance, 'application') +""" + result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def test_derivation_and_recovery_verifier_keep_the_published_bytes(): + module = delivery_module() + instance = module.TokenDelivery(metadata=None, token_secret=b'k'*32, + clock=lambda: NOW, recovery_window=lambda: 900) + request = request_for('identity.token_create', {'nonce': b64(b'n'*32)}, + 'https://example.invalid', request_id='stable-delivery', contract_version=2) + expected = hmac.digest(b'k'*32, b'issued-token-v1\0' + canonical({ + 'subject': 'u_subject', 'request_id': 'stable-delivery', 'nonce': b64(b'n'*32)}), 'sha256') + assert instance.issued_token(request, 'u_subject') == expected + assert instance.issued_token(request, 'u_other') != expected + assert module.recovery_verifier(b64(b'r'*32)) == hashlib.sha256( + b'token-recovery-v1\0' + b'r'*32).hexdigest() + with pytest.raises(Failure, match='invalid_recovery_secret'): + module.recovery_verifier(b64(b'r'*31)) + + +def test_authentication_uses_the_same_verifier_owner(): + from msg.security import authentication + module = delivery_module() + assert authentication.recovery_verifier is module.recovery_verifier + source = Path(authentication.__file__).read_text() + assert 'token-recovery-v1' not in source + + +async def test_real_service_binds_one_delivery_instance_for_all_executors(installed): + app, _ = installed + module = delivery_module() + assert isinstance(app.token_delivery, module.TokenDelivery) + assert app.executor.response_hook == app.token_delivery.release + assert app.new_executor().response_hook == app.token_delivery.release + + +async def test_standalone_delivery_claim_commits_once_without_persisting_secrets(installed): + app, _ = installed + module = delivery_module() + args, rid, _, _ = temporary_v3_args(request_id='domain-claim-once') + request = request_for('identity.temporary', args, app.settings.service_url, + request_id=rid, contract_version=3, + expires_at=NOW.replace(minute=1)) + app.executor.response_hook = None + committed = await app.executor.execute(request) + assert committed.status == 'ok' and 'token' not in committed.data, wire(committed) + delivery = module.TokenDelivery(metadata=app.metadata, token_secret=app._token_secret, + clock=lambda: NOW, recovery_window=lambda: app.settings.credential_delivery_recovery_window) + answers = await asyncio.gather(delivery.release(request, committed), + delivery.release(request, committed), return_exceptions=True) + successes = [value for value in answers if not isinstance(value, BaseException)] + failures = [value for value in answers if isinstance(value, Failure)] + assert len(successes) == len(failures) == 1 + assert failures[0].code == 'token_delivery_unavailable' + released = successes[0] + assert unb64(released.data['token']) == app.issued_token(request, committed.subject) + assert released.receipt == committed.receipt + verify(app.receipt_signer.public_key, receipt_bytes(released), released.receipt, purpose='receipt') + async with app.metadata.transaction(write=False) as tx: + saved = await tx.request_result(committed.subject, rid, request.payload_digest) + assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes(committed) + assert saved.data == committed.data and 'token' not in saved.data + row = tx.one('SELECT claimed_at,recovery_verifier FROM token_deliveries WHERE credential_id=?', + (committed.data['credential_id'],)) + assert row[0] is not None and row[1] == module.recovery_verifier(args['recovery_secret']) + assert released.data['token'] not in str(row) and args['recovery_secret'] not in str(row) + # Event fields are stored in canonical JSON, not separate SQL columns. + events = [loads(row[0]) for row in tx.rows('SELECT body FROM events')] + matching = [event for event in events if event.get('request_id') == rid] + assert len(matching) == 1 + assert matching[0]['subject'] == committed.subject + assert released.data['token'] not in canonical(matching[0]).decode() + assert args['recovery_secret'] not in canonical(matching[0]).decode() + + +async def test_delivery_failure_returns_no_new_secret_or_claim(installed): + app, _ = installed + module = delivery_module() + args, rid, _, _ = temporary_v3_args(request_id='domain-expired-claim') + request = request_for('identity.temporary', args, app.settings.service_url, + request_id=rid, contract_version=3, expires_at=NOW.replace(minute=1)) + app.executor.response_hook = None + committed = await app.executor.execute(request) + assert committed.status == 'ok', wire(committed) + async with app.metadata.transaction(write=False) as tx: + credential = await tx.credential(committed.data['credential_id']) + delivery = module.TokenDelivery(metadata=app.metadata, token_secret=app._token_secret, + clock=lambda: credential.expires_at, recovery_window=lambda: 900) + with pytest.raises(Failure, match='credential_expired'): + await delivery.release(request, committed) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT claimed_at FROM token_deliveries WHERE credential_id=?', + (credential.id,))[0] is None + saved = await tx.request_result(committed.subject, rid, request.payload_digest) + assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes(committed) + assert saved.data == committed.data and 'token' not in saved.data