From b68526267a4aaea2df8418ad4d400f64a845fa63 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:29:52 +0800 Subject: [PATCH 01/14] test(identity): expose credential delivery ownership without changing its protocol Refs #162 #83 #85. Add independent-construction and real one-time-claim regressions before extracting the existing Application policy. Keep existing credential, client recovery, batch and security suites in the cloud gate. --- .github/workflows/token-delivery-boundary.yml | 82 ++++++++++ docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md | 9 ++ tests/test_token_delivery_boundary.py | 147 ++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 .github/workflows/token-delivery-boundary.yml create mode 100644 docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md create mode 100644 tests/test_token_delivery_boundary.py diff --git a/.github/workflows/token-delivery-boundary.yml b/.github/workflows/token-delivery-boundary.yml new file mode 100644 index 00000000..5178b251 --- /dev/null +++ b/.github/workflows/token-delivery-boundary.yml @@ -0,0 +1,82 @@ +name: Credential delivery boundary +on: + push: + branches: [fix/token-delivery-boundary-20260929] + pull_request: + paths: + - 'src/msg/**' + - 'tests/test_token_delivery_boundary.py' + - '.github/workflows/token-delivery-boundary.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: token-delivery-${{ github.event.pull_request.head.sha || github.sha }} + cancel-in-progress: false +jobs: + focused: + runs-on: ubuntu-latest + timeout-minutes: 12 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest" + --health-interval 5s --health-timeout 5s --health-retries 10 + valkey: + image: valkey/valkey:9.0.6 + ports: ['6379:6379'] + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + MSG_TEST_VALKEY_URL: redis://127.0.0.1:6379/0 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install isolated test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Record source identity + env: + EVENT_HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, platform, subprocess + from pathlib import Path + def git(ref): + return subprocess.check_output(['git', 'rev-parse', ref], text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({ + 'head': os.environ['EVENT_HEAD'], 'checkout': git('HEAD'), 'tree': git('HEAD^{tree}'), + 'run': os.environ['GITHUB_RUN_ID'], 'attempt': os.environ['GITHUB_RUN_ATTEMPT'], + 'python': platform.python_version(), 'scope': 'isolated-regression-not-production' + }, indent=2) + '\n') + PY + - name: Exercise boundaries and unchanged security contracts + run: >- + python -m pytest tests/test_token_delivery_boundary.py tests/test_token_delivery.py + tests/test_credential_delivery_contract.py tests/test_credential_delivery_matrix.py + tests/test_credential_recovery_window.py tests/test_client_token_delivery.py + tests/test_batch_secret_release.py tests/test_security_patch_regressions.py + --junitxml=artifacts/token-delivery-boundary.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: token-delivery-boundary-evidence + path: artifacts/ + if-no-files-found: error diff --git a/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md b/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md new file mode 100644 index 00000000..171cc9f8 --- /dev/null +++ b/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md @@ -0,0 +1,9 @@ +# Credential delivery boundary progress + +Issue #162, supplemental to #167; base2d1cc443c828dd2f735995cc6a5d20021fee6ac7 / tree6171f6041ffd0addaf72eadd35acf562f478281f. Branchfix/token-delivery-boundary-20260929. Coordination #83 comment5879008128. No other source branch is overwritten. + +Regression-first: independent import/construction, one verifier owner, compatibility forwarding without business policy in Application, unchanged derivation/receipt bytes, actual PostgreSQL concurrent one-time claim and expired refusal without consuming a claim. Existing full token/credential/recovery/client/batch security suites remain. Project execution is cloud-only; source editing and artifact inspection are local. + +Pending implementation: one narrow TokenDelivery owner, explicit metadata/key/clock/current-window dependencies, same service for identity forwarding and post-commit response release. Keep transaction-local recovery binding separate from the later claim commit; only then may a token enter a response. Persisted results/events/receipts remain secret-free. Preserve current settings/clock providers rather than freeze them accidentally. + +Final focused/full four-shard/node-ID/conformance/build/recovery outcomes are pending. No deployment, actual credentials, Root/PIN, backup destruction, external delivery or recovery promotion. #67 stays completed and its original functional regressions remain binding; this change is ownership-only, not a new token protocol. diff --git a/tests/test_token_delivery_boundary.py b/tests/test_token_delivery_boundary.py new file mode 100644 index 00000000..86c91522 --- /dev/null +++ b/tests/test_token_delivery_boundary.py @@ -0,0 +1,147 @@ +"""Credential delivery has one owner, separate from service composition.""" +import ast +import asyncio +import hashlib +import hmac +import importlib +import importlib.util +import inspect +from pathlib import Path +import subprocess +import sys + +import pytest + +from msg.core.codec import b64, canonical, unb64, wire +from msg.core.errors import Failure +from msg.core.requests import request_for, receipt_bytes +from msg.security.crypto import verify +from test_service import NOW, temporary_v3_args + + +def delivery_module(): + assert importlib.util.find_spec('msg.security.token_delivery') is not None + return importlib.import_module('msg.security.token_delivery') + + +def test_composition_only_delegates_the_existing_delivery_api(): + from msg.application import Application + module = delivery_module() + tree = ast.parse(inspect.getsource(Application)) + methods = {node.name: node for node in tree.body[0].body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))} + for name in ('issued_token', 'record_token_delivery', '_secrets_for_caller'): + method = methods[name] + assert len(method.body) == 1 and isinstance(method.body[0], ast.Return) + assert 'token_delivery' in ast.unparse(method) + assert Application.recovery_verifier is module.recovery_verifier + assert module.TokenDelivery.recovery_verifier is module.recovery_verifier + + +def test_domain_import_and_constructor_need_no_application_or_plugins(): + module = delivery_module() + assert set(inspect.signature(module.TokenDelivery).parameters) == { + 'metadata', 'token_secret', 'clock', 'recovery_window'} + code = """ +import importlib.abc +import sys +class NoApplication(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.startswith(('msg.application', 'msg.plugins', 'msg.transports', 'msg.storage')): + raise AssertionError('delivery imports composition/adapter: ' + fullname) +sys.meta_path.insert(0, NoApplication()) +from msg.security.token_delivery import TokenDelivery +instance = TokenDelivery(metadata=None, token_secret=b'x'*32, + clock=lambda: None, recovery_window=lambda: 900) +assert not hasattr(instance, 'application') +""" + result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def test_derivation_and_recovery_verifier_keep_the_published_bytes(): + module = delivery_module() + instance = module.TokenDelivery(metadata=None, token_secret=b'k'*32, + clock=lambda: NOW, recovery_window=lambda: 900) + request = request_for('identity.token_create', {'nonce': b64(b'n'*32)}, + 'https://example.invalid', request_id='stable-delivery', contract_version=2) + expected = hmac.digest(b'k'*32, b'issued-token-v1\0' + canonical({ + 'subject': 'u_subject', 'request_id': 'stable-delivery', 'nonce': b64(b'n'*32)}), 'sha256') + assert instance.issued_token(request, 'u_subject') == expected + assert instance.issued_token(request, 'u_other') != expected + assert module.recovery_verifier(b64(b'r'*32)) == hashlib.sha256( + b'token-recovery-v1\0' + b'r'*32).hexdigest() + with pytest.raises(Failure, match='invalid_recovery_secret'): + module.recovery_verifier(b64(b'r'*31)) + + +def test_authentication_uses_the_same_verifier_owner(): + from msg.security import authentication + module = delivery_module() + assert authentication.recovery_verifier is module.recovery_verifier + source = Path(authentication.__file__).read_text() + assert 'token-recovery-v1' not in source + + +async def test_real_service_binds_one_delivery_instance_for_all_executors(installed): + app, _ = installed + module = delivery_module() + assert isinstance(app.token_delivery, module.TokenDelivery) + assert app.executor.response_hook == app.token_delivery.release + assert app.new_executor().response_hook == app.token_delivery.release + + +async def test_standalone_delivery_claim_commits_once_without_persisting_secrets(installed): + app, _ = installed + module = delivery_module() + args, rid, _, _ = temporary_v3_args(request_id='domain-claim-once') + request = request_for('identity.temporary', args, app.settings.service_url, + request_id=rid, contract_version=3, + expires_at=NOW.replace(minute=1)) + app.executor.response_hook = None + committed = await app.executor.execute(request) + assert committed.status == 'ok' and 'token' not in committed.data, wire(committed) + delivery = module.TokenDelivery(metadata=app.metadata, token_secret=app._token_secret, + clock=lambda: NOW, recovery_window=lambda: app.settings.credential_delivery_recovery_window) + answers = await asyncio.gather(delivery.release(request, committed), + delivery.release(request, committed), return_exceptions=True) + successes = [value for value in answers if not isinstance(value, BaseException)] + failures = [value for value in answers if isinstance(value, Failure)] + assert len(successes) == len(failures) == 1 + assert failures[0].code == 'token_delivery_unavailable' + released = successes[0] + assert unb64(released.data['token']) == app.issued_token(request, committed.subject) + assert released.receipt == committed.receipt + verify(app.receipt_signer.public_key, receipt_bytes(released), released.receipt, purpose='receipt') + async with app.metadata.transaction(write=False) as tx: + saved = await tx.request_result(committed.subject, rid, request.payload_digest) + assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes(committed) + assert saved.data == committed.data and 'token' not in saved.data + row = tx.one('SELECT claimed_at,recovery_verifier FROM token_deliveries WHERE credential_id=?', + (committed.data['credential_id'],)) + assert row[0] is not None and row[1] == module.recovery_verifier(args['recovery_secret']) + assert released.data['token'] not in str(row) and args['recovery_secret'] not in str(row) + assert tx.one('SELECT COUNT(*) FROM events WHERE request_id=?', (rid,))[0] == 1 + + +async def test_delivery_failure_returns_no_new_secret_or_claim(installed): + app, _ = installed + module = delivery_module() + args, rid, _, _ = temporary_v3_args(request_id='domain-expired-claim') + request = request_for('identity.temporary', args, app.settings.service_url, + request_id=rid, contract_version=3, expires_at=NOW.replace(minute=1)) + app.executor.response_hook = None + committed = await app.executor.execute(request) + assert committed.status == 'ok', wire(committed) + async with app.metadata.transaction(write=False) as tx: + credential = await tx.credential(committed.data['credential_id']) + delivery = module.TokenDelivery(metadata=app.metadata, token_secret=app._token_secret, + clock=lambda: credential.expires_at, recovery_window=lambda: 900) + with pytest.raises(Failure, match='credential_expired'): + await delivery.release(request, committed) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT claimed_at FROM token_deliveries WHERE credential_id=?', + (credential.id,))[0] is None + saved = await tx.request_result(committed.subject, rid, request.payload_digest) + assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes(committed) + assert saved.data == committed.data and 'token' not in saved.data From f015c60f05469f0058255bd645afef084e87840d Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:33:00 +0800 Subject: [PATCH 02/14] refactor(identity): give one-time credential delivery a single domain owner Refs #162 #83 #85. Preserve issuance/claim transaction timing, legacy derivation and receipt bytes, expiry and one-time response semantics. Application only composes/delegates; authentication shares the exact verifier. Narrow current clock/window providers preserve live settings changes. All three native source blobs match the reviewed local text. --- docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md | 18 ++++- src/msg/application.py | 71 ++++-------------- src/msg/security/authentication.py | 5 +- src/msg/security/token_delivery.py | 93 ++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 64 deletions(-) create mode 100644 src/msg/security/token_delivery.py diff --git a/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md b/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md index 171cc9f8..32116115 100644 --- a/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md +++ b/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md @@ -2,8 +2,20 @@ Issue #162, supplemental to #167; base2d1cc443c828dd2f735995cc6a5d20021fee6ac7 / tree6171f6041ffd0addaf72eadd35acf562f478281f. Branchfix/token-delivery-boundary-20260929. Coordination #83 comment5879008128. No other source branch is overwritten. -Regression-first: independent import/construction, one verifier owner, compatibility forwarding without business policy in Application, unchanged derivation/receipt bytes, actual PostgreSQL concurrent one-time claim and expired refusal without consuming a claim. Existing full token/credential/recovery/client/batch security suites remain. Project execution is cloud-only; source editing and artifact inspection are local. +## Source ownership -Pending implementation: one narrow TokenDelivery owner, explicit metadata/key/clock/current-window dependencies, same service for identity forwarding and post-commit response release. Keep transaction-local recovery binding separate from the later claim commit; only then may a token enter a response. Persisted results/events/receipts remain secret-free. Preserve current settings/clock providers rather than freeze them accidentally. +Before: Application owns composition plus issued_token/recovery_verifier/record_token_delivery/_secrets_for_caller. Authentication contains a second recovery-verifier formula. -Final focused/full four-shard/node-ID/conformance/build/recovery outcomes are pending. No deployment, actual credentials, Root/PIN, backup destruction, external delivery or recovery promotion. #67 stays completed and its original functional regressions remain binding; this change is ownership-only, not a new token protocol. +After: security.token_delivery owns TokenDelivery and the one recovery_verifier function. Application constructs it, retains compatibility forwarding only, and binds all executors directly to the same release method. Authentication consumes the same pure verifier. Domain dependencies are explicitly metadata, the existing token derivation key, clock and a current-window provider; there is no Root signer, vault, Registry, plugin or HTTP object. + +Current providers deliberately preserve later settings/clock changes. Static text/AST review found the issued_token and release bodies identical to the original, recovery_verifier body identical, and record_token_delivery identical except access to the narrow current-window provider. No project code/test was executed locally. + +## Timing and compatibility + +The existing handler records recovery binding inside its issuance transaction. After it commits, release opens its own write transaction and commits the one-time claim before adding the token to response data. A failed/dropped response still requires the existing independent recovery operation. Results, events, original receipts, IDs, purpose strings, deadlines and operation versions retain their meaning. No new generic response hook, protocol version, identity or recovery mechanism. + +## Regression and cloud evidence + +Tests-first headb68526267a4aaea2df8418ad4d400f64a845fa63 / tree0f653206afd064f3e752cbb3442c202fbd72a4bb contains seven new ownership/construction/wire/concurrent-claim/expired-refusal cases. Focused cloud workflow retains all existing token, credential, recovery-window, client recovery, batch secret and security regressions. The red focused run is not cancelled by the later implementation head; both are bound to exact source identity. Results remain pending until actual artifacts are read. + +Final focused/full four-shard/node-ID/conformance/build/recovery outcomes are pending. No deployment, actual credentials, Root/PIN, backup destruction, external delivery or recovery promotion. #67 stays completed and its original functional regressions remain binding; this change is ownership-only, not a new token protocol. #167 must finish its own combined main gate separately. diff --git a/src/msg/application.py b/src/msg/application.py index 1862a579..752c312f 100644 --- a/src/msg/application.py +++ b/src/msg/application.py @@ -1,20 +1,17 @@ """Explicit composition root for network services. Contains no root signer.""" from __future__ import annotations -from dataclasses import replace -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from pathlib import Path -import hashlib import hmac import importlib from msg.constants import ROOT_SPACE, ROOT_SUBJECT -from msg.core.codec import loads, decode, unb64, b64, canonical, wire +from msg.core.codec import loads, decode, unb64 from msg.core.cursors import CursorCodec -from msg.core.errors import Failure, require +from msg.core.errors import require from msg.core.executor import OperationExecutor from msg.core.models import Certificate, Scope -from msg.core.requests import SECRET_DELIVERY_MIN_VERSION from msg.core.registry import Registry from msg.security.authentication import AuthenticationService from msg.security.authorization import AuthorizationService @@ -22,6 +19,7 @@ from msg.security.certificates import CertificateValidator from msg.security.crypto import Ed25519Signer from msg.security.quarantine import active as quarantine_active +from msg.security.token_delivery import TokenDelivery, recovery_verifier from msg.storage.git import GitContentStore @@ -39,6 +37,7 @@ def __init__(self,settings, *, clock=None, selftest_run_id=None): self.metadata=None self.contents=None self.executor=None + self.token_delivery=None self._loaded=False # Plugins are installed code, never resources, posts, or configuration expressions. implemented=('identity','content','discussion','communication','discovery','achievements','recovery','sharing','money','offers','store','bounty','orders','delivery') @@ -97,6 +96,9 @@ async def load(self): require(len(self._token_secret)==32,'invalid_service_key') self._vault_key=hmac.digest(self._token_secret,b'custodial-vault-aesgcm-v1','sha256') self.cursors=CursorCodec(hmac.digest(self._token_secret,b'cursor-key-v1','sha256')) + self.token_delivery=TokenDelivery(metadata=self.metadata,token_secret=self._token_secret, + clock=lambda: self.clock(), + recovery_window=lambda: self.settings.credential_delivery_recovery_window) self.certificates=CertificateValidator(self.registry,root_certificate,root_public,self.settings.service_url,self.clock) async with self.metadata.transaction(write=False) as tx: quarantined=quarantine_active(tx) @@ -133,7 +135,7 @@ def new_executor(self, authenticator=None): max_request_bytes=self.settings.server.limits.max_request_bytes, result_projection=self._result_projection, event_notifications=self._event_notifications) - executor.response_hook=self._secrets_for_caller + executor.response_hook=self.token_delivery.release executor.recovery_drill_marker=self.settings.config_dir/'recovery-drill.json' if self.executor is not None: executor.recovery_quarantined=self.executor.recovery_quarantined @@ -156,62 +158,15 @@ async def online_issuer(self,tx): return cert def issued_token(self,request,subject): - # Nonces are high-entropy claims, not random tokens persisted in results. - return hmac.digest(self._token_secret,b'issued-token-v1\0'+canonical({ - 'subject':subject,'request_id':request.request_id,'nonce':request.arguments['nonce']}),'sha256') + return self.token_delivery.issued_token(request,subject) - @staticmethod - def recovery_verifier(value): - secret=unb64(value,limit=64) - require(len(secret)>=32,'invalid_recovery_secret') - return hashlib.sha256(b'token-recovery-v1\0'+secret).hexdigest() + recovery_verifier=staticmethod(recovery_verifier) def record_token_delivery(self,tx,request,credential,now,*,recovery_deadline=None): - """Bind recovery before commit; neither token nor recovery secret is stored.""" - require(request.operation in SECRET_DELIVERY_MIN_VERSION and - request.contract_version>=SECRET_DELIVERY_MIN_VERSION[request.operation], - 'credential_delivery_upgrade_required') - field='new_recovery_secret' if request.operation=='identity.token_recover' else 'recovery_secret' - secret=unb64(request.arguments[field],limit=64) - require(secret!=unb64(request.arguments['nonce'],limit=64),'recovery_secret_not_independent') - if request.operation=='identity.token_recover': - require(secret!=unb64(request.arguments['recovery_secret'],limit=64), - 'recovery_secret_not_independent') - expires=min(credential.expires_at, - recovery_deadline if recovery_deadline is not None else - now+timedelta(seconds=self.settings.credential_delivery_recovery_window)) - require(expires>now,'recovery_unavailable') - tx.execute('''INSERT INTO token_deliveries - (credential_id,subject,request_id,request_digest,recovery_verifier,recovery_expires_at, - claimed_at,consumed_at) VALUES (?,?,?,?,?,?,NULL,NULL)''', - (credential.id,credential.subject_id,request.request_id,request.payload_digest, - self.recovery_verifier(request.arguments[field]),wire(expires)),write=True) + return self.token_delivery.record_token_delivery(tx,request,credential,now,recovery_deadline=recovery_deadline) async def _secrets_for_caller(self,request,result): - if result.status!='ok' or request.operation not in SECRET_DELIVERY_MIN_VERSION: - return result - require(request.contract_version>=SECRET_DELIVERY_MIN_VERSION[request.operation], - 'credential_delivery_upgrade_required') - token=self.issued_token(request,result.subject) - async with self.metadata.transaction(write=True) as tx: - credential=await tx.credential(result.data['credential_id']) - require(credential.revoked_at is None and credential.expires_at>self.clock(),'credential_expired') - require(hmac.compare_digest(credential.verifier,hashlib.sha256(token).digest()),'invalid_token_result') - row=tx.one('''SELECT subject,request_id,request_digest,claimed_at,consumed_at - FROM token_deliveries WHERE credential_id=?''',(credential.id,)) - require(row is not None and row[0]==result.subject and row[1]==request.request_id - and row[2]==request.payload_digest and row[4] is None,'token_delivery_unavailable', - details={'committed_result':wire(result,compact=True)}) - require(row[3] is None,'token_delivery_unavailable', - details={'committed_result':wire(result,compact=True)}) - updated=tx.execute('''UPDATE token_deliveries SET claimed_at=? WHERE credential_id=? - AND claimed_at IS NULL AND consumed_at IS NULL''', - (wire(self.clock()),credential.id),write=True) - require(updated.rowcount==1,'token_delivery_unavailable', - details={'committed_result':wire(result,compact=True)}) - # The claim commits before the adapter sees the token. A dropped response - # must use a separate, pre-bound recovery secret to rotate the credential. - return replace(result,data=dict(result.data,token=b64(token))) + return await self.token_delivery.release(request,result) async def close(self): if self.metadata is not None: diff --git a/src/msg/security/authentication.py b/src/msg/security/authentication.py index 3c2281ee..3a7291b1 100644 --- a/src/msg/security/authentication.py +++ b/src/msg/security/authentication.py @@ -11,6 +11,7 @@ from msg.core.requests import payload_fields,signing_bytes from msg.security.crypto import key_id,subject_id,verify from msg.security.quarantine import active as quarantine_active +from msg.security.token_delivery import recovery_verifier CUSTODIAL_SIGNED_WRITES=frozenset({ @@ -64,9 +65,7 @@ async def authenticate(self,request,session, *, entry): require(old.kind=='token' and old.subject_id==request.subject,'recovery_unavailable') actor=await session.subject(old.subject_id) require(not actor.local_only and actor.resource_id!=ROOT_SUBJECT,'local_only') - raw=unb64(request.arguments['recovery_secret'],limit=64) - require(len(raw)>=32,'invalid_recovery_secret') - verifier=hashlib.sha256(b'token-recovery-v1\0'+raw).hexdigest() + verifier=recovery_verifier(request.arguments['recovery_secret']) row=session.one('''SELECT recovery_verifier,recovery_expires_at,consumed_at,request_id FROM token_deliveries WHERE credential_id=? AND subject=?''', (old.id,actor.resource_id)) diff --git a/src/msg/security/token_delivery.py b/src/msg/security/token_delivery.py new file mode 100644 index 00000000..e60bc6b0 --- /dev/null +++ b/src/msg/security/token_delivery.py @@ -0,0 +1,93 @@ +"""One-time credential delivery, distinct from service composition. + +The handler records the verifier in its existing issuance transaction. After that +transaction commits, release claims the delivery in a separate write transaction; +only a committed claim can produce a response-only token. Lost responses must use +the pre-bound recovery operation, never replay a persistent secret. This module +owns no signer, vault, root capability, HTTP adapter or plugin registry. + +Clock and window providers preserve the configured current values without giving +the domain access to the whole settings/Application object. These are trusted +in-process dependencies, not a Python sandbox or an additional authorization path. +""" +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import replace +from datetime import datetime, timedelta +import hashlib +import hmac + +from msg.core.codec import b64, canonical, unb64, wire +from msg.core.contracts import MetadataStore +from msg.core.errors import require +from msg.core.requests import SECRET_DELIVERY_MIN_VERSION + + +def recovery_verifier(value): + secret=unb64(value,limit=64) + require(len(secret)>=32,'invalid_recovery_secret') + return hashlib.sha256(b'token-recovery-v1\0'+secret).hexdigest() + +class TokenDelivery: + def __init__(self, *, metadata: MetadataStore, token_secret: bytes, + clock: Callable[[], datetime], recovery_window: Callable[[], int]): + require(len(token_secret)==32,'invalid_service_key') + self.metadata=metadata + self._token_secret=token_secret + self.clock=clock + self.recovery_window=recovery_window + + def issued_token(self,request,subject): + # Nonces are high-entropy claims, not random tokens persisted in results. + return hmac.digest(self._token_secret,b'issued-token-v1\0'+canonical({ + 'subject':subject,'request_id':request.request_id,'nonce':request.arguments['nonce']}),'sha256') + + recovery_verifier=staticmethod(recovery_verifier) + + def record_token_delivery(self,tx,request,credential,now,*,recovery_deadline=None): + """Bind recovery before commit; neither token nor recovery secret is stored.""" + require(request.operation in SECRET_DELIVERY_MIN_VERSION and + request.contract_version>=SECRET_DELIVERY_MIN_VERSION[request.operation], + 'credential_delivery_upgrade_required') + field='new_recovery_secret' if request.operation=='identity.token_recover' else 'recovery_secret' + secret=unb64(request.arguments[field],limit=64) + require(secret!=unb64(request.arguments['nonce'],limit=64),'recovery_secret_not_independent') + if request.operation=='identity.token_recover': + require(secret!=unb64(request.arguments['recovery_secret'],limit=64), + 'recovery_secret_not_independent') + expires=min(credential.expires_at, + recovery_deadline if recovery_deadline is not None else + now+timedelta(seconds=self.recovery_window())) + require(expires>now,'recovery_unavailable') + tx.execute('''INSERT INTO token_deliveries + (credential_id,subject,request_id,request_digest,recovery_verifier,recovery_expires_at, + claimed_at,consumed_at) VALUES (?,?,?,?,?,?,NULL,NULL)''', + (credential.id,credential.subject_id,request.request_id,request.payload_digest, + self.recovery_verifier(request.arguments[field]),wire(expires)),write=True) + + async def release(self,request,result): + if result.status!='ok' or request.operation not in SECRET_DELIVERY_MIN_VERSION: + return result + require(request.contract_version>=SECRET_DELIVERY_MIN_VERSION[request.operation], + 'credential_delivery_upgrade_required') + token=self.issued_token(request,result.subject) + async with self.metadata.transaction(write=True) as tx: + credential=await tx.credential(result.data['credential_id']) + require(credential.revoked_at is None and credential.expires_at>self.clock(),'credential_expired') + require(hmac.compare_digest(credential.verifier,hashlib.sha256(token).digest()),'invalid_token_result') + row=tx.one('''SELECT subject,request_id,request_digest,claimed_at,consumed_at + FROM token_deliveries WHERE credential_id=?''',(credential.id,)) + require(row is not None and row[0]==result.subject and row[1]==request.request_id + and row[2]==request.payload_digest and row[4] is None,'token_delivery_unavailable', + details={'committed_result':wire(result,compact=True)}) + require(row[3] is None,'token_delivery_unavailable', + details={'committed_result':wire(result,compact=True)}) + updated=tx.execute('''UPDATE token_deliveries SET claimed_at=? WHERE credential_id=? + AND claimed_at IS NULL AND consumed_at IS NULL''', + (wire(self.clock()),credential.id),write=True) + require(updated.rowcount==1,'token_delivery_unavailable', + details={'committed_result':wire(result,compact=True)}) + # The claim commits before the adapter sees the token. A dropped response + # must use a separate, pre-bound recovery secret to rotate the credential. + return replace(result,data=dict(result.data,token=b64(token))) From 26e43fcde63d6d5f50feec7d676f16f3924bfbfe Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:44:04 +0800 Subject: [PATCH 03/14] test(identity): inspect persisted event JSON for one-time delivery evidence Refs #162 #83. Correct the new test's nonexistent events.request_id column after independently verified cloud failure. Keep the exact one-event assertion and strengthen subject/secret checks against real stored Event JSON. Production code and schema are unchanged. --- docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md | 22 ++++++++++++---------- tests/test_token_delivery_boundary.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md b/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md index 32116115..d4855a63 100644 --- a/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md +++ b/docs/TOKEN_DELIVERY_BOUNDARY_PROGRESS.md @@ -1,21 +1,23 @@ # Credential delivery boundary progress -Issue #162, supplemental to #167; base2d1cc443c828dd2f735995cc6a5d20021fee6ac7 / tree6171f6041ffd0addaf72eadd35acf562f478281f. Branchfix/token-delivery-boundary-20260929. Coordination #83 comment5879008128. No other source branch is overwritten. +Issue #162; PR #172, stacked on #167 base2d1cc443c828dd2f735995cc6a5d20021fee6ac7 / tree6171f6041ffd0addaf72eadd35acf562f478281f. Branchfix/token-delivery-boundary-20260929. Coordination #83 comment5879008128. No other source branch is overwritten. -## Source ownership +## Single source and transaction timing -Before: Application owns composition plus issued_token/recovery_verifier/record_token_delivery/_secrets_for_caller. Authentication contains a second recovery-verifier formula. +security.token_delivery owns TokenDelivery and the one recovery_verifier function. Application constructs it, retains compatibility forwarding only, and binds HTTP/SSH executors directly to the same release method. Authentication consumes that verifier, not a second formula. Dependencies: metadata, the existing token derivation key, clock and current-window providers. No Root signer, vault, Registry, plugin or HTTP object. -After: security.token_delivery owns TokenDelivery and the one recovery_verifier function. Application constructs it, retains compatibility forwarding only, and binds all executors directly to the same release method. Authentication consumes the same pure verifier. Domain dependencies are explicitly metadata, the existing token derivation key, clock and a current-window provider; there is no Root signer, vault, Registry, plugin or HTTP object. +The original issued_token/release/verifier bodies are unchanged; record_token_delivery changes only current-window access. Live clock/settings changes are preserved. The handler still records recovery binding inside issuance. After commit, release independently commits the one-time claim before exposing the response token. Lost responses still require recovery. IDs, operation versions, purpose strings, deadlines, persistent results/events/receipts retain their existing meaning. No project code/test executed locally. -Current providers deliberately preserve later settings/clock changes. Static text/AST review found the issued_token and release bodies identical to the original, recovery_verifier body identical, and record_token_delivery identical except access to the narrow current-window provider. No project code/test was executed locally. +## Independently verified cloud evidence -## Timing and compatibility +RED b68526267a4aaea2df8418ad4d400f64a845fa63 / tree0f653206afd064f3e752cbb3442c202fbd72a4bb, run36486490339, artifact10999622677: **7 failed /115 passed /0 errors /0 skipped**. All seven new ownership checks failed; all existing suites passed. ZIP SHA2560fed854c2890019a5892fbabdde7a6fe3093b05d6805b64d701504f9f44de958; source.json confirms head/tree/attempt1/Python3.15.0rc2. -The existing handler records recovery binding inside its issuance transaction. After it commits, release opens its own write transaction and commits the one-time claim before adding the token to response data. A failed/dropped response still requires the existing independent recovery operation. Results, events, original receipts, IDs, purpose strings, deadlines and operation versions retain their meaning. No new generic response hook, protocol version, identity or recovery mechanism. +First implementation f015c60f05469f0058255bd645afef084e87840d / tree46af80b065a8e4eaea319b87a70bb5ddb281f966, push run36486854337, artifact10998634353: **1 failed /121 passed /0 errors /0 skipped**. ZIP SHA2560186be18b2ed3393af3fe5eaa5ee11eb1caa315f941cd0a1ddd469fb39fdc189 and source.json independently verified. -## Regression and cloud evidence +The sole failure is a new test's SQL query for events.request_id, which does not exist: events stores seq/id/body and canonical Event JSON includes request_id. All earlier assertions in that case (concurrent one-time release, receipt verification, secret-free persistent result, recovery binding) passed. The corrected test reads real body JSON, still requires exactly one matching event, and additionally checks its subject and absence of both token and recovery secret. No production schema/code or old assertion is changed to accommodate a fabricated column. -Tests-first headb68526267a4aaea2df8418ad4d400f64a845fa63 / tree0f653206afd064f3e752cbb3442c202fbd72a4bb contains seven new ownership/construction/wire/concurrent-claim/expired-refusal cases. Focused cloud workflow retains all existing token, credential, recovery-window, client recovery, batch secret and security regressions. The red focused run is not cancelled by the later implementation head; both are bound to exact source identity. Results remain pending until actual artifacts are read. +The corrected head must rerun the full credential selection and full four-shard/node-ID/conformance/build/recovery gates. Neither RED nor first implementation is a passing merge candidate. Latest final status belongs to #172 comments/actual Actions, not historical evidence. -Final focused/full four-shard/node-ID/conformance/build/recovery outcomes are pending. No deployment, actual credentials, Root/PIN, backup destruction, external delivery or recovery promotion. #67 stays completed and its original functional regressions remain binding; this change is ownership-only, not a new token protocol. #167 must finish its own combined main gate separately. +## Boundaries + +#67 remains completed with every original token/credential/recovery-window/client/batch/security regression retained. This is ownership repair, not a new token protocol. No deployment, actual credentials, Root/PIN, funds, external delivery, backup destruction or promotion. #167 finishes its own stable combined-main gate separately. diff --git a/tests/test_token_delivery_boundary.py b/tests/test_token_delivery_boundary.py index 86c91522..e8e4743f 100644 --- a/tests/test_token_delivery_boundary.py +++ b/tests/test_token_delivery_boundary.py @@ -12,7 +12,7 @@ import pytest -from msg.core.codec import b64, canonical, unb64, wire +from msg.core.codec import b64, canonical, loads, unb64, wire from msg.core.errors import Failure from msg.core.requests import request_for, receipt_bytes from msg.security.crypto import verify @@ -121,7 +121,13 @@ async def test_standalone_delivery_claim_commits_once_without_persisting_secrets (committed.data['credential_id'],)) assert row[0] is not None and row[1] == module.recovery_verifier(args['recovery_secret']) assert released.data['token'] not in str(row) and args['recovery_secret'] not in str(row) - assert tx.one('SELECT COUNT(*) FROM events WHERE request_id=?', (rid,))[0] == 1 + # Event fields are stored in canonical JSON, not separate SQL columns. + events = [loads(row[0]) for row in tx.rows('SELECT body FROM events')] + matching = [event for event in events if event.get('request_id') == rid] + assert len(matching) == 1 + assert matching[0]['subject'] == committed.subject + assert released.data['token'] not in canonical(matching[0]).decode() + assert args['recovery_secret'] not in canonical(matching[0]).decode() async def test_delivery_failure_returns_no_new_secret_or_claim(installed): From 094225154b3f13ca1967a6d3b16d9ce123a2e978 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:08:21 +0800 Subject: [PATCH 04/14] test(hosting): pin read-only runtime acceptance and cloud regression workflow --- .github/workflows/hosting-readonly.yml | 72 +++++++ docs/HOSTING_RUNTIME_PROGRESS.md | 25 +++ tests/test_hosting_runtime.py | 285 +++++++++++++++++++++++++ 3 files changed, 382 insertions(+) create mode 100644 .github/workflows/hosting-readonly.yml create mode 100644 docs/HOSTING_RUNTIME_PROGRESS.md create mode 100644 tests/test_hosting_runtime.py diff --git a/.github/workflows/hosting-readonly.yml b/.github/workflows/hosting-readonly.yml new file mode 100644 index 00000000..a670d54a --- /dev/null +++ b/.github/workflows/hosting-readonly.yml @@ -0,0 +1,72 @@ +name: Hosting read-only boundary +on: + push: + branches: ['fix/hosting-readonly-20260929'] + pull_request: + paths: + - 'src/msg/**' + - 'tests/**' + - 'deploy/**' + - '.github/workflows/hosting-readonly.yml' + - 'pyproject.toml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: hosting-readonly-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + hosting: + runs-on: ubuntu-latest + timeout-minutes: 15 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd 'pg_isready -U msgtest' + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install isolated test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Bind evidence to checkout and requested head + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, platform, subprocess + from pathlib import Path + git=lambda *args: subprocess.check_output(['git',*args],text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({ + 'head':os.environ['HEAD_SHA'], 'checkout':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'), 'python':platform.python_version(), + 'run_id':os.environ['GITHUB_RUN_ID'], 'attempt':os.environ['GITHUB_RUN_ATTEMPT']},indent=2)) + PY + - name: Exercise isolated hosting and retain established web regressions + run: >- + python -m pytest tests/test_hosting_runtime.py tests/test_hosting_same_origin.py + tests/test_cli_hosting.py --junitxml=artifacts/hosting.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: hosting-readonly-evidence + path: artifacts/ + if-no-files-found: error diff --git a/docs/HOSTING_RUNTIME_PROGRESS.md b/docs/HOSTING_RUNTIME_PROGRESS.md new file mode 100644 index 00000000..43fc9fda --- /dev/null +++ b/docs/HOSTING_RUNTIME_PROGRESS.md @@ -0,0 +1,25 @@ +# Hosting read-only runtime — #161 + +Base: `261c8816cd581deaf69fc0c810b8fa6dad5ead5e`, tree `6171f6041ffd0addaf72eadd35acf562f478281f` (#167 merged). +Branch: `fix/hosting-readonly-20260929`. Coordination: #83. + +## Scope and authority + +Only #161: independent hosting process composition, read-only metadata/content, necessary public trust, deployment identity and regression evidence. Keep the current same-origin/CSP rules. Do not change signing bytes, IDs, grants, operation versions or the main/worker behavior. #80/#84 production acceptance remains separate. + +Read the current design and completed-contract archive on 2026-09-29. Existing source is the #167 build artifact `10999494535` (SHA256 `cff2bbc5beb0575f79e1eaf069494a0cbdcfe5862b64c15401f974a1791340f0`); its checkout tree matches this base. No production probes or key reads. + +## Execution log + +1. Added failing acceptance tests before implementation: no Application/signers/installer; actual unreadable service-key directory; SELECT-only PostgreSQL account; current certificate/preview checks; HEAD/Range/304; recovery gates; stale/missing prerequisites; distinct systemd identity. +2. Added `hosting-readonly.yml`: real PostgreSQL 16/Git/age/Python 3.15, source identity + JUnit artifacts. Keep prior hosting/CLI regressions in the same job; full existing four-shard CI is unchanged. +3. RED run pending. No test result or implementation completion claimed yet. + +## Next gates + +- Reuse one Registry installer and the existing authentication/authorization; hosting retains contracts but no captured business handlers or Application pointer. +- Read-only storage opens without migration, rule publication, directory creation or writer keys. Use a separate OS/DB identity, not only a wrapper flag. +- Retain signed private previews, immediate revocation, quarantine, streaming/range and CSP. +- Inspect RED then GREEN artifacts against head/tree, complete full cloud CI and reconcile current main before merge. + +All project execution runs in GitHub Actions. Local work is source text/Git inspection and artifact parsing only. No deployment, real Root/PIN, external delivery, destructive backup operation or restore promotion. diff --git a/tests/test_hosting_runtime.py b/tests/test_hosting_runtime.py new file mode 100644 index 00000000..e1c11a26 --- /dev/null +++ b/tests/test_hosting_runtime.py @@ -0,0 +1,285 @@ +"""The hosting process reads current authority without owning a write service.""" +from dataclasses import replace +from datetime import timedelta +from pathlib import Path +import importlib +import uuid + +import httpx +import psycopg +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict, make_conninfo +import pytest + +from msg.core.codec import b64, canonical, wire +from msg.core.errors import Failure +from msg.core.requests import request_for +from test_service import NOW, call, register + +# Independent acceptance inventory: no ledger, token delivery, vault or job access. +READ_TABLES = ( + 'schema_version', 'resources', 'revisions', 'identities', 'memberships', + 'credentials', 'certificates', 'settings', 'share_grants', 'share_grants_v2', + 'dm_conversations', 'system_sources', +) + + +def runtime_class(): + # Import inside tests so the pre-implementation baseline fails tests, not collection. + return importlib.import_module('msg.hosting_runtime').HostingRuntime + + +@pytest.fixture +async def reader_settings(installed): + app, _ = installed + role = 'hosting_' + uuid.uuid4().hex + password = uuid.uuid4().hex + admin_dsn = app.settings.server.postgres_dsn + connection_info = conninfo_to_dict(admin_dsn) + with psycopg.connect(admin_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('CREATE ROLE {} LOGIN PASSWORD {} NOSUPERUSER NOCREATEDB ' + 'NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS').format( + sql.Identifier(role), sql.Literal(password))) + conn.execute(sql.SQL('GRANT CONNECT ON DATABASE {} TO {}').format( + sql.Identifier(connection_info['dbname']), sql.Identifier(role))) + conn.execute(sql.SQL('GRANT USAGE ON SCHEMA public TO {}').format(sql.Identifier(role))) + conn.execute(sql.SQL('GRANT SELECT ON {} TO {}').format( + sql.SQL(', ').join(sql.Identifier('public', name) for name in READ_TABLES), + sql.Identifier(role))) + dsn = make_conninfo(admin_dsn, user=role, password=password) + settings = replace(app.settings, server=replace(app.settings.server, + postgres_dsn=dsn, valkey_url=None, mail=None), + public_web_origin=app.settings.service_url) + try: + yield settings + finally: + with psycopg.connect(admin_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('DROP OWNED BY {}').format(sql.Identifier(role))) + conn.execute(sql.SQL('DROP ROLE {}').format(sql.Identifier(role))) + + +async def website(app): + key, subject, certificate = await register(app, 'readonly-host') + site = await call(app, 'hosting.create', {'parent': '/@readonly-host', 'name': 'web'}, + key=key, subject=subject) + source = await call(app, 'content.file_put', {'parent': '/@readonly-host/files', + 'name': 'index.html', 'data': b64(b'

published

'), 'media_type': 'text/html'}, + key=key, subject=subject) + assert site.status == source.status == 'ok' + entries = [{'path': 'index.html', 'source': wire(source.resources[0])}] + published = await call(app, 'hosting.deploy', {'id': site.resources[0].id, 'entries': entries}, + key=key, subject=subject, expected=((site.resources[0].id, site.data['generation']),)) + assert published.status == 'ok', wire(published) + preview = await call(app, 'hosting.preview', {'id': site.resources[0].id, 'entries': entries}, + key=key, subject=subject, expected=((site.resources[0].id, published.data['generation']),)) + assert preview.status == 'ok', wire(preview) + return key, subject, certificate, site, preview + + +def isolated(response): + assert 'sandbox' in response.headers['content-security-policy'] + assert 'allow-same-origin' not in response.headers['content-security-policy'] + assert response.headers['cache-control'] == 'no-store' + assert response.headers['x-content-type-options'] == 'nosniff' + assert 'set-cookie' not in response.headers + + +async def test_hosting_load_has_no_application_signers_or_startup_writes(installed, reader_settings, monkeypatch): + app, _ = installed + await website(app) + cls = runtime_class() + from msg.application import Application + import msg.bootstrap + from msg.security.crypto import Ed25519Signer + + def forbidden(*args, **kwargs): + pytest.fail('read-only hosting assembled a signer, full Application or installer') + + monkeypatch.setattr(Application, '__init__', forbidden) + monkeypatch.setattr(Ed25519Signer, 'from_bytes', forbidden) + monkeypatch.setattr(msg.bootstrap, 'sync_system_sources', forbidden) + before = sorted(str(p.relative_to(reader_settings.server.content_dir)) + for p in reader_settings.server.content_dir.rglob('*')) + keys = reader_settings.service_keys + original_mode = keys.stat().st_mode & 0o777 + keys.chmod(0) + reader = cls(reader_settings, clock=lambda: NOW) + try: + # This is an actual OS denial on the unprivileged CI runner, not a mock. + with pytest.raises(PermissionError): + (keys / 'online.key').read_bytes() + await reader.load() + for name in ('executor', 'online_signer', 'receipt_signer', '_token_secret', + '_vault_key', 'token_delivery', 'online_issuer', 'application', 'app'): + assert not hasattr(reader, name), name + assert not hasattr(reader.contents, 'put') + assert not hasattr(reader.contents, 'pin') + from msg.extensions.hosting import hosting_app + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(reader)), + base_url=reader_settings.service_url) as http: + response = await http.get('/@readonly-host/web/') + assert response.status_code == 200 and response.content == b'

published

' + isolated(response) + assert before == sorted(str(p.relative_to(reader_settings.server.content_dir)) + for p in reader_settings.server.content_dir.rglob('*')) + finally: + keys.chmod(original_mode) + await reader.close() + + +async def test_hosting_reuses_exact_registry_contracts_but_no_business_callbacks(installed, reader_settings): + app, _ = installed + reader = runtime_class()(reader_settings, clock=lambda: NOW) + await reader.load() + try: + assert reader.registry.catalog() == app.registry.catalog() + assert reader.registry.capabilities() == app.registry.capabilities() + assert reader.registry.resource_types() == app.registry.resource_types() + for operation in reader.registry.operations(): + assert reader.registry.schema(operation.input_schema) == app.registry.schema(operation.input_schema) + assert operation.handler.__closure__ is None + assert operation.requirements.__closure__ is None + with pytest.raises(Failure, match='readonly_registry'): + await operation.handler(None, None, None) + with pytest.raises(Failure, match='readonly_registry'): + await operation.requirements(None, None) + for manifest in reader.registry._plugins.values(): + assert all(spec is reader.registry.operation(spec.name, spec.version) + for spec in manifest.operations) + finally: + await reader.close() + + +async def test_hosting_database_has_both_transaction_and_role_write_denial(reader_settings): + reader = runtime_class()(reader_settings, clock=lambda: NOW) + await reader.load() + try: + with pytest.raises(Failure, match='readonly_transaction'): + async with reader.metadata.transaction(write=True): + pytest.fail('write transaction was entered') + async with reader.metadata.transaction(write=False) as tx: + assert tx.one('SHOW transaction_read_only') == ('on',) + with pytest.raises(Failure, match='readonly_transaction'): + tx.execute("UPDATE settings SET value=value", write=True) + # Even bypassing the wrapper and READ ONLY transaction, DB grants deny it. + with psycopg.connect(reader_settings.server.postgres_dsn) as conn: + with pytest.raises(psycopg.errors.InsufficientPrivilege): + conn.execute('UPDATE public.settings SET value=value') + with psycopg.connect(reader_settings.server.postgres_dsn) as conn: + with pytest.raises(psycopg.errors.InsufficientPrivilege): + conn.execute('SELECT * FROM public.token_deliveries') + finally: + await reader.close() + + +async def test_hosting_rejects_writer_database_account(installed): + app, _ = installed + reader = runtime_class()(replace(app.settings, public_web_origin=app.settings.service_url), + clock=lambda: NOW) + try: + with pytest.raises(Failure, match='hosting_database_not_readonly'): + await reader.load() + finally: + await reader.close() + + +@pytest.mark.parametrize('damage', ['trust', 'release', 'content']) +async def test_hosting_missing_or_stale_installation_fails_without_repair(installed, reader_settings, damage): + app, _ = installed + expected = {'trust': 'root_not_initialized', 'release': 'hosting_installation_stale', + 'content': 'content_store_not_initialized'}[damage] + settings = reader_settings + if damage == 'trust': + settings.trust_file.unlink() + elif damage == 'release': + async with app.metadata.transaction(write=True) as tx: + tx.execute("UPDATE system_sources SET source_digest='sha256:stale' WHERE resource_id='r_agents'", + write=True) + else: + settings = replace(settings, server=replace(settings.server, + content_dir=settings.server.content_dir.parent / 'missing-content')) + reader = runtime_class()(settings, clock=lambda: NOW) + try: + with pytest.raises(Failure, match=expected): + await reader.load() + if damage == 'trust': + assert not settings.trust_file.exists() + elif damage == 'content': + assert not settings.server.content_dir.exists() + else: + async with app.metadata.transaction(write=False) as tx: + assert tx.one("SELECT source_digest FROM system_sources WHERE resource_id='r_agents'") == ('sha256:stale',) + finally: + await reader.close() + + +async def test_hosting_private_preview_head_range_revocation_and_zero_effects(installed, reader_settings): + app, _ = installed + key, subject, certificate, site, preview = await website(app) + reader = runtime_class()(reader_settings, clock=lambda: NOW) + await reader.load() + from msg.extensions.hosting import hosting_app + candidate = preview.resources[0].id + path = f'/@readonly-host/web/_preview/{candidate}/index.html' + packet = request_for('discovery.raw', {'id': candidate}, app.settings.service_url, + signer=key, subject=subject, certificates=(certificate,), expires_at=NOW+timedelta(seconds=120)) + headers = {'X-Msg-Request': b64(canonical(wire(packet)))} + async with app.metadata.transaction(write=False) as tx: + before = {table: tuple(tx.rows(f'SELECT * FROM {table}')) for table in + ('resources', 'revisions', 'events', 'results', 'credentials', 'jobs')} + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(reader)), + base_url=app.settings.service_url) as http: + assert (await http.get(path)).status_code == 403 + page = await http.get(path, headers=headers) + assert page.status_code == 200 and page.content == b'

published

' + isolated(page) + assert (await http.head(path, headers=headers)).content == b'' + assert (await http.get(path, headers={**headers, 'Range': 'bytes=0-3'})).content == b'

' + assert (await http.get(path, headers={**headers, 'If-None-Match': page.headers['etag']})).status_code == 304 + assert (await http.get('/@readonly-host/web/missing.html')).status_code == 404 + assert (await http.post(path, headers=headers, content=b'write')).status_code == 405 + assert (await http.post('/-/p/content.post_create', content=b'{}')).status_code == 404 + assert (await http.get('/@readonly-host/web/', headers={'Host': 'other.invalid'})).status_code == 404 + async with app.metadata.transaction(write=False) as tx: + assert before == {table: tuple(tx.rows(f'SELECT * FROM {table}')) for table in before} + revoked = await call(app, 'content.chmod', {'id': candidate, 'mode': '0000'}, + key=key, subject=subject, expected=((candidate, preview.data['generation']),)) + assert revoked.status == 'ok' + assert (await http.get(path, headers=headers)).status_code == 403 + finally: + await reader.close() + + +@pytest.mark.parametrize('gate', ['database', 'marker']) +async def test_hosting_checks_quarantine_before_disclosing_bytes(installed, reader_settings, gate): + app, _ = installed + await website(app) + reader = runtime_class()(reader_settings, clock=lambda: NOW) + await reader.load() + from msg.extensions.hosting import hosting_app + try: + if gate == 'database': + async with app.metadata.transaction(write=True) as tx: + tx.execute("INSERT INTO settings(key,value) VALUES ('recovery_quarantine','{}')", write=True) + else: + (app.settings.config_dir / 'recovery-drill.json').write_text('{}') + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(reader)), + base_url=app.settings.service_url) as http: + response = await http.get('/@readonly-host/web/') + assert response.status_code != 200 + assert b'

published

' not in response.content + assert b'recovery_quarantined' in response.content + isolated(response) + finally: + await reader.close() + + +def test_hosting_unit_has_distinct_identity_and_no_data_write_grant(): + service = (Path(__file__).resolve().parents[1] / 'deploy/msgd-hosting.service').read_text() + assert 'User=msgd-hosting\n' in service + assert 'Group=msgd-hosting\n' in service + assert 'ReadWritePaths=' not in service + assert '--config-dir /etc/msgd-hosting' in service + assert 'ProtectSystem=strict' in service + assert '/var/lib/msgd/service-keys' in service From 6d24ee847c2e4c0349c71ce667d45b4ac04393f1 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:13:22 +0800 Subject: [PATCH 05/14] ci(hosting): quote Docker health command for runner argument parsing --- .github/workflows/hosting-readonly.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/hosting-readonly.yml b/.github/workflows/hosting-readonly.yml index a670d54a..9a34ac2b 100644 --- a/.github/workflows/hosting-readonly.yml +++ b/.github/workflows/hosting-readonly.yml @@ -28,7 +28,7 @@ jobs: POSTGRES_DB: postgres ports: ['5432:5432'] options: >- - --health-cmd 'pg_isready -U msgtest' + --health-cmd "pg_isready -U msgtest" --health-interval 10s --health-timeout 5s --health-retries 5 env: MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} @@ -52,13 +52,14 @@ jobs: run: | mkdir -p artifacts python - <<'PY' - import json, os, platform, subprocess + import json, os, platform, shutil, subprocess from pathlib import Path git=lambda *args: subprocess.check_output(['git',*args],text=True).strip() Path('artifacts/source.json').write_text(json.dumps({ 'head':os.environ['HEAD_SHA'], 'checkout':git('rev-parse','HEAD'), 'tree':git('rev-parse','HEAD^{tree}'), 'python':platform.python_version(), 'run_id':os.environ['GITHUB_RUN_ID'], 'attempt':os.environ['GITHUB_RUN_ATTEMPT']},indent=2)) + shutil.copyfile('tests/test_hosting_runtime.py','artifacts/test_hosting_runtime.py') PY - name: Exercise isolated hosting and retain established web regressions run: >- From 20399c0eacffece853d2d0e2035986e30b0636a4 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:20:11 +0800 Subject: [PATCH 06/14] test(hosting): require opt-in read-group publication without silent permission changes --- .github/workflows/hosting-readonly.yml | 7 ++-- docs/HOSTING_RUNTIME_PROGRESS.md | 26 +++++++------ tests/test_hosting_read_permissions.py | 52 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 tests/test_hosting_read_permissions.py diff --git a/.github/workflows/hosting-readonly.yml b/.github/workflows/hosting-readonly.yml index 9a34ac2b..c3b2132e 100644 --- a/.github/workflows/hosting-readonly.yml +++ b/.github/workflows/hosting-readonly.yml @@ -59,12 +59,13 @@ jobs: 'head':os.environ['HEAD_SHA'], 'checkout':git('rev-parse','HEAD'), 'tree':git('rev-parse','HEAD^{tree}'), 'python':platform.python_version(), 'run_id':os.environ['GITHUB_RUN_ID'], 'attempt':os.environ['GITHUB_RUN_ATTEMPT']},indent=2)) - shutil.copyfile('tests/test_hosting_runtime.py','artifacts/test_hosting_runtime.py') + for source in Path('tests').glob('test_hosting*.py'): + shutil.copyfile(source,Path('artifacts')/source.name) PY - name: Exercise isolated hosting and retain established web regressions run: >- - python -m pytest tests/test_hosting_runtime.py tests/test_hosting_same_origin.py - tests/test_cli_hosting.py --junitxml=artifacts/hosting.xml + python -m pytest tests/test_hosting*.py tests/test_cli_hosting.py + --junitxml=artifacts/hosting.xml - uses: actions/upload-artifact@v4 if: always() with: diff --git a/docs/HOSTING_RUNTIME_PROGRESS.md b/docs/HOSTING_RUNTIME_PROGRESS.md index 43fc9fda..fe09e269 100644 --- a/docs/HOSTING_RUNTIME_PROGRESS.md +++ b/docs/HOSTING_RUNTIME_PROGRESS.md @@ -1,25 +1,27 @@ # Hosting read-only runtime — #161 Base: `261c8816cd581deaf69fc0c810b8fa6dad5ead5e`, tree `6171f6041ffd0addaf72eadd35acf562f478281f` (#167 merged). -Branch: `fix/hosting-readonly-20260929`. Coordination: #83. +Branch: `fix/hosting-readonly-20260929`; PR #173; coordination #83. ## Scope and authority -Only #161: independent hosting process composition, read-only metadata/content, necessary public trust, deployment identity and regression evidence. Keep the current same-origin/CSP rules. Do not change signing bytes, IDs, grants, operation versions or the main/worker behavior. #80/#84 production acceptance remains separate. +Only #161: independent hosting process composition, read-only metadata/content, necessary public trust, deployment identity and regression evidence. Keep the current same-origin/CSP rules. Do not change signing bytes, IDs, grants, operation versions or main/worker behavior. #80/#84 production acceptance remains separate. -Read the current design and completed-contract archive on 2026-09-29. Existing source is the #167 build artifact `10999494535` (SHA256 `cff2bbc5beb0575f79e1eaf069494a0cbdcfe5862b64c15401f974a1791340f0`); its checkout tree matches this base. No production probes or key reads. +Read the current design and completed-contract archive on 2026-09-29. Initial source: #167 build artifact `10999494535` (SHA256 `cff2bbc5beb0575f79e1eaf069494a0cbdcfe5862b64c15401f974a1791340f0`), independently hashed; its source tree matches this base. No production probes or key reads. -## Execution log +## Cloud evidence -1. Added failing acceptance tests before implementation: no Application/signers/installer; actual unreadable service-key directory; SELECT-only PostgreSQL account; current certificate/preview checks; HEAD/Range/304; recovery gates; stale/missing prerequisites; distinct systemd identity. -2. Added `hosting-readonly.yml`: real PostgreSQL 16/Git/age/Python 3.15, source identity + JUnit artifacts. Keep prior hosting/CLI regressions in the same job; full existing four-shard CI is unchanged. -3. RED run pending. No test result or implementation completion claimed yet. +- `0942251`: first acceptance tests + dedicated workflow. Run `36547254507` failed at Docker initialization because the runner parsed single quotes in `--health-cmd` incorrectly. No tests ran; this is not RED evidence. +- `6d24ee847c2e4c0349c71ce667d45b4ac04393f1`: corrected runner quoting. Valid RED run `36547778158`: **17 tests, 11 failures, 6 passed, 0 errors, 0 skipped**. Ten failures are the missing read-only runtime and one is the existing shared-user/read-write unit. Existing hosting/CLI regressions passed. +- RED artifact `11023785174`, independently verified SHA256 `c4f67c186d03cd57df2a85668dcd5487ac5f261a91f913dfb047432eb6b7f2c0`. Manifest: head `6d24ee8`, merge checkout `90744666cc37d156366041d63ee4b7d33a4e26eb`, tree `201784072a47b7b036e400b769c9e9dab225cd89`, Python `3.15.0rc2`. +- Added separate permission acceptance before implementation: newly published text/CAS/index files must be group-readable but never group-writable; staging remains private; the default remains private; enabling sharing must refuse rather than chmod an existing installation silently. Cloud run pending. -## Next gates +## Implementation in progress / remaining gates -- Reuse one Registry installer and the existing authentication/authorization; hosting retains contracts but no captured business handlers or Application pointer. -- Read-only storage opens without migration, rule publication, directory creation or writer keys. Use a separate OS/DB identity, not only a wrapper flag. -- Retain signed private previews, immediate revocation, quarantine, streaming/range and CSP. -- Inspect RED then GREEN artifacts against head/tree, complete full cloud CI and reconcile current main before merge. +- Reuse one Registry installer and existing authentication/authorization; hosting retains metadata but no captured business handlers or Application pointer. +- Open read-only storage without migration, rule publication, directory creation or writer keys. Separate OS and DB accounts; verify actual privileges, not just a wrapper flag. +- Opt-in content read group needs real publication modes, not a unit file that cannot read future files. Offline operator preparation only; no automatic historical chmod. +- Retain signed private previews, current revocation, quarantine, streaming/range and CSP. +- Inspect GREEN artifacts against head/tree, complete full cloud CI and reconcile current main before merge. No implementation completion or GREEN result claimed yet. All project execution runs in GitHub Actions. Local work is source text/Git inspection and artifact parsing only. No deployment, real Root/PIN, external delivery, destructive backup operation or restore promotion. diff --git a/tests/test_hosting_read_permissions.py b/tests/test_hosting_read_permissions.py new file mode 100644 index 00000000..dba21082 --- /dev/null +++ b/tests/test_hosting_read_permissions.py @@ -0,0 +1,52 @@ +"""Optional content-group sharing grants read, never staging or write authority.""" +import importlib +import os + +import pytest + +from msg.core.errors import Failure +from msg.storage.git import GitContentStore + + +@pytest.mark.parametrize('group_read', [False, True]) +async def test_new_publication_uses_exact_opt_in_read_group_mode(tmp_path, group_read): + previous = os.umask(0o077) + try: + store = GitContentStore(tmp_path / 'content', binary_dir=tmp_path / 'blobs', + staging_dir=tmp_path / 'staging', group_read=group_read) + text = await store.put_bytes(b'

group reader

', 'text/html') + binary = await store.put_bytes(b'\0\x01\x02\xff', 'application/octet-stream') + finally: + os.umask(previous) + expected = 0o040 if group_read else 0 + for path in (store.index / text.digest[7:], store.index / binary.digest[7:], + store.binary / binary.digest[7:]): + assert path.stat().st_mode & 0o077 == expected + assert store.staging.stat().st_mode & 0o077 == 0 + if group_read: + for path in (store.path, store.index, store.binary): + assert path.stat().st_mode & 0o2777 == 0o2750 + entry = store._entry(text) + obj = store.repo / 'objects' / entry['oid'][:2] / entry['oid'][2:] + assert obj.stat().st_mode & 0o077 == 0o040 + reader_class = importlib.import_module('msg.storage.read_only').GitContentReader + reader = reader_class(store.path, binary_dir=store.binary) + assert reader_class.read is GitContentStore.read + assert await reader.read_bytes(text) == b'

group reader

' + assert await reader.read_bytes(binary) == b'\0\x01\x02\xff' + assert not hasattr(reader, 'put_bytes') + assert not hasattr(reader, 'staging') + + +def test_enabling_group_reads_never_silently_changes_existing_permissions(tmp_path): + previous = os.umask(0o077) + try: + private = GitContentStore(tmp_path / 'content', binary_dir=tmp_path / 'blobs', + staging_dir=tmp_path / 'staging') + finally: + os.umask(previous) + before = {str(p): p.stat().st_mode for p in tmp_path.rglob('*')} + with pytest.raises(Failure, match='content_sharing_not_prepared'): + GitContentStore(private.path, binary_dir=private.binary, + staging_dir=private.staging, group_read=True) + assert before == {str(p): p.stat().st_mode for p in tmp_path.rglob('*')} From 246f0fd83f41185dde4e796d0f96f1cca8db7934 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:32:12 +0800 Subject: [PATCH 07/14] refactor(hosting): compose an independently read-only runtime and deployment role --- .github/workflows/hosting-readonly.yml | 1 + deploy/hosting-role.sql | 16 +++ deploy/hosting.example.toml | 18 ++++ deploy/msgd-hosting.service | 21 ++-- deploy/prepare-hosting-content.sh | 38 +++++++ docs/HOSTING_RUNTIME.md | 91 ++++++++++++++++ docs/HOSTING_RUNTIME_PROGRESS.md | 26 ++--- src/msg/application.py | 20 +--- src/msg/config.py | 15 ++- src/msg/daemon.py | 37 ++++--- src/msg/extensions/hosting.py | 7 ++ src/msg/hosting_runtime.py | 138 +++++++++++++++++++++++++ src/msg/plugins/__init__.py | 28 +++++ src/msg/storage/git.py | 43 ++++++-- src/msg/storage/read_only.py | 106 +++++++++++++++++++ tests/test_extensions.py | 34 +++--- tests/test_hosting_runtime.py | 26 ++++- 17 files changed, 589 insertions(+), 76 deletions(-) create mode 100644 deploy/hosting-role.sql create mode 100644 deploy/hosting.example.toml create mode 100755 deploy/prepare-hosting-content.sh create mode 100644 docs/HOSTING_RUNTIME.md create mode 100644 src/msg/hosting_runtime.py create mode 100644 src/msg/storage/read_only.py diff --git a/.github/workflows/hosting-readonly.yml b/.github/workflows/hosting-readonly.yml index c3b2132e..6ed927d9 100644 --- a/.github/workflows/hosting-readonly.yml +++ b/.github/workflows/hosting-readonly.yml @@ -65,6 +65,7 @@ jobs: - name: Exercise isolated hosting and retain established web regressions run: >- python -m pytest tests/test_hosting*.py tests/test_cli_hosting.py + tests/test_extensions.py::test_hosting_publish_is_explicit_versioned_on_service_origin --junitxml=artifacts/hosting.xml - uses: actions/upload-artifact@v4 if: always() diff --git a/deploy/hosting-role.sql b/deploy/hosting-role.sql new file mode 100644 index 00000000..3962fe74 --- /dev/null +++ b/deploy/hosting-role.sql @@ -0,0 +1,16 @@ +-- Review and run once as the installation database owner, in the chosen DB. +-- This file neither creates a database nor changes existing PUBLIC grants. +-- A password/peer map belongs in the operator's secret setup, never this file. +CREATE ROLE msgd_hosting LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE + NOINHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('GRANT CONNECT ON DATABASE %I TO msgd_hosting', current_database()) \gexec +GRANT USAGE ON SCHEMA public TO msgd_hosting; +GRANT SELECT ON public.schema_version, public.resources, public.revisions, + public.identities, public.memberships, public.credentials, public.certificates, + public.settings, public.share_grants, public.share_grants_v2, + public.dm_conversations, public.system_sources TO msgd_hosting; +ALTER ROLE msgd_hosting SET default_transaction_read_only = on; +ALTER ROLE msgd_hosting SET search_path = pg_catalog, public, pg_temp; +-- No role membership, ownership, CREATE, sequence use, other-table SELECT, +-- column/table writes or callable application SECURITY DEFINER functions. +-- If PUBLIC already grants those, startup refuses; review that DB separately. diff --git a/deploy/hosting.example.toml b/deploy/hosting.example.toml new file mode 100644 index 00000000..b8d6c54b --- /dev/null +++ b/deploy/hosting.example.toml @@ -0,0 +1,18 @@ +# Install as /etc/msgd-hosting/msgd.toml, root:msgd-hosting 0640. +# This is NOT a copy of the writer configuration or its PostgreSQL credentials. +[server] +service_url = "https://msg.example.org" +public_web_origin = "https://msg.example.org" + +[hosting] +# The same marker checked by the write service. Never select a private copy. +recovery_marker = "/etc/msgd/recovery-drill.json" + +[storage] +postgres_dsn = "service=msgd-hosting" +content = "/var/lib/msgd/git/content" +blobs = "/var/lib/msgd/blobs/sha256" +# These parser-compatible paths are deliberately inaccessible and never opened. +service_keys = "/var/lib/msgd/service" +staging = "/var/lib/msgd/transfers/staging" +repositories = "/var/lib/msgd/git/repos" diff --git a/deploy/msgd-hosting.service b/deploy/msgd-hosting.service index 16b12bd7..a7f26a0d 100644 --- a/deploy/msgd-hosting.service +++ b/deploy/msgd-hosting.service @@ -1,21 +1,28 @@ [Unit] -Description=msg.lmm.best separate-origin static hosting +Description=msg.lmm.best read-only hosted-content process After=network.target [Service] Type=simple -User=msgd -Group=msgd -WorkingDirectory=/var/lib/msgd -ExecStart=/opt/msgd/venv/bin/msgd hosting --listen 127.0.0.1 --port 8043 +User=msgd-hosting +Group=msgd-hosting +SupplementaryGroups=msgd-content-read +WorkingDirectory=/ +ExecStart=/opt/msgd/venv/bin/msgd --config-dir /etc/msgd-hosting hosting --listen 127.0.0.1 --port 8043 +Environment=PGSERVICEFILE=/etc/msgd-hosting/pg_service.conf +Environment=PGPASSFILE=/etc/msgd-hosting/pgpass Restart=on-failure UMask=0077 NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict ProtectHome=true -ReadWritePaths=/var/lib/msgd -InaccessiblePaths=/var/lib/msgd-root /etc/msgd/root +ReadOnlyPaths=/etc/msgd-hosting /var/lib/msgd/git/content /var/lib/msgd/blobs/sha256 +InaccessiblePaths=/var/lib/msgd-root -/etc/msgd/root /var/lib/msgd/service -/var/lib/msgd/service-keys /var/lib/msgd/git/repos /var/lib/msgd/transfers -/etc/msgd/msgd.toml -/etc/msgd/server.toml -/etc/msgd/mail.toml +RestrictSUIDSGID=true +CapabilityBoundingSet= +AmbientCapabilities= +LimitCORE=0 [Install] WantedBy=multi-user.target diff --git a/deploy/prepare-hosting-content.sh b/deploy/prepare-hosting-content.sh new file mode 100755 index 00000000..b4f910b8 --- /dev/null +++ b/deploy/prepare-hosting-content.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Explicit offline OS-administrator preparation, never called by hosting.load(). +# Stop writers first. This changes only OS identities and content permissions. +set -eu +[ "$(id -u)" = 0 ] || { echo 'OS administrator required' >&2; exit 1; } +if systemctl is-active --quiet msgd.service || systemctl is-active --quiet msgd-worker.service; then + echo 'Stop all writers, including any non-systemd instances, before preparation.' >&2 + exit 1 +fi +getent passwd msgd >/dev/null || { echo 'Prepare the write service first.' >&2; exit 1; } +for directory in /var/lib/msgd /var/lib/msgd/git /var/lib/msgd/blobs /var/lib/msgd/git/content /var/lib/msgd/blobs/sha256; do + [ -d "$directory" ] && [ ! -L "$directory" ] || { echo "Missing or symlinked directory: $directory" >&2; exit 1; } +done +for directory in /var/lib/msgd/git/content /var/lib/msgd/blobs/sha256; do + [ -z "$(find -P "$directory" -type l -print -quit)" ] || { echo 'Content symlinks require operator review.' >&2; exit 1; } +done +getent group msgd-content-read >/dev/null || groupadd --system msgd-content-read +if ! getent passwd msgd-hosting >/dev/null; then + useradd --system --user-group --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin msgd-hosting +fi +usermod -a -G msgd-content-read msgd +usermod -a -G msgd-content-read msgd-hosting +for directory in /var/lib/msgd /var/lib/msgd/git /var/lib/msgd/blobs; do + chgrp msgd-content-read "$directory" + chmod 0710 "$directory" +done +for directory in /var/lib/msgd/git/content /var/lib/msgd/blobs/sha256; do + chgrp -R msgd-content-read "$directory" + find "$directory" -type d -exec chmod 2750 {} + + find "$directory" -type f -exec chmod u+rw,g+r,g-w,o-rwx {} + +done +git -c safe.directory=/var/lib/msgd/git/content/private.git --git-dir=/var/lib/msgd/git/content/private.git config core.sharedRepository 0640 +install -d -o root -g msgd-hosting -m 0750 /etc/msgd-hosting +install -d -o root -g msgd-hosting -m 0750 /etc/msgd-hosting/trust +[ -e /etc/msgd/trust/root.json ] || { echo 'Public trust is missing.' >&2; exit 1; } +[ -e /etc/msgd-hosting/trust/root.json ] || ln -s /etc/msgd/trust/root.json /etc/msgd-hosting/trust/root.json +printf '%s\n' 'Now enable hosting.content_group_read=true in the WRITER config and review the separate reader config/DB role.' +printf '%s\n' 'No configuration secrets were copied, no service started, and no database permissions changed.' diff --git a/docs/HOSTING_RUNTIME.md b/docs/HOSTING_RUNTIME.md new file mode 100644 index 00000000..09227028 --- /dev/null +++ b/docs/HOSTING_RUNTIME.md @@ -0,0 +1,91 @@ +# Hosting runtime boundary (#161) + +## Runtime roles + +| Role | Composition | Runtime secrets | Metadata / content | Startup work | +|---|---|---|---|---| +| `msgd serve` | Application, full registered operations and executor | Online CA, receipt and token keys | Existing write authority | Verify installation and sync changed release rules | +| `msgd hosting` | HostingRuntime, callback-free Registry, existing authenticator/authorizer | Public trust only; separate read-only DB credential | Allowlisted SELECT, READ ONLY transactions; Git/CAS reads only | Verify trust, current authority, schema accessibility and exact installed release; never migrate/sync | +| `msgd worker` | Existing Application and registered effect worker | Existing channel/tool/issuer dependencies | Existing controlled write/effect authority | Existing lifecycle; not narrowed by this change | + +Hosting is a process/OS/DB capability boundary, not a sandbox for Python plugins. +The installed plugin order remains in `msg.plugins.install_registry`. Hosting uses +the identical schema, capability and operation metadata, discarding both handlers +and requirement closures. It holds no parent Application, issuer, signer, token +secret, cursor MAC, OperationExecutor, Valkey signal or outbound sender. + +`GitContentReader` binds the existing streamed Git/CAS read methods without a writer +instance or staging path. PostgreSQL uses the existing canonical transaction/session +implementation without initialization. The wrapper rejects write transactions; each +transaction also rejects a superuser, inherited role, owner/write/CREATE privileges, +unneeded table reads and callable application SECURITY DEFINER routines. This checks +configuration rather than repairing it. READ ONLY alone is not a role boundary. + +## Separate deployment configuration + +Review `deploy/msgd-hosting.service`, `deploy/hosting.example.toml` and +`deploy/hosting-role.sql`. Use a distinct `msgd-hosting` OS user and `msgd_hosting` +database login. The host config contains the same public service URL, plugin list +and content locations but **not** a copy of the writer DSN or mail/token keys. +A PostgreSQL service entry `msgd-hosting` plus a reader-owned 0600 pgpass/peer mapping +supplies the dedicated database credential. Protect those files and the config. + +The SQL template grants only SELECT on schema_version, resources, revisions, +identities, memberships, credentials, certificates, settings, share_grants, +share_grants_v2, dm_conversations and system_sources. It does not grant table +ownership, sequence use, CREATE, role membership or token delivery/vault/job/ledger +reads. Existing PUBLIC grants must be reviewed separately; the template does not +revoke other applications' permissions. The runtime refuses an overpowered login. +New schema/authorization dependencies require review plus regression coverage before +adding a table. Column/row data returned by public HTTP still uses current authority. + +Do not copy a possibly stale public trust file or recovery marker. Link the public +`trust/root.json` to the installed public trust, keep its directory traversable, and +set `hosting.recovery_marker` to the writer's real `recovery-drill.json` path. The +hosting account must be able to stat the marker's parent without reading writer +configuration or secrets. The sample marker is `/etc/msgd/recovery-drill.json`. +Missing/untraversable parents fail closed. An observed marker latches quarantine +until restart, even if someone subsequently removes it; the database gate is also +checked for every request. Root rotation requires restarting readers after the +approved trust update. The role cannot clear either recovery condition. + +## Existing and future content permissions + +Content sharing is **opt-in**: `[hosting] content_group_read=false` remains the +writer default. Do not simply add hosting to the writer's private group. + +After stopping every writer, review `deploy/prepare-hosting-content.sh` and run it +as an OS administrator only for the standard paths it names. This is an explicit +permission migration, not an automatic startup action. It grants the dedicated +`msgd-content-read` group traversal and read access only to internal content and +CAS; it does not copy secrets, start services or change database grants. It refuses +symlinked content. Review nonstandard layouts manually rather than widening paths. +The database, service keys, repositories and transfer staging remain out of scope. +All future writer processes need the new supplementary group after restart. + +Then explicitly set `content_group_read=true` in the **writer** configuration. New +content/index/CAS files are group-readable but not group-writable or world-readable; +setgid content directories retain the read group's ownership. New Git repositories +use `--shared=0640`; existing repositories must already have the reviewed mode. +Binary staging remains private; only verified publication adopts the destination's +read group. A private existing installation fails with `content_sharing_not_prepared` +instead of silently chmod'ing history. Backups/restores must preserve these modes +and restore quarantine; repeat the operator review before read service promotion. + +## Transport and acceptance + +This does not change the current **same-origin** contract. The edge proxy sends only +hosted-content paths to the loopback hosting listener; ordinary API/operation routes +remain on the main service. Response-level CSP sandbox, no same-origin allowance, +no-store/nosniff, signed private preview, current revocation, history, HEAD/Range/304 +and explicit publish/rollback semantics are preserved. Unknown host/path/methods do +not open a write route. Startup never generates a key, creates content directories, +repairs missing tables or publishes release revisions. + +`tests/test_hosting_runtime.py` exercises the actual SELECT-only PostgreSQL login, +an OS-unreadable service-key directory, unchanged business facts, denied writes, +missing/stale prerequisites and current authorization/quarantine. Permission tests +check new text/binary/index publication under umask 0077 and refusal to alter an +existing private installation. Existing transport/CLI checks and full four-shard +CI remain required. Evidence and outstanding checks: `HOSTING_RUNTIME_PROGRESS.md`. +Green isolated CI is not a production deployment or completion of #80/#84. diff --git a/docs/HOSTING_RUNTIME_PROGRESS.md b/docs/HOSTING_RUNTIME_PROGRESS.md index fe09e269..1be7d966 100644 --- a/docs/HOSTING_RUNTIME_PROGRESS.md +++ b/docs/HOSTING_RUNTIME_PROGRESS.md @@ -5,23 +5,23 @@ Branch: `fix/hosting-readonly-20260929`; PR #173; coordination #83. ## Scope and authority -Only #161: independent hosting process composition, read-only metadata/content, necessary public trust, deployment identity and regression evidence. Keep the current same-origin/CSP rules. Do not change signing bytes, IDs, grants, operation versions or main/worker behavior. #80/#84 production acceptance remains separate. +Only #161: independent hosting composition, read-only metadata/content, public trust, deployment identity and regression evidence. Keep current same-origin/CSP rules. Do not change signing bytes, IDs, grants, operation versions or main/worker behavior. #80/#84 production acceptance remains separate. Capability matrix and operator guidance: `HOSTING_RUNTIME.md`. -Read the current design and completed-contract archive on 2026-09-29. Initial source: #167 build artifact `10999494535` (SHA256 `cff2bbc5beb0575f79e1eaf069494a0cbdcfe5862b64c15401f974a1791340f0`), independently hashed; its source tree matches this base. No production probes or key reads. +Read the current design and completed-contract archive on 2026-09-29. Initial source: #167 artifact `10999494535` (independently verified SHA256 `cff2bbc5beb0575f79e1eaf069494a0cbdcfe5862b64c15401f974a1791340f0`); source tree matches this base. ## Cloud evidence -- `0942251`: first acceptance tests + dedicated workflow. Run `36547254507` failed at Docker initialization because the runner parsed single quotes in `--health-cmd` incorrectly. No tests ran; this is not RED evidence. -- `6d24ee847c2e4c0349c71ce667d45b4ac04393f1`: corrected runner quoting. Valid RED run `36547778158`: **17 tests, 11 failures, 6 passed, 0 errors, 0 skipped**. Ten failures are the missing read-only runtime and one is the existing shared-user/read-write unit. Existing hosting/CLI regressions passed. -- RED artifact `11023785174`, independently verified SHA256 `c4f67c186d03cd57df2a85668dcd5487ac5f261a91f913dfb047432eb6b7f2c0`. Manifest: head `6d24ee8`, merge checkout `90744666cc37d156366041d63ee4b7d33a4e26eb`, tree `201784072a47b7b036e400b769c9e9dab225cd89`, Python `3.15.0rc2`. -- Added separate permission acceptance before implementation: newly published text/CAS/index files must be group-readable but never group-writable; staging remains private; the default remains private; enabling sharing must refuse rather than chmod an existing installation silently. Cloud run pending. +- `0942251`: first tests/workflow. Run `36547254507` failed at Docker initialization because runner health-command parsing requires double quotes. No test execution; not RED evidence. +- `6d24ee847c2e4c0349c71ce667d45b4ac04393f1`: valid RED run `36547778158`: **17 tests, 11 failures, 6 passed, 0 errors, 0 skipped**. Ten missing-runtime failures and one existing shared-user/read-write unit failure. Old hosting/CLI tests passed. Artifact `11023785174`, independently verified SHA256 `c4f67c186d03cd57df2a85668dcd5487ac5f261a91f913dfb047432eb6b7f2c0`; merge checkout `90744666cc37d156366041d63ee4b7d33a4e26eb`, tree `201784072a47b7b036e400b769c9e9dab225cd89`, Python 3.15.0rc2. +- `20399c0eacffece853d2d0e2035986e30b0636a4`: permission tests added before implementation. Valid RED run `36548533968`: **20 tests, 14 failures, 6 passed, 0 errors, 0 skipped**. Three additional failures are the missing `group_read` option. Artifact `11023512357`, independently verified SHA256 `33ff93271408f4e2006891154bf4621aa660dce6c26324b9945fdc4f54faac60`; checkout `475335d453cb3be925c6ddf5ce48c0769b83aaf1`, tree `2763aed9a71f7cefa6b68bc3214a15c3d43aaa36`. -## Implementation in progress / remaining gates +## Implemented; GREEN/full verification pending -- Reuse one Registry installer and existing authentication/authorization; hosting retains metadata but no captured business handlers or Application pointer. -- Open read-only storage without migration, rule publication, directory creation or writer keys. Separate OS and DB accounts; verify actual privileges, not just a wrapper flag. -- Opt-in content read group needs real publication modes, not a unit file that cannot read future files. Offline operator preparation only; no automatic historical chmod. -- Retain signed private previews, current revocation, quarantine, streaming/range and CSP. -- Inspect GREEN artifacts against head/tree, complete full cloud CI and reconcile current main before merge. No implementation completion or GREEN result claimed yet. +- Shared ordered Registry installer; HostingRuntime retains contracts but replaces every handler/requirement closure, including plugin manifests. No full Application, issuer, executor, private service key, cursor MAC or outbound sender. +- Read-only PostgreSQL composition opens with initialization disabled, uses real READ ONLY transactions and validates the independent login's actual grants. Content reader binds the canonical streaming implementations without a writer/staging instance. +- Startup verifies public trust, current authority and exact release source records. It never creates missing storage, migrates schema or synchronizes release rules. Shared recovery marker plus per-request database quarantine remain fail-closed. +- Dedicated OS/DB configuration and operator-only preparation script. Opt-in writer `hosting.content_group_read=true` publishes group-readable/non-writable Git/index/CAS files under umask 0077; existing private permissions are never silently rewritten. +- Daemon chooses hosting before normal Application construction. Existing publication/CSP test now uses the read-only fixture without deleting assertions. New assertions use the existing canonical error `read_only_transaction` and existing forbidden-host 403 status, rather than changing those contracts. +- Added daemon composition regression; dedicated workflow also executes the established publication-origin regression. -All project execution runs in GitHub Actions. Local work is source text/Git inspection and artifact parsing only. No deployment, real Root/PIN, external delivery, destructive backup operation or restore promotion. +All project execution runs in GitHub Actions. Local work is source text/Git/AST inspection and artifact parsing only. No production probes, deployments, real Root/PIN/key reads, outbound delivery, destructive backups or restore promotion. Do not close #161 before inspecting GREEN and complete four-shard evidence against the current head/tree; reconcile live main before merge. diff --git a/src/msg/application.py b/src/msg/application.py index 1862a579..84eae16c 100644 --- a/src/msg/application.py +++ b/src/msg/application.py @@ -6,7 +6,6 @@ from pathlib import Path import hashlib import hmac -import importlib from msg.constants import ROOT_SPACE, ROOT_SUBJECT from msg.core.codec import loads, decode, unb64, b64, canonical, wire @@ -18,7 +17,8 @@ from msg.core.registry import Registry from msg.security.authentication import AuthenticationService from msg.security.authorization import AuthorizationService -from msg.security.capabilities import install_capabilities, primary_ceiling, base_grants, temporary_ceiling +from msg.security.capabilities import primary_ceiling, base_grants, temporary_ceiling +from msg.plugins import install_registry from msg.security.certificates import CertificateValidator from msg.security.crypto import Ed25519Signer from msg.security.quarantine import active as quarantine_active @@ -40,18 +40,7 @@ def __init__(self,settings, *, clock=None, selftest_run_id=None): self.contents=None self.executor=None self._loaded=False - # Plugins are installed code, never resources, posts, or configuration expressions. - implemented=('identity','content','discussion','communication','discovery','achievements','recovery','sharing','money','offers','store','bounty','orders','delivery') - configured=set(settings.server.plugins) - require(configured<=set(implemented)|{'transfer','extensions','system','batch'},'unknown_plugin') - for plugin in implemented: - if plugin in configured: - importlib.import_module('msg.plugins.'+plugin).install(self) - for plugin in ('transfer','extensions','system','batch'): - if plugin in configured: - importlib.import_module('msg.plugins.'+plugin).install(self) - install_capabilities(self.registry) - self.registry.freeze() + install_registry(self, settings.server.plugins) def primary_ceiling(self): return primary_ceiling(self.registry,self.default_scope()) @@ -79,7 +68,8 @@ async def open_storage(self): if self.contents is None: self.contents=GitContentStore(self.settings.server.content_dir, binary_dir=self.settings.server.blob_dir, - staging_dir=self.settings.server.staging_dir) + staging_dir=self.settings.server.staging_dir, + group_read=self.settings.hosting_content_group_read) async def load(self): require(self.settings.trust_file.is_file(),'root_not_initialized') diff --git a/src/msg/config.py b/src/msg/config.py index 21caaebd..54d8a974 100644 --- a/src/msg/config.py +++ b/src/msg/config.py @@ -70,6 +70,8 @@ class Settings: recovery_custodians: tuple[RecoveryCustodian,...]=() money: MoneyConfig=MoneyConfig() hosting_base_capacity_bytes: int=10*1024*1024 + hosting_recovery_marker: Path | None=None + hosting_content_group_read: bool=False @property def config_dir(self): @@ -108,11 +110,17 @@ def load_settings(config_dir=Path('/etc/msgd')): window_minutes=int(window[:-1]) require(1<=window_minutes<=60,'invalid_credential_delivery_recovery_window') hosting=data.get('hosting',{}) - require(isinstance(hosting,dict) and set(hosting)<={'base_capacity_bytes'}, + require(isinstance(hosting,dict) and set(hosting)<={'base_capacity_bytes','recovery_marker','content_group_read'}, 'unknown_hosting_configuration') hosting_base=hosting.get('base_capacity_bytes',10*1024*1024) require(type(hosting_base) is int and 0 < hosting_base <= 1024**4, 'invalid_hosting_capacity') + content_group_read=hosting.get('content_group_read',False) + require(type(content_group_read) is bool,'invalid_content_group_read') + recovery_marker=hosting.get('recovery_marker') + require(recovery_marker is None or (isinstance(recovery_marker,str) and + Path(recovery_marker).is_absolute() and '..' not in Path(recovery_marker).parts and + not any(ord(c)<32 for c in recovery_marker)), 'invalid_hosting_recovery_marker') money=data.get('money',{}) require(isinstance(money,dict) and set(money)<=set(MoneyConfig.__dataclass_fields__), 'unknown_money_configuration') @@ -255,7 +263,9 @@ def load_settings(config_dir=Path('/etc/msgd')): tool_methods=tuple(tools.get('methods',('GET','HEAD'))),tool_ports=tuple(tools.get('ports',(80,443))), recovery_custodians=tuple(custodians), money=MoneyConfig(display_name=display_name,code=code), - hosting_base_capacity_bytes=hosting_base) + hosting_base_capacity_bytes=hosting_base, + hosting_recovery_marker=Path(recovery_marker) if recovery_marker is not None else None, + hosting_content_group_read=content_group_read) def write_example(config_dir,data_dir,service_url='https://msg.lmm.best',*,postgres_dsn='service=msgd',valkey_url=None): @@ -283,6 +293,7 @@ def write_example(config_dir,data_dir,service_url='https://msg.lmm.best',*,postg [hosting] base_capacity_bytes = 10485760 +content_group_read = false [storage] postgres_dsn = {json.dumps(postgres_dsn)} diff --git a/src/msg/daemon.py b/src/msg/daemon.py index 637715da..5b0423c9 100644 --- a/src/msg/daemon.py +++ b/src/msg/daemon.py @@ -208,6 +208,26 @@ def main(argv=None): if args.command=='ssh-session': from msg.extensions.ssh import forced_session return asyncio.run(forced_session(args.config_dir,args.credential)) + if args.command=='hosting': + from contextlib import asynccontextmanager + from msg.config import load_settings + from msg.hosting_runtime import HostingRuntime + from msg.extensions.hosting import hosting_app + import uvicorn + settings=load_settings(args.config_dir) + network_runtime(settings) + app=HostingRuntime(settings) + @asynccontextmanager + async def lifespan(asgi): + try: + await app.load() + yield + finally: + await app.close() + asgi=hosting_app(app) + asgi.router.lifespan_context=lifespan + uvicorn.run(asgi,host=args.listen,port=args.port,access_log=False,ws='none') + return 0 app=load_application(args.config_dir) if args.command=='backup': from msg.admin.backups import backup @@ -222,21 +242,8 @@ async def save(): if result is not None:emit(result) return 0 import uvicorn - if args.command=='hosting': - require(app.settings.public_web_origin is not None,'hosting_origin_not_configured') - from msg.extensions.hosting import hosting_app - # The hosted origin exposes no operations, cookies, MCP or root routes. - from contextlib import asynccontextmanager - @asynccontextmanager - async def lifespan(asgi): - await app.load() - yield - await app.close() - asgi=hosting_app(app);asgi.router.lifespan_context=lifespan - uvicorn.run(asgi,host=args.listen,port=args.port,access_log=False,ws='none') - else: - from msg.transports.http import create_app - uvicorn.run(create_app(app),host=app.settings.listen,port=app.settings.port,access_log=False,ws='none') + from msg.transports.http import create_app + uvicorn.run(create_app(app),host=app.settings.listen,port=app.settings.port,access_log=False,ws='none') return 0 except (Failure,OSError,ValueError) as exc: code=exc.code if isinstance(exc,Failure) else 'local_operation_failed' diff --git a/src/msg/extensions/hosting.py b/src/msg/extensions/hosting.py index 380ddee4..47f9f7b9 100644 --- a/src/msg/extensions/hosting.py +++ b/src/msg/extensions/hosting.py @@ -161,6 +161,8 @@ async def serve_hosted(service,request): site_path='/'+parts[0]+'/'+parts[1] try: async with service.metadata.transaction(write=False) as tx: + from msg.security.quarantine import require_live_authority + require_live_authority(tx) try: rid=await tx.resolve(site_path) except Failure as exc: @@ -243,7 +245,12 @@ async def serve_hosted(service,request): def hosting_app(service): + require(getattr(service,'readonly_hosting',False),'hosting_runtime_required') async def dispatch(request): + try: + service.require_ready() + except Failure as exc: + return hosted_error(exc.code) expected=urlsplit(service.settings.service_url) supplied=urlsplit('//'+request.headers.get('host','')) if supplied.hostname!=expected.hostname: diff --git a/src/msg/hosting_runtime.py b/src/msg/hosting_runtime.py new file mode 100644 index 00000000..bcf9ef20 --- /dev/null +++ b/src/msg/hosting_runtime.py @@ -0,0 +1,138 @@ +"""Explicit read-only composition for the independent hosting HTTP process.""" +from dataclasses import replace +from datetime import UTC, datetime +from functools import partial +from types import SimpleNamespace +import os + +import psycopg + +from msg.constants import ROOT_SUBJECT +from msg.core.codec import decode, digest, loads, unb64 +from msg.core.errors import Failure, require +from msg.core.models import Certificate, ResourceRef +from msg.core.registry import Registry +from msg.plugins import install_registry +from msg.security.authentication import AuthenticationService +from msg.security.authorization import AuthorizationService +from msg.security.capabilities import primary_ceiling, temporary_ceiling +from msg.security.certificates import CertificateValidator +from msg.security.quarantine import require_live_authority +from msg.storage.read_only import GitContentReader, ReadOnlyPostgresStore + + +async def _no_execution(*args, **kwargs): + raise Failure('readonly_registry') + + +def _contract_only(spec): + if spec.handler is _no_execution and spec.requirements is _no_execution: + return spec + return replace(spec, handler=_no_execution, requirements=_no_execution) + + +class _ContractRegistry(Registry): + """Use the original contracts/validation but retain no business closures.""" + def add_operation(self, spec): + super().add_operation(_contract_only(spec)) + + def add(self, manifest): + super().add(replace(manifest, operations=tuple( + _contract_only(spec) for spec in manifest.operations))) + + +async def require_prepared_sources(tx): + """The installer owns publication; this role only compares its release inputs.""" + from msg.bootstrap import RULE_SPECS, SOURCE_PATH_OVERRIDES, SOURCE_HEADER, system_source_root + root = system_source_root() + expected = set() + for rule_id, rid, _, _, default_path, _ in RULE_SPECS: + path = SOURCE_PATH_OVERRIDES.get(rule_id, default_path) + raw = root.joinpath(path).read_bytes() + header = SOURCE_HEADER.match(raw.decode('utf-8')) + require(header is not None and header.group(1) == rule_id, + 'hosting_installation_stale') + row = tx.one('''SELECT source_path,rule_id,source_kind,source_version,source_digest,revision_id + FROM system_sources WHERE resource_id=?''', (rid,)) + require(row is not None and row[:5] == ( + 'docs/system/' + path, rule_id, 'release', int(header.group(2)), digest(raw)), + 'hosting_installation_stale') + resource = await tx.resource(rid) + require(resource.revision == row[5], 'hosting_installation_stale') + revision = await tx.revision(ResourceRef(id=rid, revision=row[5])) + require(revision.source_kind == 'release' and revision.source_digest == row[4] + and revision.source_version == row[3], 'hosting_installation_stale') + expected.add(rid) + require({row[0] for row in tx.rows('SELECT resource_id FROM system_sources')} == expected, + 'hosting_installation_stale') + + +class HostingRuntime: + """Current verification + existing bytes. No Application, issuer or executor.""" + readonly_hosting = True + + def __init__(self, settings, *, clock=None): + self.settings = settings + self.clock = clock or (lambda: datetime.now(UTC)) + self.registry = _ContractRegistry() + # Installers only register trusted code. The throw-away context has no + # storage, signer, policy service or parent Application to capture. + install_registry(SimpleNamespace(registry=self.registry), settings.server.plugins) + self.metadata = None + self.contents = None + self._loaded = False + self._quarantined = False + self._marker = settings.hosting_recovery_marker or settings.config_dir / 'recovery-drill.json' + + def require_ready(self, *, loading=False): + require(loading or self._loaded, 'hosting_not_ready') + try: + self._marker.lstat() # includes a dangling symlink: never a bypass + except FileNotFoundError: + require(self._marker.parent.is_dir() and os.access(self._marker.parent, os.X_OK), + 'hosting_recovery_state_unavailable') + except OSError as exc: + raise Failure('hosting_recovery_state_unavailable') from exc + else: + self._quarantined = True + require(not self._quarantined, 'recovery_quarantined') + + async def load(self): + require(not self._loaded, 'hosting_already_loaded') + require(self.settings.public_web_origin is not None, 'hosting_origin_not_configured') + self.require_ready(loading=True) + require(self.settings.trust_file.is_file(), 'root_not_initialized') + trust = loads(self.settings.trust_file.read_bytes()) + require(type(trust) is dict and set(trust) == {'version', 'public_key', 'certificate'} + and trust['version'] == 1, 'invalid_trust_anchor') + root = decode(Certificate, trust['certificate']) + public = unb64(trust['public_key'], limit=32) + require(len(public) == 32, 'invalid_trust_anchor') + self.contents = GitContentReader(self.settings.server.content_dir, + binary_dir=self.settings.server.blob_dir) + self.metadata = ReadOnlyPostgresStore(self.settings.server.postgres_dsn) + self.certificates = CertificateValidator(self.registry, root, public, + self.settings.service_url, self.clock) + try: + async with self.metadata.transaction(write=False) as tx: + require_live_authority(tx) + require(tx.setting('active_root_certificate', root.resource_id) == root.resource_id, + 'service_restart_required') + await self.certificates.validate(root.resource_id, tx) + require((await tx.subject(ROOT_SUBJECT)).local_only, 'root_policy_corrupt') + await require_prepared_sources(tx) + except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn, + psycopg.errors.InsufficientPrivilege) as exc: + raise Failure('hosting_installation_stale') from exc + self.authenticator = AuthenticationService(self.registry, self.certificates, + self.settings.service_url, self.clock, + partial(primary_ceiling, self.registry), partial(temporary_ceiling, self.registry)) + self.authorizer = AuthorizationService(self.registry, self.certificates) + self.require_ready(loading=True) + self._loaded = True + return self + + async def close(self): + self._loaded = False + if self.metadata is not None: + await self.metadata.close() diff --git a/src/msg/plugins/__init__.py b/src/msg/plugins/__init__.py index e69de29b..6f1c8086 100644 --- a/src/msg/plugins/__init__.py +++ b/src/msg/plugins/__init__.py @@ -0,0 +1,28 @@ +"""The installed plugin order has one owner for every runtime role.""" +import importlib + +from msg.core.errors import require + + +BUILTINS = ( + 'identity', 'content', 'discussion', 'communication', 'discovery', + 'achievements', 'recovery', 'sharing', 'money', 'offers', 'store', + 'bounty', 'orders', 'delivery', 'transfer', 'extensions', 'system', 'batch', +) + + +def install_registry(context, plugins): + """Register installed code only; the context determines available capabilities. + + The write Application supplies its own context. A read-only host supplies a + disposable registration context and a registry which discards callbacks. + No storage connection, runtime secret or business operation is needed here. + """ + from msg.security.capabilities import install_capabilities + configured = set(plugins) + require(configured <= set(BUILTINS), 'unknown_plugin') + for name in BUILTINS: + if name in configured: + importlib.import_module('msg.plugins.' + name).install(context) + install_capabilities(context.registry) + context.registry.freeze() diff --git a/src/msg/storage/git.py b/src/msg/storage/git.py index 9846a21a..8c0223eb 100644 --- a/src/msg/storage/git.py +++ b/src/msg/storage/git.py @@ -218,20 +218,43 @@ def collect_unlinked(self, *, older_than: float, index: Path) -> tuple[int,int]: class GitContentStore: def __init__(self,path: Path, *, binary_dir: Path | None = None, - staging_dir: Path | None = None): + staging_dir: Path | None = None, group_read: bool = False): + require(type(group_read) is bool,'invalid_content_group_read') + self.group_read=group_read self.path=Path(path) self.repo=self.path/'private.git' self.index=self.path/'index' self.binary=Path(binary_dir) if binary_dir is not None else self.path/'binary' self.staging=Path(staging_dir) if staging_dir is not None else self.path/'staging' - for p in (self.path,self.index,self.binary,self.staging): - p.mkdir(parents=True,exist_ok=True) - if not self.repo.exists(): - subprocess.run(['git','init','--bare',str(self.repo)],check=True,capture_output=True) self.env={"PATH":os.environ.get('PATH','/usr/bin:/bin'),"LANG":"C.UTF-8", "HOME":str(self.path),"GIT_CONFIG_NOSYSTEM":"1","GIT_CONFIG_GLOBAL":"/dev/null", "GIT_AUTHOR_NAME":"msg","GIT_AUTHOR_EMAIL":"msg@localhost", "GIT_COMMITTER_NAME":"msg","GIT_COMMITTER_EMAIL":"msg@localhost"} + # An existing installation needs an explicit offline permission review. + # Validate before creating/chmod'ing anything, including staging paths. + if group_read: + for p in (self.path,self.index,self.binary): + if p.exists() or p.is_symlink(): + require(p.is_dir() and not p.is_symlink() and + p.stat().st_mode & 0o2077 == 0o2050, + 'content_sharing_not_prepared') + if self.repo.exists() or self.repo.is_symlink(): + require(self.repo.is_dir() and not self.repo.is_symlink(), + 'content_sharing_not_prepared') + config=subprocess.run(['git','--git-dir',str(self.repo),'config', + '--local','core.sharedRepository'],capture_output=True,env=self.env,timeout=30) + require(config.returncode==0 and config.stdout.strip()==b'0640', + 'content_sharing_not_prepared') + for p in (self.path,self.index,self.binary): + existed=p.exists() + p.mkdir(parents=True,exist_ok=True) + if group_read and not existed: + p.chmod(0o2750) + self.staging.mkdir(parents=True,exist_ok=True) + if not self.repo.exists(): + shared=['--shared=0640'] if group_read else [] + subprocess.run(['git','init','--bare',*shared,str(self.repo)], + check=True,capture_output=True,env=self.env,timeout=30) def _run(self,*args,input=None,stdin=None): result=subprocess.run(['git','--git-dir',str(self.repo),'-c','core.fsync=all', @@ -281,6 +304,13 @@ def save_git(): else: destination=self.binary/self._key(ref) if not destination.exists(): + if self.group_read: + # Rename retains the staging GID, so explicitly use the + # prepared destination group. Staging itself stays private. + os.chown(name,-1,self.binary.stat().st_gid) + os.chmod(name,0o640) + with open(name,'rb') as staged: + os.fsync(staged.fileno()) os.replace(name,destination) directory=os.open(destination.parent,os.O_RDONLY|os.O_DIRECTORY) try: @@ -288,7 +318,8 @@ def save_git(): finally: os.close(directory) entry={'kind':'binary','size':size} - durable_write(self.index/self._key(ref),canonical(entry)) + durable_write(self.index/self._key(ref),canonical(entry), + mode=0o640 if self.group_read else 0o600) return ref finally: if os.path.exists(name): diff --git a/src/msg/storage/read_only.py b/src/msg/storage/read_only.py new file mode 100644 index 00000000..00625262 --- /dev/null +++ b/src/msg/storage/read_only.py @@ -0,0 +1,106 @@ +"""Storage capabilities for a process which must never initialize or publish. + +PostgreSQL grants are an independent boundary, not replaced by READ ONLY or a +Python wrapper. Content reads share the exact existing Git/CAS implementation, +without constructing a writer or exposing its publication methods. +""" +from contextlib import asynccontextmanager +from pathlib import Path +import os + +from msg.core.errors import require +from msg.storage.git import GitContentStore +from msg.storage.postgres import PostgresMetadataStore + + +HOSTING_READ_TABLES = frozenset({ + 'schema_version', 'resources', 'revisions', 'identities', 'memberships', + 'credentials', 'certificates', 'settings', 'share_grants', 'share_grants_v2', + 'dm_conversations', 'system_sources', +}) + + +def require_readonly_role(tx): + """Refuse owner/writer/SET ROLE/secret-table authority; never repair grants.""" + role = tx.one('''SELECT current_user, session_user, rolsuper, rolcreaterole, + rolcreatedb, rolreplication, rolbypassrls FROM pg_catalog.pg_roles + WHERE rolname=current_user''') + require(role is not None and role[0] == role[1] and not any(role[2:]), + 'hosting_database_not_readonly') + require(tx.one('''SELECT 1 FROM pg_catalog.pg_roles + WHERE rolname<>current_user AND pg_has_role(oid,'MEMBER') LIMIT 1''') is None, + 'hosting_database_not_readonly') + require(not tx.one("SELECT has_database_privilege(current_database(),'CREATE')")[0], + 'hosting_database_not_readonly') + require(tx.one('''SELECT 1 FROM pg_catalog.pg_namespace + WHERE left(nspname,3)<>'pg_' AND nspname<>'information_schema' + AND has_schema_privilege(oid,'CREATE') LIMIT 1''') is None, + 'hosting_database_not_readonly') + for schema, name, readable, writable in tx.rows(''' + SELECT n.nspname,c.relname,has_any_column_privilege(c.oid,'SELECT'), + (has_table_privilege(c.oid,'INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER') + OR has_any_column_privilege(c.oid,'INSERT,UPDATE,REFERENCES')) + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE c.relkind IN ('r','p','v','m','f') + AND left(n.nspname,3)<>'pg_' AND n.nspname<>'information_schema' + '''): + require(not writable and (not readable or + (schema == 'public' and name in HOSTING_READ_TABLES)), + 'hosting_database_not_readonly') + require(tx.one('''SELECT 1 FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE c.relkind='S' AND left(n.nspname,3)<>'pg_' + AND has_sequence_privilege(c.oid,'USAGE,UPDATE') LIMIT 1''') is None, + 'hosting_database_not_readonly') + # A callable SECURITY DEFINER routine can reintroduce its owner's privileges. + require(tx.one('''SELECT 1 FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace + WHERE p.prosecdef AND left(n.nspname,3)<>'pg_' + AND n.nspname<>'information_schema' + AND has_function_privilege(p.oid,'EXECUTE') LIMIT 1''') is None, + 'hosting_database_not_readonly') + + +class ReadOnlyPostgresStore: + """Reuse canonical sessions/transactions, without migration or a write API.""" + def __init__(self, dsn): + self._store = PostgresMetadataStore(dsn, initialize=False) + + @asynccontextmanager + async def transaction(self, *, write=False): + require(not write, 'read_only_transaction') + async with self._store.transaction(write=False) as tx: + # Explicit ordering prevents writable/temp schemas from shadowing + # the installed public schema or PostgreSQL's privilege functions. + tx.execute('SET LOCAL search_path = pg_catalog, public, pg_temp') + require_readonly_role(tx) + yield tx + + async def close(self): + await self._store.close() + + +class GitContentReader: + """Read existing content; no mkdir, init, staging, pin, write or writer pointer.""" + def __init__(self, path, *, binary_dir=None): + self.path = Path(path) + self.repo = self.path / 'private.git' + self.index = self.path / 'index' + self.binary = Path(binary_dir) if binary_dir is not None else self.path / 'binary' + require(all(p.is_dir() for p in (self.path, self.repo, self.index, self.binary)), + 'content_store_not_initialized') + self.env = { + 'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'LANG': 'C.UTF-8', + 'HOME': str(self.path), 'GIT_CONFIG_NOSYSTEM': '1', + 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_OPTIONAL_LOCKS': '0', + # The approved content repository is owned by the writer, not this UID. + 'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'safe.directory', + 'GIT_CONFIG_VALUE_0': str(self.repo), + } + + # Unbound implementations, not a wrapped GitContentStore instance. Keep the + # original limits, references, streaming, range and truncation behavior. + _key = staticmethod(GitContentStore._key) + _entry = GitContentStore._entry + read = GitContentStore.read + read_bytes = GitContentStore.read_bytes diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 749c3b16..823f7895 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -6,6 +6,7 @@ from msg.extensions.hosting import hosting_app from msg.extensions.repositories import NativeGitStore from test_service import register,call +from test_hosting_runtime import reader_settings, runtime_class def test_public_key_encryption_recipient_and_tamper_checks(): @@ -44,7 +45,7 @@ async def test_keystore_stores_only_ciphertext_and_has_common_acl(installed): @pytest.mark.asyncio -async def test_hosting_publish_is_explicit_versioned_on_service_origin(installed): +async def test_hosting_publish_is_explicit_versioned_on_service_origin(installed,reader_settings): app,_=installed key,uid,_=await register(app,'site-owner') private_file=await call(app,'content.file_put',{'parent':'/@site-owner/files','name':'index.html', @@ -56,19 +57,24 @@ async def test_hosting_publish_is_explicit_versioned_on_service_origin(installed 'entries':[{'path':'index.html','source':wire(private_file.resources[0])}]},key=key,subject=uid, expected=((site.resources[0].id,site.data['generation']),)) assert published.status=='ok',wire(published) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(app)),base_url=app.settings.service_url) as http: - result=await http.get('/@site-owner/w/') - assert result.status_code==200 and result.content==b'

Published explicitly

' - assert 'set-cookie' not in result.headers - assert 'sandbox' in result.headers['content-security-policy'] - assert 'allow-same-origin' not in result.headers['content-security-policy'] - head=await http.head('/@site-owner/w/') - assert head.headers['content-security-policy']==result.headers['content-security-policy'] - cached=await http.get('/@site-owner/w/',headers={'If-None-Match':result.headers['etag']}) - assert cached.status_code==304 - assert cached.headers['content-security-policy']==result.headers['content-security-policy'] - denied=await http.get('/@site-owner/w/../../root',headers={'Host':'pages.example.test'}) - assert denied.status_code==403 + reader=runtime_class()(reader_settings,clock=app.clock) + await reader.load() + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(reader)),base_url=app.settings.service_url) as http: + result=await http.get('/@site-owner/w/') + assert result.status_code==200 and result.content==b'

Published explicitly

' + assert 'set-cookie' not in result.headers + assert 'sandbox' in result.headers['content-security-policy'] + assert 'allow-same-origin' not in result.headers['content-security-policy'] + head=await http.head('/@site-owner/w/') + assert head.headers['content-security-policy']==result.headers['content-security-policy'] + cached=await http.get('/@site-owner/w/',headers={'If-None-Match':result.headers['etag']}) + assert cached.status_code==304 + assert cached.headers['content-security-policy']==result.headers['content-security-policy'] + denied=await http.get('/@site-owner/w/../../root',headers={'Host':'pages.example.test'}) + assert denied.status_code==403 + finally: + await reader.close() source=await call(app,'discovery.get',{'id':private_file.resources[0].id}) assert source.error.code=='permission_denied' traversal=await call(app,'hosting.deploy',{'id':site.resources[0].id, diff --git a/tests/test_hosting_runtime.py b/tests/test_hosting_runtime.py index e1c11a26..2f432f05 100644 --- a/tests/test_hosting_runtime.py +++ b/tests/test_hosting_runtime.py @@ -154,12 +154,12 @@ async def test_hosting_database_has_both_transaction_and_role_write_denial(reade reader = runtime_class()(reader_settings, clock=lambda: NOW) await reader.load() try: - with pytest.raises(Failure, match='readonly_transaction'): + with pytest.raises(Failure, match='read_only_transaction'): async with reader.metadata.transaction(write=True): pytest.fail('write transaction was entered') async with reader.metadata.transaction(write=False) as tx: assert tx.one('SHOW transaction_read_only') == ('on',) - with pytest.raises(Failure, match='readonly_transaction'): + with pytest.raises(Failure, match='read_only_transaction'): tx.execute("UPDATE settings SET value=value", write=True) # Even bypassing the wrapper and READ ONLY transaction, DB grants deny it. with psycopg.connect(reader_settings.server.postgres_dsn) as conn: @@ -240,7 +240,7 @@ async def test_hosting_private_preview_head_range_revocation_and_zero_effects(in assert (await http.get('/@readonly-host/web/missing.html')).status_code == 404 assert (await http.post(path, headers=headers, content=b'write')).status_code == 405 assert (await http.post('/-/p/content.post_create', content=b'{}')).status_code == 404 - assert (await http.get('/@readonly-host/web/', headers={'Host': 'other.invalid'})).status_code == 404 + assert (await http.get('/@readonly-host/web/', headers={'Host': 'other.invalid'})).status_code == 403 async with app.metadata.transaction(write=False) as tx: assert before == {table: tuple(tx.rows(f'SELECT * FROM {table}')) for table in before} revoked = await call(app, 'content.chmod', {'id': candidate, 'mode': '0000'}, @@ -282,4 +282,22 @@ def test_hosting_unit_has_distinct_identity_and_no_data_write_grant(): assert 'ReadWritePaths=' not in service assert '--config-dir /etc/msgd-hosting' in service assert 'ProtectSystem=strict' in service - assert '/var/lib/msgd/service-keys' in service + assert '/var/lib/msgd/service' in service + + +def test_hosting_subcommand_never_assembles_main_application(reader_settings, monkeypatch): + from msg import daemon + import msg.config + import uvicorn + called = [] + + def forbidden(*args, **kwargs): + pytest.fail('hosting constructed the normal write Application') + + monkeypatch.setattr(daemon, 'load_application', forbidden) + monkeypatch.setattr(daemon, 'network_runtime', lambda settings: called.append(settings)) + monkeypatch.setattr(msg.config, 'load_settings', lambda directory: reader_settings) + monkeypatch.setattr(uvicorn, 'run', lambda asgi, **options: called.append((asgi, options))) + assert daemon.main(['--config-dir', str(reader_settings.config_dir), 'hosting']) == 0 + assert called[0] == reader_settings + assert len(called) == 2 and called[1][1]['access_log'] is False From df7efdbaa637e7f316d0eab1e0ce3dc156f83954 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:39:19 +0800 Subject: [PATCH 08/14] fix(hosting): check sequence permissions only after resolving sequence OIDs --- src/msg/storage/read_only.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/msg/storage/read_only.py b/src/msg/storage/read_only.py index 00625262..22521688 100644 --- a/src/msg/storage/read_only.py +++ b/src/msg/storage/read_only.py @@ -47,11 +47,13 @@ def require_readonly_role(tx): require(not writable and (not readable or (schema == 'public' and name in HOSTING_READ_TABLES)), 'hosting_database_not_readonly') - require(tx.one('''SELECT 1 FROM pg_catalog.pg_class c + # Fetch sequence OIDs first: SQL predicates do not impose evaluation order, + # and has_sequence_privilege must never receive an index/table OID. + for (oid,) in tx.rows('''SELECT c.oid FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace - WHERE c.relkind='S' AND left(n.nspname,3)<>'pg_' - AND has_sequence_privilege(c.oid,'USAGE,UPDATE') LIMIT 1''') is None, - 'hosting_database_not_readonly') + WHERE c.relkind='S' AND left(n.nspname,3)<>'pg_' '''): + require(not tx.one("SELECT has_sequence_privilege(?::oid,'USAGE,UPDATE')", (oid,))[0], + 'hosting_database_not_readonly') # A callable SECURITY DEFINER routine can reintroduce its owner's privileges. require(tx.one('''SELECT 1 FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace From 6703f3de41e988977be09e509c91fc30a11738b1 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:44:59 +0800 Subject: [PATCH 09/14] test(hosting): retain parallel isolation coverage and exercise a distinct OS reader --- docs/HOSTING_RUNTIME_PROGRESS.md | 38 ++-- tests/test_hosting_runtime_isolation.py | 284 ++++++++++++++++++++++++ 2 files changed, 305 insertions(+), 17 deletions(-) create mode 100644 tests/test_hosting_runtime_isolation.py diff --git a/docs/HOSTING_RUNTIME_PROGRESS.md b/docs/HOSTING_RUNTIME_PROGRESS.md index 1be7d966..2eb57994 100644 --- a/docs/HOSTING_RUNTIME_PROGRESS.md +++ b/docs/HOSTING_RUNTIME_PROGRESS.md @@ -1,27 +1,31 @@ # Hosting read-only runtime — #161 -Base: `261c8816cd581deaf69fc0c810b8fa6dad5ead5e`, tree `6171f6041ffd0addaf72eadd35acf562f478281f` (#167 merged). -Branch: `fix/hosting-readonly-20260929`; PR #173; coordination #83. +Base `261c8816cd581deaf69fc0c810b8fa6dad5ead5e`, tree `6171f6041ffd0addaf72eadd35acf562f478281f` (#167). Branch `fix/hosting-readonly-20260929`, PR #173; coordination #83. -## Scope and authority +## Scope and implementation -Only #161: independent hosting composition, read-only metadata/content, public trust, deployment identity and regression evidence. Keep current same-origin/CSP rules. Do not change signing bytes, IDs, grants, operation versions or main/worker behavior. #80/#84 production acceptance remains separate. Capability matrix and operator guidance: `HOSTING_RUNTIME.md`. +Independent HostingRuntime, canonical Registry metadata without business callbacks, original authentication/authorization, real SELECT-only PostgreSQL login, no initialization/migration/write execution, read-only canonical Git/CAS streaming, public trust/recovery gates and dedicated OS/DB deployment. Explicit content-group sharing preserves future publication readability without exposing staging or changing default private permissions. Details: `HOSTING_RUNTIME.md`. -Read the current design and completed-contract archive on 2026-09-29. Initial source: #167 artifact `10999494535` (independently verified SHA256 `cff2bbc5beb0575f79e1eaf069494a0cbdcfe5862b64c15401f974a1791340f0`); source tree matches this base. +Keep current same-origin/CSP, signing bytes, IDs, grants and operation versions. #80/#84 production acceptance is separate. Read the authoritative design/completed archive on 2026-09-29. Baseline source artifact10999494535 has independently verified SHA256 `cff2bbc5beb0575f79e1eaf069494a0cbdcfe5862b64c15401f974a1791340f0`; source tree matches base. -## Cloud evidence +## Verified cloud evidence -- `0942251`: first tests/workflow. Run `36547254507` failed at Docker initialization because runner health-command parsing requires double quotes. No test execution; not RED evidence. -- `6d24ee847c2e4c0349c71ce667d45b4ac04393f1`: valid RED run `36547778158`: **17 tests, 11 failures, 6 passed, 0 errors, 0 skipped**. Ten missing-runtime failures and one existing shared-user/read-write unit failure. Old hosting/CLI tests passed. Artifact `11023785174`, independently verified SHA256 `c4f67c186d03cd57df2a85668dcd5487ac5f261a91f913dfb047432eb6b7f2c0`; merge checkout `90744666cc37d156366041d63ee4b7d33a4e26eb`, tree `201784072a47b7b036e400b769c9e9dab225cd89`, Python 3.15.0rc2. -- `20399c0eacffece853d2d0e2035986e30b0636a4`: permission tests added before implementation. Valid RED run `36548533968`: **20 tests, 14 failures, 6 passed, 0 errors, 0 skipped**. Three additional failures are the missing `group_read` option. Artifact `11023512357`, independently verified SHA256 `33ff93271408f4e2006891154bf4621aa660dce6c26324b9945fdc4f54faac60`; checkout `475335d453cb3be925c6ddf5ce48c0769b83aaf1`, tree `2763aed9a71f7cefa6b68bc3214a15c3d43aaa36`. +| Head | Run / artifact | JUnit | Independent ZIP SHA256 | +|---|---|---|---| +| `0942251` | 36547254507 / none | Docker setup failed; NOT RED | none | +| `6d24ee847c2e4c0349c71ce667d45b4ac04393f1` | 36547778158 / 11023785174 | 17: 11 failed, 6 passed, 0 errors/skips | `c4f67c186d03cd57df2a85668dcd5487ac5f261a91f913dfb047432eb6b7f2c0` | +| `20399c0eacffece853d2d0e2035986e30b0636a4` | 36548533968 / 11023512357 | 20: 14 failed, 6 passed, 0 errors/skips | `33ff93271408f4e2006891154bf4621aa660dce6c26324b9945fdc4f54faac60` | +| `246f0fd83f41185dde4e796d0f96f1cca8db7934` | 36549829088 / 11024435325 | 22: 8 failed, 14 passed, 0 errors/skips | `f308a1c1dd112339cd6fb6912350c4f63f9964e6e4bd79acc37bd739c9789fe9` | +| `df7efdbaa637e7f316d0eab1e0ce3dc156f83954` | 36550574815 / 11024546594 | **22 passed, 0 failures/errors/skips** | `6dd107a22546b392c6ce136d1eab61f9c7927c387731e73d73a4cbb9add2fd9f` | -## Implemented; GREEN/full verification pending +Source manifests checked against each head/tree. First implementation tree0941aa2, checkout05e7b3b; GREEN tree `d974b049886a93e3cf784cba251231fcb88e6ac5`, checkout `f976bb8bbe91ca4afd88473334b9db2940dcf07b`, Python3.15.0rc2, attempt1. Initial eight implementation failures shared a PostgreSQL evaluation-order bug: sequence privilege checks received index OIDs. `df7efd` resolves sequence OIDs before privilege calls; no assertion removed or weakened. Existing error name `read_only_transaction` and Host403 remain unchanged. -- Shared ordered Registry installer; HostingRuntime retains contracts but replaces every handler/requirement closure, including plugin manifests. No full Application, issuer, executor, private service key, cursor MAC or outbound sender. -- Read-only PostgreSQL composition opens with initialization disabled, uses real READ ONLY transactions and validates the independent login's actual grants. Content reader binds the canonical streaming implementations without a writer/staging instance. -- Startup verifies public trust, current authority and exact release source records. It never creates missing storage, migrates schema or synchronizes release rules. Shared recovery marker plus per-request database quarantine remain fail-closed. -- Dedicated OS/DB configuration and operator-only preparation script. Opt-in writer `hosting.content_group_read=true` publishes group-readable/non-writable Git/index/CAS files under umask 0077; existing private permissions are never silently rewritten. -- Daemon chooses hosting before normal Application construction. Existing publication/CSP test now uses the read-only fixture without deleting assertions. New assertions use the existing canonical error `read_only_transaction` and existing forbidden-host 403 status, rather than changing those contracts. -- Added daemon composition regression; dedicated workflow also executes the established publication-origin regression. +## Parallel convergence and remaining acceptance -All project execution runs in GitHub Actions. Local work is source text/Git/AST inspection and artifact parsing only. No production probes, deployments, real Root/PIN/key reads, outbound delivery, destructive backups or restore promotion. Do not close #161 before inspecting GREEN and complete four-shard evidence against the current head/tree; reconcile live main before merge. +Discovered overlapping #174 after implementation; coordinated in #83 comments5887561813 and #174 comment5887616225. Do not merge both runtime implementations. Preserve #174's independent UID and failure-containment acceptance, adapted from head `eafea15af0b8ff26c0208d5c0fcc039b4d32fb56` into `tests/test_hosting_runtime_isolation.py`. + +The retained tests check post-revocation HEAD/Range/304, cached quarantine and dangling markers, missing schema, excess DB privileges, and an actual UID65534 reader of fresh text/binary publications. Adaptations use the #173 runtime/error names, explicit group-read opt-in and a disposable libpq service file. No default ACL/post-publication chmod may hide unreadable new files. Added wrong schema version, missing SELECT, sequence/secret-table privileges and latched quarantine cases. This test-only extension is not yet verified; required-schema checks may need correction before GREEN. + +Full four-shard/gate/conformance/build run36550574826 and ancillary boundary/recovery runs still require inspection; focused GREEN is not final completion. Reconcile current main and both PR discussions before merge; do not overwrite #155/#168/#170/#171/#172. No #161 closure yet. + +All project execution runs in GitHub Actions. Local work is source text/Git/AST inspection and artifact parsing only. No production probes, deployments, Root/PIN/private-key access, real outbound delivery, funds, destructive backup operations or restore promotion. diff --git a/tests/test_hosting_runtime_isolation.py b/tests/test_hosting_runtime_isolation.py new file mode 100644 index 00000000..f861a0ce --- /dev/null +++ b/tests/test_hosting_runtime_isolation.py @@ -0,0 +1,284 @@ +"""Independent-UID and failure-containment cases retained from PR #174. + +Adapted from eafea15af0b8ff26c0208d5c0fcc039b4d32fb56 to the #173 reader API. +These are disposable CI installs, not production permission changes. +""" +from datetime import timedelta +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +import httpx +import psycopg +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict +import pytest + +from msg.core.codec import b64, canonical, wire +from msg.core.errors import Failure +from msg.core.requests import request_for +from test_hosting_runtime import reader_settings, runtime_class, isolated +from test_service import NOW, call, register + + +async def preview(app, name='readonly-preview', *, binary=False): + key, subject, _ = await register(app, name) + site = await call(app, 'hosting.create', {'parent': '/@'+name, 'name': 'web'}, + key=key, subject=subject) + assert site.status == 'ok', wire(site) + payload = b'\x00readonly-live-binary\xff' if binary else b'

private read-only preview

' + filename = 'asset.bin' if binary else 'index.html' + source = await call(app, 'content.file_put', {'parent': '/@'+name+'/files', + 'name': filename, 'data': b64(payload), + 'media_type': 'application/octet-stream' if binary else 'text/html'}, key=key, subject=subject) + assert source.status == 'ok', wire(source) + candidate = await call(app, 'hosting.preview', {'id': site.resources[0].id, + 'entries': [{'path': filename, 'source': wire(source.resources[0])}]}, + key=key, subject=subject, expected=((site.resources[0].id, site.data['generation']),)) + assert candidate.status == 'ok', wire(candidate) + packet = request_for('discovery.raw', {'id': candidate.resources[0].id}, app.settings.service_url, + signer=key, subject=subject, expires_at=NOW+timedelta(seconds=120)) + return key, subject, candidate, payload, { + 'path': f'/@{name}/web/_preview/{candidate.resources[0].id}/{filename}', + 'header': b64(canonical(wire(packet))), 'payload': b64(payload)} + + +async def test_preview_rechecks_acl_before_head_range_and_cached_response(installed, reader_settings): + app, _ = installed + key, subject, candidate, payload, request = await preview(app) + service = runtime_class()(reader_settings, clock=app.clock) + await service.load() + try: + from msg.extensions.hosting import hosting_app + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(service)), + base_url=app.settings.service_url) as http: + path, headers = request['path'], {'X-Msg-Request': request['header']} + assert (await http.get(path)).status_code == 403 + accepted = await http.get(path, headers=headers) + assert accepted.status_code == 200 and accepted.content == payload + isolated(accepted) + assert (await http.get('/-/schema')).status_code == 404 + assert (await http.get(path, headers={**headers, 'Host': 'other.example'})).status_code == 403 + denied = await call(app, 'content.chmod', {'id': candidate.resources[0].id, 'mode': '0000'}, + key=key, subject=subject, expected=((candidate.resources[0].id, candidate.data['generation']),)) + assert denied.status == 'ok', wire(denied) + for method, extra in (('GET', {}), ('HEAD', {}), ('GET', {'Range': 'bytes=0-7'}), + ('GET', {'If-None-Match': accepted.headers['etag']})): + response = await http.request(method, path, headers={**headers, **extra}) + assert response.status_code == 403 and payload not in response.content + finally: + await service.close() + + +@pytest.mark.parametrize('gate', ['database', 'marker', 'marker_symlink']) +async def test_quarantine_revokes_cached_reads_and_new_startup(installed, reader_settings, gate): + app, _ = installed + service = runtime_class()(reader_settings, clock=app.clock) + await service.load() + marker = reader_settings.config_dir/'recovery-drill.json' + try: + from msg.extensions.hosting import hosting_app + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(service)), + base_url=app.settings.service_url) as http: + original = await http.get('/@root/web/index.html') + assert original.status_code == 200 + if gate == 'database': + async with app.metadata.transaction(write=True) as tx: + tx.set_setting('recovery_quarantine', {'version': 1, 'test': True}) + elif gate == 'marker': + marker.write_text('{}') + else: + marker.symlink_to(marker.with_name('missing-marker-target')) + for method in ('GET', 'HEAD'): + response = await http.request(method, '/@root/web/index.html', + headers={'If-None-Match': original.headers['etag']}) + assert response.status_code not in (200, 206, 304) + assert original.content not in response.content + fresh = runtime_class()(reader_settings, clock=app.clock) + try: + with pytest.raises(Failure, match='recovery_quarantined'): + await fresh.load() + finally: + await fresh.close() + if gate != 'database': + marker.unlink() + assert (await http.get('/@root/web/index.html')).status_code not in (200, 206, 304) + finally: + marker.unlink(missing_ok=True) + await service.close() + + +@pytest.mark.parametrize('damage', ['missing', 'wrong_version', 'missing_read_grant']) +async def test_required_schema_is_verified_without_repair(installed, reader_settings, damage): + app, _ = installed + reader = conninfo_to_dict(reader_settings.server.postgres_dsn)['user'] + with psycopg.connect(app.settings.server.postgres_dsn, autocommit=True) as conn: + if damage == 'missing': + conn.execute('DROP TABLE schema_version') + elif damage == 'wrong_version': + conn.execute('UPDATE schema_version SET version=999') + else: + conn.execute(sql.SQL('REVOKE SELECT ON share_grants_v2 FROM {}').format(sql.Identifier(reader))) + service = runtime_class()(reader_settings, clock=app.clock) + try: + with pytest.raises(Failure, match='hosting_installation_stale'): + await service.load() + finally: + await service.close() + with psycopg.connect(app.settings.server.postgres_dsn) as conn: + if damage == 'missing': + assert conn.execute("SELECT to_regclass('public.schema_version')").fetchone() == (None,) + elif damage == 'wrong_version': + assert conn.execute('SELECT version FROM schema_version').fetchall() == [(999,)] + else: + assert not conn.execute('SELECT has_table_privilege(%s,%s,%s)', + (reader, 'public.share_grants_v2', 'SELECT')).fetchone()[0] + + +@pytest.mark.parametrize('extra', ['write', 'membership', 'definer', 'sequence', 'secret_read']) +async def test_effective_privilege_escalations_fail_startup(installed, reader_settings, extra): + app, _ = installed + reader = conninfo_to_dict(reader_settings.server.postgres_dsn)['user'] + with psycopg.connect(app.settings.server.postgres_dsn, autocommit=True) as conn: + if extra == 'write': + conn.execute(sql.SQL('GRANT UPDATE ON settings TO {}').format(sql.Identifier(reader))) + elif extra == 'membership': + conn.execute(sql.SQL('GRANT {} TO {}').format(sql.Identifier(conn.info.user), sql.Identifier(reader))) + elif extra == 'definer': + conn.execute('CREATE FUNCTION hosting_test_definer() RETURNS integer LANGUAGE sql ' + "SECURITY DEFINER AS 'SELECT 1'") + elif extra == 'sequence': + conn.execute('CREATE SEQUENCE hosting_test_sequence') + conn.execute(sql.SQL('GRANT USAGE ON SEQUENCE hosting_test_sequence TO {}').format(sql.Identifier(reader))) + else: + conn.execute(sql.SQL('GRANT SELECT ON token_deliveries TO {}').format(sql.Identifier(reader))) + service = runtime_class()(reader_settings, clock=app.clock) + try: + with pytest.raises(Failure, match='hosting_database_not_readonly'): + await service.load() + finally: + await service.close() + + +READ_PROCESS = r''' +import asyncio, json, os, sys +from datetime import UTC, datetime +from pathlib import Path +import httpx +from msg.config import load_settings +from msg.core.codec import unb64 +from msg.hosting_runtime import HostingRuntime +from msg.extensions.hosting import hosting_app + +async def main(): + assert os.geteuid() == 65534, os.geteuid() + settings = load_settings(Path(sys.argv[1])) + try: + Path(sys.argv[2]).read_bytes() + except PermissionError: + pass + else: + raise AssertionError('reader can read service signing material') + try: + (settings.server.content_dir/'forbidden-write').write_bytes(b'no') + except PermissionError: + pass + else: + raise AssertionError('reader can write business files') + service = HostingRuntime(settings, clock=lambda: datetime(2026, 9, 27, tzinfo=UTC)) + await service.load() + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(service)), + base_url=settings.service_url) as http: + assert (await http.get('/@root/web/index.html')).status_code == 200 + for request in json.loads(Path(sys.argv[3]).read_text()): + assert (await http.get(request['path'])).status_code == 403 + response = await http.get(request['path'], headers={'X-Msg-Request': request['header']}) + assert response.status_code == 200, response.status_code + assert response.content == unb64(request['payload']) + print(json.dumps({'uid': os.geteuid(), 'private_key_denied': True, + 'business_write_denied': True, 'live_previews': 2})) + finally: + await service.close() +asyncio.run(main()) +''' + + +async def test_real_other_uid_reads_new_text_and_binary_without_keys(installed, reader_settings): + assert shutil.which('sudo'), 'Linux isolation prerequisite missing' + app, _ = installed + base = Path(tempfile.mkdtemp(prefix='msg-hosting-reader-')) + changed_ancestors = {} + try: + shutil.copytree(Path(__file__).resolve().parents[1]/'src'/'msg', base/'src'/'msg') + from msg.config import write_example + config = write_example(base/'etc', base/'unused', app.settings.service_url, + postgres_dsn='service=hosting-test') + config_file = config.config_dir/'msgd.toml' + text = config_file.read_text().replace('[server]', '[server]\npublic_web_origin = '+json.dumps(app.settings.service_url)) + text = text.replace('[hosting]', '[hosting]\nrecovery_marker = '+json.dumps(str(app.settings.config_dir/'recovery-drill.json'))) + for previous, actual in ((config.server.content_dir, app.contents.path), + (config.server.blob_dir, app.contents.binary), + (config.server.staging_dir, app.contents.staging)): + text = text.replace(str(previous), str(actual)) + config_file.write_text(text) + config.trust_file.parent.mkdir(parents=True) + shutil.copyfile(reader_settings.trust_file, config.trust_file) + service_file = base/'pg_service.conf' + service_file.write_text('[hosting-test]\n'+''.join( + f'{key}={value}\n' for key, value in conninfo_to_dict(reader_settings.server.postgres_dsn).items())) + # Explicitly prepare existing content before opt-in. No post-publication + # chmod/default ACL can hide a mode bug in newly published objects. + for root in (app.contents.path, app.contents.binary): + for parent in root.parents: + if parent in (Path('/tmp'), Path('/')): + break + if parent not in changed_ancestors: + changed_ancestors[parent] = parent.stat().st_mode & 0o7777 + parent.chmod(changed_ancestors[parent] | 0o010) + for path in [root, *root.rglob('*')]: + path.chmod(0o2750 if path.is_dir() else 0o640) + subprocess.run(['git', '--git-dir', str(app.contents.repo), 'config', 'core.sharedRepository', '0640'], + check=True, capture_output=True) + from msg.storage.git import GitContentStore + old_umask = os.umask(0o077) + try: + app.contents = GitContentStore(app.contents.path, binary_dir=app.contents.binary, + staging_dir=app.contents.staging, group_read=True) + first = await preview(app, 'live-text-reader') + second = await preview(app, 'live-binary-reader', binary=True) + finally: + os.umask(old_umask) + requests = base/'requests.json' + requests.write_text(json.dumps([first[-1], second[-1]])) + script = base/'reader.py' + script.write_text(READ_PROCESS) + for path in [base, *base.rglob('*')]: + path.chmod(0o550 if path.is_dir() else 0o440) + def snapshots(): + return {str(path): hashlib.sha256(path.read_bytes()).hexdigest() + for root in (app.contents.path, app.contents.binary) + for path in root.rglob('*') if path.is_file()} + before = snapshots() + result = subprocess.run(['sudo', '-n', '-u', '#65534', '-g', '#'+str(os.getgid()), '--', + 'env', 'PYTHONDONTWRITEBYTECODE=1', 'PYTHONPATH='+str(base/'src'), + 'PGSERVICEFILE='+str(service_file), sys.executable, str(script), str(config.config_dir), + str(app.settings.service_keys/'online.key'), str(requests)], + capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stderr[-6000:] + assert json.loads(result.stdout) == {'uid': 65534, 'private_key_denied': True, + 'business_write_denied': True, 'live_previews': 2} + assert before == snapshots() + assert not (app.contents.path/'forbidden-write').exists() + finally: + for path in [base, *base.rglob('*')]: + if path.is_dir(): + path.chmod(0o700) + shutil.rmtree(base) + for parent, mode in reversed(tuple(changed_ancestors.items())): + parent.chmod(mode) From c314d1b2ff0c892e274d731d5c424154a44f2784 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:49:40 +0800 Subject: [PATCH 10/14] test(hosting): retain key revocation and restorable owner privilege regressions --- .github/workflows/hosting-authority.yml | 68 +++++++++++++++ docs/HOSTING_AUTHORITY_PROGRESS.md | 30 +++++++ tests/test_hosting_runtime_authority.py | 106 ++++++++++++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 .github/workflows/hosting-authority.yml create mode 100644 docs/HOSTING_AUTHORITY_PROGRESS.md create mode 100644 tests/test_hosting_runtime_authority.py diff --git a/.github/workflows/hosting-authority.yml b/.github/workflows/hosting-authority.yml new file mode 100644 index 00000000..ad426a98 --- /dev/null +++ b/.github/workflows/hosting-authority.yml @@ -0,0 +1,68 @@ +name: Hosting authority regression +on: + push: + branches: ['fix/hosting-authority-*'] + pull_request: + paths: + - 'src/msg/hosting_runtime.py' + - 'src/msg/storage/read_only.py' + - 'tests/test_hosting_runtime_authority.py' + - '.github/workflows/hosting-authority.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: hosting-authority-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + authority: + runs-on: ubuntu-latest + timeout-minutes: 15 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd 'pg_isready -U msgtest -d postgres' + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Bind evidence to source + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, pathlib, platform, subprocess + git=lambda *args: subprocess.check_output(['git',*args],text=True).strip() + pathlib.Path('artifacts/source.json').write_text(json.dumps({ + 'head':os.environ['HEAD_SHA'], 'checkout':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'), 'python':platform.python_version(), + 'run_id':os.environ['GITHUB_RUN_ID'], 'attempt':os.environ['GITHUB_RUN_ATTEMPT']},indent=2)+'\n') + PY + - name: Install disposable test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Exercise revoked keys and restorable ownership rights + run: python -m pytest tests/test_hosting_runtime_authority.py --junitxml=artifacts/authority.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: hosting-authority-evidence + path: artifacts/ + if-no-files-found: error diff --git a/docs/HOSTING_AUTHORITY_PROGRESS.md b/docs/HOSTING_AUTHORITY_PROGRESS.md new file mode 100644 index 00000000..e9912bbc --- /dev/null +++ b/docs/HOSTING_AUTHORITY_PROGRESS.md @@ -0,0 +1,30 @@ +# Hosting authority supplement (#161 / #173 / #174) + +## Consolidation + +#173 is the single product integration path. #174 is not to be merged: its +runtime/storage/publication implementation is superseded, not stacked. Original +#174 RED evidence and notes remain linked in that PR (comment5887238998 and +comment5887734281). The clean historical source ends at +51af07f4835c37a00cf438055ca41a19f7cd6246; no production deployment occurred. + +#173 head6703f3de41e988977be09e509c91fc30a11738b1 already retains #174's distinct-UID, +ACL/cache, missing-schema and quarantine cases. This separate child branch adds +only the not-yet-transferred authority cases, without overwriting #173. Coordination: +#173 comments5887748079 and5887771169. Base tree bc9358a0cdc3a9dc9e771ae8c748c815a86475ee. + +## Test-first change + +Real key replacement/revocation rejects previously accepted private-preview +signatures before GET/HEAD/Range/ETag. Column-only UPDATE is rejected. For table, +schema, sequence and database ownership, tests first prove a login can GRANT its +own rights back after REVOKE, remove those ACL bits again, then require hosting to +reject the owner. Database ownership is restored in fixture cleanup. + +All project execution is GitHub Actions. The dedicated workflow saves exact head, +checkout/tree, runtime and JUnit. Tests are not yet claimed green or red until +artifacts are read. Full combined CI and #173's other existing failures remain +that branch's merge gate; this supplement cannot close #161 by itself. + +No production credentials, permission changes, live delivery, Root/PIN, payment +or recovery promotion. PostgreSQL objects and signatures are disposable fixtures. diff --git a/tests/test_hosting_runtime_authority.py b/tests/test_hosting_runtime_authority.py new file mode 100644 index 00000000..4b2fae0e --- /dev/null +++ b/tests/test_hosting_runtime_authority.py @@ -0,0 +1,106 @@ +"""Retained #174 authority cases on #173's single hosting implementation. + +These tests use disposable PostgreSQL objects and real signed calls. No second +runtime, shared installer or alternative publication policy is introduced. +""" +import httpx +import psycopg +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict +import pytest + +from msg.core.codec import b64, canonical, wire +from msg.core.errors import Failure +from msg.security.crypto import Ed25519Signer +from test_hosting_runtime import reader_settings, runtime_class +from test_hosting_runtime_isolation import preview +from test_service import call + + +async def test_revoked_signing_key_cannot_reuse_a_warm_private_preview(installed, reader_settings): + app, _ = installed + key, subject, _, payload, request = await preview(app, 'revocable-reader') + replacement = Ed25519Signer.generate() + public = b64(replacement.public_key) + proof = replacement.sign(canonical({'subject_id': subject, 'public_key': public}), purpose='key-add') + added = await call(app, 'identity.key_add', {'public_key': public, + 'possession_proof': wire(proof), 'ceiling': wire(app.primary_ceiling())}, key=key, subject=subject) + assert added.status == 'ok', wire(added) + service = runtime_class()(reader_settings, clock=app.clock) + await service.load() + try: + from msg.extensions.hosting import hosting_app + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=hosting_app(service)), + base_url=app.settings.service_url) as http: + headers = {'X-Msg-Request': request['header']} + accepted = await http.get(request['path'], headers=headers) + assert accepted.status_code == 200 and accepted.content == payload + revoked = await call(app, 'identity.key_revoke', {'key_id': key.key_id}, + key=replacement, subject=subject) + assert revoked.status == 'ok', wire(revoked) + for method, extra in (('GET', {}), ('HEAD', {}), ('GET', {'Range': 'bytes=0-3'}), + ('GET', {'If-None-Match': accepted.headers['etag']})): + denied = await http.request(method, request['path'], headers={**headers, **extra}) + assert denied.status_code == 403 and payload not in denied.content + finally: + await service.close() + + +async def test_column_only_update_is_not_a_readonly_database_role(installed, reader_settings): + app, _ = installed + reader = conninfo_to_dict(reader_settings.server.postgres_dsn)['user'] + with psycopg.connect(app.settings.server.postgres_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('GRANT UPDATE(value) ON settings TO {}').format(sql.Identifier(reader))) + service = runtime_class()(reader_settings, clock=app.clock) + try: + with pytest.raises(Failure, match='hosting_database_not_readonly'): + await service.load() + finally: + await service.close() + + +@pytest.mark.parametrize('kind', ['table', 'schema', 'sequence', 'database']) +async def test_owner_can_restore_revoked_rights_and_must_not_be_a_reader(installed, reader_settings, kind): + app, _ = installed + reader = conninfo_to_dict(reader_settings.server.postgres_dsn)['user'] + name = 'hosting_owned_probe' + owner_dsn = app.settings.server.postgres_dsn + service = runtime_class()(reader_settings, clock=app.clock) + with psycopg.connect(owner_dsn, autocommit=True) as conn: + original_owner, database = conn.info.user, conn.info.dbname + if kind == 'table': + conn.execute('CREATE TABLE public.hosting_owned_probe (value integer)') + conn.execute(sql.SQL('ALTER TABLE public.hosting_owned_probe OWNER TO {}').format(sql.Identifier(reader))) + grant, privilege = 'TABLE public.hosting_owned_probe', 'UPDATE' + elif kind == 'schema': + conn.execute(sql.SQL('CREATE SCHEMA hosting_owned_probe AUTHORIZATION {}').format(sql.Identifier(reader))) + grant, privilege = 'SCHEMA hosting_owned_probe', 'CREATE' + elif kind == 'sequence': + conn.execute('CREATE SEQUENCE public.hosting_owned_probe') + conn.execute(sql.SQL('ALTER SEQUENCE public.hosting_owned_probe OWNER TO {}').format(sql.Identifier(reader))) + grant, privilege = 'SEQUENCE public.hosting_owned_probe', 'USAGE' + else: + conn.execute(sql.SQL('ALTER DATABASE {} OWNER TO {}').format(sql.Identifier(database), sql.Identifier(reader))) + grant = sql.SQL('DATABASE {}').format(sql.Identifier(database)).as_string(conn) + privilege = 'CREATE, TEMPORARY' + # Keep CONNECT for a database owner; revoke every object ACL otherwise. + revoked = privilege if kind == 'database' else 'ALL' + query = sql.SQL('REVOKE {} ON {} FROM {}').format(sql.SQL(revoked), sql.SQL(grant), sql.Identifier(reader)) + conn.execute(query) + try: + # Demonstrate why current ACL bits alone cannot establish a read-only + # boundary: the owner can recover rights without an administrator. + with psycopg.connect(reader_settings.server.postgres_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('GRANT {} ON {} TO {}').format( + sql.SQL(privilege), sql.SQL(grant), sql.Identifier(reader))) + conn.execute(query) + with pytest.raises(Failure, match='hosting_database_not_readonly'): + await service.load() + finally: + await service.close() + if kind == 'database': + # DROP OWNED does not restore database ownership; restore it before + # the shared role fixture removes the disposable reader login. + with psycopg.connect(owner_dsn, autocommit=True) as conn: + conn.execute(sql.SQL('ALTER DATABASE {} OWNER TO {}').format( + sql.Identifier(database), sql.Identifier(original_owner))) From 99a0242bf9fdc089f61b7b127c21853e66b20a7d Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:51:12 +0800 Subject: [PATCH 11/14] fix(ci): quote the PostgreSQL health command for the Actions process runner --- .github/workflows/hosting-authority.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hosting-authority.yml b/.github/workflows/hosting-authority.yml index ad426a98..d5756fc3 100644 --- a/.github/workflows/hosting-authority.yml +++ b/.github/workflows/hosting-authority.yml @@ -27,7 +27,7 @@ jobs: POSTGRES_DB: postgres ports: ['5432:5432'] options: >- - --health-cmd 'pg_isready -U msgtest -d postgres' + --health-cmd "pg_isready -U msgtest -d postgres" --health-interval 10s --health-timeout 5s --health-retries 5 env: MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} From 2b97ee9a3a3e7b01fbd60f7b19c84d3578ab47b7 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:51:39 +0800 Subject: [PATCH 12/14] fix(hosting): verify complete reader schema and launch UID test after dropping privilege --- src/msg/storage/read_only.py | 13 +++++++++++-- tests/test_hosting_runtime_isolation.py | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/msg/storage/read_only.py b/src/msg/storage/read_only.py index 22521688..1236c3b8 100644 --- a/src/msg/storage/read_only.py +++ b/src/msg/storage/read_only.py @@ -36,10 +36,12 @@ def require_readonly_role(tx): WHERE left(nspname,3)<>'pg_' AND nspname<>'information_schema' AND has_schema_privilege(oid,'CREATE') LIMIT 1''') is None, 'hosting_database_not_readonly') - for schema, name, readable, writable in tx.rows(''' + available = set() + for schema, name, readable, writable, full_read in tx.rows(''' SELECT n.nspname,c.relname,has_any_column_privilege(c.oid,'SELECT'), (has_table_privilege(c.oid,'INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER') - OR has_any_column_privilege(c.oid,'INSERT,UPDATE,REFERENCES')) + OR has_any_column_privilege(c.oid,'INSERT,UPDATE,REFERENCES')), + has_table_privilege(c.oid,'SELECT') FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE c.relkind IN ('r','p','v','m','f') AND left(n.nspname,3)<>'pg_' AND n.nspname<>'information_schema' @@ -47,6 +49,13 @@ def require_readonly_role(tx): require(not writable and (not readable or (schema == 'public' and name in HOSTING_READ_TABLES)), 'hosting_database_not_readonly') + if schema == 'public' and full_read: + available.add(name) + require(available == HOSTING_READ_TABLES, 'hosting_installation_stale') + # The installed PostgreSQL metadata format is version 1. A reader cannot + # initialize or migrate it, or infer readiness from a few working tables. + require(tx.rows('SELECT version FROM public.schema_version') == [(1,)], + 'hosting_installation_stale') # Fetch sequence OIDs first: SQL predicates do not impose evaluation order, # and has_sequence_privilege must never receive an index/table OID. for (oid,) in tx.rows('''SELECT c.oid FROM pg_catalog.pg_class c diff --git a/tests/test_hosting_runtime_isolation.py b/tests/test_hosting_runtime_isolation.py index f861a0ce..9d67c7e7 100644 --- a/tests/test_hosting_runtime_isolation.py +++ b/tests/test_hosting_runtime_isolation.py @@ -210,7 +210,7 @@ async def main(): async def test_real_other_uid_reads_new_text_and_binary_without_keys(installed, reader_settings): - assert shutil.which('sudo'), 'Linux isolation prerequisite missing' + assert shutil.which('sudo') and shutil.which('setpriv'), 'Linux isolation prerequisites missing' app, _ = installed base = Path(tempfile.mkdtemp(prefix='msg-hosting-reader-')) changed_ancestors = {} @@ -265,8 +265,8 @@ def snapshots(): for root in (app.contents.path, app.contents.binary) for path in root.rglob('*') if path.is_file()} before = snapshots() - result = subprocess.run(['sudo', '-n', '-u', '#65534', '-g', '#'+str(os.getgid()), '--', - 'env', 'PYTHONDONTWRITEBYTECODE=1', 'PYTHONPATH='+str(base/'src'), + result = subprocess.run(['sudo', '-n', '--', 'setpriv', '--reuid=65534', + '--regid='+str(os.getgid()), '--clear-groups', '--no-new-privs', '--', 'env', 'PYTHONDONTWRITEBYTECODE=1', 'PYTHONPATH='+str(base/'src'), 'PGSERVICEFILE='+str(service_file), sys.executable, str(script), str(config.config_dir), str(app.settings.service_keys/'online.key'), str(requests)], capture_output=True, text=True, timeout=120) From d17a2875986b9f897c266f3632f3e8681ce58a96 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:54:11 +0800 Subject: [PATCH 13/14] fix(hosting): reject database object owners even after their ACLs are revoked --- docs/HOSTING_AUTHORITY_PROGRESS.md | 73 ++++++++++++++++++------------ src/msg/storage/read_only.py | 9 ++++ 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/docs/HOSTING_AUTHORITY_PROGRESS.md b/docs/HOSTING_AUTHORITY_PROGRESS.md index e9912bbc..fa1005db 100644 --- a/docs/HOSTING_AUTHORITY_PROGRESS.md +++ b/docs/HOSTING_AUTHORITY_PROGRESS.md @@ -1,30 +1,47 @@ # Hosting authority supplement (#161 / #173 / #174) -## Consolidation - -#173 is the single product integration path. #174 is not to be merged: its -runtime/storage/publication implementation is superseded, not stacked. Original -#174 RED evidence and notes remain linked in that PR (comment5887238998 and -comment5887734281). The clean historical source ends at -51af07f4835c37a00cf438055ca41a19f7cd6246; no production deployment occurred. - -#173 head6703f3de41e988977be09e509c91fc30a11738b1 already retains #174's distinct-UID, -ACL/cache, missing-schema and quarantine cases. This separate child branch adds -only the not-yet-transferred authority cases, without overwriting #173. Coordination: -#173 comments5887748079 and5887771169. Base tree bc9358a0cdc3a9dc9e771ae8c748c815a86475ee. - -## Test-first change - -Real key replacement/revocation rejects previously accepted private-preview -signatures before GET/HEAD/Range/ETag. Column-only UPDATE is rejected. For table, -schema, sequence and database ownership, tests first prove a login can GRANT its -own rights back after REVOKE, remove those ACL bits again, then require hosting to -reject the owner. Database ownership is restored in fixture cleanup. - -All project execution is GitHub Actions. The dedicated workflow saves exact head, -checkout/tree, runtime and JUnit. Tests are not yet claimed green or red until -artifacts are read. Full combined CI and #173's other existing failures remain -that branch's merge gate; this supplement cannot close #161 by itself. - -No production credentials, permission changes, live delivery, Root/PIN, payment -or recovery promotion. PostgreSQL objects and signatures are disposable fixtures. +## Single integration path + +#173 is the only product integration path. #174 is not to be merged: its duplicate +runtime/storage/publication implementation is superseded. Original #174 RED and +failure-correction evidence remain in that PR (comments5887238998,5887734281). +Historical clean source51af07f4835c37a00cf438055ca41a19f7cd6246 is retained, not deployed. + +#173 head6703f3de41e988977be09e509c91fc30a11738b1 already retained #174's real UID, +ACL/cache, missing-schema and quarantine tests. This independent child branch +`fix/hosting-authority-174-to-173` adds only key-revocation/owner-authority cases +and one narrow ownership guard. It never overwrites #173. Coordination is in +#173 comments5887748079,5887771169,5887792936 and5887839120; #83 comment5887825824. + +## Actual cloud evidence + +Test commit c314d1b2ff0c892e274d731d5c424154a44f2784 added six tests. Its first run +36551691097 failed container initialization because Actions did not group a +single-quoted Docker health command. No test ran and no artifact existed; this +is NOT a functional RED. Commit99a0242bf9fdc089f61b7b127c21853e66b20a7d corrected +the workflow to the established double-quoted form without changing assertions. + +Valid RED: head=checkout99a0242bf9fdc089f61b7b127c21853e66b20a7d, +tree70361477198cf11e08ec9d6876ac6ccdb08415c8, run36551843100/attempt1, +artifact11024911828, Python3.15.0rc2. JUnit **6 tests:3 passed/3 failed/0 errors/0 skipped**. +ZIP downloaded and SHA256 verified against provider: +5664e005eb1cd92ba8d7f1f4e97b980d823793639426a24fb94b1aed891a9146. + +The three failures are table/schema/sequence owners: the actual reader login +successfully restores its own permissions with GRANT, removes ACLs again, and +hosting.load incorrectly succeeds. Database ownership, real key revocation +(GET/HEAD/Range/ETag), and column-only UPDATE cases already passed. + +## Minimal repair + +`storage/read_only.py::require_readonly_role` additionally rejects current-database, +schema, relation and routine ownership before checking effective ACLs. The existing +read-table allowlist, canonical transactions, signature/authentication, source +validation and GitContentReader are unchanged. The same six assertions must turn +green; no test is skipped or converted into an expected failure. + +Project execution is only in GitHub Actions. This commit has no green claim until +its report is read. #173's full combined CI, remaining schema/UID repairs and +exact merge-tree verification are still mandatory. This supplement alone cannot +close #161, #80 or #84. No production secrets, deployment, permissions, real +outbound delivery, Root/PIN, funds or recovery promotion were touched. diff --git a/src/msg/storage/read_only.py b/src/msg/storage/read_only.py index 22521688..b1a9679e 100644 --- a/src/msg/storage/read_only.py +++ b/src/msg/storage/read_only.py @@ -27,6 +27,15 @@ def require_readonly_role(tx): WHERE rolname=current_user''') require(role is not None and role[0] == role[1] and not any(role[2:]), 'hosting_database_not_readonly') + # An owner can GRANT its own rights back after REVOKE ALL. Current ACL bits + # cannot establish read-only authority even when every write check is false. + require(tx.one('''SELECT 1 FROM ( + SELECT datdba AS owner_id FROM pg_catalog.pg_database WHERE datname=current_database() + UNION ALL SELECT nspowner FROM pg_catalog.pg_namespace + UNION ALL SELECT relowner FROM pg_catalog.pg_class + UNION ALL SELECT proowner FROM pg_catalog.pg_proc + ) AS owned WHERE owner_id=(SELECT oid FROM pg_catalog.pg_roles + WHERE rolname=current_user) LIMIT 1''') is None, 'hosting_database_not_readonly') require(tx.one('''SELECT 1 FROM pg_catalog.pg_roles WHERE rolname<>current_user AND pg_has_role(oid,'MEMBER') LIMIT 1''') is None, 'hosting_database_not_readonly') From 1138c27e1229e580cc7c61d13b00cf0e2bc30e18 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 17:58:42 +0800 Subject: [PATCH 14/14] docs: record verified red-to-green hosting ownership regression evidence --- docs/HOSTING_AUTHORITY_PROGRESS.md | 82 +++++++++++++++++------------- 1 file changed, 48 insertions(+), 34 deletions(-) diff --git a/docs/HOSTING_AUTHORITY_PROGRESS.md b/docs/HOSTING_AUTHORITY_PROGRESS.md index fa1005db..b6800247 100644 --- a/docs/HOSTING_AUTHORITY_PROGRESS.md +++ b/docs/HOSTING_AUTHORITY_PROGRESS.md @@ -2,29 +2,29 @@ ## Single integration path -#173 is the only product integration path. #174 is not to be merged: its duplicate -runtime/storage/publication implementation is superseded. Original #174 RED and -failure-correction evidence remain in that PR (comments5887238998,5887734281). -Historical clean source51af07f4835c37a00cf438055ca41a19f7cd6246 is retained, not deployed. - -#173 head6703f3de41e988977be09e509c91fc30a11738b1 already retained #174's real UID, -ACL/cache, missing-schema and quarantine tests. This independent child branch -`fix/hosting-authority-174-to-173` adds only key-revocation/owner-authority cases -and one narrow ownership guard. It never overwrites #173. Coordination is in -#173 comments5887748079,5887771169,5887792936 and5887839120; #83 comment5887825824. - -## Actual cloud evidence - -Test commit c314d1b2ff0c892e274d731d5c424154a44f2784 added six tests. Its first run -36551691097 failed container initialization because Actions did not group a -single-quoted Docker health command. No test ran and no artifact existed; this -is NOT a functional RED. Commit99a0242bf9fdc089f61b7b127c21853e66b20a7d corrected -the workflow to the established double-quoted form without changing assertions. - -Valid RED: head=checkout99a0242bf9fdc089f61b7b127c21853e66b20a7d, +#173 is the only product integration path. #174 is closed as superseded, not +merged or accepted as a completed #161. Its duplicate runtime/storage/publication +implementation is not stacked. Original #174 RED and failure-correction evidence +remain in that PR (comments5887238998,5887734281). Historical source +51af07f4835c37a00cf438055ca41a19f7cd6246 is retained, not deployed. + +#173 head6703f3de41e988977be09e509c91fc30a11738b1 retained #174's real UID, +ACL/cache, missing-schema and quarantine tests. The independent child branch +`fix/hosting-authority-174-to-173` adds only key-revocation/owner-authority tests +and nine lines of ownership checking; it never overwrites #173. Coordination: +#173 comments5887748079,5887771169,5887792936,5887839120; #83 comment5887825824. + +## Verified cloud evidence + +Test commit c314d1b2ff0c892e274d731d5c424154a44f2784 added six tests. Run36551691097 +failed container initialization because Actions did not group a single-quoted +Docker health command. No tests/artifacts: NOT a functional RED. Commit +99a0242bf9fdc089f61b7b127c21853e66b20a7d corrected only the workflow quoting. + +**RED**: head=checkout99a0242bf9fdc089f61b7b127c21853e66b20a7d, tree70361477198cf11e08ec9d6876ac6ccdb08415c8, run36551843100/attempt1, artifact11024911828, Python3.15.0rc2. JUnit **6 tests:3 passed/3 failed/0 errors/0 skipped**. -ZIP downloaded and SHA256 verified against provider: +Downloaded ZIP SHA256 verified against provider: 5664e005eb1cd92ba8d7f1f4e97b980d823793639426a24fb94b1aed891a9146. The three failures are table/schema/sequence owners: the actual reader login @@ -32,16 +32,30 @@ successfully restores its own permissions with GRANT, removes ACLs again, and hosting.load incorrectly succeeds. Database ownership, real key revocation (GET/HEAD/Range/ETag), and column-only UPDATE cases already passed. -## Minimal repair - -`storage/read_only.py::require_readonly_role` additionally rejects current-database, -schema, relation and routine ownership before checking effective ACLs. The existing -read-table allowlist, canonical transactions, signature/authentication, source -validation and GitContentReader are unchanged. The same six assertions must turn -green; no test is skipped or converted into an expected failure. - -Project execution is only in GitHub Actions. This commit has no green claim until -its report is read. #173's full combined CI, remaining schema/UID repairs and -exact merge-tree verification are still mandatory. This supplement alone cannot -close #161, #80 or #84. No production secrets, deployment, permissions, real -outbound delivery, Root/PIN, funds or recovery promotion were touched. +**GREEN**: head=checkoutd17a2875986b9f897c266f3632f3e8681ce58a96, +treeeef4fef2bf5a19908a4a8fb8be690c80bbd2dc01, run36552196870/attempt1, +artifact11024289895, Python3.15.0rc2. JUnit **the same 6 tests:6 passed/0 failures/0 errors/0 skipped**. +Downloaded ZIP SHA256 verified against provider: +47981a0025c74219454aef9f47f8b2cb396255f8e682f39e30a125f650c5b4db. +Source manifest and every test name/result were independently inspected. This +is a focused regression result, not a claim that canonical #173 or the full +project has passed. This later note-only commit is not the tested d17 head. + +## Repair and integration handoff + +`storage/read_only.py::require_readonly_role` rejects current-database, schema, +relation and routine ownership before checking effective ACLs. Source diff is +**9 added/0 removed lines**. Existing table-read allowlist, canonical transactions, +authentication, signatures, release validation and GitContentReader are unchanged. +No assertions were weakened, skipped or marked as expected failures. + +For integration into #173, retain the three source commits c314d1b (tests), +99a0242 (health quoting) and d17a287 (guard), plus this evidence note, or apply +their exact narrow diff with traceable attribution. Do not copy the old #174 +Application/Git implementation. Reconcile against the latest #173 head and run +its full combined four-shard/node-ID/conformance/build and isolation workflows. +The canonical branch owner retains schema/marker/real-UID fixes and serial merge. + +All project execution is in Actions. No production secrets, deployment, +permissions, real outbound delivery, Root/PIN, funds or recovery promotion. +#161 is not closed by this supplement; #80/#84 full acceptance remains separate.