From 6ea463d6fa8b256cb07d0ede2d7795a5336eda4f Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:00:55 +0800 Subject: [PATCH 1/8] test(client): verify a dependency-clean wheel and four signed transports Refs #158 #82 #83 #85. Add strict import, helper compatibility and server preflight regressions before changing implementation; retain all full-server CI gates. --- .github/workflows/client-boundary.yml | 72 +++++++++++++++ docs/CLIENT_BOUNDARY_PROGRESS.md | 21 +++++ scripts/check_client_install.py | 124 ++++++++++++++++++++++++++ tests/test_client_boundary.py | 123 +++++++++++++++++++++++++ 4 files changed, 340 insertions(+) create mode 100644 .github/workflows/client-boundary.yml create mode 100644 docs/CLIENT_BOUNDARY_PROGRESS.md create mode 100644 scripts/check_client_install.py create mode 100644 tests/test_client_boundary.py diff --git a/.github/workflows/client-boundary.yml b/.github/workflows/client-boundary.yml new file mode 100644 index 00000000..383239b6 --- /dev/null +++ b/.github/workflows/client-boundary.yml @@ -0,0 +1,72 @@ +name: Client boundary +on: + push: + branches: ['fix/client-boundary-*'] + pull_request: + paths: + - 'src/msg/**' + - 'pyproject.toml' + - 'tests/test_client_boundary.py' + - 'scripts/check_client_install.py' + - '.github/workflows/client-boundary.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: client-boundary-${{ github.ref }} + cancel-in-progress: true +jobs: + installed-client: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Record tested source + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, pathlib, subprocess + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + pathlib.Path('artifacts/source.json').write_text(json.dumps({ + 'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}'), + 'run_id': os.environ['GITHUB_RUN_ID'], 'event': os.environ['GITHUB_EVENT_NAME'], + }, indent=2) + '\n') + PY + - name: Install full development dependencies + run: python -m pip install -e '.[dev]' + - name: Verify helper ownership, exact proof context and fail-closed assembly + run: python -m pytest tests/test_client_boundary.py --junitxml=artifacts/client-boundary.xml + - name: Build the distributable wheel and source archive + if: always() + run: | + python -m build + python scripts/check_package_artifacts.py dist + - name: Install a clean client-only wheel outside the checkout + if: always() + run: | + python -m venv "$RUNNER_TEMP/msg-client-only" + client_python="$RUNNER_TEMP/msg-client-only/bin/python" + "$client_python" -m pip install dist/*.whl + "$client_python" -m pip check + mkdir -p "$RUNNER_TEMP/msg-client-outside" + cd "$RUNNER_TEMP/msg-client-outside" + "$RUNNER_TEMP/msg-client-only/bin/msg" --help > "$GITHUB_WORKSPACE/artifacts/client-help.txt" + "$RUNNER_TEMP/msg-client-only/bin/msgd" --help > "$GITHUB_WORKSPACE/artifacts/daemon-help.txt" + "$client_python" -I "$GITHUB_WORKSPACE/scripts/check_client_install.py" --minimal-install \ + > "$GITHUB_WORKSPACE/artifacts/client-install.json" + "$client_python" -m pip list --format=json > "$GITHUB_WORKSPACE/artifacts/client-packages.json" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: client-boundary + path: | + artifacts/ + dist/ diff --git a/docs/CLIENT_BOUNDARY_PROGRESS.md b/docs/CLIENT_BOUNDARY_PROGRESS.md new file mode 100644 index 00000000..11966c65 --- /dev/null +++ b/docs/CLIENT_BOUNDARY_PROGRESS.md @@ -0,0 +1,21 @@ +# Client dependency boundary progress — 2026-09-29 + +Refs #158/#82/#83/#85. Branch `fix/client-boundary-20260929` starts at tool-boundary head `f14fd1533f70983b046df4b4c813993e8a536d12` (#165). It does not write the #155 recovery, #156 executor or #166 metadata-session branches. + +## Regression-first work + +This commit adds seven boundary/compatibility/installation checks and a standalone fresh-interpreter client probe. It contains no implementation fix yet. The probe imports client/CLI/TUI modules under a strict server-import guard, creates/reloads protected client key files, and sends the same genuinely signed request through HTTP, path GET, GraphQL and MCP using an in-process mock HTTP endpoint. It never sends to a real service. + +The independent workflow builds the real wheel and installs it with normal dependencies into a fresh virtualenv outside the source checkout. It runs pip check, both command help entrypoints and the probe with explicit rejection of server-only packages. No PostgreSQL/Valkey service is configured for this workflow; full repository CI remains unchanged and mandatory. + +## Intended ownership and compatibility + +- Compact result serialization belongs to protocol codec; executor keeps only a compatibility import. +- Atomic protected local-file replacement has one narrowly named owner, not the Git/CAS backend. Existing permissions/fsync/replacement semantics remain unchanged. +- Custodial proof input/derivation and signed migration statements share one state-free owner; server-held vault and retirement verification stay server-side. Existing exports remain aliases, not copied security code. +- MCP version negotiation constants must not require importing the server implementation from the network client. +- Base installation expresses client dependencies; the server extra is the complete server install. The existing dev extra retains server dependencies and msg/msgd command names are unchanged. Documentation must explicitly migrate full server installation to the server extra; no automatic package installation. + +## Evidence still required + +Read the RED run before implementation. Then verify helper compatibility, four real client transport calls, a dependency-clean wheel environment, all existing credential journal/recovery and server tests, full exact-node-ID/conformance/build gates and current integration tree. No size, speed, Windows/Android field result or production readiness is claimed by this refactor. Keep #158 open until those facts are proved. diff --git a/scripts/check_client_install.py b/scripts/check_client_install.py new file mode 100644 index 00000000..f12ed662 --- /dev/null +++ b/scripts/check_client_install.py @@ -0,0 +1,124 @@ +"""Exercise real client imports and signed envelopes without a server runtime. + +Run with --minimal-install in a fresh wheel-only virtualenv, outside the source +checkout. Network traffic uses httpx.MockTransport, not a remote service. +""" +from __future__ import annotations + +import argparse +import asyncio +import importlib +import importlib.abc +import importlib.util +import json +from pathlib import Path +import stat +import sys +import tempfile + +SERVER_DEPENDENCIES = ('starlette', 'uvicorn', 'psycopg', 'valkey', 'aiohttp', 'dns', 'graphql') +FORBIDDEN = ('msg.application', 'msg.core.executor', 'msg.storage', 'msg.admin', + 'msg.workers', 'msg.security.vault', 'msg.security.custodial_migration', + 'msg.transports.http', 'msg.transports.graphql', 'msg.transports.mcp') + + +class ClientBoundary(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if any(fullname == name or fullname.startswith(name + '.') for name in FORBIDDEN): + raise RuntimeError('client imported a server implementation: ' + fullname) + return None + + +async def check(directory): + import httpx + from msg.client import ClientState + from msg.core.codec import canonical, loads, wire + from msg.core.requests import request_for, signing_bytes + from msg.security.crypto import Ed25519Signer, verify + from msg.transports.client import TRANSPORTS + from msg.transports.packet import decode_packet, path_packet + + for name in ('cli', 'tui', 'client_certificates', 'client_content', 'client_custodial', + 'client_market', 'client_recovery', 'client_secrets', 'client_tokens', 'client_upgrade'): + importlib.import_module('msg.' + name) + state = ClientState(directory, server='https://unit.invalid') + signer = Ed25519Signer.from_bytes(bytes(range(32))) + state.save_signer(signer) + state.ensure_encryption_key() + restored = ClientState(directory) + assert restored.signer.public_key == signer.public_key + assert restored.encryption_recipient == state.encryption_recipient + for path in (state.path, state.key_path, state.age_key_path): + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert stat.S_IMODE(directory.stat().st_mode) == 0o700 + request = request_for('content.read', {'id': 'r_test'}, state.server, + subject='u_test', signer=signer, request_id='client-boundary') + calls = [] + + def respond(incoming): + path = incoming.url.path + if path == '/_transports': + return httpx.Response(200, json={'version': 1, 'target_service': state.server, + 'operations': {'content.read': 'read'}, 'limits': { + 'max_request_bytes': 1048576, 'max_response_bytes': 1048576, + 'max_path_bytes': 8192, 'encodings': ['j', 'gz']}}) + if path.startswith('/-/g/'): + _, _, _, operation, encoding, encoded = path.split('/') + packet = path_packet(encoded, encoding, 1048576) + assert packet.operation == operation and incoming.method == 'GET' + kind = 'path_get' + else: + data = loads(incoming.content) + if path == '/_read/graphql': + data = data['variables']['packet'] + kind = 'graphql' + elif path == '/-/mcp': + assert incoming.headers['MCP-Protocol-Version'] == '2025-11-25' + data = data['params']['arguments']['packet'] + kind = 'mcp_http' + else: + assert path == '/-/p/content.read' + kind = 'http' + assert incoming.method == 'POST' + packet = decode_packet(data) + assert canonical(packet) == canonical(request) + verify(signer.public_key, signing_bytes(packet), packet.proof.signature, purpose='request') + calls.append(kind) + result = {'request_id': packet.request_id, 'operation': packet.operation, 'status': 'ok', + 'actor': 'u_test', 'subject': 'u_test', 'resources': [{'id': 'r_test'}], + 'data': {'content': 'signed client response'}} + if kind == 'graphql': + result = {'data': {'call': result}} + elif kind == 'mcp_http': + result = {'jsonrpc': '2.0', 'id': packet.request_id, + 'result': {'structuredContent': result}} + return httpx.Response(200, json=result) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: + for name, transport in TRANSPORTS.items(): + result = await transport(state.server, http=http).call(request) + assert result.status == 'ok' and result.resources[0].id == 'r_test', (name, wire(result)) + assert result.data['content'] == 'signed client response' + assert sorted(calls) == sorted(TRANSPORTS) + assert not any(any(name == prefix or name.startswith(prefix + '.') for prefix in FORBIDDEN) + for name in sys.modules) + return calls + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--minimal-install', action='store_true') + args = parser.parse_args() + if args.minimal_install: + present = [name for name in SERVER_DEPENDENCIES if importlib.util.find_spec(name) is not None] + assert not present, 'server-only Python dependencies in client environment: ' + ', '.join(present) + sys.meta_path.insert(0, ClientBoundary()) + with tempfile.TemporaryDirectory(prefix='msg-client-install-') as folder: + calls = asyncio.run(check(Path(folder) / 'client')) + print(json.dumps({'status': 'ok', 'transports': sorted(calls), + 'server_implementation_imported': False, + 'minimal_install': args.minimal_install})) + + +if __name__ == '__main__': + main() diff --git a/tests/test_client_boundary.py b/tests/test_client_boundary.py new file mode 100644 index 00000000..02e46139 --- /dev/null +++ b/tests/test_client_boundary.py @@ -0,0 +1,123 @@ +"""Public helper ownership, unchanged bytes and supported client/server installs.""" +from __future__ import annotations + +import hashlib +import hmac +import importlib.util +from pathlib import Path +import subprocess +import sys +import tomllib + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def test_result_serialization_has_one_protocol_owner(): + from msg.core.codec import result_wire + from msg.core.executor import result_wire as compatibility_export + from msg.core.models import OperationResult + from msg.cli import result_wire as client_export + + assert result_wire is compatibility_export is client_export + assert result_wire.__module__ == 'msg.core.codec' + result = OperationResult(request_id='r', operation='content.read', status='ok', actor=None, + subject=None, resources=(), data={'text': 'plain'}) + value = result_wire(result) + assert value['data'] == {'text': 'plain'} + assert 'replayed' not in value and 'prefer_cli' not in value + + +def test_atomic_file_owner_keeps_compatibility_and_failed_replace_protection(tmp_path, monkeypatch): + from msg.atomic_file import durable_write + from msg.storage.git import durable_write as compatibility_export + import msg.atomic_file as atomic + + assert durable_write is compatibility_export + path = tmp_path / 'state' / 'identity' + durable_write(path, b'first') + assert path.read_bytes() == b'first' and path.stat().st_mode & 0o777 == 0o600 + def fail_replace(*args): + raise OSError('simulated replace refusal') + monkeypatch.setattr(atomic.os, 'replace', fail_replace) + with pytest.raises(OSError, match='simulated replace refusal'): + durable_write(path, b'not committed') + assert path.read_bytes() == b'first' + assert list(path.parent.iterdir()) == [path] + + +def test_custodial_proof_and_statements_are_shared_without_vault_state(): + from msg.security.custodial_protocol import ( + UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof, stage_statement, ack_statement) + from msg.security.vault import client_upgrade_proof as old_proof + from msg.security.custodial_migration import stage_statement as old_stage, ack_statement as old_ack + from msg.core.codec import b64, canonical + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + assert client_upgrade_proof is old_proof + assert stage_statement is old_stage and ack_statement is old_ack + assert client_upgrade_proof.__module__ == 'msg.security.custodial_protocol' + expected_fields = ('subject_id', 'challenge_id', 'request_id', 'handle', 'public_key', + 'encryption_recipient', 'server_public', 'nonce', 'expires_at') + assert UPGRADE_CONTEXT_FIELDS == expected_fields + challenge = {name: name + '-fixture' for name in expected_fields} + challenge['nonce'] = b64(bytes(range(32))) + challenge['not_signed'] = 'not part of this published proof context' + context = {name: challenge[name] for name in expected_fields} + shared = bytes(range(32, 64)) + key = HKDF(algorithm=hashes.SHA256(), length=32, salt=bytes(range(32)), + info=b'msg-custodial-upgrade-pop-v1').derive(shared) + expected = b64(hmac.digest(key, canonical(context), 'sha256')) + assert _proof(shared, challenge) == expected + assert stage_statement('u', {'choice': 'rewrap', 'migration_ack': 'omit'}, 'r') == { + 'domain': 'msg-custodial-decision-v1', 'subject_id': 'u', 'request_id': 'r', + 'decision': {'choice': 'rewrap'}} + assert ack_statement('u', 'c', 'd', {'id': 'old'}, {'id': 'new'}, 'p', 'r') == { + 'domain': 'msg-custodial-history-ack-v1', 'subject_id': 'u', 'challenge_id': 'c', + 'inventory_digest': 'd', 'source': {'id': 'old'}, 'target': {'id': 'new'}, + 'method': 'rewrap', 'plaintext_digest': 'p', 'request_id': 'r'} + + +def test_mcp_client_uses_neutral_version_declaration(): + from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS + from msg.transports import mcp + assert PROTOCOL_VERSION == mcp.PROTOCOL_VERSION == '2025-11-25' + assert SUPPORTED_VERSIONS is mcp.SUPPORTED_VERSIONS + assert SUPPORTED_VERSIONS == frozenset({'2025-11-25', '2025-06-18', '2025-03-26'}) + + +def test_fresh_client_import_guard_covers_actual_four_transport_calls(): + completed = subprocess.run([sys.executable, str(ROOT / 'scripts/check_client_install.py')], + capture_output=True, text=True, timeout=30) + assert completed.returncode == 0, completed.stdout + completed.stderr + import json + report = json.loads(completed.stdout) + assert report['transports'] == ['graphql', 'http', 'mcp_http', 'path_get'] + assert report['server_implementation_imported'] is False + + +def test_install_metadata_separates_roles_but_dev_keeps_full_server(): + project = tomllib.loads((ROOT / 'pyproject.toml').read_text())['project'] + def name(requirement): + return requirement.split('>=')[0].split('[')[0] + server_names = {'starlette', 'uvicorn', 'psycopg', 'valkey', 'aiohttp', 'dnspython', 'graphql-core'} + assert not server_names & {name(value) for value in project['dependencies']} + extras = project['optional-dependencies'] + assert server_names <= {name(value) for value in extras['server']} + assert 'msg-lmm-best[server]' in extras['dev'] + assert project['scripts'] == {'msg': 'msg.cli:main', 'msgd': 'msg.daemon:main'} + + +def test_missing_server_dependencies_are_reported_before_local_state(tmp_path, monkeypatch, capsys): + import msg.daemon as daemon + original = importlib.util.find_spec + monkeypatch.setattr(importlib.util, 'find_spec', + lambda name, *args, **kwargs: None if name == 'psycopg' else original(name, *args, **kwargs)) + directory = tmp_path / 'not-created' + assert daemon.main(['--config-dir', str(directory), 'serve']) == 2 + captured = capsys.readouterr() + assert 'server_dependencies_required' in captured.err + assert 'msg-lmm-best[server]' in captured.err + assert not directory.exists() From 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:02:02 +0800 Subject: [PATCH 2/8] test: define market ownership and protected posting contracts --- .github/workflows/market-boundaries.yml | 66 ++++++++++ docs/MARKET_BOUNDARY_PROGRESS.md | 57 +++++++++ tests/test_market_boundaries.py | 159 ++++++++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 .github/workflows/market-boundaries.yml create mode 100644 docs/MARKET_BOUNDARY_PROGRESS.md create mode 100644 tests/test_market_boundaries.py diff --git a/.github/workflows/market-boundaries.yml b/.github/workflows/market-boundaries.yml new file mode 100644 index 00000000..572838b3 --- /dev/null +++ b/.github/workflows/market-boundaries.yml @@ -0,0 +1,66 @@ +name: Market boundary contracts +on: + push: + branches: [fix/market-boundaries-20260929] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: market-boundaries-${{ github.ref }} + cancel-in-progress: false +jobs: + contracts: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest" + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install isolated test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Record exact source + env: + SOURCE_SHA: ${{ github.sha }} + run: | + mkdir -p artifacts + test "$(git rev-parse HEAD)" = "$SOURCE_SHA" + python - <<'PY' + import json,os,subprocess + from pathlib import Path + git=lambda *args:subprocess.check_output(['git',*args],text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({'commit':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'),'event_sha':os.environ['SOURCE_SHA'], + 'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') + PY + - name: Run ownership and actual ledger contracts + run: python -m pytest tests/test_market_boundaries.py --junitxml=artifacts/market-boundaries.xml + - name: Preserve all published checkout and recovery semantics + run: | + python -m pytest tests/test_orders.py tests/test_managed_checkout.py tests/test_checkout_decision_integrity.py tests/test_market_contracts.py tests/test_market_integration.py tests/test_market_recovery.py tests/test_market_account_scope.py tests/test_market_71.py tests/test_market_72.py tests/test_money_core.py tests/test_market_redemption.py tests/test_bounty.py tests/test_store_foundation.py --junitxml=artifacts/market-regressions.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: market-boundaries-${{ github.sha }} + path: artifacts/ + if-no-files-found: error diff --git a/docs/MARKET_BOUNDARY_PROGRESS.md b/docs/MARKET_BOUNDARY_PROGRESS.md new file mode 100644 index 00000000..c67cd19d --- /dev/null +++ b/docs/MARKET_BOUNDARY_PROGRESS.md @@ -0,0 +1,57 @@ +# Market boundary progress + +## Scope and coordination + +Issue #159; references #85/#83, not completion of #71/#72 or production acceptance. +Base: `476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7`, tree +`40ca207a3c49be347120e058e47ed328ec4e3098`. Branch: +`fix/market-boundaries-20260929`. Coordination is #83 comment5878359290. + +This work does not modify #165/#167 core/tool/executor ownership, #155 recovery, +or #166 metadata session extraction. Main and all active claims were refreshed +before starting. No production, real funds, Root/PIN or outward delivery action. + +## Regression first + +The first commit changes no production implementation. It adds ten tests: +seven ownership/import/unique-guard checks and three real PostgreSQL behavior +contracts. Current expected failures concern the new module boundaries; expected +passing characterizations cover buyer/seller privacy, forged escrow authority +with no ledger changes, receipt verification/byte preservation and replay. +No test result is claimed until the exact cloud run is inspected. + +## Observed dependency graph + +`market.orders/email/arbitration/delivery -> plugins.orders` for order records, +subject checks and existing projections. `market -> plugins.store` for listing +and package reads. `market.escrow/orders -> plugins.money` for protected posting. +`market.policy/targets/notifications/escrow/delivery -> plugins.delivery` for +managed delivery records and verification. `plugins.orders/delivery -> market.escrow` +closes the reverse dependency. Function-local imports are included in this audit. + +## Intended ownership and invariants + +- `market.order_records`: signed subject/viewer checks, order IDs, authorized + record lookup and explicitly legacy-compatible projections. +- `market.catalog`: existing authorized listing/body/package reads, not plugin + registration or version-specific catalog schema/defaults. +- `market.ledger`: existing policy constants, account requirements and one posting + implementation. The in-process escrow guard has one owner; moving it does not + make it a credential or a network operation. Local-only Root issuance is unchanged. +- `market.managed_delivery`: existing bounded managed-package verification, + local preparation and read helpers. Their published version semantics remain; + they do not replace v3 service/manual delivery or arbitration state machines. + +Old plugin imports re-export these same functions, never wrappers with new policy +or copied implementations. Version-specific schema, request signing and response +adaptation stay at their existing entrypoints. Existing transactions, account +locks, receipt purposes, ID generation and recovery/lease fences remain intact. +No new Manager, Factory, Repository, ORM or alternate money/escrow path is added. + +## Verification and integration + +Focused cloud Actions run the new contracts and existing checkout v1/v2/v3, +account-scope, deterministic clearing, Bounty, redemption, storage and recovery +suites. Full four-shard exact node-ID/conformance/build and applicable recovery +checks remain required on the final synchronized head. No skips, weakened +assertions, changed published versions or historical fact/schema migration. diff --git a/tests/test_market_boundaries.py b/tests/test_market_boundaries.py new file mode 100644 index 00000000..0955e8ed --- /dev/null +++ b/tests/test_market_boundaries.py @@ -0,0 +1,159 @@ +"""Cross-version market ownership and unchanged real-ledger behavior.""" +from __future__ import annotations + +import ast +import importlib +from pathlib import Path +import subprocess +import sys + +import pytest + +from msg.core.codec import canonical, decode, wire +from msg.core.errors import Failure +from msg.core.models import Signature +from msg.security.crypto import verify +from test_managed_checkout import setup_sale, buy, ok + + +ENTRYPOINTS = {'msg.plugins.orders','msg.plugins.store','msg.plugins.money','msg.plugins.delivery'} +OWNERS = ( + ('msg.plugins.orders','msg.market.order_records',{ + '_subject':'require_signed_subject','_viewer':'require_order_viewer', + '_order_id':'new_order_id','_row':'read_order','_view':'legacy_order_view'}), + ('msg.plugins.store','msg.market.catalog',{ + '_listing':'read_listing','_body':'read_listing_body','_package_row':'read_package_record'}), + ('msg.plugins.money','msg.market.ledger',{ + 'account_requirements':'account_requirements','_owner':'require_money_subject', + '_amount':'checked_amount','_balance':'balance','_supply':'total_supply', + 'clearing_decision':'clearing_decision','_post_entry':'append_entry','_post_transfer':'post_transfer'}), + ('msg.plugins.delivery','msg.market.managed_delivery',{ + '_delivery':'read_delivery','_buyer_order':'read_buyer_order', + '_verified_delivery':'verify_managed_delivery','_package':'read_managed_package', + '_payload':'read_managed_payloads','prepare_managed':'prepare_managed', + 'delivery_summary':'delivery_summary'}), +) + + +def test_market_has_no_business_entrypoint_imports_even_inside_functions(): + import msg.market + violations=[] + for path in Path(msg.market.__file__).parent.rglob('*.py'): + for node in ast.walk(ast.parse(path.read_text())): + names=([node.module] if isinstance(node,ast.ImportFrom) else + [item.name for item in node.names] if isinstance(node,ast.Import) else []) + for name in names: + if name in ENTRYPOINTS: + violations.append((path.name,node.lineno,name)) + assert not violations, violations + + +@pytest.mark.parametrize('entrypoint,owner,names',OWNERS) +def test_compatibility_imports_are_the_same_owned_functions(entrypoint,owner,names): + adapter=importlib.import_module(entrypoint) + shared=importlib.import_module(owner) + assert not hasattr(shared,'install') + for old,new in names.items(): + function=getattr(shared,new) + assert function is getattr(adapter,old), (old,new) + assert function.__module__==owner + records=importlib.import_module('msg.market.order_records') + store=importlib.import_module('msg.plugins.store') + assert store._subject is records.require_signed_subject + + +def test_shared_market_and_versioned_use_cases_import_without_loading_entrypoints(): + code=''' +import importlib, importlib.abc, sys +forbidden={'msg.plugins.orders','msg.plugins.store','msg.plugins.money','msg.plugins.delivery'} +class NoEntrypoints(importlib.abc.MetaPathFinder): + def find_spec(self,fullname,path=None,target=None): + if fullname in forbidden: + raise AssertionError('shared market loaded business entrypoint '+fullname) +sys.meta_path.insert(0,NoEntrypoints()) +for name in ('order_records','catalog','ledger','managed_delivery','orders','delivery', + 'escrow','arbitration','email','policy','rationale','targets','delivery_notifications'): + importlib.import_module('msg.market.'+name) +assert not forbidden.intersection(sys.modules) +''' + result=subprocess.run([sys.executable,'-c',code],capture_output=True,text=True) + assert result.returncode==0,result.stderr + + +def test_ledger_guard_has_one_owner_and_does_not_depend_on_escrow_or_handlers(): + ledger=importlib.import_module('msg.market.ledger') + escrow=importlib.import_module('msg.market.escrow') + assert ledger._ESCROW_WRITE is escrow._ESCROW_WRITE + imports=[n.module for n in ast.walk(ast.parse(Path(ledger.__file__).read_text())) + if isinstance(n,ast.ImportFrom)] + assert 'msg.market.escrow' not in imports + assert not ENTRYPOINTS.intersection(imports) + money=importlib.import_module('msg.plugins.money') + for name in ('CURRENCY_ID','CODE','SCALE','MAX_MINOR','POLICY_VERSION','POLICY_DIGEST'): + assert getattr(money,name)==getattr(ledger,name) + + +async def test_shared_order_views_preserve_buyer_privacy_and_zero_write_semantics(installed): + from msg.plugins.orders import _row,_view + app,seller,buyer,_,args=await setup_sale(installed,policy='escrow-v1') + args['auto_accept']=False + order=ok(await buy(app,buyer,args,rid='shared-order-views')).data['order'] + async with app.metadata.transaction(write=False) as tx: + counts=lambda: tuple(tx.one(f'SELECT COUNT(*) FROM {name}')[0] + for name in ('money_ledger','store_orders','store_deliveries','results','events','jobs')) + before=counts() + record=_row(tx,order['id'],buyer[1]) + buyer_view=_view(record,buyer[1]) + seller_view=_view(_row(tx,order['id'],seller[1]),seller[1]) + assert buyer_view['delivery_target']=={'subject_id':buyer[1],'channel':'site'} + assert {'delivery_target','payment_intent_digest','receipt_refs'}<=set(buyer_view) + assert not {'delivery_target','payment_intent_digest','receipt_refs'}.intersection(seller_view) + assert buyer_view['state']==seller_view['state']=='delivered' + with pytest.raises(Failure,match='order_not_found'): + _row(tx,order['id'],'unrelated-viewer') + with pytest.raises(Failure,match='order_not_found'): + _row(tx,'ord_does_not_exist',buyer[1]) + assert counts()==before + + +async def test_shared_posting_path_rejects_forged_escrow_authority_without_writes(installed): + from msg.plugins.money import _balance,_post_transfer + from msg.plugins.orders import _row + app,seller,buyer,_,args=await setup_sale(installed,policy='escrow-v1') + args['auto_accept']=False + order=ok(await buy(app,buyer,args,rid='shared-posting-guard')).data['order'] + async with app.metadata.transaction(write=True) as tx: + record=_row(tx,order['id'],buyer[1]) + before=tuple(tx.one(f'SELECT COUNT(*) FROM {name}')[0] + for name in ('money_ledger','money_accounts','order_escrow_decisions','order_settlements')) + for authority in (None,object()): + with pytest.raises(Failure,match='escrow_release_forbidden'): + _post_transfer(tx,sender=record['escrow_subject'],recipient=seller[1], + amount=record['total_price_minor'],actor=buyer[1], + request_id='forged-shared-leg',now=app.clock(),receipt_signer=app.receipt_signer, + escrow_authority=authority) + after=tuple(tx.one(f'SELECT COUNT(*) FROM {name}')[0] + for name in ('money_ledger','money_accounts','order_escrow_decisions','order_settlements')) + assert before==after + assert _balance(tx,record['escrow_subject'])==record['total_price_minor'] + assert _balance(tx,seller[1])==0 + + +async def test_alias_reads_and_replay_preserve_signed_receipt_bytes(installed): + from msg.plugins.delivery import _delivery,_verified_delivery + from msg.plugins.orders import _row + app,_,buyer,_,args=await setup_sale(installed) + first=ok(await buy(app,buyer,args,rid='shared-receipt-bytes')) + payment=first.data['payment'] + verify(app.receipt_signer.public_key,canonical(payment['body']), + decode(Signature,payment['signature']),purpose='money-receipt') + async with app.metadata.transaction(write=False) as tx: + before=tuple(tx.rows('SELECT seq,receipt FROM money_ledger ORDER BY seq')) + record=_row(tx,first.data['order']['id'],buyer[1]) + await _verified_delivery(app,tx,record,_delivery(tx,record['id'])) + replay=ok(await buy(app,buyer,args,rid='shared-receipt-bytes')) + assert replay.replayed and replay.data==first.data + assert canonical(replay.data['payment'])==canonical(payment) + async with app.metadata.transaction(write=False) as tx: + assert tuple(tx.rows('SELECT seq,receipt FROM money_ledger ORDER BY seq'))==before + assert canonical(payment).decode() in [row[1] for row in before] From d0d7f4f61f138b0ced266f2324f4a90c1689b6b5 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:05:41 +0800 Subject: [PATCH 3/8] ci: assemble and audit the exact market ownership extraction --- .github/tools/assemble_market_boundaries.py | 313 ++++++++++++++++++ .../workflows/assemble-market-boundaries.yml | 60 ++++ 2 files changed, 373 insertions(+) create mode 100644 .github/tools/assemble_market_boundaries.py create mode 100644 .github/workflows/assemble-market-boundaries.yml diff --git a/.github/tools/assemble_market_boundaries.py b/.github/tools/assemble_market_boundaries.py new file mode 100644 index 00000000..79fac431 --- /dev/null +++ b/.github/tools/assemble_market_boundaries.py @@ -0,0 +1,313 @@ +"""Exact-base source extraction for PR #168; never imports project modules.""" +from pathlib import Path +import ast +import copy +import difflib +import hashlib +import io +import json +import re +import subprocess +import tokenize + +ROOT = Path.cwd() +BASE = '8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8' +EXPECTED = { + 'orders':'f9abe3da981c548a60d0e1b1630d82e02d0e8491', + 'store':'09a1e14328d512e91913bf00e987cf234816348e', + 'money':'424a03f1e6de882aba425f857ab602df818f90c1', + 'delivery':'4500a65ed55e5ea06426c9ac3011f4317ecdc70b', +} +NAMES = { + 'orders':('order_records',{ + '_subject':'require_signed_subject','_viewer':'require_order_viewer', + '_order_id':'new_order_id','_row':'read_order','_view':'legacy_order_view'}), + 'store':('catalog',{ + '_listing':'read_listing','_body':'read_listing_body','_package_row':'read_package_record'}), + 'money':('ledger',{ + 'account_requirements':'account_requirements','_owner':'require_money_subject', + '_amount':'checked_amount','_balance':'balance','_supply':'total_supply', + 'clearing_decision':'clearing_decision','_post_entry':'append_entry','_post_transfer':'post_transfer'}), + 'delivery':('managed_delivery',{ + '_delivery':'read_delivery','_buyer_order':'read_buyer_order', + '_verified_delivery':'verify_managed_delivery','_package':'read_managed_package', + '_payload':'read_managed_payloads','prepare_managed':'prepare_managed', + 'delivery_summary':'delivery_summary'}), +} +CONSTANTS = {'orders':('_COLUMNS',),'store':(), + 'money':('CURRENCY_ID','CODE','SCALE','MAX_MINOR','POLICY_VERSION','POLICY_DIGEST'), + 'delivery':('MAX_INLINE_BYTES','_COLUMNS')} +HEADERS = { + 'orders':'''"""Shared order identity, authorized records and legacy-compatible projections. + +These functions own record access, not operation registration or settlement. +""" +from __future__ import annotations +import base64 +import os +from msg.core.codec import loads +from msg.core.errors import require + +''', + 'store':'''"""Authorized catalog reads shared by published checkout versions. + +Version-specific schemas, creation and updates stay at catalog entrypoints. +""" +from __future__ import annotations +from msg.core.codec import loads +from msg.core.errors import Failure, require +from msg.core.models import ResourceRef +from msg.plugins.common import check_access, resolve +from msg.market.order_records import require_signed_subject + +''', + 'money':'''"""One append-only ledger posting path in the caller's transaction. + +This module grants no commit rights. Public transfers cannot supply the internal +escrow guard; Root issuance remains a separate local-console use case. +""" +from __future__ import annotations +from uuid import uuid4 +from msg.constants import ROOT_SUBJECT +from msg.core.codec import canonical, digest, wire +from msg.core.errors import require +from msg.core.models import AccessRequirement, SignatureProof + +''', + 'delivery':'''"""Shared bounded managed-package verification and local preparation. + +These preserve the managed checkout contracts; versioned manual/service delivery, +arbitration and operation registration retain their existing entrypoints. +""" +from __future__ import annotations +from msg.core.codec import b64, canonical, decode, digest, loads, wire +from msg.core.errors import require +from msg.core.models import BlobRef +from msg.plugins.common import new_id +from msg.market.ledger import CURRENCY_ID, balance +from msg.market.catalog import read_package_record +from msg.market.order_records import read_order +from msg.market.delivery_targets import validate_target + +''', +} +EXTRA_RENAMES = {'store':{'_subject':'require_signed_subject'}, + 'delivery':{'order_row':'read_order','_package_row':'read_package_record','_balance':'balance'}} +before = {} +after = {} +audit = [] + + +def read(path): + text = (ROOT/path).read_text() + before.setdefault(path,text) + return text + + +def nodes(text): + return ast.parse(text).body + + +def functions(text): + return {node.name:node for node in nodes(text) + if isinstance(node,(ast.FunctionDef,ast.AsyncFunctionDef))} + + +def assignments(text): + return {target.id:node for node in nodes(text) if isinstance(node,ast.Assign) + for target in node.targets if isinstance(target,ast.Name)} + + +def segment(text,node): + return ast.get_source_segment(text,node) + + +def rename(text,mapping): + tokens = tokenize.generate_tokens(io.StringIO(text).readline) + return tokenize.untokenize(token._replace(string=mapping.get(token.string,token.string)) + if token.type==tokenize.NAME else token for token in tokens) + + +def splice(text,edits): + starts=[0] + for line in text.splitlines(True): + starts.append(starts[-1]+len(line)) + indexed=[] + for node,value in edits: + indexed.append((starts[node.lineno-1]+node.col_offset, + starts[node.end_lineno-1]+node.end_col_offset,value)) + for start,end,value in sorted(indexed,reverse=True): + text=text[:start]+value+text[end:] + return text + + +def clean(text): + return re.sub(r'\n{4,}','\n\n\n',text).rstrip()+'\n' + + +sources={name:read('src/msg/plugins/'+name+'.py') for name in NAMES} +for name,expected in EXPECTED.items(): + actual=subprocess.check_output(['git','hash-object','src/msg/plugins/'+name+'.py'],text=True).strip() + assert actual==expected,(name,actual,expected) +assert segment(sources['orders'],functions(sources['orders'])['_subject']) == segment( + sources['store'],functions(sources['store'])['_subject']) + +# Move the unique in-process guard with the posting invariant, not with an API version. +escrow_path='src/msg/market/escrow.py' +escrow=read(escrow_path) +assert subprocess.check_output(['git','hash-object',escrow_path],text=True).strip() == '92e0bb595c8fec8665285f0d131630a52500d9fa' +guard_start=escrow.index('# Not a credential, account or public capability.') +guard_end=escrow.index('TRANSITIONS =') +guard=escrow[guard_start:guard_end] +assert '_ESCROW_WRITE = object()' in guard +assert guard.count('object()')==1 +after[escrow_path]=escrow[:guard_start]+'from msg.market.ledger import _ESCROW_WRITE\n\n'+escrow[guard_end:] + +providers={} +for name,(owner,mapping) in NAMES.items(): + exports={old:('msg.market.'+owner,new) for old,new in mapping.items()} + exports.update({constant:('msg.market.'+owner,constant) for constant in CONSTANTS[name]}) + if name=='store': + exports['_subject']=('msg.market.order_records','require_signed_subject') + providers['msg.plugins.'+name]=exports + +for name,(owner,mapping) in NAMES.items(): + text=sources[name] + fs,cs=functions(text),assignments(text) + renames={**mapping,**EXTRA_RENAMES.get(name,{})} + pieces=[HEADERS[name]] + pieces.extend(segment(text,cs[constant])+'\n' for constant in CONSTANTS[name]) + if name=='money': + pieces.append('\n'+guard) + for old,new in mapping.items(): + body=segment(text,fs[old]) + if name=='money' and old=='_post_transfer': + line=' from msg.market.escrow import _ESCROW_WRITE\n' + assert body.count(line)==1 + body=body.replace(line,'') + pieces.append('\n\n'+rename(body,renames)+'\n') + target='src/msg/market/'+owner+'.py' + assert not (ROOT/target).exists(),target + before[target]='' + after[target]=clean(''.join(pieces)) + for old,new in mapping.items(): + original=segment(text,fs[old]) + if name=='money' and old=='_post_transfer': + original=original.replace(' from msg.market.escrow import _ESCROW_WRITE\n','') + expected=ast.dump(ast.parse(rename(original,renames)),include_attributes=False) + actual=ast.dump(ast.parse(segment(after[target],functions(after[target])[new])),include_attributes=False) + assert actual==expected,(name,old) + audit.append(name+'.'+old+' -> '+owner+'.'+new) + edits=[(fs[old],'') for old in mapping]+[(cs[c],'') for c in CONSTANTS[name]] + if name=='store': + edits.append((fs['_subject'],'')) + adapter=splice(text,edits) + # Existing entrypoint code still names the compatibility imports. Each alias + # is the exact owned function, not a wrapper or a second implementation. + imports=['# Compatibility imports; shared implementation has one market owner.'] + for old,(module,new) in providers['msg.plugins.'+name].items(): + imports.append('from '+module+' import '+new+(' as '+old if new!=old else '')) + adapter=adapter.replace('def install(app):','\n'.join(imports)+'\n\n\ndef install(app):',1) + after['src/msg/plugins/'+name+'.py']=clean(adapter) + + +def rewire(text): + edits=[] + for node in ast.walk(ast.parse(text)): + if not isinstance(node,ast.ImportFrom) or node.module not in providers: + continue + groups={} + for item in node.names: + target,new=providers[node.module].get(item.name,(node.module,item.name)) + local=item.asname or item.name + groups.setdefault(target,[]).append(new+(' as '+local if new!=local else '')) + replacement=('\n'+' '*node.col_offset).join( + 'from '+module+' import '+', '.join(names) for module,names in groups.items()) + edits.append((node,replacement)) + return splice(text,edits) + + +# Only import wiring changes in existing market use cases and versioned handlers. +# Source outside these four entrypoints and market remains untouched. +paths={str(p.relative_to(ROOT)) for p in (ROOT/'src/msg/market').rglob('*.py')} +paths.update('src/msg/plugins/'+name+'.py' for name in NAMES) +paths.update(after) +for path in sorted(paths): + text=after[path] if path in after else read(path) + candidate=rewire(text) + if candidate!=before.get(path,''): + after[path]=clean(candidate) + + +class WithoutImports(ast.NodeTransformer): + def visit_ImportFrom(self,node): + return None + def visit_Import(self,node): + return None + + +def stable(node): + return ast.dump(WithoutImports().visit(copy.deepcopy(node)),include_attributes=False) + + +for name in NAMES: + old=functions(sources[name]) + new=functions(after['src/msg/plugins/'+name+'.py']) + removed=set(NAMES[name][1])|({'_subject'} if name=='store' else set()) + assert set(new)==set(old)-removed,(name,set(new),set(old)-removed) + for function in new: + assert stable(old[function])==stable(new[function]),(name,function) + audit.append(name+'.'+function+' body/schema unchanged') +for path,text in after.items(): + if not before.get(path) or not path.startswith('src/msg/market/'): + continue + left=ast.parse(before[path]);right=ast.parse(text) + if path==escrow_path: + left.body=[n for n in left.body if not (isinstance(n,ast.Assign) and + any(isinstance(t,ast.Name) and t.id=='_ESCROW_WRITE' for t in n.targets))] + assert stable(left)==stable(right),path + audit.append(path+' business AST unchanged') +for path,text in after.items(): + if path.startswith('src/msg/market/'): + assert not [n.module for n in ast.walk(ast.parse(text)) + if isinstance(n,ast.ImportFrom) and n.module in providers],path +# All old constants are moved verbatim; the changed authority import is singular. +assert after['src/msg/market/ledger.py'].count('_ESCROW_WRITE = object()')==1 +assert '_ESCROW_WRITE = object()' not in after[escrow_path] + +notes='docs/MARKET_BOUNDARY_PROGRESS.md' +after[notes]=read(notes)+''' + +## Implementation and source audit + +PR #168 extracts the existing shared implementation into `order_records`, +`catalog`, `ledger` and `managed_delivery`, with canonical public function names. +Old plugin names re-export those same function objects. All existing market use +cases, including function-local imports, and legacy plugin handlers use these +owners. The duplicate store subject check becomes the same order-subject check. + +The exact-base cloud assembly checks source blob identities, compares every +moved function after identifier-only renaming, and checks every remaining handler +and market state machine after ignoring import statements. SQL strings, schemas, +IDs, receipt purposes, positional order and existing algorithm bodies are not +rewritten. The only guard change is its owner: `ledger` defines `_ESCROW_WRITE` +once, escrow re-exports it, and protected posting reads that same object directly. +The guard is still supplied only by the existing checked escrow release path. + +No generated source is imported or executed during assembly. Cloud test jobs run +both new real-database contracts and existing published-version/recovery suites. +The assembly artifact contains the exact changed paths, audit and patch. Its +one-shot helper files are removed from the submitted tree before final validation. +No successful final-head result is claimed by this source-only audit. +''' + +out=Path('/tmp/market-boundary-assembly');out.mkdir(parents=True,exist_ok=True) +changed={path:text for path,text in after.items() if text!=before.get(path,'')} +patch=''.join(''.join(difflib.unified_diff(before.get(path,'').splitlines(True),text.splitlines(True), + fromfile='a/'+path if before.get(path) else '/dev/null',tofile='b/'+path)) for path,text in sorted(changed.items())) +for path,text in changed.items(): + (ROOT/path).write_text(text) +(out/'changes.patch').write_text(patch) +(out/'audit.json').write_text(json.dumps({'baseline_commit':BASE,'baseline_blobs':EXPECTED, + 'checked_invariants':audit,'changed_paths':sorted(changed), + 'patch_sha256':hashlib.sha256(patch.encode()).hexdigest()},indent=2)+'\n') diff --git a/.github/workflows/assemble-market-boundaries.yml b/.github/workflows/assemble-market-boundaries.yml new file mode 100644 index 00000000..732be59a --- /dev/null +++ b/.github/workflows/assemble-market-boundaries.yml @@ -0,0 +1,60 @@ +name: Assemble audited market boundary extraction +on: + push: + branches: [build/market-boundaries-168] + paths: + - .github/tools/assemble_market_boundaries.py + - .github/workflows/assemble-market-boundaries.yml +permissions: + contents: write +jobs: + assemble: + if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/build/market-boundaries-168' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Require exact regression source + env: + INPUT_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$INPUT_SHA" + git merge-base --is-ancestor 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 HEAD + git diff --exit-code 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 -- src tests docs + - name: Assemble and audit shared functions and unchanged state machines + run: python .github/tools/assemble_market_boundaries.py + - name: Commit only audited source and notes + run: | + git add src/msg/market src/msg/plugins/orders.py src/msg/plugins/money.py src/msg/plugins/delivery.py src/msg/plugins/store.py docs/MARKET_BOUNDARY_PROGRESS.md + git diff --cached --check + python - <<'PY' + import json,subprocess + from pathlib import Path + expected=set(json.loads(Path('/tmp/market-boundary-assembly/audit.json').read_text())['changed_paths']) + actual=set(subprocess.check_output(['git','diff','--cached','--name-only'],text=True).splitlines()) + assert actual==expected,(actual-expected,expected-actual) + PY + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' commit -m 'refactor(market): own shared records posting and managed delivery independently of entrypoints' + - name: Publish only the dedicated build branch without force + env: + INPUT_SHA: ${{ github.sha }} + run: | + test "$(git ls-remote origin refs/heads/build/market-boundaries-168 | cut -f1)" = "$INPUT_SHA" + git push origin HEAD:refs/heads/build/market-boundaries-168 + python - <<'PY' + import json,os,subprocess + from pathlib import Path + git=lambda *args:subprocess.check_output(['git',*args],text=True).strip() + Path('/tmp/market-boundary-assembly/source.json').write_text(json.dumps({ + 'input':os.environ['INPUT_SHA'],'commit':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') + PY + git archive --format=tar.gz -o /tmp/market-boundary-assembly/source.tar.gz HEAD + - uses: actions/upload-artifact@v4 + if: always() + with: + name: market-boundary-assembly + path: /tmp/market-boundary-assembly/ + if-no-files-found: error From f92390fe8f54fbf63fe21d770f37d6fdbb56190e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:06:54 +0000 Subject: [PATCH 4/8] refactor(market): own shared records posting and managed delivery independently of entrypoints --- docs/MARKET_BOUNDARY_PROGRESS.md | 23 ++++ src/msg/market/arbitration.py | 2 +- src/msg/market/catalog.py | 39 ++++++ src/msg/market/delivery.py | 6 +- src/msg/market/delivery_notifications.py | 4 +- src/msg/market/email.py | 2 +- src/msg/market/escrow.py | 13 +- src/msg/market/ledger.py | 145 +++++++++++++++++++++++ src/msg/market/managed_delivery.py | 137 +++++++++++++++++++++ src/msg/market/order_records.py | 69 +++++++++++ src/msg/market/orders.py | 6 +- src/msg/market/policy.py | 2 +- src/msg/market/rationale.py | 2 +- src/msg/market/targets.py | 4 +- src/msg/plugins/delivery.py | 142 +++------------------- src/msg/plugins/money.py | 145 +++-------------------- src/msg/plugins/orders.py | 72 ++--------- src/msg/plugins/store.py | 43 ++----- 18 files changed, 480 insertions(+), 376 deletions(-) create mode 100644 src/msg/market/catalog.py create mode 100644 src/msg/market/ledger.py create mode 100644 src/msg/market/managed_delivery.py create mode 100644 src/msg/market/order_records.py diff --git a/docs/MARKET_BOUNDARY_PROGRESS.md b/docs/MARKET_BOUNDARY_PROGRESS.md index c67cd19d..6f6e1f0a 100644 --- a/docs/MARKET_BOUNDARY_PROGRESS.md +++ b/docs/MARKET_BOUNDARY_PROGRESS.md @@ -55,3 +55,26 @@ account-scope, deterministic clearing, Bounty, redemption, storage and recovery suites. Full four-shard exact node-ID/conformance/build and applicable recovery checks remain required on the final synchronized head. No skips, weakened assertions, changed published versions or historical fact/schema migration. + + +## Implementation and source audit + +PR #168 extracts the existing shared implementation into `order_records`, +`catalog`, `ledger` and `managed_delivery`, with canonical public function names. +Old plugin names re-export those same function objects. All existing market use +cases, including function-local imports, and legacy plugin handlers use these +owners. The duplicate store subject check becomes the same order-subject check. + +The exact-base cloud assembly checks source blob identities, compares every +moved function after identifier-only renaming, and checks every remaining handler +and market state machine after ignoring import statements. SQL strings, schemas, +IDs, receipt purposes, positional order and existing algorithm bodies are not +rewritten. The only guard change is its owner: `ledger` defines `_ESCROW_WRITE` +once, escrow re-exports it, and protected posting reads that same object directly. +The guard is still supplied only by the existing checked escrow release path. + +No generated source is imported or executed during assembly. Cloud test jobs run +both new real-database contracts and existing published-version/recovery suites. +The assembly artifact contains the exact changed paths, audit and patch. Its +one-shot helper files are removed from the submitted tree before final validation. +No successful final-head result is claimed by this source-only audit. diff --git a/src/msg/market/arbitration.py b/src/msg/market/arbitration.py index 03e9d3a3..ba05d0c0 100644 --- a/src/msg/market/arbitration.py +++ b/src/msg/market/arbitration.py @@ -15,7 +15,7 @@ from msg.market.rationale import PINNED_REF from msg.market.rationale import verified as verify_rationale from msg.plugins.common import check_access, new_id, no_requirements -from msg.plugins.orders import _row, _subject +from msg.market.order_records import read_order as _row, require_signed_subject as _subject from msg.plugins.schemas import IDENTIFIER, REF, SIGNATURE, obj from msg.security.crypto import verify diff --git a/src/msg/market/catalog.py b/src/msg/market/catalog.py new file mode 100644 index 00000000..78fa2cb4 --- /dev/null +++ b/src/msg/market/catalog.py @@ -0,0 +1,39 @@ +"""Authorized catalog reads shared by published checkout versions. + +Version-specific schemas, creation and updates stay at catalog entrypoints. +""" +from __future__ import annotations +from msg.core.codec import loads +from msg.core.errors import Failure, require +from msg.core.models import ResourceRef +from msg.plugins.common import check_access, resolve +from msg.market.order_records import require_signed_subject + + +async def read_listing(app, ctx, request, tx, value, *, seller=False): + try: + rid = await resolve(tx, value) + resource = await tx.resource(rid) + require(resource.type == 'listing' and resource.parent == 't_store', 'listing_not_found') + await check_access(app, ctx, request, tx, rid, 'read') + if seller: + require(resource.owner == require_signed_subject(ctx), 'listing_not_found') + return resource + except Failure as exc: + if exc.code in {'not_found', 'permission_denied', 'credential_ceiling', + 'certificate_gate', 'ancestor_inactive'}: + raise Failure('listing_not_found') from None + raise + + +async def read_listing_body(app, tx, resource, revision=None): + rev = await tx.revision(ResourceRef(id=resource.id, revision=revision)) + body = loads(await app.contents.read_bytes(rev.content)) + # Listings deposited before the sale/bounty split were all sale listings. + return {'mode': 'sale', **body}, rev + + +async def read_package_record(tx, package_id): + return tx.one('''SELECT id,listing_id,listing_revision,seller,revision,kind,manifest, + payload_refs,digest,total_size,delivery_mode,deposited_at FROM store_packages WHERE id=?''', + (package_id,)) diff --git a/src/msg/market/delivery.py b/src/msg/market/delivery.py index 940468c9..2c916ad7 100644 --- a/src/msg/market/delivery.py +++ b/src/msg/market/delivery.py @@ -11,10 +11,10 @@ from msg.market.policy import contract from msg.market.targets import enqueue_notification, validate_target from msg.plugins.common import check_access, new_id -from msg.plugins.delivery import _buyer_order, _delivery -from msg.plugins.orders import _row, _subject, _viewer +from msg.market.managed_delivery import read_buyer_order as _buyer_order, read_delivery as _delivery +from msg.market.order_records import read_order as _row, require_signed_subject as _subject, require_order_viewer as _viewer from msg.plugins.schemas import IDENTIFIER, REF, obj -from msg.plugins.store import _package_row +from msg.market.catalog import read_package_record as _package_row INLINE = 65536 PART = 65536 diff --git a/src/msg/market/delivery_notifications.py b/src/msg/market/delivery_notifications.py index 9a2b4f0c..5b2d0726 100644 --- a/src/msg/market/delivery_notifications.py +++ b/src/msg/market/delivery_notifications.py @@ -85,8 +85,8 @@ async def queue_notification(app, ctx, request, tx, order, *, enabled): async def project_notification(app, tx, job, principal): """Reconstruct the allowlisted message only after every live owner check.""" - from msg.plugins.orders import _row - from msg.plugins.delivery import _delivery, _verified_delivery + from msg.market.order_records import read_order as _row + from msg.market.managed_delivery import read_delivery as _delivery, verify_managed_delivery as _verified_delivery require(mail_enabled(app), 'mail_disabled') require((job.operation, job.arguments.get('contract_version')) in { ('orders.buy', 2), ('delivery.notify', 1)}, 'invalid_delivery_notification') diff --git a/src/msg/market/email.py b/src/msg/market/email.py index b1000832..0902b251 100644 --- a/src/msg/market/email.py +++ b/src/msg/market/email.py @@ -15,7 +15,7 @@ from msg.core.models import EffectJob, HandlerOutput from msg.market.targets import enqueue_notification, mail_enabled, save_target, validate_target from msg.plugins.common import new_id -from msg.plugins.orders import _row, _subject +from msg.market.order_records import read_order as _row, require_signed_subject as _subject from msg.plugins.schemas import IDENTIFIER, obj diff --git a/src/msg/market/escrow.py b/src/msg/market/escrow.py index 92e0bb59..39c4237d 100644 --- a/src/msg/market/escrow.py +++ b/src/msg/market/escrow.py @@ -9,12 +9,11 @@ from msg.core.requests import signing_bytes from msg.market.policy import contract, delivery_snapshot from msg.plugins.common import new_id -from msg.plugins.money import CURRENCY_ID, _balance, _post_transfer +from msg.market.ledger import CURRENCY_ID, balance as _balance, post_transfer as _post_transfer from msg.security.crypto import verify -# Not a credential, account or public capability. Only trusted in-process market -# code can reach this token; ordinary signed money.transfer never receives it. -_ESCROW_WRITE = object() +from msg.market.ledger import _ESCROW_WRITE + TRANSITIONS = { 'created': {'funded', 'cancelled'}, 'funded': {'delivered', 'refunded', 'disputed'}, @@ -132,7 +131,7 @@ async def resolve_due(app, *, limit=100): services/manual ciphertext refund; subjective complaints and expired panels remain held. This function has no external side effect. """ - from msg.plugins.orders import _row + from msg.market.order_records import read_order as _row now = app.clock() changed = [] async with app.metadata.transaction(write=True) as tx: @@ -200,8 +199,8 @@ def __init__(self, app): async def settle(self, ctx, request, tx, *, reason, order_id=None): # Import handlers' read-only projections, never their write entry points. - from msg.plugins.delivery import _delivery, _verified_delivery - from msg.plugins.orders import _row, _subject + from msg.market.managed_delivery import read_delivery as _delivery, verify_managed_delivery as _verified_delivery + from msg.market.order_records import read_order as _row, require_signed_subject as _subject buyer=_subject(ctx) require(order_id is None or reason == 'checkout_accept', 'escrow_decision_mismatch') order=_row(tx,order_id or request.arguments['order_id'],buyer) diff --git a/src/msg/market/ledger.py b/src/msg/market/ledger.py new file mode 100644 index 00000000..93b0901c --- /dev/null +++ b/src/msg/market/ledger.py @@ -0,0 +1,145 @@ +"""One append-only ledger posting path in the caller's transaction. + +This module grants no commit rights. Public transfers cannot supply the internal +escrow guard; Root issuance remains a separate local-console use case. +""" +from __future__ import annotations +from uuid import uuid4 +from msg.constants import ROOT_SUBJECT +from msg.core.codec import canonical, digest, wire +from msg.core.errors import require +from msg.core.models import AccessRequirement, SignatureProof + +CURRENCY_ID = 'primary' +CODE = 'MSG' +SCALE = 6 +MAX_MINOR = 2**63 - 1 +POLICY_VERSION = 2 +POLICY_DIGEST = digest({'version': POLICY_VERSION, 'currency_id': CURRENCY_ID, + 'scale': SCALE, 'transfer_fee': 0, 'allow_overdraft': False, + 'posting_identity': 'actor/request_id/entry_key'}) + +# Not a credential, account or public capability. Only trusted in-process market +# code can reach this token; ordinary signed money.transfer never receives it. +_ESCROW_WRITE = object() + + +async def account_requirements(request, tx): + """Require the credential's scope to cover its account, even on replay. + + An operation name in a listing-scoped key is not permission to spend the + owner's balance or read unrelated private orders. Anonymous calls still + reach the handler's uniform not-found/signature response. + """ + if request.proof is None: + return () + key_id = (request.proof.signature.key_id if isinstance(request.proof, SignatureProof) + else request.proof.credential_id) + credential = await tx.credential(key_id) + return (AccessRequirement(resource_id=request.subject or credential.subject_id, + operation=f'{request.operation}@{request.contract_version}', check='read'),) + + +def require_money_subject(ctx): + require(ctx.principal.subject is not None and + ctx.principal.actor == ctx.principal.subject, 'money_subject_required') + return ctx.principal.subject + + +def checked_amount(value): + require(type(value) is int and 0 < value <= MAX_MINOR, 'invalid_money_amount') + return value + + +def balance(tx, subject): + debit = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?", + (subject,))[0] + credit = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE credit_account=?", + (subject,))[0] + return int(credit) - int(debit) + + +def total_supply(tx): + issued = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE kind='mint'")[0] + burned = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE kind='burn'")[0] + return int(issued) - int(burned) + + +def clearing_decision(*, kind, sender, recipient, amount, sender_balance, + recipient_balance, supply, local_issuer=False): + """Pure policy v2: no clock, UUID, subject privileges or discretionary input.""" + amount = checked_amount(amount) + require(all(type(value) is int and 0 <= value <= MAX_MINOR + for value in (sender_balance, recipient_balance, supply)), 'invalid_money_state') + require(kind in {'mint', 'burn', 'transfer', 'redeem', 'refund'}, 'invalid_money_kind') + if kind in {'mint', 'burn'}: + require(local_issuer, 'root_local_only') + require((kind == 'mint' and sender is None and recipient == ROOT_SUBJECT) or + (kind == 'burn' and sender == ROOT_SUBJECT and recipient is None), + 'invalid_money_accounts') + else: + require(sender is not None and recipient is not None, 'invalid_money_accounts') + require(sender != recipient, 'self_transfer_forbidden') + if sender in {ROOT_SUBJECT, '@root'}: + require(local_issuer, 'root_local_only') + if sender is not None: + require(sender_balance >= amount, 'insufficient_funds') + if recipient is not None: + require(recipient_balance + amount <= MAX_MINOR, 'money_overflow') + if kind == 'burn': + require(supply >= amount, 'invalid_money_state') + if kind == 'mint': + require(supply + amount <= MAX_MINOR, 'money_overflow') + return {'kind': kind, 'debit_account': sender, 'credit_account': recipient, + 'amount_minor': amount, 'currency_id': CURRENCY_ID, + 'policy_version': POLICY_VERSION, 'policy_digest': POLICY_DIGEST} + + +def append_entry(tx, *, sender, recipient, amount, actor, request_id, now, + receipt_signer, reference=None, kind='transfer', entry_key='primary', + _local_issuer=False): + """Append one named leg in the caller's serialized transaction, never commit. + + Named legs permit atomic fund+release without manufacturing request IDs. + Historical receipts remain byte-for-byte untouched by the schema upgrade. + Only the separate local RootAdmin use case supplies _local_issuer. + """ + require(not _local_issuer or actor == ROOT_SUBJECT, 'root_local_only') + clearing_decision(kind=kind, sender=sender, recipient=recipient, amount=amount, + sender_balance=balance(tx, sender) if sender else 0, + recipient_balance=balance(tx, recipient) if recipient else 0, + supply=total_supply(tx), local_issuer=_local_issuer) + for account in {sender, recipient} - {None}: + tx.execute('INSERT INTO money_accounts(subject_id,currency_id) VALUES (?,?) ' + 'ON CONFLICT(subject_id,currency_id) DO NOTHING', + (account, CURRENCY_ID), write=True) + transaction_id = 'lt_' + uuid4().hex + sequence = tx.one("SELECT nextval(pg_get_serial_sequence('money_ledger','seq'))")[0] + body = {'transaction_id': transaction_id, 'kind': kind, 'currency_id': CURRENCY_ID, + 'amount_minor': amount, 'from_subject': sender, 'to_subject': recipient, + 'actor': actor, 'request_id': request_id, 'entry_key': entry_key, + 'reference': reference, 'committed_at': wire(now), + 'policy_version': POLICY_VERSION, 'policy_digest': POLICY_DIGEST, + 'ledger_sequence': sequence} + receipt = {'body': body, + 'signature': wire(receipt_signer.sign(canonical(body), purpose='money-receipt'))} + tx.execute('''INSERT INTO money_ledger + (seq,id,kind,currency_id,amount_minor,debit_account,credit_account,actor,request_id, + reference,committed_at,policy_version,policy_digest,receipt,entry_key) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', + (sequence,transaction_id,kind,CURRENCY_ID,amount,sender,recipient,actor, + request_id,reference,wire(now),POLICY_VERSION,POLICY_DIGEST, + canonical(receipt).decode(),entry_key), write=True) + return receipt + + +def post_transfer(tx, *, sender, recipient, amount, actor, request_id, now, + receipt_signer, reference=None, kind='transfer', entry_key='primary', + escrow_authority=None): + require(kind in {'transfer', 'redeem', 'refund'}, 'invalid_money_kind') + account = tx.one('SELECT kind FROM ledger_accounts WHERE id=?', (sender,)) + if account and account[0] == 'order_escrow': + require(escrow_authority is _ESCROW_WRITE, 'escrow_release_forbidden') + return append_entry(tx, sender=sender, recipient=recipient, amount=amount, + actor=actor, request_id=request_id, now=now, receipt_signer=receipt_signer, + reference=reference, kind=kind, entry_key=entry_key) diff --git a/src/msg/market/managed_delivery.py b/src/msg/market/managed_delivery.py new file mode 100644 index 00000000..e04b8c9e --- /dev/null +++ b/src/msg/market/managed_delivery.py @@ -0,0 +1,137 @@ +"""Shared bounded managed-package verification and local preparation. + +These preserve the managed checkout contracts; versioned manual/service delivery, +arbitration and operation registration retain their existing entrypoints. +""" +from __future__ import annotations +from msg.core.codec import b64, canonical, decode, digest, loads, wire +from msg.core.errors import require +from msg.core.models import BlobRef +from msg.plugins.common import new_id +from msg.market.ledger import CURRENCY_ID, balance +from msg.market.catalog import read_package_record +from msg.market.order_records import read_order +from msg.market.delivery_targets import validate_target + +MAX_INLINE_BYTES = 1024 * 1024 +_COLUMNS = ('id', 'order_id', 'recipient_subject', 'kind', 'payload_refs', + 'manifest', 'package_digest', 'delivery_digest', 'channel', + 'state', 'prepared_at', 'claimed_at', 'receipt') + + +def read_delivery(tx, order_id): + row = tx.one('''SELECT id,order_id,recipient_subject,kind,payload_refs, + manifest,package_digest,delivery_digest,channel,state,prepared_at, + claimed_at,receipt FROM store_deliveries WHERE order_id=?''', (order_id,)) + if row is None: + return None + result = dict(zip(_COLUMNS, row)) + result['payload_refs'] = loads(result['payload_refs']) + result['manifest'] = loads(result['manifest']) + result['receipt'] = loads(result['receipt']) if result['receipt'] else None + return result + + +def read_buyer_order(tx, order_id, buyer): + row = read_order(tx, order_id, buyer) + require(row['buyer'] == buyer, 'order_not_found') + return row + + +async def verify_managed_delivery(app, tx, order, delivery): + validate_target(order, delivery) + require(delivery is not None and + delivery['recipient_subject'] == order['buyer'] and + delivery['package_digest'] == order['package_digest'] and + delivery['channel'] == 'site', 'delivery_mismatch') + kind, manifest, refs = await read_managed_package(app, tx, order) + body = {'delivery_id': delivery['id'], 'order_id': order['id'], + 'recipient_subject': order['buyer'], 'kind': kind, + 'manifest': manifest, 'payload_refs': refs, + 'package_digest': order['package_digest'], 'channel': 'site'} + require(delivery['kind'] == kind and delivery['manifest'] == manifest and + delivery['payload_refs'] == refs and + delivery['delivery_digest'] == digest(body), 'delivery_mismatch') + + +async def read_managed_package(app, tx, order): + require(order['package_id'] and order['package_revision'] and + order['package_digest'], 'delivery_package_mismatch') + row = await read_package_record(tx, order['package_id']) + require(row is not None and row[1] == order['listing_id'] and + row[3] == order['seller'] and row[4] == order['package_revision'] and + row[8] == order['package_digest'] and row[10] == 'managed_instant' and + row[5] in {'file', 'bundle'}, 'delivery_package_mismatch') + manifest, refs = loads(row[6]), loads(row[7]) + snapshot = {'seller': row[3], 'listing_id': row[1], + 'listing_revision': row[2], 'revision': row[4], 'kind': row[5], + 'manifest': manifest, 'payload_refs': refs, + 'total_size': row[9], 'delivery_mode': row[10]} + require(digest(snapshot) == order['package_digest'] and + row[9] <= MAX_INLINE_BYTES and row[9] >= 0, + 'delivery_package_mismatch') + require(refs and (row[5] == 'bundle' or len(refs) == 1), + 'delivery_package_mismatch') + total = 0 + for ref in refs: + require(isinstance(ref, dict) and isinstance(ref.get('blob'), dict), + 'delivery_package_mismatch') + blob = decode(BlobRef, ref['blob']) + require(blob.size >= 0 and blob.size <= MAX_INLINE_BYTES, + 'delivery_package_mismatch') + data = await app.contents.read_bytes(blob, limit=MAX_INLINE_BYTES) + require(len(data) == blob.size and digest(data) == blob.digest, + 'delivery_package_mismatch') + total += blob.size + require(total == row[9], 'delivery_package_mismatch') + return row[5], manifest, refs + + +async def read_managed_payloads(app, delivery): + files = [] + for ref in delivery['payload_refs']: + blob = decode(BlobRef, ref['blob']) + data = await app.contents.read_bytes(blob, limit=MAX_INLINE_BYTES) + require(len(data) == blob.size and digest(data) == blob.digest, + 'delivery_package_mismatch') + files.append({'id': ref['id'], 'revision': ref['revision'], + 'media_type': blob.media_type, 'digest': blob.digest, + 'size': blob.size, 'data': b64(data)}) + return files + + +async def prepare_managed(app, ctx, tx, order): + """Prepare inside the caller's transaction; never authenticate or commit here.""" + buyer = order['buyer'] + require(order['state'] == 'funded' and order['delivered_at'] is None, + 'order_not_deliverable') + validate_target(order) + require(order['currency_id'] == CURRENCY_ID and + balance(tx, order['escrow_subject']) == order['total_price_minor'], + 'escrow_balance_mismatch') + kind, manifest, refs = await read_managed_package(app, tx, order) + delivery_id = new_id('dlv') + body = {'delivery_id': delivery_id, 'order_id': order['id'], + 'recipient_subject': buyer, 'kind': kind, + 'manifest': manifest, 'payload_refs': refs, + 'package_digest': order['package_digest'], 'channel': 'site'} + delivery_digest = digest(body) + tx.execute('''INSERT INTO store_deliveries + (id,order_id,recipient_subject,kind,payload_refs,manifest, + package_digest,delivery_digest,channel,state,prepared_at,claimed_at,receipt) + VALUES (?,?,?,?,?,?,?,?,?,'prepared',?,NULL,NULL)''', + (delivery_id, order['id'], buyer, kind, canonical(refs).decode(), + canonical(manifest).decode(), order['package_digest'], + delivery_digest, 'site', wire(ctx.now)), write=True) + changed = tx.execute('''UPDATE store_orders SET state='delivered',delivered_at=? + WHERE id=? AND buyer=? AND state='funded' AND delivered_at IS NULL''', + (wire(ctx.now), order['id'], buyer), write=True) + require(changed.rowcount == 1, 'order_not_deliverable') + return read_delivery(tx, order['id']) + + +def delivery_summary(delivery): + return {'delivery_id': delivery['id'], 'order_id': delivery['order_id'], + 'state': delivery['state'], 'channel': delivery['channel'], + 'package_digest': delivery['package_digest'], + 'delivery_digest': delivery['delivery_digest']} diff --git a/src/msg/market/order_records.py b/src/msg/market/order_records.py new file mode 100644 index 00000000..3f3e3136 --- /dev/null +++ b/src/msg/market/order_records.py @@ -0,0 +1,69 @@ +"""Shared order identity, authorized records and legacy-compatible projections. + +These functions own record access, not operation registration or settlement. +""" +from __future__ import annotations +import base64 +import os +from msg.core.codec import loads +from msg.core.errors import require + +_COLUMNS = ('id', 'buyer', 'seller', 'listing_id', 'listing_revision', + 'package_id', 'package_revision', 'package_digest', 'quantity', + 'unit_price_minor', 'total_price_minor', 'currency_id', + 'escrow_subject', 'escrow_policy', 'dispute_policy', 'terms_digest', + 'delivery_target', 'payment_intent_digest', 'payment_transaction_id', + 'state', 'created_at', 'funded_at', 'delivered_at', 'settled_at', + 'receipt_refs') + + +def require_signed_subject(ctx): + subject = ctx.principal.subject + require(subject is not None and ctx.principal.actor == subject and + ctx.principal.method == 'signature', 'signature_required') + return subject + + +def require_order_viewer(ctx): + subject = ctx.principal.subject + require(subject is not None and ctx.principal.actor == subject, + 'order_not_found') + return subject + + +def new_order_id(): + # 160 random bits, unguessable even if an attacker sees other order IDs. + return 'ord_' + base64.b32encode(os.urandom(20)).decode('ascii').rstrip('=').lower() + + +def read_order(tx, order_id, viewer): + row = tx.one('''SELECT id,buyer,seller,listing_id,listing_revision, + package_id,package_revision,package_digest,quantity,unit_price_minor, + total_price_minor,currency_id,escrow_subject,escrow_policy, + dispute_policy,terms_digest,delivery_target,payment_intent_digest, + payment_transaction_id,state,created_at,funded_at,delivered_at, + settled_at,receipt_refs FROM store_orders WHERE id=?''', (order_id,)) + # A valid ID is not an access grant. Keep nonexistent and unauthorized alike. + require(row is not None and viewer in row[1:3], 'order_not_found') + result = dict(zip(_COLUMNS, row)) + result['delivery_target'] = loads(result['delivery_target']) + result['receipt_refs'] = loads(result['receipt_refs']) + return result + + +def legacy_order_view(row, viewer): + keys = ('id', 'buyer', 'seller', 'listing_id', 'listing_revision', + 'package_revision', 'package_digest', 'quantity', + 'unit_price_minor', 'total_price_minor', 'currency_id', + 'escrow_policy', 'dispute_policy', 'terms_digest', 'state', + 'created_at', 'funded_at', 'delivered_at', 'settled_at') + result = {key: row[key] for key in keys} + result['payment_status'] = ('refunded' if row['state'] == 'refunded' else + 'funded' if row['payment_transaction_id'] else + 'pending') + result['delivery_channel'] = row['delivery_target']['channel'] + if viewer == row['buyer']: + result['delivery_target'] = row['delivery_target'] + result['payment_intent_digest'] = row['payment_intent_digest'] + result['receipt_refs'] = row['receipt_refs'] + return result diff --git a/src/msg/market/orders.py b/src/msg/market/orders.py index 308c2d9f..d7b7c7c7 100644 --- a/src/msg/market/orders.py +++ b/src/msg/market/orders.py @@ -11,10 +11,10 @@ from msg.market.targets import ( enqueue_notification, notification_view, save_target, target_for, validate_target, ) -from msg.plugins.money import CURRENCY_ID, MAX_MINOR, _post_transfer -from msg.plugins.orders import _order_id, _row, _subject, _view, _viewer +from msg.market.ledger import CURRENCY_ID, MAX_MINOR, post_transfer as _post_transfer +from msg.market.order_records import new_order_id as _order_id, read_order as _row, require_signed_subject as _subject, legacy_order_view as _view, require_order_viewer as _viewer from msg.plugins.schemas import IDENTIFIER, obj -from msg.plugins.store import _body, _listing, _package_row +from msg.market.catalog import read_listing_body as _body, read_listing as _listing, read_package_record as _package_row HASH = {'type': 'string', 'pattern': '^sha256:[a-f0-9]{64}$'} ORDER = obj({'order_id': IDENTIFIER}, ('order_id',)) diff --git a/src/msg/market/policy.py b/src/msg/market/policy.py index 3b414cca..8d7c25e1 100644 --- a/src/msg/market/policy.py +++ b/src/msg/market/policy.py @@ -79,7 +79,7 @@ def contract(tx, order_id): def delivery_snapshot(tx, order_id): """Commit delivery identity and prepared bytes, excluding subsequent buyer ACK.""" - from msg.plugins.delivery import _delivery + from msg.market.managed_delivery import read_delivery as _delivery delivery = _delivery(tx, order_id) return digest(None if delivery is None else { key: value for key, value in delivery.items() diff --git a/src/msg/market/rationale.py b/src/msg/market/rationale.py index 291da1d2..ecd18b4c 100644 --- a/src/msg/market/rationale.py +++ b/src/msg/market/rationale.py @@ -10,7 +10,7 @@ from msg.core.models import BlobRef, HandlerOutput, ResourceRef from msg.market.delivery import verify_blob from msg.plugins.common import check_access -from msg.plugins.orders import _subject +from msg.market.order_records import require_signed_subject as _subject from msg.plugins.schemas import IDENTIFIER, obj PINNED_REF = obj({'id': IDENTIFIER, 'revision': IDENTIFIER}, ('id', 'revision')) diff --git a/src/msg/market/targets.py b/src/msg/market/targets.py index 1833d0aa..a4226be6 100644 --- a/src/msg/market/targets.py +++ b/src/msg/market/targets.py @@ -106,8 +106,8 @@ async def enqueue_notification(app, tx, ctx, request, order): async def render_notification(app, tx, job): - from msg.plugins.delivery import _delivery - from msg.plugins.orders import _row + from msg.market.managed_delivery import read_delivery as _delivery + from msg.market.order_records import read_order as _row order = _row(tx, job.arguments['order_id'], job.principal.subject) require(order['buyer'] == job.principal.subject and job.principal.actor == order['buyer'], 'delivery_recipient_mismatch') diff --git a/src/msg/plugins/delivery.py b/src/msg/plugins/delivery.py index 4500a65e..fd01f408 100644 --- a/src/msg/plugins/delivery.py +++ b/src/msg/plugins/delivery.py @@ -12,136 +12,24 @@ from msg.core.errors import require from msg.core.models import BlobRef, HandlerOutput from msg.plugins.common import new_id, registration -from msg.plugins.money import CURRENCY_ID, _balance, account_requirements +from msg.market.ledger import CURRENCY_ID, balance as _balance, account_requirements from msg.market.escrow import EscrowEngine from msg.market.delivery_targets import validate_target -from msg.plugins.orders import _row as order_row, _subject +from msg.market.order_records import read_order as order_row, require_signed_subject as _subject from msg.plugins.schemas import IDENTIFIER, obj -from msg.plugins.store import _package_row - - -MAX_INLINE_BYTES = 1024 * 1024 -_COLUMNS = ('id', 'order_id', 'recipient_subject', 'kind', 'payload_refs', - 'manifest', 'package_digest', 'delivery_digest', 'channel', - 'state', 'prepared_at', 'claimed_at', 'receipt') - - -def _delivery(tx, order_id): - row = tx.one('''SELECT id,order_id,recipient_subject,kind,payload_refs, - manifest,package_digest,delivery_digest,channel,state,prepared_at, - claimed_at,receipt FROM store_deliveries WHERE order_id=?''', (order_id,)) - if row is None: - return None - result = dict(zip(_COLUMNS, row)) - result['payload_refs'] = loads(result['payload_refs']) - result['manifest'] = loads(result['manifest']) - result['receipt'] = loads(result['receipt']) if result['receipt'] else None - return result - - -def _buyer_order(tx, order_id, buyer): - row = order_row(tx, order_id, buyer) - require(row['buyer'] == buyer, 'order_not_found') - return row - - -async def _verified_delivery(app, tx, order, delivery): - validate_target(order, delivery) - require(delivery is not None and - delivery['recipient_subject'] == order['buyer'] and - delivery['package_digest'] == order['package_digest'] and - delivery['channel'] == 'site', 'delivery_mismatch') - kind, manifest, refs = await _package(app, tx, order) - body = {'delivery_id': delivery['id'], 'order_id': order['id'], - 'recipient_subject': order['buyer'], 'kind': kind, - 'manifest': manifest, 'payload_refs': refs, - 'package_digest': order['package_digest'], 'channel': 'site'} - require(delivery['kind'] == kind and delivery['manifest'] == manifest and - delivery['payload_refs'] == refs and - delivery['delivery_digest'] == digest(body), 'delivery_mismatch') - - -async def _package(app, tx, order): - require(order['package_id'] and order['package_revision'] and - order['package_digest'], 'delivery_package_mismatch') - row = await _package_row(tx, order['package_id']) - require(row is not None and row[1] == order['listing_id'] and - row[3] == order['seller'] and row[4] == order['package_revision'] and - row[8] == order['package_digest'] and row[10] == 'managed_instant' and - row[5] in {'file', 'bundle'}, 'delivery_package_mismatch') - manifest, refs = loads(row[6]), loads(row[7]) - snapshot = {'seller': row[3], 'listing_id': row[1], - 'listing_revision': row[2], 'revision': row[4], 'kind': row[5], - 'manifest': manifest, 'payload_refs': refs, - 'total_size': row[9], 'delivery_mode': row[10]} - require(digest(snapshot) == order['package_digest'] and - row[9] <= MAX_INLINE_BYTES and row[9] >= 0, - 'delivery_package_mismatch') - require(refs and (row[5] == 'bundle' or len(refs) == 1), - 'delivery_package_mismatch') - total = 0 - for ref in refs: - require(isinstance(ref, dict) and isinstance(ref.get('blob'), dict), - 'delivery_package_mismatch') - blob = decode(BlobRef, ref['blob']) - require(blob.size >= 0 and blob.size <= MAX_INLINE_BYTES, - 'delivery_package_mismatch') - data = await app.contents.read_bytes(blob, limit=MAX_INLINE_BYTES) - require(len(data) == blob.size and digest(data) == blob.digest, - 'delivery_package_mismatch') - total += blob.size - require(total == row[9], 'delivery_package_mismatch') - return row[5], manifest, refs - - -async def _payload(app, delivery): - files = [] - for ref in delivery['payload_refs']: - blob = decode(BlobRef, ref['blob']) - data = await app.contents.read_bytes(blob, limit=MAX_INLINE_BYTES) - require(len(data) == blob.size and digest(data) == blob.digest, - 'delivery_package_mismatch') - files.append({'id': ref['id'], 'revision': ref['revision'], - 'media_type': blob.media_type, 'digest': blob.digest, - 'size': blob.size, 'data': b64(data)}) - return files - - -async def prepare_managed(app, ctx, tx, order): - """Prepare inside the caller's transaction; never authenticate or commit here.""" - buyer = order['buyer'] - require(order['state'] == 'funded' and order['delivered_at'] is None, - 'order_not_deliverable') - validate_target(order) - require(order['currency_id'] == CURRENCY_ID and - _balance(tx, order['escrow_subject']) == order['total_price_minor'], - 'escrow_balance_mismatch') - kind, manifest, refs = await _package(app, tx, order) - delivery_id = new_id('dlv') - body = {'delivery_id': delivery_id, 'order_id': order['id'], - 'recipient_subject': buyer, 'kind': kind, - 'manifest': manifest, 'payload_refs': refs, - 'package_digest': order['package_digest'], 'channel': 'site'} - delivery_digest = digest(body) - tx.execute('''INSERT INTO store_deliveries - (id,order_id,recipient_subject,kind,payload_refs,manifest, - package_digest,delivery_digest,channel,state,prepared_at,claimed_at,receipt) - VALUES (?,?,?,?,?,?,?,?,?,'prepared',?,NULL,NULL)''', - (delivery_id, order['id'], buyer, kind, canonical(refs).decode(), - canonical(manifest).decode(), order['package_digest'], - delivery_digest, 'site', wire(ctx.now)), write=True) - changed = tx.execute('''UPDATE store_orders SET state='delivered',delivered_at=? - WHERE id=? AND buyer=? AND state='funded' AND delivered_at IS NULL''', - (wire(ctx.now), order['id'], buyer), write=True) - require(changed.rowcount == 1, 'order_not_deliverable') - return _delivery(tx, order['id']) - - -def delivery_summary(delivery): - return {'delivery_id': delivery['id'], 'order_id': delivery['order_id'], - 'state': delivery['state'], 'channel': delivery['channel'], - 'package_digest': delivery['package_digest'], - 'delivery_digest': delivery['delivery_digest']} +from msg.market.catalog import read_package_record as _package_row + + +# Compatibility imports; shared implementation has one market owner. +from msg.market.managed_delivery import read_delivery as _delivery +from msg.market.managed_delivery import read_buyer_order as _buyer_order +from msg.market.managed_delivery import verify_managed_delivery as _verified_delivery +from msg.market.managed_delivery import read_managed_package as _package +from msg.market.managed_delivery import read_managed_payloads as _payload +from msg.market.managed_delivery import prepare_managed +from msg.market.managed_delivery import delivery_summary +from msg.market.managed_delivery import MAX_INLINE_BYTES +from msg.market.managed_delivery import _COLUMNS def install(app): diff --git a/src/msg/plugins/money.py b/src/msg/plugins/money.py index 424a03f1..89f25df7 100644 --- a/src/msg/plugins/money.py +++ b/src/msg/plugins/money.py @@ -16,136 +16,21 @@ from msg.plugins.schemas import IDENTIFIER, obj -CURRENCY_ID = 'primary' -CODE = 'MSG' -SCALE = 6 -MAX_MINOR = 2**63 - 1 -POLICY_VERSION = 2 -POLICY_DIGEST = digest({'version': POLICY_VERSION, 'currency_id': CURRENCY_ID, - 'scale': SCALE, 'transfer_fee': 0, 'allow_overdraft': False, - 'posting_identity': 'actor/request_id/entry_key'}) - - -async def account_requirements(request, tx): - """Require the credential's scope to cover its account, even on replay. - - An operation name in a listing-scoped key is not permission to spend the - owner's balance or read unrelated private orders. Anonymous calls still - reach the handler's uniform not-found/signature response. - """ - if request.proof is None: - return () - key_id = (request.proof.signature.key_id if isinstance(request.proof, SignatureProof) - else request.proof.credential_id) - credential = await tx.credential(key_id) - return (AccessRequirement(resource_id=request.subject or credential.subject_id, - operation=f'{request.operation}@{request.contract_version}', check='read'),) - - -def _owner(ctx): - require(ctx.principal.subject is not None and - ctx.principal.actor == ctx.principal.subject, 'money_subject_required') - return ctx.principal.subject - - -def _amount(value): - require(type(value) is int and 0 < value <= MAX_MINOR, 'invalid_money_amount') - return value - - -def _balance(tx, subject): - debit = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?", - (subject,))[0] - credit = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE credit_account=?", - (subject,))[0] - return int(credit) - int(debit) - - -def _supply(tx): - issued = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE kind='mint'")[0] - burned = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE kind='burn'")[0] - return int(issued) - int(burned) - - -def clearing_decision(*, kind, sender, recipient, amount, sender_balance, - recipient_balance, supply, local_issuer=False): - """Pure policy v2: no clock, UUID, subject privileges or discretionary input.""" - amount = _amount(amount) - require(all(type(value) is int and 0 <= value <= MAX_MINOR - for value in (sender_balance, recipient_balance, supply)), 'invalid_money_state') - require(kind in {'mint', 'burn', 'transfer', 'redeem', 'refund'}, 'invalid_money_kind') - if kind in {'mint', 'burn'}: - require(local_issuer, 'root_local_only') - require((kind == 'mint' and sender is None and recipient == ROOT_SUBJECT) or - (kind == 'burn' and sender == ROOT_SUBJECT and recipient is None), - 'invalid_money_accounts') - else: - require(sender is not None and recipient is not None, 'invalid_money_accounts') - require(sender != recipient, 'self_transfer_forbidden') - if sender in {ROOT_SUBJECT, '@root'}: - require(local_issuer, 'root_local_only') - if sender is not None: - require(sender_balance >= amount, 'insufficient_funds') - if recipient is not None: - require(recipient_balance + amount <= MAX_MINOR, 'money_overflow') - if kind == 'burn': - require(supply >= amount, 'invalid_money_state') - if kind == 'mint': - require(supply + amount <= MAX_MINOR, 'money_overflow') - return {'kind': kind, 'debit_account': sender, 'credit_account': recipient, - 'amount_minor': amount, 'currency_id': CURRENCY_ID, - 'policy_version': POLICY_VERSION, 'policy_digest': POLICY_DIGEST} - - -def _post_entry(tx, *, sender, recipient, amount, actor, request_id, now, - receipt_signer, reference=None, kind='transfer', entry_key='primary', - _local_issuer=False): - """Append one named leg in the caller's serialized transaction, never commit. - - Named legs permit atomic fund+release without manufacturing request IDs. - Historical receipts remain byte-for-byte untouched by the schema upgrade. - Only the separate local RootAdmin use case supplies _local_issuer. - """ - require(not _local_issuer or actor == ROOT_SUBJECT, 'root_local_only') - clearing_decision(kind=kind, sender=sender, recipient=recipient, amount=amount, - sender_balance=_balance(tx, sender) if sender else 0, - recipient_balance=_balance(tx, recipient) if recipient else 0, - supply=_supply(tx), local_issuer=_local_issuer) - for account in {sender, recipient} - {None}: - tx.execute('INSERT INTO money_accounts(subject_id,currency_id) VALUES (?,?) ' - 'ON CONFLICT(subject_id,currency_id) DO NOTHING', - (account, CURRENCY_ID), write=True) - transaction_id = 'lt_' + uuid4().hex - sequence = tx.one("SELECT nextval(pg_get_serial_sequence('money_ledger','seq'))")[0] - body = {'transaction_id': transaction_id, 'kind': kind, 'currency_id': CURRENCY_ID, - 'amount_minor': amount, 'from_subject': sender, 'to_subject': recipient, - 'actor': actor, 'request_id': request_id, 'entry_key': entry_key, - 'reference': reference, 'committed_at': wire(now), - 'policy_version': POLICY_VERSION, 'policy_digest': POLICY_DIGEST, - 'ledger_sequence': sequence} - receipt = {'body': body, - 'signature': wire(receipt_signer.sign(canonical(body), purpose='money-receipt'))} - tx.execute('''INSERT INTO money_ledger - (seq,id,kind,currency_id,amount_minor,debit_account,credit_account,actor,request_id, - reference,committed_at,policy_version,policy_digest,receipt,entry_key) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', - (sequence,transaction_id,kind,CURRENCY_ID,amount,sender,recipient,actor, - request_id,reference,wire(now),POLICY_VERSION,POLICY_DIGEST, - canonical(receipt).decode(),entry_key), write=True) - return receipt - - -def _post_transfer(tx, *, sender, recipient, amount, actor, request_id, now, - receipt_signer, reference=None, kind='transfer', entry_key='primary', - escrow_authority=None): - require(kind in {'transfer', 'redeem', 'refund'}, 'invalid_money_kind') - account = tx.one('SELECT kind FROM ledger_accounts WHERE id=?', (sender,)) - if account and account[0] == 'order_escrow': - from msg.market.escrow import _ESCROW_WRITE - require(escrow_authority is _ESCROW_WRITE, 'escrow_release_forbidden') - return _post_entry(tx, sender=sender, recipient=recipient, amount=amount, - actor=actor, request_id=request_id, now=now, receipt_signer=receipt_signer, - reference=reference, kind=kind, entry_key=entry_key) +# Compatibility imports; shared implementation has one market owner. +from msg.market.ledger import account_requirements +from msg.market.ledger import require_money_subject as _owner +from msg.market.ledger import checked_amount as _amount +from msg.market.ledger import balance as _balance +from msg.market.ledger import total_supply as _supply +from msg.market.ledger import clearing_decision +from msg.market.ledger import append_entry as _post_entry +from msg.market.ledger import post_transfer as _post_transfer +from msg.market.ledger import CURRENCY_ID +from msg.market.ledger import CODE +from msg.market.ledger import SCALE +from msg.market.ledger import MAX_MINOR +from msg.market.ledger import POLICY_VERSION +from msg.market.ledger import POLICY_DIGEST def install(app): diff --git a/src/msg/plugins/orders.py b/src/msg/plugins/orders.py index f9abe3da..4e70aa16 100644 --- a/src/msg/plugins/orders.py +++ b/src/msg/plugins/orders.py @@ -14,70 +14,18 @@ from msg.core.errors import require from msg.core.models import HandlerOutput from msg.plugins.common import registration -from msg.plugins.money import CURRENCY_ID, MAX_MINOR, _balance, _post_transfer, account_requirements +from msg.market.ledger import CURRENCY_ID, MAX_MINOR, balance as _balance, post_transfer as _post_transfer, account_requirements from msg.plugins.schemas import IDENTIFIER, obj -from msg.plugins.store import _body, _listing, _package_row +from msg.market.catalog import read_listing_body as _body, read_listing as _listing, read_package_record as _package_row -_COLUMNS = ('id', 'buyer', 'seller', 'listing_id', 'listing_revision', - 'package_id', 'package_revision', 'package_digest', 'quantity', - 'unit_price_minor', 'total_price_minor', 'currency_id', - 'escrow_subject', 'escrow_policy', 'dispute_policy', 'terms_digest', - 'delivery_target', 'payment_intent_digest', 'payment_transaction_id', - 'state', 'created_at', 'funded_at', 'delivered_at', 'settled_at', - 'receipt_refs') - - -def _subject(ctx): - subject = ctx.principal.subject - require(subject is not None and ctx.principal.actor == subject and - ctx.principal.method == 'signature', 'signature_required') - return subject - - -def _viewer(ctx): - subject = ctx.principal.subject - require(subject is not None and ctx.principal.actor == subject, - 'order_not_found') - return subject - - -def _order_id(): - # 160 random bits, unguessable even if an attacker sees other order IDs. - return 'ord_' + base64.b32encode(os.urandom(20)).decode('ascii').rstrip('=').lower() - - -def _row(tx, order_id, viewer): - row = tx.one('''SELECT id,buyer,seller,listing_id,listing_revision, - package_id,package_revision,package_digest,quantity,unit_price_minor, - total_price_minor,currency_id,escrow_subject,escrow_policy, - dispute_policy,terms_digest,delivery_target,payment_intent_digest, - payment_transaction_id,state,created_at,funded_at,delivered_at, - settled_at,receipt_refs FROM store_orders WHERE id=?''', (order_id,)) - # A valid ID is not an access grant. Keep nonexistent and unauthorized alike. - require(row is not None and viewer in row[1:3], 'order_not_found') - result = dict(zip(_COLUMNS, row)) - result['delivery_target'] = loads(result['delivery_target']) - result['receipt_refs'] = loads(result['receipt_refs']) - return result - - -def _view(row, viewer): - keys = ('id', 'buyer', 'seller', 'listing_id', 'listing_revision', - 'package_revision', 'package_digest', 'quantity', - 'unit_price_minor', 'total_price_minor', 'currency_id', - 'escrow_policy', 'dispute_policy', 'terms_digest', 'state', - 'created_at', 'funded_at', 'delivered_at', 'settled_at') - result = {key: row[key] for key in keys} - result['payment_status'] = ('refunded' if row['state'] == 'refunded' else - 'funded' if row['payment_transaction_id'] else - 'pending') - result['delivery_channel'] = row['delivery_target']['channel'] - if viewer == row['buyer']: - result['delivery_target'] = row['delivery_target'] - result['payment_intent_digest'] = row['payment_intent_digest'] - result['receipt_refs'] = row['receipt_refs'] - return result +# Compatibility imports; shared implementation has one market owner. +from msg.market.order_records import require_signed_subject as _subject +from msg.market.order_records import require_order_viewer as _viewer +from msg.market.order_records import new_order_id as _order_id +from msg.market.order_records import read_order as _row +from msg.market.order_records import legacy_order_view as _view +from msg.market.order_records import _COLUMNS def install(app): @@ -188,7 +136,7 @@ async def buy(ctx, request, tx): canonical([receipt_id]).decode()), write=True) data = {'payment': receipt} if instant: - from msg.plugins.delivery import prepare_managed, delivery_summary + from msg.market.managed_delivery import prepare_managed, delivery_summary from msg.market.delivery_notifications import initialize_notification order = _row(tx, order_id, buyer) delivery = await prepare_managed(app, ctx, tx, order) diff --git a/src/msg/plugins/store.py b/src/msg/plugins/store.py index 09a1e143..e5da5477 100644 --- a/src/msg/plugins/store.py +++ b/src/msg/plugins/store.py @@ -17,36 +17,6 @@ MAX_MANIFEST_BYTES = 65536 -def _subject(ctx): - subject = ctx.principal.subject - require(subject is not None and ctx.principal.actor == subject and - ctx.principal.method == 'signature', 'signature_required') - return subject - - -async def _listing(app, ctx, request, tx, value, *, seller=False): - try: - rid = await resolve(tx, value) - resource = await tx.resource(rid) - require(resource.type == 'listing' and resource.parent == 't_store', 'listing_not_found') - await check_access(app, ctx, request, tx, rid, 'read') - if seller: - require(resource.owner == _subject(ctx), 'listing_not_found') - return resource - except Failure as exc: - if exc.code in {'not_found', 'permission_denied', 'credential_ceiling', - 'certificate_gate', 'ancestor_inactive'}: - raise Failure('listing_not_found') from None - raise - - -async def _body(app, tx, resource, revision=None): - rev = await tx.revision(ResourceRef(id=resource.id, revision=revision)) - body = loads(await app.contents.read_bytes(rev.content)) - # Listings deposited before the sale/bounty split were all sale listings. - return {'mode': 'sale', **body}, rev - - def _public(resource, body): # A package identifier is not a capability; package payloads are seller-only. return {'listing_id': resource.id, 'listing_revision': resource.revision, @@ -74,12 +44,6 @@ def _validate(body, now): require(body['package_ref'] is not None, 'package_required') -async def _package_row(tx, package_id): - return tx.one('''SELECT id,listing_id,listing_revision,seller,revision,kind,manifest, - payload_refs,digest,total_size,delivery_mode,deposited_at FROM store_packages WHERE id=?''', - (package_id,)) - - def _package_public(row): return dict(zip(('id', 'listing_id', 'listing_revision', 'seller', 'revision', 'kind', 'manifest', 'payload_refs', 'digest', 'total_size', @@ -87,6 +51,13 @@ def _package_public(row): (*row[:6], loads(row[6]), loads(row[7]), *row[8:]))) +# Compatibility imports; shared implementation has one market owner. +from msg.market.catalog import read_listing as _listing +from msg.market.catalog import read_listing_body as _body +from msg.market.catalog import read_package_record as _package_row +from msg.market.order_records import require_signed_subject as _subject + + def install(app): op, finish = registration(app, 'store', ('content',)) listing_fields = { From 65b7330d248f4af96ac9ecd1f1fad3fdc6f59782 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:09:22 +0800 Subject: [PATCH 5/8] ci: remove market assembly tooling and preserve verified red evidence --- .github/tools/assemble_market_boundaries.py | 313 ------------------ .../workflows/assemble-market-boundaries.yml | 60 ---- docs/MARKET_BOUNDARY_PROGRESS.md | 132 ++++---- 3 files changed, 64 insertions(+), 441 deletions(-) delete mode 100644 .github/tools/assemble_market_boundaries.py delete mode 100644 .github/workflows/assemble-market-boundaries.yml diff --git a/.github/tools/assemble_market_boundaries.py b/.github/tools/assemble_market_boundaries.py deleted file mode 100644 index 79fac431..00000000 --- a/.github/tools/assemble_market_boundaries.py +++ /dev/null @@ -1,313 +0,0 @@ -"""Exact-base source extraction for PR #168; never imports project modules.""" -from pathlib import Path -import ast -import copy -import difflib -import hashlib -import io -import json -import re -import subprocess -import tokenize - -ROOT = Path.cwd() -BASE = '8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8' -EXPECTED = { - 'orders':'f9abe3da981c548a60d0e1b1630d82e02d0e8491', - 'store':'09a1e14328d512e91913bf00e987cf234816348e', - 'money':'424a03f1e6de882aba425f857ab602df818f90c1', - 'delivery':'4500a65ed55e5ea06426c9ac3011f4317ecdc70b', -} -NAMES = { - 'orders':('order_records',{ - '_subject':'require_signed_subject','_viewer':'require_order_viewer', - '_order_id':'new_order_id','_row':'read_order','_view':'legacy_order_view'}), - 'store':('catalog',{ - '_listing':'read_listing','_body':'read_listing_body','_package_row':'read_package_record'}), - 'money':('ledger',{ - 'account_requirements':'account_requirements','_owner':'require_money_subject', - '_amount':'checked_amount','_balance':'balance','_supply':'total_supply', - 'clearing_decision':'clearing_decision','_post_entry':'append_entry','_post_transfer':'post_transfer'}), - 'delivery':('managed_delivery',{ - '_delivery':'read_delivery','_buyer_order':'read_buyer_order', - '_verified_delivery':'verify_managed_delivery','_package':'read_managed_package', - '_payload':'read_managed_payloads','prepare_managed':'prepare_managed', - 'delivery_summary':'delivery_summary'}), -} -CONSTANTS = {'orders':('_COLUMNS',),'store':(), - 'money':('CURRENCY_ID','CODE','SCALE','MAX_MINOR','POLICY_VERSION','POLICY_DIGEST'), - 'delivery':('MAX_INLINE_BYTES','_COLUMNS')} -HEADERS = { - 'orders':'''"""Shared order identity, authorized records and legacy-compatible projections. - -These functions own record access, not operation registration or settlement. -""" -from __future__ import annotations -import base64 -import os -from msg.core.codec import loads -from msg.core.errors import require - -''', - 'store':'''"""Authorized catalog reads shared by published checkout versions. - -Version-specific schemas, creation and updates stay at catalog entrypoints. -""" -from __future__ import annotations -from msg.core.codec import loads -from msg.core.errors import Failure, require -from msg.core.models import ResourceRef -from msg.plugins.common import check_access, resolve -from msg.market.order_records import require_signed_subject - -''', - 'money':'''"""One append-only ledger posting path in the caller's transaction. - -This module grants no commit rights. Public transfers cannot supply the internal -escrow guard; Root issuance remains a separate local-console use case. -""" -from __future__ import annotations -from uuid import uuid4 -from msg.constants import ROOT_SUBJECT -from msg.core.codec import canonical, digest, wire -from msg.core.errors import require -from msg.core.models import AccessRequirement, SignatureProof - -''', - 'delivery':'''"""Shared bounded managed-package verification and local preparation. - -These preserve the managed checkout contracts; versioned manual/service delivery, -arbitration and operation registration retain their existing entrypoints. -""" -from __future__ import annotations -from msg.core.codec import b64, canonical, decode, digest, loads, wire -from msg.core.errors import require -from msg.core.models import BlobRef -from msg.plugins.common import new_id -from msg.market.ledger import CURRENCY_ID, balance -from msg.market.catalog import read_package_record -from msg.market.order_records import read_order -from msg.market.delivery_targets import validate_target - -''', -} -EXTRA_RENAMES = {'store':{'_subject':'require_signed_subject'}, - 'delivery':{'order_row':'read_order','_package_row':'read_package_record','_balance':'balance'}} -before = {} -after = {} -audit = [] - - -def read(path): - text = (ROOT/path).read_text() - before.setdefault(path,text) - return text - - -def nodes(text): - return ast.parse(text).body - - -def functions(text): - return {node.name:node for node in nodes(text) - if isinstance(node,(ast.FunctionDef,ast.AsyncFunctionDef))} - - -def assignments(text): - return {target.id:node for node in nodes(text) if isinstance(node,ast.Assign) - for target in node.targets if isinstance(target,ast.Name)} - - -def segment(text,node): - return ast.get_source_segment(text,node) - - -def rename(text,mapping): - tokens = tokenize.generate_tokens(io.StringIO(text).readline) - return tokenize.untokenize(token._replace(string=mapping.get(token.string,token.string)) - if token.type==tokenize.NAME else token for token in tokens) - - -def splice(text,edits): - starts=[0] - for line in text.splitlines(True): - starts.append(starts[-1]+len(line)) - indexed=[] - for node,value in edits: - indexed.append((starts[node.lineno-1]+node.col_offset, - starts[node.end_lineno-1]+node.end_col_offset,value)) - for start,end,value in sorted(indexed,reverse=True): - text=text[:start]+value+text[end:] - return text - - -def clean(text): - return re.sub(r'\n{4,}','\n\n\n',text).rstrip()+'\n' - - -sources={name:read('src/msg/plugins/'+name+'.py') for name in NAMES} -for name,expected in EXPECTED.items(): - actual=subprocess.check_output(['git','hash-object','src/msg/plugins/'+name+'.py'],text=True).strip() - assert actual==expected,(name,actual,expected) -assert segment(sources['orders'],functions(sources['orders'])['_subject']) == segment( - sources['store'],functions(sources['store'])['_subject']) - -# Move the unique in-process guard with the posting invariant, not with an API version. -escrow_path='src/msg/market/escrow.py' -escrow=read(escrow_path) -assert subprocess.check_output(['git','hash-object',escrow_path],text=True).strip() == '92e0bb595c8fec8665285f0d131630a52500d9fa' -guard_start=escrow.index('# Not a credential, account or public capability.') -guard_end=escrow.index('TRANSITIONS =') -guard=escrow[guard_start:guard_end] -assert '_ESCROW_WRITE = object()' in guard -assert guard.count('object()')==1 -after[escrow_path]=escrow[:guard_start]+'from msg.market.ledger import _ESCROW_WRITE\n\n'+escrow[guard_end:] - -providers={} -for name,(owner,mapping) in NAMES.items(): - exports={old:('msg.market.'+owner,new) for old,new in mapping.items()} - exports.update({constant:('msg.market.'+owner,constant) for constant in CONSTANTS[name]}) - if name=='store': - exports['_subject']=('msg.market.order_records','require_signed_subject') - providers['msg.plugins.'+name]=exports - -for name,(owner,mapping) in NAMES.items(): - text=sources[name] - fs,cs=functions(text),assignments(text) - renames={**mapping,**EXTRA_RENAMES.get(name,{})} - pieces=[HEADERS[name]] - pieces.extend(segment(text,cs[constant])+'\n' for constant in CONSTANTS[name]) - if name=='money': - pieces.append('\n'+guard) - for old,new in mapping.items(): - body=segment(text,fs[old]) - if name=='money' and old=='_post_transfer': - line=' from msg.market.escrow import _ESCROW_WRITE\n' - assert body.count(line)==1 - body=body.replace(line,'') - pieces.append('\n\n'+rename(body,renames)+'\n') - target='src/msg/market/'+owner+'.py' - assert not (ROOT/target).exists(),target - before[target]='' - after[target]=clean(''.join(pieces)) - for old,new in mapping.items(): - original=segment(text,fs[old]) - if name=='money' and old=='_post_transfer': - original=original.replace(' from msg.market.escrow import _ESCROW_WRITE\n','') - expected=ast.dump(ast.parse(rename(original,renames)),include_attributes=False) - actual=ast.dump(ast.parse(segment(after[target],functions(after[target])[new])),include_attributes=False) - assert actual==expected,(name,old) - audit.append(name+'.'+old+' -> '+owner+'.'+new) - edits=[(fs[old],'') for old in mapping]+[(cs[c],'') for c in CONSTANTS[name]] - if name=='store': - edits.append((fs['_subject'],'')) - adapter=splice(text,edits) - # Existing entrypoint code still names the compatibility imports. Each alias - # is the exact owned function, not a wrapper or a second implementation. - imports=['# Compatibility imports; shared implementation has one market owner.'] - for old,(module,new) in providers['msg.plugins.'+name].items(): - imports.append('from '+module+' import '+new+(' as '+old if new!=old else '')) - adapter=adapter.replace('def install(app):','\n'.join(imports)+'\n\n\ndef install(app):',1) - after['src/msg/plugins/'+name+'.py']=clean(adapter) - - -def rewire(text): - edits=[] - for node in ast.walk(ast.parse(text)): - if not isinstance(node,ast.ImportFrom) or node.module not in providers: - continue - groups={} - for item in node.names: - target,new=providers[node.module].get(item.name,(node.module,item.name)) - local=item.asname or item.name - groups.setdefault(target,[]).append(new+(' as '+local if new!=local else '')) - replacement=('\n'+' '*node.col_offset).join( - 'from '+module+' import '+', '.join(names) for module,names in groups.items()) - edits.append((node,replacement)) - return splice(text,edits) - - -# Only import wiring changes in existing market use cases and versioned handlers. -# Source outside these four entrypoints and market remains untouched. -paths={str(p.relative_to(ROOT)) for p in (ROOT/'src/msg/market').rglob('*.py')} -paths.update('src/msg/plugins/'+name+'.py' for name in NAMES) -paths.update(after) -for path in sorted(paths): - text=after[path] if path in after else read(path) - candidate=rewire(text) - if candidate!=before.get(path,''): - after[path]=clean(candidate) - - -class WithoutImports(ast.NodeTransformer): - def visit_ImportFrom(self,node): - return None - def visit_Import(self,node): - return None - - -def stable(node): - return ast.dump(WithoutImports().visit(copy.deepcopy(node)),include_attributes=False) - - -for name in NAMES: - old=functions(sources[name]) - new=functions(after['src/msg/plugins/'+name+'.py']) - removed=set(NAMES[name][1])|({'_subject'} if name=='store' else set()) - assert set(new)==set(old)-removed,(name,set(new),set(old)-removed) - for function in new: - assert stable(old[function])==stable(new[function]),(name,function) - audit.append(name+'.'+function+' body/schema unchanged') -for path,text in after.items(): - if not before.get(path) or not path.startswith('src/msg/market/'): - continue - left=ast.parse(before[path]);right=ast.parse(text) - if path==escrow_path: - left.body=[n for n in left.body if not (isinstance(n,ast.Assign) and - any(isinstance(t,ast.Name) and t.id=='_ESCROW_WRITE' for t in n.targets))] - assert stable(left)==stable(right),path - audit.append(path+' business AST unchanged') -for path,text in after.items(): - if path.startswith('src/msg/market/'): - assert not [n.module for n in ast.walk(ast.parse(text)) - if isinstance(n,ast.ImportFrom) and n.module in providers],path -# All old constants are moved verbatim; the changed authority import is singular. -assert after['src/msg/market/ledger.py'].count('_ESCROW_WRITE = object()')==1 -assert '_ESCROW_WRITE = object()' not in after[escrow_path] - -notes='docs/MARKET_BOUNDARY_PROGRESS.md' -after[notes]=read(notes)+''' - -## Implementation and source audit - -PR #168 extracts the existing shared implementation into `order_records`, -`catalog`, `ledger` and `managed_delivery`, with canonical public function names. -Old plugin names re-export those same function objects. All existing market use -cases, including function-local imports, and legacy plugin handlers use these -owners. The duplicate store subject check becomes the same order-subject check. - -The exact-base cloud assembly checks source blob identities, compares every -moved function after identifier-only renaming, and checks every remaining handler -and market state machine after ignoring import statements. SQL strings, schemas, -IDs, receipt purposes, positional order and existing algorithm bodies are not -rewritten. The only guard change is its owner: `ledger` defines `_ESCROW_WRITE` -once, escrow re-exports it, and protected posting reads that same object directly. -The guard is still supplied only by the existing checked escrow release path. - -No generated source is imported or executed during assembly. Cloud test jobs run -both new real-database contracts and existing published-version/recovery suites. -The assembly artifact contains the exact changed paths, audit and patch. Its -one-shot helper files are removed from the submitted tree before final validation. -No successful final-head result is claimed by this source-only audit. -''' - -out=Path('/tmp/market-boundary-assembly');out.mkdir(parents=True,exist_ok=True) -changed={path:text for path,text in after.items() if text!=before.get(path,'')} -patch=''.join(''.join(difflib.unified_diff(before.get(path,'').splitlines(True),text.splitlines(True), - fromfile='a/'+path if before.get(path) else '/dev/null',tofile='b/'+path)) for path,text in sorted(changed.items())) -for path,text in changed.items(): - (ROOT/path).write_text(text) -(out/'changes.patch').write_text(patch) -(out/'audit.json').write_text(json.dumps({'baseline_commit':BASE,'baseline_blobs':EXPECTED, - 'checked_invariants':audit,'changed_paths':sorted(changed), - 'patch_sha256':hashlib.sha256(patch.encode()).hexdigest()},indent=2)+'\n') diff --git a/.github/workflows/assemble-market-boundaries.yml b/.github/workflows/assemble-market-boundaries.yml deleted file mode 100644 index 732be59a..00000000 --- a/.github/workflows/assemble-market-boundaries.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Assemble audited market boundary extraction -on: - push: - branches: [build/market-boundaries-168] - paths: - - .github/tools/assemble_market_boundaries.py - - .github/workflows/assemble-market-boundaries.yml -permissions: - contents: write -jobs: - assemble: - if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/build/market-boundaries-168' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Require exact regression source - env: - INPUT_SHA: ${{ github.sha }} - run: | - test "$(git rev-parse HEAD)" = "$INPUT_SHA" - git merge-base --is-ancestor 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 HEAD - git diff --exit-code 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 -- src tests docs - - name: Assemble and audit shared functions and unchanged state machines - run: python .github/tools/assemble_market_boundaries.py - - name: Commit only audited source and notes - run: | - git add src/msg/market src/msg/plugins/orders.py src/msg/plugins/money.py src/msg/plugins/delivery.py src/msg/plugins/store.py docs/MARKET_BOUNDARY_PROGRESS.md - git diff --cached --check - python - <<'PY' - import json,subprocess - from pathlib import Path - expected=set(json.loads(Path('/tmp/market-boundary-assembly/audit.json').read_text())['changed_paths']) - actual=set(subprocess.check_output(['git','diff','--cached','--name-only'],text=True).splitlines()) - assert actual==expected,(actual-expected,expected-actual) - PY - git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' commit -m 'refactor(market): own shared records posting and managed delivery independently of entrypoints' - - name: Publish only the dedicated build branch without force - env: - INPUT_SHA: ${{ github.sha }} - run: | - test "$(git ls-remote origin refs/heads/build/market-boundaries-168 | cut -f1)" = "$INPUT_SHA" - git push origin HEAD:refs/heads/build/market-boundaries-168 - python - <<'PY' - import json,os,subprocess - from pathlib import Path - git=lambda *args:subprocess.check_output(['git',*args],text=True).strip() - Path('/tmp/market-boundary-assembly/source.json').write_text(json.dumps({ - 'input':os.environ['INPUT_SHA'],'commit':git('rev-parse','HEAD'), - 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') - PY - git archive --format=tar.gz -o /tmp/market-boundary-assembly/source.tar.gz HEAD - - uses: actions/upload-artifact@v4 - if: always() - with: - name: market-boundary-assembly - path: /tmp/market-boundary-assembly/ - if-no-files-found: error diff --git a/docs/MARKET_BOUNDARY_PROGRESS.md b/docs/MARKET_BOUNDARY_PROGRESS.md index 6f6e1f0a..cb88a4b9 100644 --- a/docs/MARKET_BOUNDARY_PROGRESS.md +++ b/docs/MARKET_BOUNDARY_PROGRESS.md @@ -2,79 +2,75 @@ ## Scope and coordination -Issue #159; references #85/#83, not completion of #71/#72 or production acceptance. -Base: `476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7`, tree -`40ca207a3c49be347120e058e47ed328ec4e3098`. Branch: -`fix/market-boundaries-20260929`. Coordination is #83 comment5878359290. +Issue #159, PR #168; references #85/#83, not completion of #71/#72 or production acceptance. +Base main: `476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7`, tree +`40ca207a3c49be347120e058e47ed328ec4e3098`. Branch: `fix/market-boundaries-20260929`. +Coordination: #83 comments5878359290 and5878591036; #159 comment5878570249. -This work does not modify #165/#167 core/tool/executor ownership, #155 recovery, -or #166 metadata session extraction. Main and all active claims were refreshed -before starting. No production, real funds, Root/PIN or outward delivery action. +This change does not modify #165/#167 core/tool/executor ownership, #155 recovery, +or #166 metadata sessions. No production, real funds, Root/PIN or outward delivery. -## Regression first +## Verified regression-first evidence -The first commit changes no production implementation. It adds ten tests: -seven ownership/import/unique-guard checks and three real PostgreSQL behavior -contracts. Current expected failures concern the new module boundaries; expected -passing characterizations cover buyer/seller privacy, forged escrow authority -with no ledger changes, receipt verification/byte preservation and replay. -No test result is claimed until the exact cloud run is inspected. +Red run **36483360971**, job109134178838, tested exact commit +`8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8` / tree +`907f51806164625749e57bf754713ae9f365f9b6`: **7 failed, 3 passed, +0 errors, 0 skipped**. All failures were new ownership/import assertions. +The real PostgreSQL buyer/seller privacy, forged-escrow zero-write guard, +and signed-receipt byte preservation/idempotent replay tests passed. -## Observed dependency graph +Artifact10998081065 was independently downloaded and hashed; source.json and JUnit +were read. SHA-256: `07f566e15500142c37223782d5cc10086bb9e92a967e5a204c52c374abae3600`. +No project tests ran locally. -`market.orders/email/arbitration/delivery -> plugins.orders` for order records, -subject checks and existing projections. `market -> plugins.store` for listing -and package reads. `market.escrow/orders -> plugins.money` for protected posting. -`market.policy/targets/notifications/escrow/delivery -> plugins.delivery` for -managed delivery records and verification. `plugins.orders/delivery -> market.escrow` -closes the reverse dependency. Function-local imports are included in this audit. +## Implemented ownership -## Intended ownership and invariants +Before: market use cases imported records/catalog/posting/managed-delivery from +`plugins.orders/store/money/delivery`, including function-local imports, while +those entrypoints depended on market escrow and policies. -- `market.order_records`: signed subject/viewer checks, order IDs, authorized +After: + +- `market.order_records` owns signed subject/viewer checks, order IDs, authorized record lookup and explicitly legacy-compatible projections. -- `market.catalog`: existing authorized listing/body/package reads, not plugin - registration or version-specific catalog schema/defaults. -- `market.ledger`: existing policy constants, account requirements and one posting - implementation. The in-process escrow guard has one owner; moving it does not - make it a credential or a network operation. Local-only Root issuance is unchanged. -- `market.managed_delivery`: existing bounded managed-package verification, - local preparation and read helpers. Their published version semantics remain; - they do not replace v3 service/manual delivery or arbitration state machines. - -Old plugin imports re-export these same functions, never wrappers with new policy -or copied implementations. Version-specific schema, request signing and response -adaptation stay at their existing entrypoints. Existing transactions, account -locks, receipt purposes, ID generation and recovery/lease fences remain intact. -No new Manager, Factory, Repository, ORM or alternate money/escrow path is added. - -## Verification and integration - -Focused cloud Actions run the new contracts and existing checkout v1/v2/v3, -account-scope, deterministic clearing, Bounty, redemption, storage and recovery -suites. Full four-shard exact node-ID/conformance/build and applicable recovery -checks remain required on the final synchronized head. No skips, weakened -assertions, changed published versions or historical fact/schema migration. - - -## Implementation and source audit - -PR #168 extracts the existing shared implementation into `order_records`, -`catalog`, `ledger` and `managed_delivery`, with canonical public function names. -Old plugin names re-export those same function objects. All existing market use -cases, including function-local imports, and legacy plugin handlers use these -owners. The duplicate store subject check becomes the same order-subject check. - -The exact-base cloud assembly checks source blob identities, compares every -moved function after identifier-only renaming, and checks every remaining handler -and market state machine after ignoring import statements. SQL strings, schemas, -IDs, receipt purposes, positional order and existing algorithm bodies are not -rewritten. The only guard change is its owner: `ledger` defines `_ESCROW_WRITE` -once, escrow re-exports it, and protected posting reads that same object directly. -The guard is still supplied only by the existing checked escrow release path. - -No generated source is imported or executed during assembly. Cloud test jobs run -both new real-database contracts and existing published-version/recovery suites. -The assembly artifact contains the exact changed paths, audit and patch. Its -one-shot helper files are removed from the submitted tree before final validation. -No successful final-head result is claimed by this source-only audit. +- `market.catalog` owns existing authorized listing/body/package reads, not + version-specific schemas, defaults or registration. +- `market.ledger` owns policy constants, account requirements and the single + posting implementation. It alone defines the existing `_ESCROW_WRITE` object; + escrow re-exports that same object. It is not a public/network credential. +- `market.managed_delivery` owns existing bounded managed-package verification, + preparation and read helpers; it does not replace manual/service/arbitration flows. + +Old plugin paths re-export the same function objects, never wrappers or copies. +Existing market use cases and versioned handlers import the neutral owners. +The duplicate store subject check is the same function as the order subject check. +No new Manager, Factory, Repository, ORM, ledger or escrow state machine exists. + +## Verified source assembly and review + +Cloud assembly **36483781552** succeeded from input +`d0d7f4f61f138b0ced266f2324f4a90c1689b6b5`. Output commit +`f92390fe8f54fbf63fe21d770f37d6fdbb56190e`, tree +`f3f8a2c20c031e0fdf0e1129b2123e7350e7cfaa`. + +Its audit checks exact input blobs and all **23** moved functions after identifier-only +renaming, every remaining handler/schema, and the complete existing market state +machines after excluding import statements. The only guard change is its owner; +its identity check and the checked escrow-release call path remain unchanged. +SQL strings, account locks, signatures, IDs, receipt purposes and transactions were +not rewritten. Shared posting still cannot commit or grant remote Root issuance. + +Artifact10997307359 was downloaded, its audit/patch/source read, and its SHA-256 +verified as `2dbcbf167086480216596757ffd32b42fa8ccc61cdea3d767e2862d1aba3ae74`. +The contained patch independently hashes to +`0f3ee2de9d5cb04c50ebf4b573c44652fa7a50a225f3e006f50944432e9173c9`. +The two one-shot assembly files are removed from the submitted tree. + +## Final-head gates + +Source review/assembly is not green test evidence. The new boundary contracts, +existing v1/v2/v3 checkout, clearing, Bounty, redemption and recovery regressions, +and complete four-shard exact-node-ID/conformance/build gate must run on the final +synchronized head. No skip, weakened assertion or published-version migration. +Actual final results and merge SHA are recorded in PR #168 and coordination #83; +this source commit does not predict those outcomes or close umbrella issues. From 93d864b3b003310c6a139a918e5e14e65482e404 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:14:48 +0800 Subject: [PATCH 6/8] ci(client): apply the reviewed dependency-boundary patch on its fixed cloud branch Refs #158 #82 #83 #85. Patch SHA-256 244778f572724b991a8b14ff3334b810a3af14329de999bf148c5445950c8a02. Restrict publishing to 19 enumerated files and this feature branch; remove the one-shot publisher in the generated source commit. Record RED and explicit full-server installation compatibility. --- .github/workflows/prepare-client-boundary.yml | 82 ++++ docs/CLIENT_BOUNDARY_PROGRESS.md | 24 +- docs/CLIENT_INSTALLATION.md | 31 ++ scripts/patches/client-boundary.patch | 456 ++++++++++++++++++ 4 files changed, 581 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/prepare-client-boundary.yml create mode 100644 docs/CLIENT_INSTALLATION.md create mode 100644 scripts/patches/client-boundary.patch diff --git a/.github/workflows/prepare-client-boundary.yml b/.github/workflows/prepare-client-boundary.yml new file mode 100644 index 00000000..34249493 --- /dev/null +++ b/.github/workflows/prepare-client-boundary.yml @@ -0,0 +1,82 @@ +name: Apply reviewed client boundary patch +on: + push: + branches: ['fix/client-boundary-20260929'] + paths: ['scripts/patches/client-boundary.patch'] +permissions: + contents: read +jobs: + apply-reviewed-patch: + if: github.repository == 'TokenNotIncluded/msg.lmm.best' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + fetch-depth: 2 + - name: Apply only the reviewed patch to its recorded parent + run: | + test "$(git rev-parse HEAD^)" = 6ea463d6fa8b256cb07d0ede2d7795a5336eda4f + git apply --index --check scripts/patches/client-boundary.patch + git apply --index scripts/patches/client-boundary.patch + git diff --cached --check + - name: Publish only this feature branch without force + env: + GH_TOKEN: ${{ github.token }} + run: | + python - <<'PY' + import hashlib, json, os, pathlib, subprocess, urllib.request + repo = 'TokenNotIncluded/msg.lmm.best' + branch = 'fix/client-boundary-20260929' + assert os.environ['GITHUB_REPOSITORY'] == repo + assert os.environ['GITHUB_REF'] == 'refs/heads/' + branch + paths = [ + 'README.md', 'docs/DEPLOYMENT.md', 'pyproject.toml', 'src/msg/atomic_file.py', + 'src/msg/cli.py', 'src/msg/client.py', 'src/msg/client_certificates.py', + 'src/msg/client_custodial.py', 'src/msg/client_upgrade.py', 'src/msg/core/codec.py', + 'src/msg/core/executor.py', 'src/msg/daemon.py', 'src/msg/security/custodial_migration.py', + 'src/msg/security/custodial_protocol.py', 'src/msg/security/vault.py', + 'src/msg/storage/git.py', 'src/msg/transports/client.py', 'src/msg/transports/mcp.py', + 'src/msg/transports/mcp_protocol.py', + ] + changed = subprocess.check_output(['git', 'diff', '--cached', '--name-only'], text=True).splitlines() + assert sorted(changed) == sorted(paths), changed + def api(method, endpoint, payload=None): + data = None if payload is None else json.dumps(payload).encode() + request = urllib.request.Request('https://api.github.com/repos/' + repo + endpoint, + data=data, method=method, headers={ + 'Authorization': 'Bearer ' + os.environ['GH_TOKEN'], + 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', + }) + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + parent = os.environ['GITHUB_SHA'] + assert api('GET', '/git/ref/heads/' + branch)['object']['sha'] == parent + old = api('GET', '/git/commits/' + parent) + assert [p['sha'] for p in old['parents']] == ['6ea463d6fa8b256cb07d0ede2d7795a5336eda4f'] + entries = [{'path': p, 'mode': '100644', 'type': 'blob', + 'content': pathlib.Path(p).read_text()} for p in paths] + for p in ['scripts/patches/client-boundary.patch', '.github/workflows/prepare-client-boundary.yml']: + entries.append({'path': p, 'mode': '100644', 'type': 'blob', 'sha': None}) + tree = api('POST', '/git/trees', {'base_tree': old['tree']['sha'], 'tree': entries}) + commit = api('POST', '/git/commits', { + 'message': 'refactor(client): isolate shared protocol helpers and client-only dependencies\n\nRefs #158 #82 #83 #85. Preserve signed bytes, credential persistence and explicit full-server installation.', + 'tree': tree['sha'], 'parents': [parent], + }) + api('PATCH', '/git/refs/heads/' + branch, {'sha': commit['sha'], 'force': False}) + pathlib.Path('artifacts').mkdir(exist_ok=True) + pathlib.Path('artifacts/patch-result.json').write_text(json.dumps({ + 'source': parent, 'commit': commit['sha'], 'tree': tree['sha'], + 'patch_sha256': hashlib.sha256(pathlib.Path('scripts/patches/client-boundary.patch').read_bytes()).hexdigest(), + 'paths': paths, + }, indent=2) + '\n') + print('Published reviewed source:', commit['sha'], tree['sha']) + PY + - uses: actions/upload-artifact@v4 + with: + name: client-patch-result + path: artifacts/ diff --git a/docs/CLIENT_BOUNDARY_PROGRESS.md b/docs/CLIENT_BOUNDARY_PROGRESS.md index 11966c65..c1f09106 100644 --- a/docs/CLIENT_BOUNDARY_PROGRESS.md +++ b/docs/CLIENT_BOUNDARY_PROGRESS.md @@ -1,21 +1,21 @@ # Client dependency boundary progress — 2026-09-29 -Refs #158/#82/#83/#85. Branch `fix/client-boundary-20260929` starts at tool-boundary head `f14fd1533f70983b046df4b4c813993e8a536d12` (#165). It does not write the #155 recovery, #156 executor or #166 metadata-session branches. +Refs #158/#82/#83/#85. Branch `fix/client-boundary-20260929` starts at tool head f14fd1533f70983b046df4b4c813993e8a536d12. #165 was subsequently merged as 08c58928ee3f6c6cb350d9e9fba75d072bd78f08 with the same exact tested tree; #160 is closed. Do not write #155 recovery, #169 executor or #166 metadata-session branches from this slice. -## Regression-first work +## Independently verified RED -This commit adds seven boundary/compatibility/installation checks and a standalone fresh-interpreter client probe. It contains no implementation fix yet. The probe imports client/CLI/TUI modules under a strict server-import guard, creates/reloads protected client key files, and sends the same genuinely signed request through HTTP, path GET, GraphQL and MCP using an in-process mock HTTP endpoint. It never sends to a real service. +- Head **6ea463d6fa8b256cb07d0ede2d7795a5336eda4f**, tree **2a89a0a36f2d87646cccd4fdbd5ed8420f31c9b9**. +- Cloud run **36483229167**, artifact **10998106395**, SHA-256 `449fc080ba7232cceea4c6f6444d1468fda03b6278112d1c9cbec96dcfdce112` was downloaded and independently verified. Source JSON matches this commit/tree; JUnit reports **7 failed / 0 errors / 0 skipped**, including the actual client import guard refusing server vault imports. This is expected regression-first failure, not a successful implementation. +- The real baseline wheel was built; minimal installation remains a failure because its base dependencies include server packages. No project code/test/build ran locally. -The independent workflow builds the real wheel and installs it with normal dependencies into a fresh virtualenv outside the source checkout. It runs pip check, both command help entrypoints and the probe with explicit rejection of server-only packages. No PostgreSQL/Valkey service is configured for this workflow; full repository CI remains unchanged and mandatory. +## Reviewed implementation -## Intended ownership and compatibility +A fixed-parent one-shot cloud patch applies only the 19 enumerated source/install-document files on this feature branch, with no force update or project execution under write credentials. The temporary publisher and patch are removed from the resulting source tree. Exact generated head/tree and artifact hash must be read before final PR verification. -- Compact result serialization belongs to protocol codec; executor keeps only a compatibility import. -- Atomic protected local-file replacement has one narrowly named owner, not the Git/CAS backend. Existing permissions/fsync/replacement semantics remain unchanged. -- Custodial proof input/derivation and signed migration statements share one state-free owner; server-held vault and retirement verification stay server-side. Existing exports remain aliases, not copied security code. -- MCP version negotiation constants must not require importing the server implementation from the network client. -- Base installation expresses client dependencies; the server extra is the complete server install. The existing dev extra retains server dependencies and msg/msgd command names are unchanged. Documentation must explicitly migrate full server installation to the server extra; no automatic package installation. +The change moves compact result encoding, atomic local-file replacement, state-free custodial proof/statement bytes and MCP version constants into their actual shared owners, retaining identity-preserving compatibility exports. Clients no longer import executor, Git storage or server vault for those helpers. Published bytes, credential file protections and server-held authority/state stay unchanged. Base metadata becomes client-only; server extra holds the original server constraints, dev still includes server, and daemon commands fail clearly before state creation when those dependencies are absent. See CLIENT_INSTALLATION.md for migration instructions. -## Evidence still required +## Remaining gates -Read the RED run before implementation. Then verify helper compatibility, four real client transport calls, a dependency-clean wheel environment, all existing credential journal/recovery and server tests, full exact-node-ID/conformance/build gates and current integration tree. No size, speed, Windows/Android field result or production readiness is claimed by this refactor. Keep #158 open until those facts are proved. +The independent workflow must prove all seven checks, real wheel outside the checkout, no server-only Python dependencies, both help entrypoints and four actual signed transport calls using an in-process mock endpoint. Full original server, journal/recovery, protocol and four-shard exact-node-ID/build gates remain mandatory. Review and verify the final combination with current main/#169 rather than treating this isolated branch as the combined result. No GREEN, merge or #158 closure is claimed at this checkpoint. + +No production deployment, Root/PIN, funds, real external notification, key destruction or recovery promotion was performed. Production and platform-field issues remain blocked on their own evidence. diff --git a/docs/CLIENT_INSTALLATION.md b/docs/CLIENT_INSTALLATION.md new file mode 100644 index 00000000..8b013e83 --- /dev/null +++ b/docs/CLIENT_INSTALLATION.md @@ -0,0 +1,31 @@ +# 客户端与服务端安装边界 + +本页对应 #158 的依赖边界调整，不表示新的协议版本或生产部署已经完成。Python 版本要求仍以 pyproject.toml 为准。 + +## 安装角色 + +仅连接远程 msg 服务的客户端，源码安装使用 `python -m pip install .` 或 `uv sync`。安装已经构建的发行文件时，使用 `python -m pip install /path/to/msg_lmm_best--py3-none-any.whl`。基础 Python 依赖只有 cryptography、httpx、jsonschema、referencing 及它们的传递依赖，不需要安装 Starlette、Uvicorn、psycopg、Valkey、aiohttp、dnspython 或 graphql-core，也不要求本机启动数据库。 + +完整服务端使用 `python -m pip install '.[server]'` 或 `uv sync --extra server`。发行包安装使用 `python -m pip install 'msg-lmm-best[server]'`，应选用已发布且与服务端部署匹配的版本。此前基础安装包含服务端依赖；升级安装脚本、镜像或虚拟环境时要显式加入 `server` extra，不能继续假定基础安装代表服务端。现有 `.[dev]` 仍拉入完整服务端依赖，原全量 CI 安装路径不变。 + +`msg`、`msgd` 两个命令入口都保留。`--help` 不要求服务端依赖。客户端环境误运行 `msgd` 的服务命令时，返回 `server_dependencies_required` 和安装提示；不会创建服务目录或偷偷安装依赖。安装 extra 只安装 Python 包，不启动数据库、不部署服务、不授予证书权限。`age`、Git、git-lfs、SSH、bubblewrap 等系统工具仍按具体功能的原要求提供；本次没有无加密、无隔离或空成功的降级实现。 + +## 共享能力归属 + +| 能力 | 唯一实现所有者 | 服务端兼容导出 | +| --- | --- | --- | +| OperationResult 的紧凑 wire 表示 | `msg.core.codec.result_wire` | `msg.core.executor.result_wire` | +| 受保护本地文件的原子替换和同步 | `msg.atomic_file.durable_write` | `msg.storage.git.durable_write` | +| 托管升级 PoP 上下文、派生及客户端证明 | `msg.security.custodial_protocol` | `msg.security.vault` 对应名称 | +| 托管迁移决策和 ACK 的签署语句 | `msg.security.custodial_protocol` | `msg.security.custodial_migration` 对应名称 | +| 已发布 MCP 协商版本集合 | `msg.transports.mcp_protocol` | `msg.transports.mcp` 对应名称 | + +兼容导出是同一函数或同一常量对象，不复制安全敏感实现。共享协议模块不持有服务器密钥、不接受 Application 或事务，也不验证销毁/退役事实。服务端 vault、权限复核、历史清单、恢复和事务发布继续留在原服务端模块。客户端签名仍绑定原 purpose、主体、请求和上下文；请求版本、签署字节、秘密一次释放、journal 与凭据权限语义不变。 + +CLI/TUI/签名客户端可以依赖 core 的协议模型、codec、requests、errors，客户端各功能模块、无服务状态的签名/证书/age/托管协议能力、atomic_file 及客户端传输。不能通过执行器、Git/CAS 存储、托管 vault、admin 或 workers 取得公共辅助函数。MCP HTTP 客户端读取版本常量也不再导入服务端适配器；stdio 的原行为与全量协议回归保留。 + +## 验证边界 + +`tests/test_client_boundary.py` 检查唯一实现、兼容导出、精确证明上下文、原子替换失败保护和缺失依赖前置错误。`scripts/check_client_install.py` 在全新解释器安装导入拦截器，再加载客户端功能、保存/重读受保护密钥，并通过四个真实传输实现发送同一个签名请求；HTTP 端点由进程内 MockTransport 提供，不访问生产站点。 + +`.github/workflows/client-boundary.yml` 构建实际 wheel/sdist，在源码目录之外的新虚拟环境中正常安装 wheel 并运行 pip check、两条 help 命令及四传输探针；`--minimal-install` 显式拒绝服务端 Python 依赖。全量 PostgreSQL/Valkey、CLI/MCP、凭据 journal/恢复、协议一致性与构建门禁仍必须通过。该验证不等于 Windows/Android 现场矩阵、外部网络、包体积/性能或生产验收；实际结果及精确提交记录在 CLIENT_BOUNDARY_PROGRESS.md 和对应 PR 中。 diff --git a/scripts/patches/client-boundary.patch b/scripts/patches/client-boundary.patch new file mode 100644 index 00000000..dbeb5aeb --- /dev/null +++ b/scripts/patches/client-boundary.patch @@ -0,0 +1,456 @@ +diff --git a/README.md b/README.md +index 74d7a1a..978ca32 100644 +--- a/README.md ++++ b/README.md +@@ -29,6 +29,15 @@ msg.lmm.best 是一个开放的交流空间。你可以发布想法、回复讨 + uv sync + ``` + ++基础安装仅包含签名客户端及传输所需的 Python 依赖，不要求本机运行 PostgreSQL 或 Valkey。 ++安装发行包时使用 `python -m pip install msg-lmm-best`。`age` 加密/备份等操作仍需相应系统工具， ++并不因为依赖分组而获得空实现或降级执行。 ++ ++部署完整服务端请使用 `uv sync --extra server`，或在源码目录安装 `python -m pip install '.[server]'`。 ++从此前默认包含服务端依赖的版本升级时，也应显式加入 `server` extra；`dev` extra 仍包含完整服务端依赖。 ++`msg`、`msgd` 的命令名和协议版本不变。仅客户端环境运行服务命令会提示缺少服务端依赖，而不会创建服务状态。 ++完整依赖与边界见[客户端安装说明](docs/CLIENT_INSTALLATION.md)。 ++ + 连接到**已启用这个版本**的服务。首次使用时将示例地址换成实际服务地址；客户端会保存服务地址和身份私钥，请妥善保管。 + + ```bash +diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md +index 32ac5ef..45db645 100644 +--- a/docs/DEPLOYMENT.md ++++ b/docs/DEPLOYMENT.md +@@ -27,7 +27,7 @@ service_keys = "/var/lib/msgd/service" + ```bash + sudo install -d -m 0755 /opt/msgd + sudo python3.15 -m venv /opt/msgd/venv +-sudo /opt/msgd/venv/bin/python -m pip install . ++sudo /opt/msgd/venv/bin/python -m pip install '.[server]' + ``` + + 在真实本机 VT 或串行控制台运行： +diff --git a/pyproject.toml b/pyproject.toml +index cfc5664..cb7cf9f 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -11,16 +11,9 @@ requires-python = ">=3.15" + license = "MIT" + dependencies = [ + "cryptography>=46,<52", +- "starlette>=0.50,<2", +- "uvicorn>=0.38,<1", + "httpx>=0.28,<1", + "jsonschema>=4.25,<5", + "referencing>=0.36,<1", +- "aiohttp>=3.13,<4", +- "dnspython>=2.8,<3", +- "graphql-core>=3.2.7,<3.3", +- "psycopg[binary]>=3.3,<4", +- "valkey>=6.1,<7", + ] + + [[project.authors]] +@@ -31,7 +24,17 @@ msg = "msg.cli:main" + msgd = "msg.daemon:main" + + [project.optional-dependencies] ++server = [ ++ "starlette>=0.50,<2", ++ "uvicorn>=0.38,<1", ++ "aiohttp>=3.13,<4", ++ "dnspython>=2.8,<3", ++ "graphql-core>=3.2.7,<3.3", ++ "psycopg[binary]>=3.3,<4", ++ "valkey>=6.1,<7", ++] + dev = [ ++ "msg-lmm-best[server]", + "pytest>=8.4", + "pytest-asyncio>=1.1", + "pytest-cov>=6", +diff --git a/src/msg/atomic_file.py b/src/msg/atomic_file.py +new file mode 100644 +index 0000000..beaf703 +--- /dev/null ++++ b/src/msg/atomic_file.py +@@ -0,0 +1,26 @@ ++"""Atomic local-file replacement with caller-selected mode and durable syncs.""" ++from __future__ import annotations ++ ++import os ++from pathlib import Path ++import tempfile ++ ++ ++def durable_write(path: Path, data: bytes, mode: int = 0o600): ++ path.parent.mkdir(parents=True,exist_ok=True) ++ fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) ++ try: ++ os.fchmod(fd,mode) ++ with os.fdopen(fd,'wb') as stream: ++ stream.write(data) ++ stream.flush() ++ os.fsync(stream.fileno()) ++ os.replace(temporary,path) ++ directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) ++ try: ++ os.fsync(directory) ++ finally: ++ os.close(directory) ++ finally: ++ if os.path.exists(temporary): ++ os.unlink(temporary) +diff --git a/src/msg/cli.py b/src/msg/cli.py +index 9913a59..2f3d74f 100644 +--- a/src/msg/cli.py ++++ b/src/msg/cli.py +@@ -10,7 +10,7 @@ from pathlib import Path + from msg.client import ClientState,MsgClient + from msg.core.codec import canonical,loads,wire + from msg.core.errors import Failure,require +-from msg.core.executor import result_wire ++from msg.core.codec import result_wire + from msg.core.models import ResourceRef,OperationResult + from msg.transports.client import TRANSPORTS + +diff --git a/src/msg/client.py b/src/msg/client.py +index 144c0ff..326ad17 100644 +--- a/src/msg/client.py ++++ b/src/msg/client.py +@@ -20,8 +20,8 @@ from msg.core.requests import request_for + from msg.security.crypto import Ed25519Signer, subject_id, key_id + from msg.security.age_keys import (generate_age_key,recipient_from_identity, + public_from_recipient,encryption_key_id) +-from msg.security.vault import client_upgrade_proof +-from msg.storage.git import durable_write ++from msg.security.custodial_protocol import client_upgrade_proof ++from msg.atomic_file import durable_write + from msg.client_tokens import read_journal, remove_journal, token_operation + from msg.transports.client import HTTPTransport, GraphQLTransport, MCPHTTPTransport + +diff --git a/src/msg/client_certificates.py b/src/msg/client_certificates.py +index d8b9c30..75fa957 100644 +--- a/src/msg/client_certificates.py ++++ b/src/msg/client_certificates.py +@@ -12,7 +12,7 @@ from msg.core.errors import require + from msg.core.models import CertificateRequest,Certificate,Signature,CapabilityGrant,IssuancePolicy + from msg.security.certificates import csr_body,verify_csr,sign_certificate + from msg.security.crypto import Ed25519Signer,key_id +-from msg.storage.git import durable_write ++from msg.atomic_file import durable_write + + + def local_key(path, *, create=False): +diff --git a/src/msg/client_custodial.py b/src/msg/client_custodial.py +index f79430e..5c162dd 100644 +--- a/src/msg/client_custodial.py ++++ b/src/msg/client_custodial.py +@@ -10,9 +10,9 @@ from msg.client_secrets import write_private + from msg.core.codec import canonical, digest, loads, wire + from msg.core.errors import Failure, require + from msg.security.age_keys import encryption_key_id, public_from_recipient, recipient_from_identity +-from msg.security.custodial_migration import ack_statement, stage_statement +-from msg.security.vault import client_upgrade_proof +-from msg.storage.git import durable_write ++from msg.security.custodial_protocol import ack_statement, stage_statement ++from msg.security.custodial_protocol import client_upgrade_proof ++from msg.atomic_file import durable_write + + + def journal_for(state): +diff --git a/src/msg/client_upgrade.py b/src/msg/client_upgrade.py +index 03c5c92..b68e887 100644 +--- a/src/msg/client_upgrade.py ++++ b/src/msg/client_upgrade.py +@@ -8,7 +8,7 @@ from uuid import uuid4 + from msg.core.codec import b64, canonical, loads, wire + from msg.core.errors import Failure, require + from msg.security.crypto import Ed25519Signer +-from msg.storage.git import durable_write ++from msg.atomic_file import durable_write + + + @contextmanager +diff --git a/src/msg/core/codec.py b/src/msg/core/codec.py +index 133d30e..c3cdd9a 100644 +--- a/src/msg/core/codec.py ++++ b/src/msg/core/codec.py +@@ -251,3 +251,11 @@ def record(**kwargs): + cls.__post_init__ = validate_record + return dataclasses.dataclass(**kwargs)(cls) + return decorate ++ ++ ++def result_wire(result): ++ value=wire(result,compact=True) ++ for name in ('replayed','prefer_cli'): ++ if not value.get(name): ++ value.pop(name,None) ++ return value +diff --git a/src/msg/core/executor.py b/src/msg/core/executor.py +index d00ac08..5eb6ba4 100644 +--- a/src/msg/core/executor.py ++++ b/src/msg/core/executor.py +@@ -4,7 +4,7 @@ import logging + import time + from dataclasses import replace + from uuid import uuid4 +-from msg.core.codec import wire,digest ++from msg.core.codec import wire,digest,result_wire + from msg.core.errors import Failure,require + from msg.core.models import ExecutionContext,HandlerOutput,OperationResult,OperationError,Event,AccessRequirement + from msg.core.requests import SECRET_DELIVERY_MIN_VERSION,receipt_bytes +@@ -186,11 +186,3 @@ class OperationExecutor: + request_id=request.request_id,actor=principal.actor,subject=principal.subject,resources=result.resources,data={'operation':spec.name})) + await tx.save_result(principal.subject,request.payload_digest,result) + return replace(result,prefer_cli=request.source!='msg',cli_url='/AGENTS.md' if request.source!='msg' else None) +- +- +-def result_wire(result): +- value=wire(result,compact=True) +- for name in ('replayed','prefer_cli'): +- if not value.get(name): +- value.pop(name,None) +- return value +diff --git a/src/msg/daemon.py b/src/msg/daemon.py +index 637715d..fc6b26b 100644 +--- a/src/msg/daemon.py ++++ b/src/msg/daemon.py +@@ -1,6 +1,7 @@ + """msgd: local installation, serving, workers, diagnostics and restricted SSH.""" + from __future__ import annotations + import argparse ++import importlib.util + import asyncio + from datetime import UTC,datetime + import json +@@ -139,6 +140,11 @@ def parser(): + + def main(argv=None): + args=parser().parse_args(argv) ++ server_modules=('starlette','uvicorn','psycopg','valkey','aiohttp','dns','graphql') ++ if any(importlib.util.find_spec(name) is None for name in server_modules): ++ print(canonical({'status':'error','error':{'code':'server_dependencies_required', ++ 'hint':"Install msg-lmm-best[server] before running server commands."}}).decode(),file=sys.stderr) ++ return 2 + try: + if args.command in {'doctor','selftest'}: + from msg.admin.diagnostics import doctor,selftest +diff --git a/src/msg/security/custodial_migration.py b/src/msg/security/custodial_migration.py +index bcff825..022d59a 100644 +--- a/src/msg/security/custodial_migration.py ++++ b/src/msg/security/custodial_migration.py +@@ -13,6 +13,7 @@ from msg.core.codec import canonical, decode, digest, loads, unb64, wire + from msg.core.errors import Failure, require + from msg.core.models import ResourceRef, Signature + from msg.security.crypto import key_id, verify ++from msg.security.custodial_protocol import stage_statement, ack_statement + from msg.security.age_keys import encryption_key_id, public_from_recipient + from msg.security.backup_retirement import BackupRetirement, verified as verified_backup + +@@ -43,22 +44,6 @@ def inventory_commitment(subject, details, challenge): + 'source_keys': details.get('source_keys', {})}) + + +-def stage_statement(subject, arguments, request_id): +- """Sign every decision field, not a caller-selected subset of the manifest.""" +- return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, +- 'request_id': request_id, +- 'decision': {key: value for key, value in arguments.items() +- if key != 'migration_ack'}} +- +- +-def ack_statement(subject, challenge_id, inventory_digest, source, target, +- plaintext_digest, request_id, *, method='rewrap'): +- return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, +- 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, +- 'source': source, 'target': target, 'method': method, +- 'plaintext_digest': plaintext_digest, 'request_id': request_id} +- +- + def keyed_source(item, details): + return dict(item, key_id=details.get('source_keys', {}).get( + item['revision'], details['old_encryption_key_id'])) +diff --git a/src/msg/security/custodial_protocol.py b/src/msg/security/custodial_protocol.py +new file mode 100644 +index 0000000..80821ca +--- /dev/null ++++ b/src/msg/security/custodial_protocol.py +@@ -0,0 +1,49 @@ ++"""Published custodial proof and statement bytes shared by both endpoints. ++ ++This module has no server-held key, transaction, current-authority or retirement ++state. Producing a proof does not verify authority or attest key destruction. ++""" ++from __future__ import annotations ++ ++import hmac ++from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey ++from cryptography.hazmat.primitives.kdf.hkdf import HKDF ++from cryptography.hazmat.primitives import hashes ++from msg.core.codec import b64, canonical, unb64 ++from msg.security.age_keys import private_from_identity ++ ++ ++UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', ++ 'encryption_recipient','server_public','nonce','expires_at') ++ ++ ++def _proof(shared,challenge): ++ context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} ++ key=HKDF(algorithm=hashes.SHA256(),length=32, ++ salt=unb64(context['nonce'],limit=64), ++ info=b'msg-custodial-upgrade-pop-v1').derive(shared) ++ return b64(hmac.digest(key,canonical(context),'sha256')) ++ ++ ++def client_upgrade_proof(age_identity,challenge): ++ private=private_from_identity(age_identity) ++ shared=private.exchange(X25519PublicKey.from_public_bytes( ++ unb64(challenge['server_public'],limit=32))) ++ return _proof(shared,challenge) ++ ++ ++ ++def stage_statement(subject, arguments, request_id): ++ """Sign every decision field, not a caller-selected subset of the manifest.""" ++ return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, ++ 'request_id': request_id, ++ 'decision': {key: value for key, value in arguments.items() ++ if key != 'migration_ack'}} ++ ++ ++def ack_statement(subject, challenge_id, inventory_digest, source, target, ++ plaintext_digest, request_id, *, method='rewrap'): ++ return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, ++ 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, ++ 'source': source, 'target': target, 'method': method, ++ 'plaintext_digest': plaintext_digest, 'request_id': request_id} +diff --git a/src/msg/security/vault.py b/src/msg/security/vault.py +index baff9a4..c61f454 100644 +--- a/src/msg/security/vault.py ++++ b/src/msg/security/vault.py +@@ -2,7 +2,6 @@ + from __future__ import annotations + + import os +-import hmac + import shutil + import subprocess + import tempfile +@@ -11,12 +10,11 @@ from pathlib import Path + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey +-from cryptography.hazmat.primitives.kdf.hkdf import HKDF +-from cryptography.hazmat.primitives import hashes + + from msg.core.codec import b64,canonical,unb64,wire + from msg.core.errors import Failure,require +-from msg.security.age_keys import recipient_from_identity,private_from_identity ++from msg.security.age_keys import recipient_from_identity ++from msg.security.custodial_protocol import UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof + from msg.security.crypto import Ed25519Signer + + +@@ -128,25 +126,6 @@ def open_upgrade_private(app,subject,challenge_id,nonce,ciphertext): + raise Failure('custodial_upgrade_challenge_corrupt') from exc + + +-UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', +- 'encryption_recipient','server_public','nonce','expires_at') +- +- +-def _proof(shared,challenge): +- context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} +- key=HKDF(algorithm=hashes.SHA256(),length=32, +- salt=unb64(context['nonce'],limit=64), +- info=b'msg-custodial-upgrade-pop-v1').derive(shared) +- return b64(hmac.digest(key,canonical(context),'sha256')) +- +- +-def client_upgrade_proof(age_identity,challenge): +- private=private_from_identity(age_identity) +- shared=private.exchange(X25519PublicKey.from_public_bytes( +- unb64(challenge['server_public'],limit=32))) +- return _proof(shared,challenge) +- +- + def server_upgrade_proof(app,subject,challenge,nonce,ciphertext): + from msg.security.age_keys import public_from_recipient + private=open_upgrade_private(app,subject,challenge['challenge_id'],nonce,ciphertext) +diff --git a/src/msg/storage/git.py b/src/msg/storage/git.py +index 9846a21..43d51ca 100644 +--- a/src/msg/storage/git.py ++++ b/src/msg/storage/git.py +@@ -17,26 +17,7 @@ from pathlib import Path + from msg.core.codec import canonical, digest, loads + from msg.core.errors import Failure, require + from msg.core.models import BlobRef +- +- +-def durable_write(path: Path, data: bytes, mode: int = 0o600): +- path.parent.mkdir(parents=True,exist_ok=True) +- fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) +- try: +- os.fchmod(fd,mode) +- with os.fdopen(fd,'wb') as stream: +- stream.write(data) +- stream.flush() +- os.fsync(stream.fileno()) +- os.replace(temporary,path) +- directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) +- try: +- os.fsync(directory) +- finally: +- os.close(directory) +- finally: +- if os.path.exists(temporary): +- os.unlink(temporary) ++from msg.atomic_file import durable_write + + + class LFSObjectStore: +diff --git a/src/msg/transports/client.py b/src/msg/transports/client.py +index dee3f63..546fbcf 100644 +--- a/src/msg/transports/client.py ++++ b/src/msg/transports/client.py +@@ -163,7 +163,7 @@ class MCPHTTPTransport(HTTPTransport): + async def call(self, request): + self._secure_delivery(request) + await self._effect(request.operation) +- from msg.transports.mcp import PROTOCOL_VERSION ++ from msg.transports.mcp_protocol import PROTOCOL_VERSION + tool_name = request.operation if request.contract_version==1 else ( + f'{request.operation}@{request.contract_version}') + value = await self._json('POST','/-/mcp',body={'jsonrpc':'2.0','id':request.request_id, +diff --git a/src/msg/transports/mcp.py b/src/msg/transports/mcp.py +index 8474f57..aa81f77 100644 +--- a/src/msg/transports/mcp.py ++++ b/src/msg/transports/mcp.py +@@ -9,11 +9,10 @@ from dataclasses import replace + from msg import __version__ + from msg.core.codec import canonical,digest,loads + from msg.core.errors import Failure,require +-from msg.core.executor import result_wire ++from msg.core.codec import result_wire + from msg.transports.packet import decode_packet,REQUEST_SCHEMA,RESULT_SCHEMA + +-PROTOCOL_VERSION='2025-11-25' +-SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) ++from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS + + + class MCPServer: +diff --git a/src/msg/transports/mcp_protocol.py b/src/msg/transports/mcp_protocol.py +new file mode 100644 +index 0000000..7eca1c5 +--- /dev/null ++++ b/src/msg/transports/mcp_protocol.py +@@ -0,0 +1,4 @@ ++"""Published MCP negotiation versions; no server or client runtime imports.""" ++ ++PROTOCOL_VERSION='2025-11-25' ++SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) From 649a85da210bdc5d13f84a4339719f9628b23b7f Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:23:27 +0800 Subject: [PATCH 7/8] fix(ci): commit client dependency metadata in the repository's original format The source archive normalizes pyproject formatting; cloud run36484923358 refused that hunk before any publish. Commit the same reviewed dependency split against the live repository file and apply the unchanged verified source patch excluding only that already-committed metadata. Refs #158 #83. --- .github/workflows/prepare-client-boundary.yml | 28 +++++++++++++------ pyproject.toml | 5 ++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/prepare-client-boundary.yml b/.github/workflows/prepare-client-boundary.yml index 34249493..5b10f23e 100644 --- a/.github/workflows/prepare-client-boundary.yml +++ b/.github/workflows/prepare-client-boundary.yml @@ -2,7 +2,9 @@ name: Apply reviewed client boundary patch on: push: branches: ['fix/client-boundary-20260929'] - paths: ['scripts/patches/client-boundary.patch'] + paths: + - 'scripts/patches/client-boundary.patch' + - '.github/workflows/prepare-client-boundary.yml' permissions: contents: read jobs: @@ -17,24 +19,25 @@ jobs: with: persist-credentials: false fetch-depth: 2 - - name: Apply only the reviewed patch to its recorded parent + - name: Apply reviewed source patch after explicit repository-format metadata commit run: | - test "$(git rev-parse HEAD^)" = 6ea463d6fa8b256cb07d0ede2d7795a5336eda4f - git apply --index --check scripts/patches/client-boundary.patch - git apply --index scripts/patches/client-boundary.patch + test "$(git rev-parse HEAD^)" = 93d864b3b003310c6a139a918e5e14e65482e404 + echo '244778f572724b991a8b14ff3334b810a3af14329de999bf148c5445950c8a02 scripts/patches/client-boundary.patch' | sha256sum --check + git apply --index --check --exclude=pyproject.toml scripts/patches/client-boundary.patch + git apply --index --exclude=pyproject.toml scripts/patches/client-boundary.patch git diff --cached --check - name: Publish only this feature branch without force env: GH_TOKEN: ${{ github.token }} run: | python - <<'PY' - import hashlib, json, os, pathlib, subprocess, urllib.request + import hashlib, json, os, pathlib, subprocess, tomllib, urllib.request repo = 'TokenNotIncluded/msg.lmm.best' branch = 'fix/client-boundary-20260929' assert os.environ['GITHUB_REPOSITORY'] == repo assert os.environ['GITHUB_REF'] == 'refs/heads/' + branch paths = [ - 'README.md', 'docs/DEPLOYMENT.md', 'pyproject.toml', 'src/msg/atomic_file.py', + 'README.md', 'docs/DEPLOYMENT.md', 'src/msg/atomic_file.py', 'src/msg/cli.py', 'src/msg/client.py', 'src/msg/client_certificates.py', 'src/msg/client_custodial.py', 'src/msg/client_upgrade.py', 'src/msg/core/codec.py', 'src/msg/core/executor.py', 'src/msg/daemon.py', 'src/msg/security/custodial_migration.py', @@ -44,6 +47,13 @@ jobs: ] changed = subprocess.check_output(['git', 'diff', '--cached', '--name-only'], text=True).splitlines() assert sorted(changed) == sorted(paths), changed + metadata = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project'] + assert metadata['dependencies'] == ['cryptography>=46,<52', 'httpx>=0.28,<1', + 'jsonschema>=4.25,<5', 'referencing>=0.36,<1'] + assert metadata['optional-dependencies']['server'] == [ + 'starlette>=0.50,<2', 'uvicorn>=0.38,<1', 'aiohttp>=3.13,<4', 'dnspython>=2.8,<3', + 'graphql-core>=3.2.7,<3.3', 'psycopg[binary]>=3.3,<4', 'valkey>=6.1,<7'] + assert metadata['optional-dependencies']['dev'][0] == 'msg-lmm-best[server]' def api(method, endpoint, payload=None): data = None if payload is None else json.dumps(payload).encode() request = urllib.request.Request('https://api.github.com/repos/' + repo + endpoint, @@ -57,7 +67,7 @@ jobs: parent = os.environ['GITHUB_SHA'] assert api('GET', '/git/ref/heads/' + branch)['object']['sha'] == parent old = api('GET', '/git/commits/' + parent) - assert [p['sha'] for p in old['parents']] == ['6ea463d6fa8b256cb07d0ede2d7795a5336eda4f'] + assert [p['sha'] for p in old['parents']] == ['93d864b3b003310c6a139a918e5e14e65482e404'] entries = [{'path': p, 'mode': '100644', 'type': 'blob', 'content': pathlib.Path(p).read_text()} for p in paths] for p in ['scripts/patches/client-boundary.patch', '.github/workflows/prepare-client-boundary.yml']: @@ -72,7 +82,7 @@ jobs: pathlib.Path('artifacts/patch-result.json').write_text(json.dumps({ 'source': parent, 'commit': commit['sha'], 'tree': tree['sha'], 'patch_sha256': hashlib.sha256(pathlib.Path('scripts/patches/client-boundary.patch').read_bytes()).hexdigest(), - 'paths': paths, + 'paths': paths, 'metadata_committed_before_patch': True, }, indent=2) + '\n') print('Published reviewed source:', commit['sha'], tree['sha']) PY diff --git a/pyproject.toml b/pyproject.toml index dac7c944..43d0c21a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,14 +10,15 @@ readme = "README.md" requires-python = ">=3.15" license = "MIT" authors = [{name="TokenNotIncluded"}] -dependencies = ["cryptography>=46,<52", "starlette>=0.50,<2", "uvicorn>=0.38,<1", "httpx>=0.28,<1", "jsonschema>=4.25,<5", "referencing>=0.36,<1", "aiohttp>=3.13,<4", "dnspython>=2.8,<3", "graphql-core>=3.2.7,<3.3", "psycopg[binary]>=3.3,<4", "valkey>=6.1,<7"] +dependencies = ["cryptography>=46,<52", "httpx>=0.28,<1", "jsonschema>=4.25,<5", "referencing>=0.36,<1"] [project.scripts] msg = "msg.cli:main" msgd = "msg.daemon:main" [project.optional-dependencies] -dev = ["pytest>=8.4", "pytest-asyncio>=1.1", "pytest-cov>=6", "ruff>=0.14", "build>=1.2", "tiktoken>=0.12"] +server = ["starlette>=0.50,<2", "uvicorn>=0.38,<1", "aiohttp>=3.13,<4", "dnspython>=2.8,<3", "graphql-core>=3.2.7,<3.3", "psycopg[binary]>=3.3,<4", "valkey>=6.1,<7"] +dev = ["msg-lmm-best[server]", "pytest>=8.4", "pytest-asyncio>=1.1", "pytest-cov>=6", "ruff>=0.14", "build>=1.2", "tiktoken>=0.12"] [tool.uv.build-backend] module-name = "msg" From 5f56bbe4ea1cddc46bed12d29026589d65485d62 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:24:52 +0000 Subject: [PATCH 8/8] refactor(client): isolate shared protocol helpers and client-only dependencies Refs #158 #82 #83 #85. Preserve signed bytes, credential persistence and explicit full-server installation. --- .github/workflows/prepare-client-boundary.yml | 92 ---- README.md | 9 + docs/DEPLOYMENT.md | 2 +- scripts/patches/client-boundary.patch | 456 ------------------ src/msg/atomic_file.py | 26 + src/msg/cli.py | 2 +- src/msg/client.py | 4 +- src/msg/client_certificates.py | 2 +- src/msg/client_custodial.py | 6 +- src/msg/client_upgrade.py | 2 +- src/msg/core/codec.py | 8 + src/msg/core/executor.py | 10 +- src/msg/daemon.py | 6 + src/msg/security/custodial_migration.py | 17 +- src/msg/security/custodial_protocol.py | 49 ++ src/msg/security/vault.py | 25 +- src/msg/storage/git.py | 21 +- src/msg/transports/client.py | 2 +- src/msg/transports/mcp.py | 5 +- src/msg/transports/mcp_protocol.py | 4 + 20 files changed, 119 insertions(+), 629 deletions(-) delete mode 100644 .github/workflows/prepare-client-boundary.yml delete mode 100644 scripts/patches/client-boundary.patch create mode 100644 src/msg/atomic_file.py create mode 100644 src/msg/security/custodial_protocol.py create mode 100644 src/msg/transports/mcp_protocol.py diff --git a/.github/workflows/prepare-client-boundary.yml b/.github/workflows/prepare-client-boundary.yml deleted file mode 100644 index 5b10f23e..00000000 --- a/.github/workflows/prepare-client-boundary.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Apply reviewed client boundary patch -on: - push: - branches: ['fix/client-boundary-20260929'] - paths: - - 'scripts/patches/client-boundary.patch' - - '.github/workflows/prepare-client-boundary.yml' -permissions: - contents: read -jobs: - apply-reviewed-patch: - if: github.repository == 'TokenNotIncluded/msg.lmm.best' - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: write - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 2 - - name: Apply reviewed source patch after explicit repository-format metadata commit - run: | - test "$(git rev-parse HEAD^)" = 93d864b3b003310c6a139a918e5e14e65482e404 - echo '244778f572724b991a8b14ff3334b810a3af14329de999bf148c5445950c8a02 scripts/patches/client-boundary.patch' | sha256sum --check - git apply --index --check --exclude=pyproject.toml scripts/patches/client-boundary.patch - git apply --index --exclude=pyproject.toml scripts/patches/client-boundary.patch - git diff --cached --check - - name: Publish only this feature branch without force - env: - GH_TOKEN: ${{ github.token }} - run: | - python - <<'PY' - import hashlib, json, os, pathlib, subprocess, tomllib, urllib.request - repo = 'TokenNotIncluded/msg.lmm.best' - branch = 'fix/client-boundary-20260929' - assert os.environ['GITHUB_REPOSITORY'] == repo - assert os.environ['GITHUB_REF'] == 'refs/heads/' + branch - paths = [ - 'README.md', 'docs/DEPLOYMENT.md', 'src/msg/atomic_file.py', - 'src/msg/cli.py', 'src/msg/client.py', 'src/msg/client_certificates.py', - 'src/msg/client_custodial.py', 'src/msg/client_upgrade.py', 'src/msg/core/codec.py', - 'src/msg/core/executor.py', 'src/msg/daemon.py', 'src/msg/security/custodial_migration.py', - 'src/msg/security/custodial_protocol.py', 'src/msg/security/vault.py', - 'src/msg/storage/git.py', 'src/msg/transports/client.py', 'src/msg/transports/mcp.py', - 'src/msg/transports/mcp_protocol.py', - ] - changed = subprocess.check_output(['git', 'diff', '--cached', '--name-only'], text=True).splitlines() - assert sorted(changed) == sorted(paths), changed - metadata = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project'] - assert metadata['dependencies'] == ['cryptography>=46,<52', 'httpx>=0.28,<1', - 'jsonschema>=4.25,<5', 'referencing>=0.36,<1'] - assert metadata['optional-dependencies']['server'] == [ - 'starlette>=0.50,<2', 'uvicorn>=0.38,<1', 'aiohttp>=3.13,<4', 'dnspython>=2.8,<3', - 'graphql-core>=3.2.7,<3.3', 'psycopg[binary]>=3.3,<4', 'valkey>=6.1,<7'] - assert metadata['optional-dependencies']['dev'][0] == 'msg-lmm-best[server]' - def api(method, endpoint, payload=None): - data = None if payload is None else json.dumps(payload).encode() - request = urllib.request.Request('https://api.github.com/repos/' + repo + endpoint, - data=data, method=method, headers={ - 'Authorization': 'Bearer ' + os.environ['GH_TOKEN'], - 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', - 'X-GitHub-Api-Version': '2022-11-28', - }) - with urllib.request.urlopen(request, timeout=60) as response: - return json.load(response) - parent = os.environ['GITHUB_SHA'] - assert api('GET', '/git/ref/heads/' + branch)['object']['sha'] == parent - old = api('GET', '/git/commits/' + parent) - assert [p['sha'] for p in old['parents']] == ['93d864b3b003310c6a139a918e5e14e65482e404'] - entries = [{'path': p, 'mode': '100644', 'type': 'blob', - 'content': pathlib.Path(p).read_text()} for p in paths] - for p in ['scripts/patches/client-boundary.patch', '.github/workflows/prepare-client-boundary.yml']: - entries.append({'path': p, 'mode': '100644', 'type': 'blob', 'sha': None}) - tree = api('POST', '/git/trees', {'base_tree': old['tree']['sha'], 'tree': entries}) - commit = api('POST', '/git/commits', { - 'message': 'refactor(client): isolate shared protocol helpers and client-only dependencies\n\nRefs #158 #82 #83 #85. Preserve signed bytes, credential persistence and explicit full-server installation.', - 'tree': tree['sha'], 'parents': [parent], - }) - api('PATCH', '/git/refs/heads/' + branch, {'sha': commit['sha'], 'force': False}) - pathlib.Path('artifacts').mkdir(exist_ok=True) - pathlib.Path('artifacts/patch-result.json').write_text(json.dumps({ - 'source': parent, 'commit': commit['sha'], 'tree': tree['sha'], - 'patch_sha256': hashlib.sha256(pathlib.Path('scripts/patches/client-boundary.patch').read_bytes()).hexdigest(), - 'paths': paths, 'metadata_committed_before_patch': True, - }, indent=2) + '\n') - print('Published reviewed source:', commit['sha'], tree['sha']) - PY - - uses: actions/upload-artifact@v4 - with: - name: client-patch-result - path: artifacts/ diff --git a/README.md b/README.md index 74d7a1a8..978ca324 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,15 @@ msg.lmm.best 是一个开放的交流空间。你可以发布想法、回复讨 uv sync ``` +基础安装仅包含签名客户端及传输所需的 Python 依赖，不要求本机运行 PostgreSQL 或 Valkey。 +安装发行包时使用 `python -m pip install msg-lmm-best`。`age` 加密/备份等操作仍需相应系统工具， +并不因为依赖分组而获得空实现或降级执行。 + +部署完整服务端请使用 `uv sync --extra server`，或在源码目录安装 `python -m pip install '.[server]'`。 +从此前默认包含服务端依赖的版本升级时，也应显式加入 `server` extra；`dev` extra 仍包含完整服务端依赖。 +`msg`、`msgd` 的命令名和协议版本不变。仅客户端环境运行服务命令会提示缺少服务端依赖，而不会创建服务状态。 +完整依赖与边界见[客户端安装说明](docs/CLIENT_INSTALLATION.md)。 + 连接到**已启用这个版本**的服务。首次使用时将示例地址换成实际服务地址；客户端会保存服务地址和身份私钥，请妥善保管。 ```bash diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 32ac5ef1..45db645c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -27,7 +27,7 @@ service_keys = "/var/lib/msgd/service" ```bash sudo install -d -m 0755 /opt/msgd sudo python3.15 -m venv /opt/msgd/venv -sudo /opt/msgd/venv/bin/python -m pip install . +sudo /opt/msgd/venv/bin/python -m pip install '.[server]' ``` 在真实本机 VT 或串行控制台运行： diff --git a/scripts/patches/client-boundary.patch b/scripts/patches/client-boundary.patch deleted file mode 100644 index dbeb5aeb..00000000 --- a/scripts/patches/client-boundary.patch +++ /dev/null @@ -1,456 +0,0 @@ -diff --git a/README.md b/README.md -index 74d7a1a..978ca32 100644 ---- a/README.md -+++ b/README.md -@@ -29,6 +29,15 @@ msg.lmm.best 是一个开放的交流空间。你可以发布想法、回复讨 - uv sync - ``` - -+基础安装仅包含签名客户端及传输所需的 Python 依赖，不要求本机运行 PostgreSQL 或 Valkey。 -+安装发行包时使用 `python -m pip install msg-lmm-best`。`age` 加密/备份等操作仍需相应系统工具， -+并不因为依赖分组而获得空实现或降级执行。 -+ -+部署完整服务端请使用 `uv sync --extra server`，或在源码目录安装 `python -m pip install '.[server]'`。 -+从此前默认包含服务端依赖的版本升级时，也应显式加入 `server` extra；`dev` extra 仍包含完整服务端依赖。 -+`msg`、`msgd` 的命令名和协议版本不变。仅客户端环境运行服务命令会提示缺少服务端依赖，而不会创建服务状态。 -+完整依赖与边界见[客户端安装说明](docs/CLIENT_INSTALLATION.md)。 -+ - 连接到**已启用这个版本**的服务。首次使用时将示例地址换成实际服务地址；客户端会保存服务地址和身份私钥，请妥善保管。 - - ```bash -diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md -index 32ac5ef..45db645 100644 ---- a/docs/DEPLOYMENT.md -+++ b/docs/DEPLOYMENT.md -@@ -27,7 +27,7 @@ service_keys = "/var/lib/msgd/service" - ```bash - sudo install -d -m 0755 /opt/msgd - sudo python3.15 -m venv /opt/msgd/venv --sudo /opt/msgd/venv/bin/python -m pip install . -+sudo /opt/msgd/venv/bin/python -m pip install '.[server]' - ``` - - 在真实本机 VT 或串行控制台运行： -diff --git a/pyproject.toml b/pyproject.toml -index cfc5664..cb7cf9f 100644 ---- a/pyproject.toml -+++ b/pyproject.toml -@@ -11,16 +11,9 @@ requires-python = ">=3.15" - license = "MIT" - dependencies = [ - "cryptography>=46,<52", -- "starlette>=0.50,<2", -- "uvicorn>=0.38,<1", - "httpx>=0.28,<1", - "jsonschema>=4.25,<5", - "referencing>=0.36,<1", -- "aiohttp>=3.13,<4", -- "dnspython>=2.8,<3", -- "graphql-core>=3.2.7,<3.3", -- "psycopg[binary]>=3.3,<4", -- "valkey>=6.1,<7", - ] - - [[project.authors]] -@@ -31,7 +24,17 @@ msg = "msg.cli:main" - msgd = "msg.daemon:main" - - [project.optional-dependencies] -+server = [ -+ "starlette>=0.50,<2", -+ "uvicorn>=0.38,<1", -+ "aiohttp>=3.13,<4", -+ "dnspython>=2.8,<3", -+ "graphql-core>=3.2.7,<3.3", -+ "psycopg[binary]>=3.3,<4", -+ "valkey>=6.1,<7", -+] - dev = [ -+ "msg-lmm-best[server]", - "pytest>=8.4", - "pytest-asyncio>=1.1", - "pytest-cov>=6", -diff --git a/src/msg/atomic_file.py b/src/msg/atomic_file.py -new file mode 100644 -index 0000000..beaf703 ---- /dev/null -+++ b/src/msg/atomic_file.py -@@ -0,0 +1,26 @@ -+"""Atomic local-file replacement with caller-selected mode and durable syncs.""" -+from __future__ import annotations -+ -+import os -+from pathlib import Path -+import tempfile -+ -+ -+def durable_write(path: Path, data: bytes, mode: int = 0o600): -+ path.parent.mkdir(parents=True,exist_ok=True) -+ fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) -+ try: -+ os.fchmod(fd,mode) -+ with os.fdopen(fd,'wb') as stream: -+ stream.write(data) -+ stream.flush() -+ os.fsync(stream.fileno()) -+ os.replace(temporary,path) -+ directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) -+ try: -+ os.fsync(directory) -+ finally: -+ os.close(directory) -+ finally: -+ if os.path.exists(temporary): -+ os.unlink(temporary) -diff --git a/src/msg/cli.py b/src/msg/cli.py -index 9913a59..2f3d74f 100644 ---- a/src/msg/cli.py -+++ b/src/msg/cli.py -@@ -10,7 +10,7 @@ from pathlib import Path - from msg.client import ClientState,MsgClient - from msg.core.codec import canonical,loads,wire - from msg.core.errors import Failure,require --from msg.core.executor import result_wire -+from msg.core.codec import result_wire - from msg.core.models import ResourceRef,OperationResult - from msg.transports.client import TRANSPORTS - -diff --git a/src/msg/client.py b/src/msg/client.py -index 144c0ff..326ad17 100644 ---- a/src/msg/client.py -+++ b/src/msg/client.py -@@ -20,8 +20,8 @@ from msg.core.requests import request_for - from msg.security.crypto import Ed25519Signer, subject_id, key_id - from msg.security.age_keys import (generate_age_key,recipient_from_identity, - public_from_recipient,encryption_key_id) --from msg.security.vault import client_upgrade_proof --from msg.storage.git import durable_write -+from msg.security.custodial_protocol import client_upgrade_proof -+from msg.atomic_file import durable_write - from msg.client_tokens import read_journal, remove_journal, token_operation - from msg.transports.client import HTTPTransport, GraphQLTransport, MCPHTTPTransport - -diff --git a/src/msg/client_certificates.py b/src/msg/client_certificates.py -index d8b9c30..75fa957 100644 ---- a/src/msg/client_certificates.py -+++ b/src/msg/client_certificates.py -@@ -12,7 +12,7 @@ from msg.core.errors import require - from msg.core.models import CertificateRequest,Certificate,Signature,CapabilityGrant,IssuancePolicy - from msg.security.certificates import csr_body,verify_csr,sign_certificate - from msg.security.crypto import Ed25519Signer,key_id --from msg.storage.git import durable_write -+from msg.atomic_file import durable_write - - - def local_key(path, *, create=False): -diff --git a/src/msg/client_custodial.py b/src/msg/client_custodial.py -index f79430e..5c162dd 100644 ---- a/src/msg/client_custodial.py -+++ b/src/msg/client_custodial.py -@@ -10,9 +10,9 @@ from msg.client_secrets import write_private - from msg.core.codec import canonical, digest, loads, wire - from msg.core.errors import Failure, require - from msg.security.age_keys import encryption_key_id, public_from_recipient, recipient_from_identity --from msg.security.custodial_migration import ack_statement, stage_statement --from msg.security.vault import client_upgrade_proof --from msg.storage.git import durable_write -+from msg.security.custodial_protocol import ack_statement, stage_statement -+from msg.security.custodial_protocol import client_upgrade_proof -+from msg.atomic_file import durable_write - - - def journal_for(state): -diff --git a/src/msg/client_upgrade.py b/src/msg/client_upgrade.py -index 03c5c92..b68e887 100644 ---- a/src/msg/client_upgrade.py -+++ b/src/msg/client_upgrade.py -@@ -8,7 +8,7 @@ from uuid import uuid4 - from msg.core.codec import b64, canonical, loads, wire - from msg.core.errors import Failure, require - from msg.security.crypto import Ed25519Signer --from msg.storage.git import durable_write -+from msg.atomic_file import durable_write - - - @contextmanager -diff --git a/src/msg/core/codec.py b/src/msg/core/codec.py -index 133d30e..c3cdd9a 100644 ---- a/src/msg/core/codec.py -+++ b/src/msg/core/codec.py -@@ -251,3 +251,11 @@ def record(**kwargs): - cls.__post_init__ = validate_record - return dataclasses.dataclass(**kwargs)(cls) - return decorate -+ -+ -+def result_wire(result): -+ value=wire(result,compact=True) -+ for name in ('replayed','prefer_cli'): -+ if not value.get(name): -+ value.pop(name,None) -+ return value -diff --git a/src/msg/core/executor.py b/src/msg/core/executor.py -index d00ac08..5eb6ba4 100644 ---- a/src/msg/core/executor.py -+++ b/src/msg/core/executor.py -@@ -4,7 +4,7 @@ import logging - import time - from dataclasses import replace - from uuid import uuid4 --from msg.core.codec import wire,digest -+from msg.core.codec import wire,digest,result_wire - from msg.core.errors import Failure,require - from msg.core.models import ExecutionContext,HandlerOutput,OperationResult,OperationError,Event,AccessRequirement - from msg.core.requests import SECRET_DELIVERY_MIN_VERSION,receipt_bytes -@@ -186,11 +186,3 @@ class OperationExecutor: - request_id=request.request_id,actor=principal.actor,subject=principal.subject,resources=result.resources,data={'operation':spec.name})) - await tx.save_result(principal.subject,request.payload_digest,result) - return replace(result,prefer_cli=request.source!='msg',cli_url='/AGENTS.md' if request.source!='msg' else None) -- -- --def result_wire(result): -- value=wire(result,compact=True) -- for name in ('replayed','prefer_cli'): -- if not value.get(name): -- value.pop(name,None) -- return value -diff --git a/src/msg/daemon.py b/src/msg/daemon.py -index 637715d..fc6b26b 100644 ---- a/src/msg/daemon.py -+++ b/src/msg/daemon.py -@@ -1,6 +1,7 @@ - """msgd: local installation, serving, workers, diagnostics and restricted SSH.""" - from __future__ import annotations - import argparse -+import importlib.util - import asyncio - from datetime import UTC,datetime - import json -@@ -139,6 +140,11 @@ def parser(): - - def main(argv=None): - args=parser().parse_args(argv) -+ server_modules=('starlette','uvicorn','psycopg','valkey','aiohttp','dns','graphql') -+ if any(importlib.util.find_spec(name) is None for name in server_modules): -+ print(canonical({'status':'error','error':{'code':'server_dependencies_required', -+ 'hint':"Install msg-lmm-best[server] before running server commands."}}).decode(),file=sys.stderr) -+ return 2 - try: - if args.command in {'doctor','selftest'}: - from msg.admin.diagnostics import doctor,selftest -diff --git a/src/msg/security/custodial_migration.py b/src/msg/security/custodial_migration.py -index bcff825..022d59a 100644 ---- a/src/msg/security/custodial_migration.py -+++ b/src/msg/security/custodial_migration.py -@@ -13,6 +13,7 @@ from msg.core.codec import canonical, decode, digest, loads, unb64, wire - from msg.core.errors import Failure, require - from msg.core.models import ResourceRef, Signature - from msg.security.crypto import key_id, verify -+from msg.security.custodial_protocol import stage_statement, ack_statement - from msg.security.age_keys import encryption_key_id, public_from_recipient - from msg.security.backup_retirement import BackupRetirement, verified as verified_backup - -@@ -43,22 +44,6 @@ def inventory_commitment(subject, details, challenge): - 'source_keys': details.get('source_keys', {})}) - - --def stage_statement(subject, arguments, request_id): -- """Sign every decision field, not a caller-selected subset of the manifest.""" -- return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, -- 'request_id': request_id, -- 'decision': {key: value for key, value in arguments.items() -- if key != 'migration_ack'}} -- -- --def ack_statement(subject, challenge_id, inventory_digest, source, target, -- plaintext_digest, request_id, *, method='rewrap'): -- return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, -- 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, -- 'source': source, 'target': target, 'method': method, -- 'plaintext_digest': plaintext_digest, 'request_id': request_id} -- -- - def keyed_source(item, details): - return dict(item, key_id=details.get('source_keys', {}).get( - item['revision'], details['old_encryption_key_id'])) -diff --git a/src/msg/security/custodial_protocol.py b/src/msg/security/custodial_protocol.py -new file mode 100644 -index 0000000..80821ca ---- /dev/null -+++ b/src/msg/security/custodial_protocol.py -@@ -0,0 +1,49 @@ -+"""Published custodial proof and statement bytes shared by both endpoints. -+ -+This module has no server-held key, transaction, current-authority or retirement -+state. Producing a proof does not verify authority or attest key destruction. -+""" -+from __future__ import annotations -+ -+import hmac -+from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey -+from cryptography.hazmat.primitives.kdf.hkdf import HKDF -+from cryptography.hazmat.primitives import hashes -+from msg.core.codec import b64, canonical, unb64 -+from msg.security.age_keys import private_from_identity -+ -+ -+UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', -+ 'encryption_recipient','server_public','nonce','expires_at') -+ -+ -+def _proof(shared,challenge): -+ context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} -+ key=HKDF(algorithm=hashes.SHA256(),length=32, -+ salt=unb64(context['nonce'],limit=64), -+ info=b'msg-custodial-upgrade-pop-v1').derive(shared) -+ return b64(hmac.digest(key,canonical(context),'sha256')) -+ -+ -+def client_upgrade_proof(age_identity,challenge): -+ private=private_from_identity(age_identity) -+ shared=private.exchange(X25519PublicKey.from_public_bytes( -+ unb64(challenge['server_public'],limit=32))) -+ return _proof(shared,challenge) -+ -+ -+ -+def stage_statement(subject, arguments, request_id): -+ """Sign every decision field, not a caller-selected subset of the manifest.""" -+ return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, -+ 'request_id': request_id, -+ 'decision': {key: value for key, value in arguments.items() -+ if key != 'migration_ack'}} -+ -+ -+def ack_statement(subject, challenge_id, inventory_digest, source, target, -+ plaintext_digest, request_id, *, method='rewrap'): -+ return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, -+ 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, -+ 'source': source, 'target': target, 'method': method, -+ 'plaintext_digest': plaintext_digest, 'request_id': request_id} -diff --git a/src/msg/security/vault.py b/src/msg/security/vault.py -index baff9a4..c61f454 100644 ---- a/src/msg/security/vault.py -+++ b/src/msg/security/vault.py -@@ -2,7 +2,6 @@ - from __future__ import annotations - - import os --import hmac - import shutil - import subprocess - import tempfile -@@ -11,12 +10,11 @@ from pathlib import Path - from cryptography.exceptions import InvalidTag - from cryptography.hazmat.primitives.ciphers.aead import AESGCM - from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey --from cryptography.hazmat.primitives.kdf.hkdf import HKDF --from cryptography.hazmat.primitives import hashes - - from msg.core.codec import b64,canonical,unb64,wire - from msg.core.errors import Failure,require --from msg.security.age_keys import recipient_from_identity,private_from_identity -+from msg.security.age_keys import recipient_from_identity -+from msg.security.custodial_protocol import UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof - from msg.security.crypto import Ed25519Signer - - -@@ -128,25 +126,6 @@ def open_upgrade_private(app,subject,challenge_id,nonce,ciphertext): - raise Failure('custodial_upgrade_challenge_corrupt') from exc - - --UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', -- 'encryption_recipient','server_public','nonce','expires_at') -- -- --def _proof(shared,challenge): -- context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} -- key=HKDF(algorithm=hashes.SHA256(),length=32, -- salt=unb64(context['nonce'],limit=64), -- info=b'msg-custodial-upgrade-pop-v1').derive(shared) -- return b64(hmac.digest(key,canonical(context),'sha256')) -- -- --def client_upgrade_proof(age_identity,challenge): -- private=private_from_identity(age_identity) -- shared=private.exchange(X25519PublicKey.from_public_bytes( -- unb64(challenge['server_public'],limit=32))) -- return _proof(shared,challenge) -- -- - def server_upgrade_proof(app,subject,challenge,nonce,ciphertext): - from msg.security.age_keys import public_from_recipient - private=open_upgrade_private(app,subject,challenge['challenge_id'],nonce,ciphertext) -diff --git a/src/msg/storage/git.py b/src/msg/storage/git.py -index 9846a21..43d51ca 100644 ---- a/src/msg/storage/git.py -+++ b/src/msg/storage/git.py -@@ -17,26 +17,7 @@ from pathlib import Path - from msg.core.codec import canonical, digest, loads - from msg.core.errors import Failure, require - from msg.core.models import BlobRef -- -- --def durable_write(path: Path, data: bytes, mode: int = 0o600): -- path.parent.mkdir(parents=True,exist_ok=True) -- fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) -- try: -- os.fchmod(fd,mode) -- with os.fdopen(fd,'wb') as stream: -- stream.write(data) -- stream.flush() -- os.fsync(stream.fileno()) -- os.replace(temporary,path) -- directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) -- try: -- os.fsync(directory) -- finally: -- os.close(directory) -- finally: -- if os.path.exists(temporary): -- os.unlink(temporary) -+from msg.atomic_file import durable_write - - - class LFSObjectStore: -diff --git a/src/msg/transports/client.py b/src/msg/transports/client.py -index dee3f63..546fbcf 100644 ---- a/src/msg/transports/client.py -+++ b/src/msg/transports/client.py -@@ -163,7 +163,7 @@ class MCPHTTPTransport(HTTPTransport): - async def call(self, request): - self._secure_delivery(request) - await self._effect(request.operation) -- from msg.transports.mcp import PROTOCOL_VERSION -+ from msg.transports.mcp_protocol import PROTOCOL_VERSION - tool_name = request.operation if request.contract_version==1 else ( - f'{request.operation}@{request.contract_version}') - value = await self._json('POST','/-/mcp',body={'jsonrpc':'2.0','id':request.request_id, -diff --git a/src/msg/transports/mcp.py b/src/msg/transports/mcp.py -index 8474f57..aa81f77 100644 ---- a/src/msg/transports/mcp.py -+++ b/src/msg/transports/mcp.py -@@ -9,11 +9,10 @@ from dataclasses import replace - from msg import __version__ - from msg.core.codec import canonical,digest,loads - from msg.core.errors import Failure,require --from msg.core.executor import result_wire -+from msg.core.codec import result_wire - from msg.transports.packet import decode_packet,REQUEST_SCHEMA,RESULT_SCHEMA - --PROTOCOL_VERSION='2025-11-25' --SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) -+from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS - - - class MCPServer: -diff --git a/src/msg/transports/mcp_protocol.py b/src/msg/transports/mcp_protocol.py -new file mode 100644 -index 0000000..7eca1c5 ---- /dev/null -+++ b/src/msg/transports/mcp_protocol.py -@@ -0,0 +1,4 @@ -+"""Published MCP negotiation versions; no server or client runtime imports.""" -+ -+PROTOCOL_VERSION='2025-11-25' -+SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) diff --git a/src/msg/atomic_file.py b/src/msg/atomic_file.py new file mode 100644 index 00000000..beaf7034 --- /dev/null +++ b/src/msg/atomic_file.py @@ -0,0 +1,26 @@ +"""Atomic local-file replacement with caller-selected mode and durable syncs.""" +from __future__ import annotations + +import os +from pathlib import Path +import tempfile + + +def durable_write(path: Path, data: bytes, mode: int = 0o600): + path.parent.mkdir(parents=True,exist_ok=True) + fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) + try: + os.fchmod(fd,mode) + with os.fdopen(fd,'wb') as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary,path) + directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + if os.path.exists(temporary): + os.unlink(temporary) diff --git a/src/msg/cli.py b/src/msg/cli.py index 9913a596..2f3d74f0 100644 --- a/src/msg/cli.py +++ b/src/msg/cli.py @@ -10,7 +10,7 @@ from msg.client import ClientState,MsgClient from msg.core.codec import canonical,loads,wire from msg.core.errors import Failure,require -from msg.core.executor import result_wire +from msg.core.codec import result_wire from msg.core.models import ResourceRef,OperationResult from msg.transports.client import TRANSPORTS diff --git a/src/msg/client.py b/src/msg/client.py index 144c0ff0..326ad17f 100644 --- a/src/msg/client.py +++ b/src/msg/client.py @@ -20,8 +20,8 @@ from msg.security.crypto import Ed25519Signer, subject_id, key_id from msg.security.age_keys import (generate_age_key,recipient_from_identity, public_from_recipient,encryption_key_id) -from msg.security.vault import client_upgrade_proof -from msg.storage.git import durable_write +from msg.security.custodial_protocol import client_upgrade_proof +from msg.atomic_file import durable_write from msg.client_tokens import read_journal, remove_journal, token_operation from msg.transports.client import HTTPTransport, GraphQLTransport, MCPHTTPTransport diff --git a/src/msg/client_certificates.py b/src/msg/client_certificates.py index d8b9c30e..75fa957c 100644 --- a/src/msg/client_certificates.py +++ b/src/msg/client_certificates.py @@ -12,7 +12,7 @@ from msg.core.models import CertificateRequest,Certificate,Signature,CapabilityGrant,IssuancePolicy from msg.security.certificates import csr_body,verify_csr,sign_certificate from msg.security.crypto import Ed25519Signer,key_id -from msg.storage.git import durable_write +from msg.atomic_file import durable_write def local_key(path, *, create=False): diff --git a/src/msg/client_custodial.py b/src/msg/client_custodial.py index f79430eb..5c162dd0 100644 --- a/src/msg/client_custodial.py +++ b/src/msg/client_custodial.py @@ -10,9 +10,9 @@ from msg.core.codec import canonical, digest, loads, wire from msg.core.errors import Failure, require from msg.security.age_keys import encryption_key_id, public_from_recipient, recipient_from_identity -from msg.security.custodial_migration import ack_statement, stage_statement -from msg.security.vault import client_upgrade_proof -from msg.storage.git import durable_write +from msg.security.custodial_protocol import ack_statement, stage_statement +from msg.security.custodial_protocol import client_upgrade_proof +from msg.atomic_file import durable_write def journal_for(state): diff --git a/src/msg/client_upgrade.py b/src/msg/client_upgrade.py index 03c5c924..b68e8872 100644 --- a/src/msg/client_upgrade.py +++ b/src/msg/client_upgrade.py @@ -8,7 +8,7 @@ from msg.core.codec import b64, canonical, loads, wire from msg.core.errors import Failure, require from msg.security.crypto import Ed25519Signer -from msg.storage.git import durable_write +from msg.atomic_file import durable_write @contextmanager diff --git a/src/msg/core/codec.py b/src/msg/core/codec.py index 133d30ef..c3cdd9a5 100644 --- a/src/msg/core/codec.py +++ b/src/msg/core/codec.py @@ -251,3 +251,11 @@ def decorate(cls): cls.__post_init__ = validate_record return dataclasses.dataclass(**kwargs)(cls) return decorate + + +def result_wire(result): + value=wire(result,compact=True) + for name in ('replayed','prefer_cli'): + if not value.get(name): + value.pop(name,None) + return value diff --git a/src/msg/core/executor.py b/src/msg/core/executor.py index d00ac08d..5eb6ba46 100644 --- a/src/msg/core/executor.py +++ b/src/msg/core/executor.py @@ -4,7 +4,7 @@ import time from dataclasses import replace from uuid import uuid4 -from msg.core.codec import wire,digest +from msg.core.codec import wire,digest,result_wire from msg.core.errors import Failure,require from msg.core.models import ExecutionContext,HandlerOutput,OperationResult,OperationError,Event,AccessRequirement from msg.core.requests import SECRET_DELIVERY_MIN_VERSION,receipt_bytes @@ -186,11 +186,3 @@ async def _independent(self,request,spec,entry): request_id=request.request_id,actor=principal.actor,subject=principal.subject,resources=result.resources,data={'operation':spec.name})) await tx.save_result(principal.subject,request.payload_digest,result) return replace(result,prefer_cli=request.source!='msg',cli_url='/AGENTS.md' if request.source!='msg' else None) - - -def result_wire(result): - value=wire(result,compact=True) - for name in ('replayed','prefer_cli'): - if not value.get(name): - value.pop(name,None) - return value diff --git a/src/msg/daemon.py b/src/msg/daemon.py index 637715da..fc6b26b8 100644 --- a/src/msg/daemon.py +++ b/src/msg/daemon.py @@ -1,6 +1,7 @@ """msgd: local installation, serving, workers, diagnostics and restricted SSH.""" from __future__ import annotations import argparse +import importlib.util import asyncio from datetime import UTC,datetime import json @@ -139,6 +140,11 @@ def parser(): def main(argv=None): args=parser().parse_args(argv) + server_modules=('starlette','uvicorn','psycopg','valkey','aiohttp','dns','graphql') + if any(importlib.util.find_spec(name) is None for name in server_modules): + print(canonical({'status':'error','error':{'code':'server_dependencies_required', + 'hint':"Install msg-lmm-best[server] before running server commands."}}).decode(),file=sys.stderr) + return 2 try: if args.command in {'doctor','selftest'}: from msg.admin.diagnostics import doctor,selftest diff --git a/src/msg/security/custodial_migration.py b/src/msg/security/custodial_migration.py index bcff8257..022d59af 100644 --- a/src/msg/security/custodial_migration.py +++ b/src/msg/security/custodial_migration.py @@ -13,6 +13,7 @@ from msg.core.errors import Failure, require from msg.core.models import ResourceRef, Signature from msg.security.crypto import key_id, verify +from msg.security.custodial_protocol import stage_statement, ack_statement from msg.security.age_keys import encryption_key_id, public_from_recipient from msg.security.backup_retirement import BackupRetirement, verified as verified_backup @@ -43,22 +44,6 @@ def inventory_commitment(subject, details, challenge): 'source_keys': details.get('source_keys', {})}) -def stage_statement(subject, arguments, request_id): - """Sign every decision field, not a caller-selected subset of the manifest.""" - return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, - 'request_id': request_id, - 'decision': {key: value for key, value in arguments.items() - if key != 'migration_ack'}} - - -def ack_statement(subject, challenge_id, inventory_digest, source, target, - plaintext_digest, request_id, *, method='rewrap'): - return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, - 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, - 'source': source, 'target': target, 'method': method, - 'plaintext_digest': plaintext_digest, 'request_id': request_id} - - def keyed_source(item, details): return dict(item, key_id=details.get('source_keys', {}).get( item['revision'], details['old_encryption_key_id'])) diff --git a/src/msg/security/custodial_protocol.py b/src/msg/security/custodial_protocol.py new file mode 100644 index 00000000..80821ca1 --- /dev/null +++ b/src/msg/security/custodial_protocol.py @@ -0,0 +1,49 @@ +"""Published custodial proof and statement bytes shared by both endpoints. + +This module has no server-held key, transaction, current-authority or retirement +state. Producing a proof does not verify authority or attest key destruction. +""" +from __future__ import annotations + +import hmac +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.primitives import hashes +from msg.core.codec import b64, canonical, unb64 +from msg.security.age_keys import private_from_identity + + +UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', + 'encryption_recipient','server_public','nonce','expires_at') + + +def _proof(shared,challenge): + context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} + key=HKDF(algorithm=hashes.SHA256(),length=32, + salt=unb64(context['nonce'],limit=64), + info=b'msg-custodial-upgrade-pop-v1').derive(shared) + return b64(hmac.digest(key,canonical(context),'sha256')) + + +def client_upgrade_proof(age_identity,challenge): + private=private_from_identity(age_identity) + shared=private.exchange(X25519PublicKey.from_public_bytes( + unb64(challenge['server_public'],limit=32))) + return _proof(shared,challenge) + + + +def stage_statement(subject, arguments, request_id): + """Sign every decision field, not a caller-selected subset of the manifest.""" + return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, + 'request_id': request_id, + 'decision': {key: value for key, value in arguments.items() + if key != 'migration_ack'}} + + +def ack_statement(subject, challenge_id, inventory_digest, source, target, + plaintext_digest, request_id, *, method='rewrap'): + return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, + 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, + 'source': source, 'target': target, 'method': method, + 'plaintext_digest': plaintext_digest, 'request_id': request_id} diff --git a/src/msg/security/vault.py b/src/msg/security/vault.py index baff9a43..c61f454e 100644 --- a/src/msg/security/vault.py +++ b/src/msg/security/vault.py @@ -2,7 +2,6 @@ from __future__ import annotations import os -import hmac import shutil import subprocess import tempfile @@ -11,12 +10,11 @@ from cryptography.exceptions import InvalidTag from cryptography.hazmat.primitives.ciphers.aead import AESGCM from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey -from cryptography.hazmat.primitives.kdf.hkdf import HKDF -from cryptography.hazmat.primitives import hashes from msg.core.codec import b64,canonical,unb64,wire from msg.core.errors import Failure,require -from msg.security.age_keys import recipient_from_identity,private_from_identity +from msg.security.age_keys import recipient_from_identity +from msg.security.custodial_protocol import UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof from msg.security.crypto import Ed25519Signer @@ -128,25 +126,6 @@ def open_upgrade_private(app,subject,challenge_id,nonce,ciphertext): raise Failure('custodial_upgrade_challenge_corrupt') from exc -UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', - 'encryption_recipient','server_public','nonce','expires_at') - - -def _proof(shared,challenge): - context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} - key=HKDF(algorithm=hashes.SHA256(),length=32, - salt=unb64(context['nonce'],limit=64), - info=b'msg-custodial-upgrade-pop-v1').derive(shared) - return b64(hmac.digest(key,canonical(context),'sha256')) - - -def client_upgrade_proof(age_identity,challenge): - private=private_from_identity(age_identity) - shared=private.exchange(X25519PublicKey.from_public_bytes( - unb64(challenge['server_public'],limit=32))) - return _proof(shared,challenge) - - def server_upgrade_proof(app,subject,challenge,nonce,ciphertext): from msg.security.age_keys import public_from_recipient private=open_upgrade_private(app,subject,challenge['challenge_id'],nonce,ciphertext) diff --git a/src/msg/storage/git.py b/src/msg/storage/git.py index 9846a21a..43d51ca0 100644 --- a/src/msg/storage/git.py +++ b/src/msg/storage/git.py @@ -17,26 +17,7 @@ from msg.core.codec import canonical, digest, loads from msg.core.errors import Failure, require from msg.core.models import BlobRef - - -def durable_write(path: Path, data: bytes, mode: int = 0o600): - path.parent.mkdir(parents=True,exist_ok=True) - fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) - try: - os.fchmod(fd,mode) - with os.fdopen(fd,'wb') as stream: - stream.write(data) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary,path) - directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) - try: - os.fsync(directory) - finally: - os.close(directory) - finally: - if os.path.exists(temporary): - os.unlink(temporary) +from msg.atomic_file import durable_write class LFSObjectStore: diff --git a/src/msg/transports/client.py b/src/msg/transports/client.py index dee3f632..546fbcf2 100644 --- a/src/msg/transports/client.py +++ b/src/msg/transports/client.py @@ -163,7 +163,7 @@ class MCPHTTPTransport(HTTPTransport): async def call(self, request): self._secure_delivery(request) await self._effect(request.operation) - from msg.transports.mcp import PROTOCOL_VERSION + from msg.transports.mcp_protocol import PROTOCOL_VERSION tool_name = request.operation if request.contract_version==1 else ( f'{request.operation}@{request.contract_version}') value = await self._json('POST','/-/mcp',body={'jsonrpc':'2.0','id':request.request_id, diff --git a/src/msg/transports/mcp.py b/src/msg/transports/mcp.py index 8474f57e..aa81f77d 100644 --- a/src/msg/transports/mcp.py +++ b/src/msg/transports/mcp.py @@ -9,11 +9,10 @@ from msg import __version__ from msg.core.codec import canonical,digest,loads from msg.core.errors import Failure,require -from msg.core.executor import result_wire +from msg.core.codec import result_wire from msg.transports.packet import decode_packet,REQUEST_SCHEMA,RESULT_SCHEMA -PROTOCOL_VERSION='2025-11-25' -SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) +from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS class MCPServer: diff --git a/src/msg/transports/mcp_protocol.py b/src/msg/transports/mcp_protocol.py new file mode 100644 index 00000000..7eca1c5b --- /dev/null +++ b/src/msg/transports/mcp_protocol.py @@ -0,0 +1,4 @@ +"""Published MCP negotiation versions; no server or client runtime imports.""" + +PROTOCOL_VERSION='2025-11-25' +SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'})