diff --git a/.github/workflows/client-boundary.yml b/.github/workflows/client-boundary.yml new file mode 100644 index 00000000..383239b6 --- /dev/null +++ b/.github/workflows/client-boundary.yml @@ -0,0 +1,72 @@ +name: Client boundary +on: + push: + branches: ['fix/client-boundary-*'] + pull_request: + paths: + - 'src/msg/**' + - 'pyproject.toml' + - 'tests/test_client_boundary.py' + - 'scripts/check_client_install.py' + - '.github/workflows/client-boundary.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: client-boundary-${{ github.ref }} + cancel-in-progress: true +jobs: + installed-client: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Record tested source + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, pathlib, subprocess + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + pathlib.Path('artifacts/source.json').write_text(json.dumps({ + 'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}'), + 'run_id': os.environ['GITHUB_RUN_ID'], 'event': os.environ['GITHUB_EVENT_NAME'], + }, indent=2) + '\n') + PY + - name: Install full development dependencies + run: python -m pip install -e '.[dev]' + - name: Verify helper ownership, exact proof context and fail-closed assembly + run: python -m pytest tests/test_client_boundary.py --junitxml=artifacts/client-boundary.xml + - name: Build the distributable wheel and source archive + if: always() + run: | + python -m build + python scripts/check_package_artifacts.py dist + - name: Install a clean client-only wheel outside the checkout + if: always() + run: | + python -m venv "$RUNNER_TEMP/msg-client-only" + client_python="$RUNNER_TEMP/msg-client-only/bin/python" + "$client_python" -m pip install dist/*.whl + "$client_python" -m pip check + mkdir -p "$RUNNER_TEMP/msg-client-outside" + cd "$RUNNER_TEMP/msg-client-outside" + "$RUNNER_TEMP/msg-client-only/bin/msg" --help > "$GITHUB_WORKSPACE/artifacts/client-help.txt" + "$RUNNER_TEMP/msg-client-only/bin/msgd" --help > "$GITHUB_WORKSPACE/artifacts/daemon-help.txt" + "$client_python" -I "$GITHUB_WORKSPACE/scripts/check_client_install.py" --minimal-install \ + > "$GITHUB_WORKSPACE/artifacts/client-install.json" + "$client_python" -m pip list --format=json > "$GITHUB_WORKSPACE/artifacts/client-packages.json" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: client-boundary + path: | + artifacts/ + dist/ diff --git a/README.md b/README.md index 74d7a1a8..978ca324 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,15 @@ msg.lmm.best 是一个开放的交流空间。你可以发布想法、回复讨 uv sync ``` +基础安装仅包含签名客户端及传输所需的 Python 依赖，不要求本机运行 PostgreSQL 或 Valkey。 +安装发行包时使用 `python -m pip install msg-lmm-best`。`age` 加密/备份等操作仍需相应系统工具， +并不因为依赖分组而获得空实现或降级执行。 + +部署完整服务端请使用 `uv sync --extra server`，或在源码目录安装 `python -m pip install '.[server]'`。 +从此前默认包含服务端依赖的版本升级时，也应显式加入 `server` extra；`dev` extra 仍包含完整服务端依赖。 +`msg`、`msgd` 的命令名和协议版本不变。仅客户端环境运行服务命令会提示缺少服务端依赖，而不会创建服务状态。 +完整依赖与边界见[客户端安装说明](docs/CLIENT_INSTALLATION.md)。 + 连接到**已启用这个版本**的服务。首次使用时将示例地址换成实际服务地址；客户端会保存服务地址和身份私钥，请妥善保管。 ```bash diff --git a/docs/CLIENT_BOUNDARY_PROGRESS.md b/docs/CLIENT_BOUNDARY_PROGRESS.md new file mode 100644 index 00000000..9e47db8d --- /dev/null +++ b/docs/CLIENT_BOUNDARY_PROGRESS.md @@ -0,0 +1,36 @@ +# Client and architecture integration progress — 2026-09-29 + +Refs #158/#156/#157/#82/#83/#85. PR #171, branch `fix/client-boundary-20260929`. #167 is the single executor/session implementation; #169 preserves alternative source/evidence, not a second implementation to merge. No other work stream's ref was changed. + +## Current checkpoint + +The preceding combined head **3ae9f8efeed9b936f851a79c29cdaab2bf84a267**, tree **6e7f84e2686170720dd408798422b9fcc5a310d4**, joined client5f56bbe4 and #167e768cbcf. After the parallel #167 branch advanced to **2d1cc443c828dd2f735995cc6a5d20021fee6ac7**, its complete diff from e768cbcf was inspected. It changes only five workflow/test/note files, retains the exact product source, expands the storage gate's migration/ledger/recovery coverage, and adopts the same complete frozen-JSON comparison correction. + +This next two-parent integration preserves 3ae9f8ef and 2d1cc443 histories, adopting those five exact blobs. All #167 source and supplemental #166 tests are retained; the #169 alternative executor is not copied. The client result helper still has one implementation: **core.codec.result_wire**, re-exported as the same object from **core.packet**; executor keeps its packet import. No packet schema/decoding or signed bytes change. + +The final combined commit/tree must pass new cloud gates; its exact ID is in the commit and PR #171. Neither an older isolated green nor pending/cancelled runs certify it. The pre-sync3ae workflows were full36486574058, architecture36486574116, storage36486574157, tool36486574150, client36486574123 and recovery36486574183; read the new head's own runs instead before merging. + +## Completed main slice + +#165 merged as **08c58928ee3f6c6cb350d9e9fba75d072bd78f08**, exact tree **8a62150d8273a2c75775872694749cb529835966**; #160 is closed. Exact-tree evidence: **1,860 full tests, 8 conformance, 38 focused**, recovery safety and wheel/sdist success. Four-shard actual-node-ID union/disjointness and artifact hashes were independently checked. Full evidence is in #165; this integration preserves that tool contract. + +## Client RED and independently verified isolated GREEN + +- RED **6ea463d6fa8b256cb07d0ede2d7795a5336eda4f**, tree2a89a0a36f2d87646cccd4fdbd5ed8420f31c9b9, run36483229167: **7 failed /0 errors /0 skips**. Artifact10998106395 SHA-256 `449fc080ba7232cceea4c6f6444d1468fda03b6278112d1c9cbec96dcfdce112`, source and JUnit verified. Actual fresh imports rejected server vault; baseline wheel failed minimal dependency acceptance. +- Reviewed source patch SHA-256 `244778f572724b991a8b14ff3334b810a3af14329de999bf148c5445950c8a02`. First cloud application36484923358 refused normalized-sdist TOML formatting before publication. Commit649a85da applied the same dependency split directly to actual repository-format metadata; no assertion was relaxed. +- Successful cloud application36485787986 produced **5f56bbe4ea1cddc46bed12d29026589d65485d62**, tree **0b339516eb31ddef8f3aa576857710706cb82a97**. Artifact10999476065 SHA-256 `b43c4f67d52fc1a27739d2ad4ee1d772bb62b4ff105419b93c3f3f84285b3617` matches source, reviewed patch and18 remaining paths. Temporary patch/publisher were removed from the resulting tree; no project code executed under its write credentials. +- **Isolated client GREEN** run36486003907, synthetic mergec4a5c34b16171333ad05803a43a145c93a4e5985, exact tree0b339516 above. Artifact11000036526 SHA-256 `5fb24ee583c5c9f76b0255475b9c7dea76d587c0c74fe8cc3fafa7f65e887810` independently downloaded and checked: **7 passed /0 failures /0 errors /0 skips**; real wheel/sdist; clean normal wheel installation outside checkout, pip check and both help entrypoints; installed package manifest excludes all seven server-only packages; strict fresh-interpreter guard reports no server imports; four actual HTTP/path-GET/GraphQL/MCP transports exchange the same verified signed packet through in-process MockTransport. This is evidence for the isolated source, not the later combined head. + +## Compatibility and retained regressions + +Atomic file replacement, state-free custodial proof/decision/ACK bytes and MCP version constants have neutral owners with same-object compatibility exports. Base installation is client-only; server extra retains original server constraints and dev retains full server. Daemon service commands fail before local state when dependencies are absent. CLIENT_INSTALLATION.md and deployment/README document the required explicit server installation migration. No authority, isolation, journal, crypto purpose, historical version, transaction or publication fence is relaxed. + +The transferred standalone assembly assertion compares the complete `wire(result.data)` because domain JSON is frozen. #169 corrected head3db973ab1784926eb4e545cabdf832e7faa0a4bc had independently verified **34 passed /0 failures/errors/skips** in focused36484427064 (artifact10999215628 SHA-256 `fcbe6d55cba0665d227cef3ae09ca5f256756cd90d86f5394ebf499fe95a91eb`). #1672d now contains that same correction, preserving batch replay and Event/business/result transaction rollback checks. + +## Remaining gates and separate work + +Current combined-head client wheel/import/four-transport checks, architecture/storage/tool/recovery suites, full four-shard exact-node-ID union, conformance/real-tokenizer and build remain required. Refresh main/#167, inspect final diff and exact merge tree before serial integration. Keep #171 draft and #158 open until those gates and review complete. Do not infer Windows/Android, external network, performance or production acceptance from isolated Linux CI. + +#155 corrected isolated head0e4dbe777053e354bcf4845b851a0c9b86ab8310 had independently verified **55 passing recovery tests**, run36480031762/artifact10995627663/hash `e200808e372cc7eeab5e688fa29948fbd0c269261727837879294969c1f838d6`. Its stale failure body was corrected, but current-main conflict/combined acceptance remain separate. #168 market and #161–#163 remain separate work. #64/#65/#84 and remaining live Root/trust/retirement/console requirements need authorized evidence, not a CI substitute. + +All project execution/tests/builds run in cloud. Local work is source text/Git-object review and downloaded artifact verification. No production deployment, real Root/PIN, funds, actual outbound delivery, key/backup destruction or recovery promotion was performed. diff --git a/docs/CLIENT_INSTALLATION.md b/docs/CLIENT_INSTALLATION.md new file mode 100644 index 00000000..8b013e83 --- /dev/null +++ b/docs/CLIENT_INSTALLATION.md @@ -0,0 +1,31 @@ +# 客户端与服务端安装边界 + +本页对应 #158 的依赖边界调整，不表示新的协议版本或生产部署已经完成。Python 版本要求仍以 pyproject.toml 为准。 + +## 安装角色 + +仅连接远程 msg 服务的客户端，源码安装使用 `python -m pip install .` 或 `uv sync`。安装已经构建的发行文件时，使用 `python -m pip install /path/to/msg_lmm_best--py3-none-any.whl`。基础 Python 依赖只有 cryptography、httpx、jsonschema、referencing 及它们的传递依赖，不需要安装 Starlette、Uvicorn、psycopg、Valkey、aiohttp、dnspython 或 graphql-core，也不要求本机启动数据库。 + +完整服务端使用 `python -m pip install '.[server]'` 或 `uv sync --extra server`。发行包安装使用 `python -m pip install 'msg-lmm-best[server]'`，应选用已发布且与服务端部署匹配的版本。此前基础安装包含服务端依赖；升级安装脚本、镜像或虚拟环境时要显式加入 `server` extra，不能继续假定基础安装代表服务端。现有 `.[dev]` 仍拉入完整服务端依赖，原全量 CI 安装路径不变。 + +`msg`、`msgd` 两个命令入口都保留。`--help` 不要求服务端依赖。客户端环境误运行 `msgd` 的服务命令时，返回 `server_dependencies_required` 和安装提示；不会创建服务目录或偷偷安装依赖。安装 extra 只安装 Python 包，不启动数据库、不部署服务、不授予证书权限。`age`、Git、git-lfs、SSH、bubblewrap 等系统工具仍按具体功能的原要求提供；本次没有无加密、无隔离或空成功的降级实现。 + +## 共享能力归属 + +| 能力 | 唯一实现所有者 | 服务端兼容导出 | +| --- | --- | --- | +| OperationResult 的紧凑 wire 表示 | `msg.core.codec.result_wire` | `msg.core.executor.result_wire` | +| 受保护本地文件的原子替换和同步 | `msg.atomic_file.durable_write` | `msg.storage.git.durable_write` | +| 托管升级 PoP 上下文、派生及客户端证明 | `msg.security.custodial_protocol` | `msg.security.vault` 对应名称 | +| 托管迁移决策和 ACK 的签署语句 | `msg.security.custodial_protocol` | `msg.security.custodial_migration` 对应名称 | +| 已发布 MCP 协商版本集合 | `msg.transports.mcp_protocol` | `msg.transports.mcp` 对应名称 | + +兼容导出是同一函数或同一常量对象，不复制安全敏感实现。共享协议模块不持有服务器密钥、不接受 Application 或事务，也不验证销毁/退役事实。服务端 vault、权限复核、历史清单、恢复和事务发布继续留在原服务端模块。客户端签名仍绑定原 purpose、主体、请求和上下文；请求版本、签署字节、秘密一次释放、journal 与凭据权限语义不变。 + +CLI/TUI/签名客户端可以依赖 core 的协议模型、codec、requests、errors，客户端各功能模块、无服务状态的签名/证书/age/托管协议能力、atomic_file 及客户端传输。不能通过执行器、Git/CAS 存储、托管 vault、admin 或 workers 取得公共辅助函数。MCP HTTP 客户端读取版本常量也不再导入服务端适配器；stdio 的原行为与全量协议回归保留。 + +## 验证边界 + +`tests/test_client_boundary.py` 检查唯一实现、兼容导出、精确证明上下文、原子替换失败保护和缺失依赖前置错误。`scripts/check_client_install.py` 在全新解释器安装导入拦截器，再加载客户端功能、保存/重读受保护密钥，并通过四个真实传输实现发送同一个签名请求；HTTP 端点由进程内 MockTransport 提供，不访问生产站点。 + +`.github/workflows/client-boundary.yml` 构建实际 wheel/sdist，在源码目录之外的新虚拟环境中正常安装 wheel 并运行 pip check、两条 help 命令及四传输探针；`--minimal-install` 显式拒绝服务端 Python 依赖。全量 PostgreSQL/Valkey、CLI/MCP、凭据 journal/恢复、协议一致性与构建门禁仍必须通过。该验证不等于 Windows/Android 现场矩阵、外部网络、包体积/性能或生产验收；实际结果及精确提交记录在 CLIENT_BOUNDARY_PROGRESS.md 和对应 PR 中。 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 32ac5ef1..45db645c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -27,7 +27,7 @@ service_keys = "/var/lib/msgd/service" ```bash sudo install -d -m 0755 /opt/msgd sudo python3.15 -m venv /opt/msgd/venv -sudo /opt/msgd/venv/bin/python -m pip install . +sudo /opt/msgd/venv/bin/python -m pip install '.[server]' ``` 在真实本机 VT 或串行控制台运行： diff --git a/docs/INTEGRATION_PROGRESS_20260929.md b/docs/INTEGRATION_PROGRESS_20260929.md new file mode 100644 index 00000000..9e71d9db --- /dev/null +++ b/docs/INTEGRATION_PROGRESS_20260929.md @@ -0,0 +1,27 @@ +# Issue integration notes — 2026-09-29 + +All project code execution, tests and builds run in GitHub Actions. Local work is source/Git/AST inspection and independent artifact parsing only. No production deployment, Root/PIN/key read, real funds/outbound delivery or restore promotion. + +## Completed on main + +- #156/#157: #167 merged at 261c8816; parallel flow independently audited 1894 real tests and conformance8 before closing both architecture issues. #169 superseded with its distinct regressions retained. +- #162: #172 originally merged into an old feature branch, not main. Promotion #175 fixed the target and merged as **edbfbae7648529c6b08a2e81a1a10bb349f6b50d**, tree **ebe1facca432803854d0c04aaa93519a589ff93f**. Independently inspected focused122 plus full1901 (465/496/483/457), exact node-ID union, no failure/error/skip/duplicate/missing; conformance8, measured token budget and wheel/sdist passed. Full36551495771; focused36551495692. Hashes/source identity: #175 final body and #162 comment5888054193. Original token RED/history: TOKEN_DELIVERY_BOUNDARY_PROGRESS.md. +- #161: #173 merged as **04ffad5834dcc959f5ec59696bb264c3d527990b**, tree **f377ce7dbdb53e0480552906a7d7529eee67171f**. Actual merged tree matches the verified combination. Focused hosting41 and same-head token122; full **36553079854 = 1935** (475/506/492/462), exact node-ID union, zero failure/error/skip/duplicate/missing. Conformance8, measured token budget, wheel/sdist and contracts gate passed. Original #174 independent UID/permission tests, its nine-line owner guard and real history are retained; only the duplicate product PR was closed. Exact final proof: #173 body and #161 comment5888274187. Historical RED/GREEN and deployment model: HOSTING_RUNTIME_PROGRESS.md, HOSTING_AUTHORITY_PROGRESS.md, HOSTING_RUNTIME.md. + +No source test result is represented as production acceptance. Main after these merges is 04ffad5834dcc959f5ec59696bb264c3d527990b. + +## Client integration — #171 / #158 + +Actual previous client head **730780abf1b25b91a972dc4a9cccf0cf17408728**, tree **758e86821ee31f59afbed22935207fad6cd5fb11**. Its old PR body referred to an earlier head. Full/client/architecture/storage/recovery runs report success. Independently downloaded full shard0 artifact **11000502466**, SHA256 **9ea50d3f09a9143e860ad6bf860e0483193b536a76e9a46dd52c488f7d6831c8**, and inspected the source distribution. Other old full shards were not independently audited by this execution flow; old status does not substitute for the new combined gate. + +Preserve real client and current-main parents. Only two runtime overlaps need combination: daemon retains both missing-server-dependency diagnostics and explicit read-only hosting; storage.git imports the canonical neutral atomic helper while retaining every group-read publication change. Do not replace TokenDelivery, current schema/grant checks or ownership guard. Other client delta files reuse the original Git blobs; pre-existing common workflows are byte-identical. + +The dev extra already includes server, so existing `.[dev]` installation is valid. uv_build intentionally includes a compatibility `pyproject.toml.orig` in its sdist; it is not a tracked leftover. Repository config must reuse original blob43d0c21a, not normalized sdist TOML. No packaging exclusion is introduced. + +Pending: push combination to existing #171; run fresh minimal wheel installation outside the checkout without PYTHONPATH/server packages, four client transports, helper-identity tests, hosting/token regressions and full four-shard/node-ID/conformance/build. Record exact head/tree/run proof before marking ready or closing #158. Original RED history remains in CLIENT_BOUNDARY_PROGRESS.md. This combination is not yet claimed GREEN. + +## Remaining queues + +Refresh #168 market, #170 Git and #155 recovery refs/comments before integrating; do not overwrite parallel work. Never count a feature-branch merge as shipping on main. Merge serially with expected-head checks and verify resulting trees. + +Keep #64/#65/#68/#69/#70/#80/#84 and broad acceptance issues open until their individual remaining criteria are met. No production permission migration, real Root/PIN/private-key operation, live funds/outbound channel, destructive backup or recovery promotion has been performed here. diff --git a/pyproject.toml b/pyproject.toml index dac7c944..43d0c21a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,14 +10,15 @@ readme = "README.md" requires-python = ">=3.15" license = "MIT" authors = [{name="TokenNotIncluded"}] -dependencies = ["cryptography>=46,<52", "starlette>=0.50,<2", "uvicorn>=0.38,<1", "httpx>=0.28,<1", "jsonschema>=4.25,<5", "referencing>=0.36,<1", "aiohttp>=3.13,<4", "dnspython>=2.8,<3", "graphql-core>=3.2.7,<3.3", "psycopg[binary]>=3.3,<4", "valkey>=6.1,<7"] +dependencies = ["cryptography>=46,<52", "httpx>=0.28,<1", "jsonschema>=4.25,<5", "referencing>=0.36,<1"] [project.scripts] msg = "msg.cli:main" msgd = "msg.daemon:main" [project.optional-dependencies] -dev = ["pytest>=8.4", "pytest-asyncio>=1.1", "pytest-cov>=6", "ruff>=0.14", "build>=1.2", "tiktoken>=0.12"] +server = ["starlette>=0.50,<2", "uvicorn>=0.38,<1", "aiohttp>=3.13,<4", "dnspython>=2.8,<3", "graphql-core>=3.2.7,<3.3", "psycopg[binary]>=3.3,<4", "valkey>=6.1,<7"] +dev = ["msg-lmm-best[server]", "pytest>=8.4", "pytest-asyncio>=1.1", "pytest-cov>=6", "ruff>=0.14", "build>=1.2", "tiktoken>=0.12"] [tool.uv.build-backend] module-name = "msg" diff --git a/scripts/check_client_install.py b/scripts/check_client_install.py new file mode 100644 index 00000000..f12ed662 --- /dev/null +++ b/scripts/check_client_install.py @@ -0,0 +1,124 @@ +"""Exercise real client imports and signed envelopes without a server runtime. + +Run with --minimal-install in a fresh wheel-only virtualenv, outside the source +checkout. Network traffic uses httpx.MockTransport, not a remote service. +""" +from __future__ import annotations + +import argparse +import asyncio +import importlib +import importlib.abc +import importlib.util +import json +from pathlib import Path +import stat +import sys +import tempfile + +SERVER_DEPENDENCIES = ('starlette', 'uvicorn', 'psycopg', 'valkey', 'aiohttp', 'dns', 'graphql') +FORBIDDEN = ('msg.application', 'msg.core.executor', 'msg.storage', 'msg.admin', + 'msg.workers', 'msg.security.vault', 'msg.security.custodial_migration', + 'msg.transports.http', 'msg.transports.graphql', 'msg.transports.mcp') + + +class ClientBoundary(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if any(fullname == name or fullname.startswith(name + '.') for name in FORBIDDEN): + raise RuntimeError('client imported a server implementation: ' + fullname) + return None + + +async def check(directory): + import httpx + from msg.client import ClientState + from msg.core.codec import canonical, loads, wire + from msg.core.requests import request_for, signing_bytes + from msg.security.crypto import Ed25519Signer, verify + from msg.transports.client import TRANSPORTS + from msg.transports.packet import decode_packet, path_packet + + for name in ('cli', 'tui', 'client_certificates', 'client_content', 'client_custodial', + 'client_market', 'client_recovery', 'client_secrets', 'client_tokens', 'client_upgrade'): + importlib.import_module('msg.' + name) + state = ClientState(directory, server='https://unit.invalid') + signer = Ed25519Signer.from_bytes(bytes(range(32))) + state.save_signer(signer) + state.ensure_encryption_key() + restored = ClientState(directory) + assert restored.signer.public_key == signer.public_key + assert restored.encryption_recipient == state.encryption_recipient + for path in (state.path, state.key_path, state.age_key_path): + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert stat.S_IMODE(directory.stat().st_mode) == 0o700 + request = request_for('content.read', {'id': 'r_test'}, state.server, + subject='u_test', signer=signer, request_id='client-boundary') + calls = [] + + def respond(incoming): + path = incoming.url.path + if path == '/_transports': + return httpx.Response(200, json={'version': 1, 'target_service': state.server, + 'operations': {'content.read': 'read'}, 'limits': { + 'max_request_bytes': 1048576, 'max_response_bytes': 1048576, + 'max_path_bytes': 8192, 'encodings': ['j', 'gz']}}) + if path.startswith('/-/g/'): + _, _, _, operation, encoding, encoded = path.split('/') + packet = path_packet(encoded, encoding, 1048576) + assert packet.operation == operation and incoming.method == 'GET' + kind = 'path_get' + else: + data = loads(incoming.content) + if path == '/_read/graphql': + data = data['variables']['packet'] + kind = 'graphql' + elif path == '/-/mcp': + assert incoming.headers['MCP-Protocol-Version'] == '2025-11-25' + data = data['params']['arguments']['packet'] + kind = 'mcp_http' + else: + assert path == '/-/p/content.read' + kind = 'http' + assert incoming.method == 'POST' + packet = decode_packet(data) + assert canonical(packet) == canonical(request) + verify(signer.public_key, signing_bytes(packet), packet.proof.signature, purpose='request') + calls.append(kind) + result = {'request_id': packet.request_id, 'operation': packet.operation, 'status': 'ok', + 'actor': 'u_test', 'subject': 'u_test', 'resources': [{'id': 'r_test'}], + 'data': {'content': 'signed client response'}} + if kind == 'graphql': + result = {'data': {'call': result}} + elif kind == 'mcp_http': + result = {'jsonrpc': '2.0', 'id': packet.request_id, + 'result': {'structuredContent': result}} + return httpx.Response(200, json=result) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: + for name, transport in TRANSPORTS.items(): + result = await transport(state.server, http=http).call(request) + assert result.status == 'ok' and result.resources[0].id == 'r_test', (name, wire(result)) + assert result.data['content'] == 'signed client response' + assert sorted(calls) == sorted(TRANSPORTS) + assert not any(any(name == prefix or name.startswith(prefix + '.') for prefix in FORBIDDEN) + for name in sys.modules) + return calls + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--minimal-install', action='store_true') + args = parser.parse_args() + if args.minimal_install: + present = [name for name in SERVER_DEPENDENCIES if importlib.util.find_spec(name) is not None] + assert not present, 'server-only Python dependencies in client environment: ' + ', '.join(present) + sys.meta_path.insert(0, ClientBoundary()) + with tempfile.TemporaryDirectory(prefix='msg-client-install-') as folder: + calls = asyncio.run(check(Path(folder) / 'client')) + print(json.dumps({'status': 'ok', 'transports': sorted(calls), + 'server_implementation_imported': False, + 'minimal_install': args.minimal_install})) + + +if __name__ == '__main__': + main() diff --git a/src/msg/atomic_file.py b/src/msg/atomic_file.py new file mode 100644 index 00000000..beaf7034 --- /dev/null +++ b/src/msg/atomic_file.py @@ -0,0 +1,26 @@ +"""Atomic local-file replacement with caller-selected mode and durable syncs.""" +from __future__ import annotations + +import os +from pathlib import Path +import tempfile + + +def durable_write(path: Path, data: bytes, mode: int = 0o600): + path.parent.mkdir(parents=True,exist_ok=True) + fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) + try: + os.fchmod(fd,mode) + with os.fdopen(fd,'wb') as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary,path) + directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + if os.path.exists(temporary): + os.unlink(temporary) diff --git a/src/msg/cli.py b/src/msg/cli.py index 9913a596..2f3d74f0 100644 --- a/src/msg/cli.py +++ b/src/msg/cli.py @@ -10,7 +10,7 @@ from msg.client import ClientState,MsgClient from msg.core.codec import canonical,loads,wire from msg.core.errors import Failure,require -from msg.core.executor import result_wire +from msg.core.codec import result_wire from msg.core.models import ResourceRef,OperationResult from msg.transports.client import TRANSPORTS diff --git a/src/msg/client.py b/src/msg/client.py index 144c0ff0..326ad17f 100644 --- a/src/msg/client.py +++ b/src/msg/client.py @@ -20,8 +20,8 @@ from msg.security.crypto import Ed25519Signer, subject_id, key_id from msg.security.age_keys import (generate_age_key,recipient_from_identity, public_from_recipient,encryption_key_id) -from msg.security.vault import client_upgrade_proof -from msg.storage.git import durable_write +from msg.security.custodial_protocol import client_upgrade_proof +from msg.atomic_file import durable_write from msg.client_tokens import read_journal, remove_journal, token_operation from msg.transports.client import HTTPTransport, GraphQLTransport, MCPHTTPTransport diff --git a/src/msg/client_certificates.py b/src/msg/client_certificates.py index d8b9c30e..75fa957c 100644 --- a/src/msg/client_certificates.py +++ b/src/msg/client_certificates.py @@ -12,7 +12,7 @@ from msg.core.models import CertificateRequest,Certificate,Signature,CapabilityGrant,IssuancePolicy from msg.security.certificates import csr_body,verify_csr,sign_certificate from msg.security.crypto import Ed25519Signer,key_id -from msg.storage.git import durable_write +from msg.atomic_file import durable_write def local_key(path, *, create=False): diff --git a/src/msg/client_custodial.py b/src/msg/client_custodial.py index f79430eb..5c162dd0 100644 --- a/src/msg/client_custodial.py +++ b/src/msg/client_custodial.py @@ -10,9 +10,9 @@ from msg.core.codec import canonical, digest, loads, wire from msg.core.errors import Failure, require from msg.security.age_keys import encryption_key_id, public_from_recipient, recipient_from_identity -from msg.security.custodial_migration import ack_statement, stage_statement -from msg.security.vault import client_upgrade_proof -from msg.storage.git import durable_write +from msg.security.custodial_protocol import ack_statement, stage_statement +from msg.security.custodial_protocol import client_upgrade_proof +from msg.atomic_file import durable_write def journal_for(state): diff --git a/src/msg/client_upgrade.py b/src/msg/client_upgrade.py index 03c5c924..b68e8872 100644 --- a/src/msg/client_upgrade.py +++ b/src/msg/client_upgrade.py @@ -8,7 +8,7 @@ from msg.core.codec import b64, canonical, loads, wire from msg.core.errors import Failure, require from msg.security.crypto import Ed25519Signer -from msg.storage.git import durable_write +from msg.atomic_file import durable_write @contextmanager diff --git a/src/msg/core/codec.py b/src/msg/core/codec.py index 133d30ef..c3cdd9a5 100644 --- a/src/msg/core/codec.py +++ b/src/msg/core/codec.py @@ -251,3 +251,11 @@ def decorate(cls): cls.__post_init__ = validate_record return dataclasses.dataclass(**kwargs)(cls) return decorate + + +def result_wire(result): + value=wire(result,compact=True) + for name in ('replayed','prefer_cli'): + if not value.get(name): + value.pop(name,None) + return value diff --git a/src/msg/core/packet.py b/src/msg/core/packet.py index 97cc4b29..aa4abc20 100644 --- a/src/msg/core/packet.py +++ b/src/msg/core/packet.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections.abc import Mapping import re -from msg.core.codec import decode, loads, canonical, wire +from msg.core.codec import decode, loads, canonical, wire, result_wire from msg.core.errors import require from msg.core.models import OperationRequest, OperationResult from msg.core.schemas import obj, IDENTIFIER, STRING, BYTES, REF, SIGNATURE @@ -61,11 +61,3 @@ def decode_result(value): # Compact results omit anonymous actor/subject; these required domain fields # are restored explicitly, not inferred from a transport connection. return decode(OperationResult,dict({'actor':None,'subject':None},**value)) - - -def result_wire(result): - value=wire(result,compact=True) - for name in ('replayed','prefer_cli'): - if not value.get(name): - value.pop(name,None) - return value diff --git a/src/msg/daemon.py b/src/msg/daemon.py index 5b0423c9..511ea1ce 100644 --- a/src/msg/daemon.py +++ b/src/msg/daemon.py @@ -1,6 +1,7 @@ """msgd: local installation, serving, workers, diagnostics and restricted SSH.""" from __future__ import annotations import argparse +import importlib.util import asyncio from datetime import UTC,datetime import json @@ -84,7 +85,7 @@ def parser(): init.add_argument('--data-dir',type=Path,default=Path('/var/lib/msgd')) init.add_argument('--service-url',default='https://msg.lmm.best') sub.add_parser('serve',help='Serve JSON/Markdown, operations, GraphQL and MCP') - hosted=sub.add_parser('hosting',help='Serve user-published files on the separate configured origin') + hosted=sub.add_parser('hosting',help='Serve user-published files with the read-only hosting runtime') hosted.add_argument('--listen',default='127.0.0.1');hosted.add_argument('--port',type=int,default=8043) worker=sub.add_parser('worker',help='Run the global durable effects/retention queue') worker.add_argument('--once',action='store_true') @@ -139,6 +140,11 @@ def parser(): def main(argv=None): args=parser().parse_args(argv) + server_modules=('starlette','uvicorn','psycopg','valkey','aiohttp','dns','graphql') + if any(importlib.util.find_spec(name) is None for name in server_modules): + print(canonical({'status':'error','error':{'code':'server_dependencies_required', + 'hint':"Install msg-lmm-best[server] before running server commands."}}).decode(),file=sys.stderr) + return 2 try: if args.command in {'doctor','selftest'}: from msg.admin.diagnostics import doctor,selftest diff --git a/src/msg/security/custodial_migration.py b/src/msg/security/custodial_migration.py index bcff8257..022d59af 100644 --- a/src/msg/security/custodial_migration.py +++ b/src/msg/security/custodial_migration.py @@ -13,6 +13,7 @@ from msg.core.errors import Failure, require from msg.core.models import ResourceRef, Signature from msg.security.crypto import key_id, verify +from msg.security.custodial_protocol import stage_statement, ack_statement from msg.security.age_keys import encryption_key_id, public_from_recipient from msg.security.backup_retirement import BackupRetirement, verified as verified_backup @@ -43,22 +44,6 @@ def inventory_commitment(subject, details, challenge): 'source_keys': details.get('source_keys', {})}) -def stage_statement(subject, arguments, request_id): - """Sign every decision field, not a caller-selected subset of the manifest.""" - return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, - 'request_id': request_id, - 'decision': {key: value for key, value in arguments.items() - if key != 'migration_ack'}} - - -def ack_statement(subject, challenge_id, inventory_digest, source, target, - plaintext_digest, request_id, *, method='rewrap'): - return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, - 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, - 'source': source, 'target': target, 'method': method, - 'plaintext_digest': plaintext_digest, 'request_id': request_id} - - def keyed_source(item, details): return dict(item, key_id=details.get('source_keys', {}).get( item['revision'], details['old_encryption_key_id'])) diff --git a/src/msg/security/custodial_protocol.py b/src/msg/security/custodial_protocol.py new file mode 100644 index 00000000..80821ca1 --- /dev/null +++ b/src/msg/security/custodial_protocol.py @@ -0,0 +1,49 @@ +"""Published custodial proof and statement bytes shared by both endpoints. + +This module has no server-held key, transaction, current-authority or retirement +state. Producing a proof does not verify authority or attest key destruction. +""" +from __future__ import annotations + +import hmac +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.primitives import hashes +from msg.core.codec import b64, canonical, unb64 +from msg.security.age_keys import private_from_identity + + +UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', + 'encryption_recipient','server_public','nonce','expires_at') + + +def _proof(shared,challenge): + context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} + key=HKDF(algorithm=hashes.SHA256(),length=32, + salt=unb64(context['nonce'],limit=64), + info=b'msg-custodial-upgrade-pop-v1').derive(shared) + return b64(hmac.digest(key,canonical(context),'sha256')) + + +def client_upgrade_proof(age_identity,challenge): + private=private_from_identity(age_identity) + shared=private.exchange(X25519PublicKey.from_public_bytes( + unb64(challenge['server_public'],limit=32))) + return _proof(shared,challenge) + + + +def stage_statement(subject, arguments, request_id): + """Sign every decision field, not a caller-selected subset of the manifest.""" + return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, + 'request_id': request_id, + 'decision': {key: value for key, value in arguments.items() + if key != 'migration_ack'}} + + +def ack_statement(subject, challenge_id, inventory_digest, source, target, + plaintext_digest, request_id, *, method='rewrap'): + return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, + 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, + 'source': source, 'target': target, 'method': method, + 'plaintext_digest': plaintext_digest, 'request_id': request_id} diff --git a/src/msg/security/vault.py b/src/msg/security/vault.py index baff9a43..c61f454e 100644 --- a/src/msg/security/vault.py +++ b/src/msg/security/vault.py @@ -2,7 +2,6 @@ from __future__ import annotations import os -import hmac import shutil import subprocess import tempfile @@ -11,12 +10,11 @@ from cryptography.exceptions import InvalidTag from cryptography.hazmat.primitives.ciphers.aead import AESGCM from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey -from cryptography.hazmat.primitives.kdf.hkdf import HKDF -from cryptography.hazmat.primitives import hashes from msg.core.codec import b64,canonical,unb64,wire from msg.core.errors import Failure,require -from msg.security.age_keys import recipient_from_identity,private_from_identity +from msg.security.age_keys import recipient_from_identity +from msg.security.custodial_protocol import UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof from msg.security.crypto import Ed25519Signer @@ -128,25 +126,6 @@ def open_upgrade_private(app,subject,challenge_id,nonce,ciphertext): raise Failure('custodial_upgrade_challenge_corrupt') from exc -UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', - 'encryption_recipient','server_public','nonce','expires_at') - - -def _proof(shared,challenge): - context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} - key=HKDF(algorithm=hashes.SHA256(),length=32, - salt=unb64(context['nonce'],limit=64), - info=b'msg-custodial-upgrade-pop-v1').derive(shared) - return b64(hmac.digest(key,canonical(context),'sha256')) - - -def client_upgrade_proof(age_identity,challenge): - private=private_from_identity(age_identity) - shared=private.exchange(X25519PublicKey.from_public_bytes( - unb64(challenge['server_public'],limit=32))) - return _proof(shared,challenge) - - def server_upgrade_proof(app,subject,challenge,nonce,ciphertext): from msg.security.age_keys import public_from_recipient private=open_upgrade_private(app,subject,challenge['challenge_id'],nonce,ciphertext) diff --git a/src/msg/storage/git.py b/src/msg/storage/git.py index 8c0223eb..865e38be 100644 --- a/src/msg/storage/git.py +++ b/src/msg/storage/git.py @@ -17,26 +17,7 @@ from msg.core.codec import canonical, digest, loads from msg.core.errors import Failure, require from msg.core.models import BlobRef - - -def durable_write(path: Path, data: bytes, mode: int = 0o600): - path.parent.mkdir(parents=True,exist_ok=True) - fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) - try: - os.fchmod(fd,mode) - with os.fdopen(fd,'wb') as stream: - stream.write(data) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary,path) - directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) - try: - os.fsync(directory) - finally: - os.close(directory) - finally: - if os.path.exists(temporary): - os.unlink(temporary) +from msg.atomic_file import durable_write class LFSObjectStore: diff --git a/src/msg/transports/client.py b/src/msg/transports/client.py index dee3f632..546fbcf2 100644 --- a/src/msg/transports/client.py +++ b/src/msg/transports/client.py @@ -163,7 +163,7 @@ class MCPHTTPTransport(HTTPTransport): async def call(self, request): self._secure_delivery(request) await self._effect(request.operation) - from msg.transports.mcp import PROTOCOL_VERSION + from msg.transports.mcp_protocol import PROTOCOL_VERSION tool_name = request.operation if request.contract_version==1 else ( f'{request.operation}@{request.contract_version}') value = await self._json('POST','/-/mcp',body={'jsonrpc':'2.0','id':request.request_id, diff --git a/src/msg/transports/mcp.py b/src/msg/transports/mcp.py index 8474f57e..aa81f77d 100644 --- a/src/msg/transports/mcp.py +++ b/src/msg/transports/mcp.py @@ -9,11 +9,10 @@ from msg import __version__ from msg.core.codec import canonical,digest,loads from msg.core.errors import Failure,require -from msg.core.executor import result_wire +from msg.core.codec import result_wire from msg.transports.packet import decode_packet,REQUEST_SCHEMA,RESULT_SCHEMA -PROTOCOL_VERSION='2025-11-25' -SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) +from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS class MCPServer: diff --git a/src/msg/transports/mcp_protocol.py b/src/msg/transports/mcp_protocol.py new file mode 100644 index 00000000..7eca1c5b --- /dev/null +++ b/src/msg/transports/mcp_protocol.py @@ -0,0 +1,4 @@ +"""Published MCP negotiation versions; no server or client runtime imports.""" + +PROTOCOL_VERSION='2025-11-25' +SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) diff --git a/tests/test_client_boundary.py b/tests/test_client_boundary.py new file mode 100644 index 00000000..02e46139 --- /dev/null +++ b/tests/test_client_boundary.py @@ -0,0 +1,123 @@ +"""Public helper ownership, unchanged bytes and supported client/server installs.""" +from __future__ import annotations + +import hashlib +import hmac +import importlib.util +from pathlib import Path +import subprocess +import sys +import tomllib + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def test_result_serialization_has_one_protocol_owner(): + from msg.core.codec import result_wire + from msg.core.executor import result_wire as compatibility_export + from msg.core.models import OperationResult + from msg.cli import result_wire as client_export + + assert result_wire is compatibility_export is client_export + assert result_wire.__module__ == 'msg.core.codec' + result = OperationResult(request_id='r', operation='content.read', status='ok', actor=None, + subject=None, resources=(), data={'text': 'plain'}) + value = result_wire(result) + assert value['data'] == {'text': 'plain'} + assert 'replayed' not in value and 'prefer_cli' not in value + + +def test_atomic_file_owner_keeps_compatibility_and_failed_replace_protection(tmp_path, monkeypatch): + from msg.atomic_file import durable_write + from msg.storage.git import durable_write as compatibility_export + import msg.atomic_file as atomic + + assert durable_write is compatibility_export + path = tmp_path / 'state' / 'identity' + durable_write(path, b'first') + assert path.read_bytes() == b'first' and path.stat().st_mode & 0o777 == 0o600 + def fail_replace(*args): + raise OSError('simulated replace refusal') + monkeypatch.setattr(atomic.os, 'replace', fail_replace) + with pytest.raises(OSError, match='simulated replace refusal'): + durable_write(path, b'not committed') + assert path.read_bytes() == b'first' + assert list(path.parent.iterdir()) == [path] + + +def test_custodial_proof_and_statements_are_shared_without_vault_state(): + from msg.security.custodial_protocol import ( + UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof, stage_statement, ack_statement) + from msg.security.vault import client_upgrade_proof as old_proof + from msg.security.custodial_migration import stage_statement as old_stage, ack_statement as old_ack + from msg.core.codec import b64, canonical + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + assert client_upgrade_proof is old_proof + assert stage_statement is old_stage and ack_statement is old_ack + assert client_upgrade_proof.__module__ == 'msg.security.custodial_protocol' + expected_fields = ('subject_id', 'challenge_id', 'request_id', 'handle', 'public_key', + 'encryption_recipient', 'server_public', 'nonce', 'expires_at') + assert UPGRADE_CONTEXT_FIELDS == expected_fields + challenge = {name: name + '-fixture' for name in expected_fields} + challenge['nonce'] = b64(bytes(range(32))) + challenge['not_signed'] = 'not part of this published proof context' + context = {name: challenge[name] for name in expected_fields} + shared = bytes(range(32, 64)) + key = HKDF(algorithm=hashes.SHA256(), length=32, salt=bytes(range(32)), + info=b'msg-custodial-upgrade-pop-v1').derive(shared) + expected = b64(hmac.digest(key, canonical(context), 'sha256')) + assert _proof(shared, challenge) == expected + assert stage_statement('u', {'choice': 'rewrap', 'migration_ack': 'omit'}, 'r') == { + 'domain': 'msg-custodial-decision-v1', 'subject_id': 'u', 'request_id': 'r', + 'decision': {'choice': 'rewrap'}} + assert ack_statement('u', 'c', 'd', {'id': 'old'}, {'id': 'new'}, 'p', 'r') == { + 'domain': 'msg-custodial-history-ack-v1', 'subject_id': 'u', 'challenge_id': 'c', + 'inventory_digest': 'd', 'source': {'id': 'old'}, 'target': {'id': 'new'}, + 'method': 'rewrap', 'plaintext_digest': 'p', 'request_id': 'r'} + + +def test_mcp_client_uses_neutral_version_declaration(): + from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS + from msg.transports import mcp + assert PROTOCOL_VERSION == mcp.PROTOCOL_VERSION == '2025-11-25' + assert SUPPORTED_VERSIONS is mcp.SUPPORTED_VERSIONS + assert SUPPORTED_VERSIONS == frozenset({'2025-11-25', '2025-06-18', '2025-03-26'}) + + +def test_fresh_client_import_guard_covers_actual_four_transport_calls(): + completed = subprocess.run([sys.executable, str(ROOT / 'scripts/check_client_install.py')], + capture_output=True, text=True, timeout=30) + assert completed.returncode == 0, completed.stdout + completed.stderr + import json + report = json.loads(completed.stdout) + assert report['transports'] == ['graphql', 'http', 'mcp_http', 'path_get'] + assert report['server_implementation_imported'] is False + + +def test_install_metadata_separates_roles_but_dev_keeps_full_server(): + project = tomllib.loads((ROOT / 'pyproject.toml').read_text())['project'] + def name(requirement): + return requirement.split('>=')[0].split('[')[0] + server_names = {'starlette', 'uvicorn', 'psycopg', 'valkey', 'aiohttp', 'dnspython', 'graphql-core'} + assert not server_names & {name(value) for value in project['dependencies']} + extras = project['optional-dependencies'] + assert server_names <= {name(value) for value in extras['server']} + assert 'msg-lmm-best[server]' in extras['dev'] + assert project['scripts'] == {'msg': 'msg.cli:main', 'msgd': 'msg.daemon:main'} + + +def test_missing_server_dependencies_are_reported_before_local_state(tmp_path, monkeypatch, capsys): + import msg.daemon as daemon + original = importlib.util.find_spec + monkeypatch.setattr(importlib.util, 'find_spec', + lambda name, *args, **kwargs: None if name == 'psycopg' else original(name, *args, **kwargs)) + directory = tmp_path / 'not-created' + assert daemon.main(['--config-dir', str(directory), 'serve']) == 2 + captured = capsys.readouterr() + assert 'server_dependencies_required' in captured.err + assert 'msg-lmm-best[server]' in captured.err + assert not directory.exists()