From aee565df7be2657c449bc16226db8f767de34acf Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:51:38 +0400 Subject: [PATCH 01/11] Mark declaration-only graph schemas as reconstructed Builds from before #1144 recorded their graph declaration but no run binding. Their execution therefore cannot be validated, and they have no Orrery artifact. Their declared graph is still exact, and is worth showing, provided nothing presents it as recorded execution. microcosm.graph.reconstruction projects the recorded declaration bytes alone. It stores a microcosm.reconstruction marker (protocol microcosm.graph.reconstruction.v1) in the schema's producer extensions. The marker names the source revision and where it came from, and binds every input file by digest, with whether a digest the build itself recorded was matched. The marker lives in graph.schema.json, so any later projection of the saved schema keeps it, and Orrery documents carry it under metadata.microcosm.extensions. The declaration must reproduce its recorded bytes through the current serializer, so the schema always describes the file that was recorded, not a reinterpretation of it. The module adds new files only. It leaves orrery.py and orrery_evidence.py untouched and imports only the declaration, schema and serializer modules. A new Orrery contract script checks that Orrery's public parser accepts the result and finds no execution claims. Part of #1291. Co-Authored-By: Claude Opus 5.5 --- ...1291-orrery-reconstruction-marker.added.md | 1 + docs/agent-guide.md | 4 +- docs/orrery-adapter.md | 62 ++++ .../src/microcosm/graph/reconstruction.py | 227 +++++++++++++++ .../shared/test_graph_reconstruction.py | 264 ++++++++++++++++++ .../orrery-contract/verify-reconstruction.mjs | 23 ++ 6 files changed, 580 insertions(+), 1 deletion(-) create mode 100644 changelog.d/1291-orrery-reconstruction-marker.added.md create mode 100644 packages/microcosm-graph/src/microcosm/graph/reconstruction.py create mode 100644 packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py create mode 100644 tools/orrery-contract/verify-reconstruction.mjs diff --git a/changelog.d/1291-orrery-reconstruction-marker.added.md b/changelog.d/1291-orrery-reconstruction-marker.added.md new file mode 100644 index 0000000000..52e26303e9 --- /dev/null +++ b/changelog.d/1291-orrery-reconstruction-marker.added.md @@ -0,0 +1 @@ +Add `microcosm.graph.reconstruction`, which rebuilds a declaration-only graph schema from a build's recorded `graph.json` and marks it with the `microcosm.reconstruction` extension (protocol `microcosm.graph.reconstruction.v1`). The marker names the source revision and every input file by digest, and it travels in `graph.schema.json` into the Orrery document's metadata. A declaration that the current serializer would not reproduce byte for byte is refused, and no operation is presented as executed. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index ba5cca2799..a93b8d74c1 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -93,7 +93,9 @@ engine-free runner installs the supported public Orrery range and locked Node dependency set under `tools/orrery-contract/`; the pytest test generates a document through Microcosm's public Python API and requires Orrery's public parser to accept it. The recorded-evidence assertion in `test_graph_evidence.py` -uses the same pinned parser for groups, statuses, activities and artifacts. +uses the same pinned parser for groups, statuses, activities and artifacts, and +the one in `test_graph_reconstruction.py` uses it for the reconstruction marker +of a declaration-only document. These CI tests perform no browser rendering. Workspace tests use an in-memory telemetry emitter by default. The shared diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index b095093260..a0823d38a5 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -268,6 +268,68 @@ mismatches, missing required bytes and corrupt files refuse export. Unavailable gate-blocked outputs, omitted operations and unconfigured summary providers are explicit. Earlier phases not supplied to the exporter remain unknown. +## Reconstructed declarations + +A build from before #1144 recorded its graph declaration, but not the run +binding that execution export validates, so its execution cannot be shown. +`microcosm.graph.reconstruction` rebuilds that build's schema from the recorded +declaration alone and marks it as reconstructed: + +```python +from microcosm.graph.reconstruction import reconstructed_schema, reconstruction_marker + +marker = reconstruction_marker( + git_commit=recorded_commit, + git_dirty=False, + revision_origin="recorded", + inputs=inputs, # one {name, role, sha256, bytes, verification} per file read +) +schema = reconstructed_schema(declaration_bytes, marker) +``` + +The marker is the producer extension `microcosm.reconstruction`: + +```json +{ + "protocol": "microcosm.graph.reconstruction.v1", + "reconstructed": true, + "basis": "declaration_only", + "source_revision": { + "git_commit": "", + "git_dirty": false, + "origin": "recorded" + }, + "inputs": [ + {"name": "graph.json", "role": "declaration", "sha256": "", + "bytes": 4096, "verification": "no_recorded_digest"}, + {"name": "rowwise_candidate_manifest.json", "role": "code_pin", + "sha256": "", "bytes": 2048, "verification": "recorded_digest_matched"} + ], + "scope": "Rebuilt after the build from its recorded graph declaration. ..." +} +``` + +- **`source_revision`:** `origin` is `recorded` when the build wrote the + revision itself, and `caller_asserted` when the build recorded none and the + caller supplied one. `git_dirty` is `null` when the build did not record it. +- **`inputs`:** every file the reconstruction read, in name order, by digest + and size. `verification` is `recorded_digest_matched` when a digest the build + itself recorded for that file matched, and `no_recorded_digest` when the + build recorded none. Exactly one input is the declaration, and its digest + equals the schema's `graph_sha256`. +- **Exact bytes:** `reconstructed_schema` refuses a declaration that the + current serializer would not write byte for byte. The schema therefore + describes the recorded file, not a reinterpretation of it. +- **Where it travels:** the marker is saved in `graph.schema.json`, so any + later projection of the saved schema keeps it. Orrery documents show it at + `metadata.microcosm.extensions["microcosm.reconstruction"]`. + `validate_reconstruction(schema)` returns the marker, or `None` for a schema + that was not reconstructed. +- **No execution claims:** a reconstructed document has no activities, + artifacts, statuses or `metadata.execution`. No operation appears as + executed, cached or gated. CI parses one with Orrery's public parser + (`tools/orrery-contract/verify-reconstruction.mjs`). + ## Orrery evidence panels | Surface | Exported evidence | diff --git a/packages/microcosm-graph/src/microcosm/graph/reconstruction.py b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py new file mode 100644 index 0000000000..3f8fe424d7 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py @@ -0,0 +1,227 @@ +"""Declaration-only graph schemas rebuilt after a build from its recorded files. + +A build that recorded its graph declaration but no run binding cannot have its +execution verified, yet its declared graph is still exact. A reconstructed +schema projects that recorded declaration alone and carries a marker in its +producer extensions: the marker says the schema was rebuilt afterwards, names +the code revision that declared the graph, and binds every input file by +digest. Because the marker lives in ``graph.schema.json``, any later +projection of the saved schema keeps it, and Orrery documents show it under +``metadata.microcosm.extensions``. Nothing here presents an operation as +executed; no receipt, cache or gate state is attached. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Mapping, Sequence + +from .decl import compile_graph +from .schema import _plain_json, graph_schema +from .serialize import graph_from_json + +__all__ = [ + "DECLARATION_ONLY", + "DECLARATION_ROLE", + "GIT_COMMIT_PATTERN", + "INPUT_VERIFICATIONS", + "RECONSTRUCTION_EXTENSION", + "RECONSTRUCTION_PROTOCOL", + "REVISION_ORIGINS", + "reconstructed_schema", + "reconstruction_marker", + "validate_reconstruction", +] + +RECONSTRUCTION_EXTENSION = "microcosm.reconstruction" +RECONSTRUCTION_PROTOCOL = "microcosm.graph.reconstruction.v1" +DECLARATION_ONLY = "declaration_only" +DECLARATION_ROLE = "declaration" +# ``recorded``: the build wrote the revision itself. ``caller_asserted``: the +# build recorded none, and whoever ran the reconstruction supplied it. +REVISION_ORIGINS = frozenset({"recorded", "caller_asserted"}) +# Whether a digest the build itself recorded for the file was matched, or the +# build recorded none and the file is bound only by the digest listed here. +INPUT_VERIFICATIONS = frozenset({"recorded_digest_matched", "no_recorded_digest"}) +RECONSTRUCTION_SCOPE = ( + "Rebuilt after the build from its recorded graph declaration. Operations " + "carry no execution, cache or gate state; nothing here was observed running." +) +_MARKER_FIELDS = frozenset( + {"protocol", "reconstructed", "basis", "source_revision", "inputs", "scope"} +) +_REVISION_FIELDS = frozenset({"git_commit", "git_dirty", "origin"}) +_INPUT_FIELDS = frozenset({"name", "role", "sha256", "bytes", "verification"}) +# A full lowercase SHA-1 or SHA-256 git object name; abbreviations are refused. +GIT_COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?") +_SHA256 = re.compile(r"[0-9a-f]{64}") + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(f"Graph reconstruction: {message}.") + + +def _text(value: object, label: str) -> str: + _require(type(value) is str and bool(value.strip()), f"{label} must be text") + return value # type: ignore[return-value] + + +def _file_name(value: object) -> str: + name = _text(value, "input name") + _require( + name not in {".", ".."} + and not any(separator in name for separator in ("/", "\\")) + and not any(ord(character) < 32 for character in name), + "input names must be plain file names", + ) + return name + + +def _validated_marker(value: object) -> dict[str, object]: + marker = _plain_json(value) + _require(type(marker) is dict, "marker must be an object") + assert isinstance(marker, dict) + _require(set(marker) == _MARKER_FIELDS, "marker fields") + _require(marker["protocol"] == RECONSTRUCTION_PROTOCOL, "marker protocol") + _require(marker["reconstructed"] is True, "marker must say reconstructed") + _require(marker["basis"] == DECLARATION_ONLY, "marker basis") + _text(marker["scope"], "marker scope") + revision = marker["source_revision"] + _require( + type(revision) is dict and set(revision) == _REVISION_FIELDS, + "source revision fields", + ) + _require( + type(revision["git_commit"]) is str + and GIT_COMMIT_PATTERN.fullmatch(revision["git_commit"]) is not None, + "source revision must be a full lowercase git commit", + ) + _require( + type(revision["git_dirty"]) in (bool, type(None)), + "source revision dirty flag must be true, false or null", + ) + _require(revision["origin"] in REVISION_ORIGINS, "source revision origin") + inputs = marker["inputs"] + _require(type(inputs) is list and bool(inputs), "marker inputs") + names = [] + for item in inputs: + _require(type(item) is dict and set(item) == _INPUT_FIELDS, "input fields") + names.append(_file_name(item["name"])) + _text(item["role"], "input role") + _require( + type(item["sha256"]) is str + and _SHA256.fullmatch(item["sha256"]) is not None, + "input digests must be lowercase SHA-256", + ) + _require( + type(item["bytes"]) is int and item["bytes"] > 0, + "input sizes must be positive integers", + ) + _require( + item["verification"] in INPUT_VERIFICATIONS, "input verification state" + ) + _require(len(set(names)) == len(names), "duplicate input name") + _require(names == sorted(names), "inputs must be listed in name order") + _require( + sum(item["role"] == DECLARATION_ROLE for item in inputs) == 1, + "exactly one input must be the declaration", + ) + return marker + + +def _declaration_input(marker: Mapping[str, object]) -> Mapping[str, object]: + inputs = marker["inputs"] + assert isinstance(inputs, list) + return next(item for item in inputs if item["role"] == DECLARATION_ROLE) + + +def reconstruction_marker( + *, + git_commit: str, + git_dirty: bool | None, + revision_origin: str, + inputs: Sequence[Mapping[str, object]], +) -> dict[str, object]: + """Return a validated marker naming the revision and every input read. + + Each input is a mapping with ``name`` (a plain file name), ``role``, + ``sha256``, ``bytes`` and ``verification``; exactly one has role + ``declaration``. Inputs are listed in name order. + """ + + ordered = sorted( + (dict(item) for item in inputs), key=lambda item: str(item.get("name")) + ) + return _validated_marker( + { + "protocol": RECONSTRUCTION_PROTOCOL, + "reconstructed": True, + "basis": DECLARATION_ONLY, + "source_revision": { + "git_commit": git_commit, + "git_dirty": git_dirty, + "origin": revision_origin, + }, + "inputs": ordered, + "scope": RECONSTRUCTION_SCOPE, + } + ) + + +def validate_reconstruction(schema: Mapping[str, object]) -> dict[str, object] | None: + """Return a schema's validated reconstruction marker, or ``None`` if absent. + + The marker must name the declaration embedded in the schema: its + declaration input's digest equals the schema's ``graph_sha256``. + """ + + extensions = schema["extensions"] + _require(isinstance(extensions, Mapping), "schema extensions must be an object") + assert isinstance(extensions, Mapping) + if RECONSTRUCTION_EXTENSION not in extensions: + return None + marker = _validated_marker(extensions[RECONSTRUCTION_EXTENSION]) + _require( + _declaration_input(marker)["sha256"] == schema["graph_sha256"], + "marker does not name this declaration", + ) + return marker + + +def reconstructed_schema( + declaration: bytes, marker: Mapping[str, object] +) -> dict[str, object]: + """Project recorded declaration bytes into a marked compiler schema. + + ``declaration`` is the exact ``graph.json`` a build wrote. The marker must + name those bytes, and the current serializer must reproduce them exactly; + otherwise the projection would describe a different declaration from the + one that was recorded, and the declaration is refused. + """ + + _require(type(declaration) is bytes, "declaration must be bytes") + checked = _validated_marker(marker) + named = _declaration_input(checked) + declaration_sha256 = hashlib.sha256(declaration).hexdigest() + _require( + named["sha256"] == declaration_sha256 and named["bytes"] == len(declaration), + "marker does not name this declaration", + ) + try: + text = declaration.decode("utf-8") + except UnicodeDecodeError as error: + raise ValueError( + "Graph reconstruction: declaration must be UTF-8 JSON." + ) from error + schema = graph_schema( + compile_graph(graph_from_json(text)), + extensions={RECONSTRUCTION_EXTENSION: checked}, + ) + _require( + schema["graph_sha256"] == declaration_sha256, + "the declaration does not reproduce its recorded bytes through the " + "current serializer", + ) + return schema diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py new file mode 100644 index 0000000000..93bd1e95f6 --- /dev/null +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py @@ -0,0 +1,264 @@ +"""Declaration-only schemas rebuilt for builds that recorded no run binding.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import subprocess + +import pytest + +from microcosm.graph import graph_schema, graph_to_json +from microcosm.graph.canonical import canonical_json +from microcosm.graph.orrery import orrery_document_from_schema, orrery_json_from_schema +from microcosm.graph.reconstruction import ( + DECLARATION_ONLY, + RECONSTRUCTION_EXTENSION, + RECONSTRUCTION_PROTOCOL, + reconstructed_schema, + reconstruction_marker, + validate_reconstruction, +) +from microcosm.graph.schema import validate_graph_schema +from test_support.microcosm_graph.schema import ( + compiled_default_population_graph, + compiled_graph, +) +from test_support.paths import paths_for + +_TEST_PATHS = paths_for("microcosm-graph") +_RECONSTRUCTION_VERIFY = ( + _TEST_PATHS.repository / "tools" / "orrery-contract" / "verify-reconstruction.mjs" +) +RECORDED_COMMIT = "0123456789abcdef0123456789abcdef01234567" + + +def recorded_declaration() -> bytes: + """The bytes a build wrote to graph.json for its declared graph.""" + + return graph_to_json(compiled_graph().graph).encode("utf-8") + + +def input_record(name: str, role: str, payload: bytes, verification: str) -> dict: + return { + "name": name, + "role": role, + "sha256": hashlib.sha256(payload).hexdigest(), + "bytes": len(payload), + "verification": verification, + } + + +def marker_for(declaration: bytes, **revision_overrides) -> dict: + revision = { + "git_commit": RECORDED_COMMIT, + "git_dirty": False, + "revision_origin": "recorded", + **revision_overrides, + } + return reconstruction_marker( + **revision, + inputs=[ + input_record( + "rowwise_candidate_manifest.json", + "code_pin", + b'{"git_commit":"recorded"}', + "recorded_digest_matched", + ), + input_record( + "graph.json", "declaration", declaration, "no_recorded_digest" + ), + ], + ) + + +def test_reconstructed_schema_carries_the_marker_into_orrery_metadata(): + declaration = recorded_declaration() + marker = marker_for(declaration) + + schema = reconstructed_schema(declaration, marker) + + assert schema["extensions"] == {RECONSTRUCTION_EXTENSION: marker} + assert schema["graph_sha256"] == hashlib.sha256(declaration).hexdigest() + assert marker["protocol"] == RECONSTRUCTION_PROTOCOL + assert marker["reconstructed"] is True + assert marker["basis"] == DECLARATION_ONLY + assert marker["source_revision"] == { + "git_commit": RECORDED_COMMIT, + "git_dirty": False, + "origin": "recorded", + } + # Inputs are listed once each, in name order, whatever order they came in. + assert [item["name"] for item in marker["inputs"]] == [ + "graph.json", + "rowwise_candidate_manifest.json", + ] + assert marker["scope"].strip() + document = orrery_document_from_schema(schema) + carried = document["metadata"]["microcosm"]["extensions"][RECONSTRUCTION_EXTENSION] + assert carried == marker + + +def test_reconstructed_document_claims_no_execution(): + declaration = recorded_declaration() + + document = orrery_document_from_schema( + reconstructed_schema(declaration, marker_for(declaration)) + ) + + assert not {"activities", "artifacts", "receipts"} & document.keys() + assert all("statuses" not in node for node in document["nodes"]) + assert "execution" not in document["metadata"] + assert sum(node["kind"] == "operation" for node in document["nodes"]) == len( + compiled_graph().graph.nodes + ) + + +def test_declaration_that_does_not_reproduce_its_recorded_bytes_is_refused(): + # The same graph, re-indented: it parses, but the current serializer would + # not write these bytes, so the projection would not be the recorded file. + reformatted = json.dumps(json.loads(recorded_declaration()), indent=2).encode() + + with pytest.raises(ValueError, match="does not reproduce its recorded bytes"): + reconstructed_schema(reformatted, marker_for(reformatted)) + + +def test_marker_must_name_the_declaration_it_was_built_from(): + other_declaration = graph_to_json(compiled_default_population_graph().graph).encode( + "utf-8" + ) + + with pytest.raises(ValueError, match="does not name this declaration"): + reconstructed_schema(other_declaration, marker_for(recorded_declaration())) + + +def _short_commit(marker: dict) -> None: + marker["source_revision"]["git_commit"] = "0123456" + + +def _uppercase_commit(marker: dict) -> None: + marker["source_revision"]["git_commit"] = RECORDED_COMMIT.upper() + + +def _guessed_origin(marker: dict) -> None: + marker["source_revision"]["origin"] = "guessed" + + +def _textual_dirty_flag(marker: dict) -> None: + marker["source_revision"]["git_dirty"] = "no" + + +def _no_declaration_input(marker: dict) -> None: + for item in marker["inputs"]: + item["role"] = "code_pin" + + +def _two_declaration_inputs(marker: dict) -> None: + for item in marker["inputs"]: + item["role"] = "declaration" + + +def _duplicate_input_names(marker: dict) -> None: + marker["inputs"][1]["name"] = marker["inputs"][0]["name"] + + +def _unsorted_inputs(marker: dict) -> None: + marker["inputs"].reverse() + + +def _nested_input_path(marker: dict) -> None: + marker["inputs"][1]["name"] = "../rowwise_candidate_manifest.json" + + +def _unknown_verification(marker: dict) -> None: + marker["inputs"][1]["verification"] = "trusted" + + +def _negative_input_size(marker: dict) -> None: + marker["inputs"][1]["bytes"] = -1 + + +def _unknown_marker_field(marker: dict) -> None: + marker["recorded_execution"] = True + + +def _recorded_basis(marker: dict) -> None: + marker["basis"] = "recorded_execution" + + +def _not_reconstructed(marker: dict) -> None: + marker["reconstructed"] = False + + +def _other_protocol(marker: dict) -> None: + marker["protocol"] = "microcosm.graph.reconstruction.v0" + + +@pytest.mark.parametrize( + "corrupt", + [ + _short_commit, + _uppercase_commit, + _guessed_origin, + _textual_dirty_flag, + _no_declaration_input, + _two_declaration_inputs, + _duplicate_input_names, + _unsorted_inputs, + _nested_input_path, + _unknown_verification, + _negative_input_size, + _unknown_marker_field, + _recorded_basis, + _not_reconstructed, + _other_protocol, + ], + ids=lambda corrupt: corrupt.__name__.strip("_"), +) +def test_marker_rejects_malformed_provenance(corrupt): + declaration = recorded_declaration() + marker = copy.deepcopy(marker_for(declaration)) + corrupt(marker) + + with pytest.raises(ValueError, match="Graph reconstruction"): + reconstructed_schema(declaration, marker) + + +def test_saved_reconstructed_schema_keeps_its_marker_when_reprojected(): + declaration = recorded_declaration() + marker = marker_for(declaration, git_dirty=None, revision_origin="caller_asserted") + schema = reconstructed_schema(declaration, marker) + + # A later projection (a collector re-projecting stored inputs, say) reads + # only the saved graph.schema.json bytes; the marker travels inside them. + restored = validate_graph_schema(json.loads(canonical_json(schema))) + + assert validate_reconstruction(restored) == marker + assert orrery_json_from_schema(restored) == orrery_json_from_schema(schema) + assert validate_reconstruction(graph_schema(compiled_graph())) is None + relabelled = graph_schema( + compiled_default_population_graph(), + extensions={RECONSTRUCTION_EXTENSION: marker}, + ) + with pytest.raises(ValueError, match="does not name this declaration"): + validate_reconstruction(relabelled) + + +def test_public_orrery_parser_accepts_reconstructed_document(): + declaration = recorded_declaration() + payload = orrery_json_from_schema( + reconstructed_schema(declaration, marker_for(declaration)), + title="INVENTED · reconstructed declaration", + ) + + result = subprocess.run( + ["node", str(_RECONSTRUCTION_VERIFY)], + input=payload, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == "Orrery accepted the reconstructed declaration.\n" diff --git a/tools/orrery-contract/verify-reconstruction.mjs b/tools/orrery-contract/verify-reconstruction.mjs new file mode 100644 index 0000000000..e21447651a --- /dev/null +++ b/tools/orrery-contract/verify-reconstruction.mjs @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { parseGraphDocument } from '@axiom-foundation/orrery'; + +const chunks = []; +for await (const chunk of process.stdin) chunks.push(chunk); +const document = parseGraphDocument(JSON.parse(Buffer.concat(chunks).toString('utf8'))); +const marker = document.metadata.microcosm.extensions['microcosm.reconstruction']; +assert.equal(marker.protocol, 'microcosm.graph.reconstruction.v1'); +assert.equal(marker.reconstructed, true); +assert.equal(marker.basis, 'declaration_only'); +assert.match(marker.source_revision.git_commit, /^[0-9a-f]{40}([0-9a-f]{24})?$/); +assert.equal(marker.inputs.filter(input => input.role === 'declaration').length, 1); +assert.equal( + marker.inputs.find(input => input.role === 'declaration').sha256, + document.metadata.microcosm.graph_sha256, +); +assert.ok(document.nodes.some(node => node.kind === 'operation')); +// A declaration-only reconstruction never presents an operation as executed. +assert.ok(!document.nodes.some(node => node.statuses?.length)); +assert.ok(!document.activities?.length); +assert.ok(!document.artifacts?.length); +assert.ok(!('execution' in document.metadata)); +process.stdout.write('Orrery accepted the reconstructed declaration.\n'); From 82ae80d358ec8d6339de37f3a967cca363d4e665 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:51:49 +0400 Subject: [PATCH 02/11] Reconstruct Orrery publications for UK builds before run bindings UK dense and national builds from before #1144 can only get an Orrery artifact by being rerun, and their inputs often no longer allow that. They did write graph.json beside their outputs, and a code pin in rowwise_candidate_manifest.json. python -m microcosm.build.uk_runtime.orrery_reconstruction reads a past build's output directory, which it only reads. It writes a new directory with the normal publication layout (graph.schema.json, graph.orrery.json, orrery.upload.json) plus a summary report. The declaration is the build's own graph.json, never one re-derived from current code. The revision is the recorded pin; --revision must match it, or supplies one, marked caller-asserted, when the build recorded none. Inputs are checked against the digests build.json and sha256sums.txt recorded. A build that already records Orrery evidence is refused and pointed at python -m microcosm.graph.orrery. The report lists what was used, what is missing, the evidence found but not bound (run manifests, the artifact index, the content store) and warnings. Output is deterministic and is moved into place only once complete. Nothing is uploaded and no emitter service starts, so publication stays a separate step. Fixes #1291. Co-Authored-By: Claude Opus 5.5 --- .../1291-uk-orrery-reconstruction.added.md | 1 + docs/orrery-adapter.md | 61 +++ .../build/uk_runtime/orrery_reconstruction.py | 497 ++++++++++++++++++ .../uk/test_uk_orrery_reconstruction.py | 431 +++++++++++++++ 4 files changed, 990 insertions(+) create mode 100644 changelog.d/1291-uk-orrery-reconstruction.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py diff --git a/changelog.d/1291-uk-orrery-reconstruction.added.md b/changelog.d/1291-uk-orrery-reconstruction.added.md new file mode 100644 index 0000000000..2499f6c656 --- /dev/null +++ b/changelog.d/1291-uk-orrery-reconstruction.added.md @@ -0,0 +1 @@ +Add `python -m microcosm.build.uk_runtime.orrery_reconstruction`, which turns the output directory of a UK dense or national build from before #1144 into a declaration-only Orrery publication. It writes a new directory with the normal publication layout (`graph.schema.json`, `graph.orrery.json`, `orrery.upload.json`) and a summary report. The declaration comes from the build's recorded `graph.json` and the revision from its recorded code pin. Every input is checked against the digests the build recorded, builds that already record Orrery evidence are refused, and nothing is uploaded. diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index a0823d38a5..e04313f9e3 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -330,6 +330,67 @@ The marker is the producer extension `microcosm.reconstruction`: executed, cached or gated. CI parses one with Orrery's public parser (`tools/orrery-contract/verify-reconstruction.mjs`). +### UK builds before #1144 + +UK dense and national builds from before #1144 wrote `graph.json` beside their +outputs, and recorded their code revision in `rowwise_candidate_manifest.json`. +This command turns such a build's output directory into a reconstructed +publication: + +```sh +uv run python -m microcosm.build.uk_runtime.orrery_reconstruction \ + --build path/to/past-build \ + --output path/to/reconstructed-publication +``` + +The declaration is the build's own `graph.json`, never one re-derived from +current code. The declaration format has not changed since the UK graph driver, +so a recorded file reproduces byte for byte; one that does not is refused. + +The build directory is only read. The command: + +- **Reads:** `graph.json`, which is required, and also `build.json`, + `rowwise_candidate_manifest.json` and `sha256sums.txt` where present. Links, + devices and inputs over 32 MiB are refused. +- **Checks recorded digests:** a file whose bytes disagree with a digest the + build recorded for it is refused. `build.json` binds the candidate manifest, + and `sha256sums.txt` binds the files it lists. +- **Resolves the revision:** it takes the candidate manifest's `git_commit` + and `git_dirty`, which must agree with the copy under `identity.code`. + `--revision` must match that commit, or supplies one when the build recorded + none; that revision is marked `caller_asserted`. With neither, it refuses. +- **Refuses later builds:** a build that records Orrery evidence + (`graph.schema.json` or `execution.evidence.json`) is exported with + `python -m microcosm.graph.orrery` instead. +- **Refuses missing declarations:** a directory without `graph.json` is refused, + and the refusal names the build records that were found. Staged bundles never + carried `graph.json`, and builds from before the UK graph driver never wrote + it. + +It writes a new directory with the layout of a normal publication, and moves the +directory into place only after every file is written: + +| File | Contents | +| --- | --- | +| `graph.schema.json` | Compiler schema of the recorded declaration, with the reconstruction marker | +| `graph.orrery.json` | Orrery document, titled `UK · reconstructed declaration ()` | +| `orrery.upload.json` | Publication inventory (roles `graph` and `schema`); its publication ID is derived from the files' digests | +| `orrery.reconstruction.json` | Summary report; not in the inventory | + +The summary report, of which the command prints a short version, lists: + +- the inputs and how each was verified; +- the expected files that are missing; +- evidence found but not bound to operations: run manifests, + `evidence-index.json`, `operations.json` and the content store; +- warnings, for a dirty or unrecorded working tree, a caller-asserted revision, + or a declaration the build recorded no digest for. Before #1144, the last one + applies to every build. + +Output is deterministic: reconstructing the same build twice gives identical +bytes and the same publication ID. The command uploads nothing and starts no +emitter service, so publishing the directory remains a separate step. + ## Orrery evidence panels | Surface | Exported evidence | diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py new file mode 100644 index 0000000000..aafa750a29 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py @@ -0,0 +1,497 @@ +"""Rebuild a declaration-only Orrery publication for a UK build before #1144. + +UK dense and national builds from before microcosm#1144 wrote their graph +declaration (``graph.json``) and their code pin, but no run binding, so their +recorded execution cannot be validated. This module projects the recorded +declaration alone into the normal publication layout, marked as reconstructed +by :mod:`microcosm.graph.reconstruction`. It reads only the build directory, +executes no kernel and uploads nothing: the publication directory is submitted +later, if at all, through the usual publication path. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import stat +import uuid +from collections.abc import Mapping +from pathlib import Path +from typing import Any, NoReturn + +from microcosm.build.staging_dataset import SHA256SUMS_FILENAME, parse_sha256sums +from microcosm.graph.canonical import canonical_json +from microcosm.graph.orrery import orrery_json_from_schema +from microcosm.graph.reconstruction import ( + DECLARATION_ONLY, + DECLARATION_ROLE, + GIT_COMMIT_PATTERN, + reconstructed_schema, + reconstruction_marker, +) + +from .orrery_publication import INVENTORY_NAME +from .rowwise_cli import MANIFEST_FILENAME, json_text + +__all__ = [ + "GRAPH_NAME", + "REPORT_NAME", + "REPORT_PROTOCOL", + "SCHEMA_NAME", + "main", + "reconstruct_uk_publication", + "summary_text", +] + +DECLARATION_NAME = "graph.json" +COMPLETION_NAME = "build.json" +SCHEMA_NAME = "graph.schema.json" +EVIDENCE_INDEX_NAME = "execution.evidence.json" +GRAPH_NAME = "graph.orrery.json" +REPORT_NAME = "orrery.reconstruction.json" +REPORT_PROTOCOL = "microcosm.uk.orrery-reconstruction-report.v1" +PUBLICATION_ID_PREFIX = "reconstructed-" +# The bound the Orrery command applies to a declaration or schema input. +_MAX_INPUT_BYTES = 32 * 1024 * 1024 +# Every build file the reconstruction reads, with its role in the marker. +_INPUT_ROLES = { + DECLARATION_NAME: DECLARATION_ROLE, + COMPLETION_NAME: "completion_record", + MANIFEST_FILENAME: "code_pin", + SHA256SUMS_FILENAME: "digest_list", +} +_ABSENT_REASONS = { + COMPLETION_NAME: ( + "completion record; without it the candidate manifest's recorded " + "digest cannot be checked" + ), + MANIFEST_FILENAME: "candidate manifest, where the build recorded its code revision", + SHA256SUMS_FILENAME: "digest list, written only when the build staged its bundle", +} +# Evidence a pre-#1144 build left that a declaration-only reconstruction finds +# but deliberately does not bind to operations. +_UNUSED_REASONS = { + ".graph-store": ( + "content store; with no run binding there is nothing to verify it against" + ), + "evidence-index.json": ( + "index of materialized artifacts; not linked to operations in a " + "declaration-only reconstruction" + ), + "operations.json": ( + "operation inventory written beside the declaration; not needed to project it" + ), +} +_RUN_MANIFEST_SUFFIX = ".graph.json" +_RUN_MANIFEST_REASON = ( + "run manifest; its receipts are not bound to operations in a " + "declaration-only reconstruction" +) +_BUILD_FIELDS = ("build_kind", "release_role", "release_id", "build_id", "created_at") + + +def _refuse(message: str) -> NoReturn: + raise ValueError(f"UK Orrery reconstruction: {message}.") + + +def _require(condition: bool, message: str) -> None: + if not condition: + _refuse(message) + + +def _read_input(directory: Path, name: str) -> bytes: + """Read one regular input file, refusing links, devices and oversized files.""" + + path = directory / name + _require(not path.is_symlink(), f"{name} must be a regular file, not a link") + descriptor = os.open( + path, os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) + ) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + _require(stat.S_ISREG(info.st_mode), f"{name} must be a regular file") + _require(info.st_size <= _MAX_INPUT_BYTES, f"{name} exceeds the input limit") + payload = stream.read(_MAX_INPUT_BYTES + 1) + _require(len(payload) <= _MAX_INPUT_BYTES, f"{name} exceeds the input limit") + return payload + + +def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + _require(key not in result, f"duplicate JSON key {key!r}") + result[key] = value + return result + + +def _reject_constant(constant: str) -> NoReturn: + _refuse(f"non-finite JSON number {constant}") + + +def _json_object(payload: bytes, name: str) -> dict[str, object]: + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError( + f"UK Orrery reconstruction: {name} is not valid JSON." + ) from error + _require(type(value) is dict, f"{name} must contain a JSON object") + return value + + +def _recorded_digests( + payloads: Mapping[str, bytes], records: Mapping[str, dict[str, object]] +) -> dict[str, list[tuple[str, str, int | None]]]: + """Digests the build itself recorded, as ``name -> [(recorder, sha256, size)]``.""" + + recorded: dict[str, list[tuple[str, str, int | None]]] = {} + if SHA256SUMS_FILENAME in payloads: + listing = payloads[SHA256SUMS_FILENAME].decode("utf-8") + for digest, name in parse_sha256sums(listing): + recorded.setdefault(name, []).append((SHA256SUMS_FILENAME, digest, None)) + completion = records.get(COMPLETION_NAME, {}) + manifest_record = completion.get("rowwise_candidate_manifest") + if isinstance(manifest_record, dict) and "sha256" in manifest_record: + _require( + manifest_record.get("filename", MANIFEST_FILENAME) == MANIFEST_FILENAME, + f"{COMPLETION_NAME} records an unexpected candidate manifest name", + ) + size = manifest_record.get("size_bytes") + recorded.setdefault(MANIFEST_FILENAME, []).append( + ( + COMPLETION_NAME, + str(manifest_record["sha256"]), + size if type(size) is int else None, + ) + ) + return recorded + + +def _recorded_code(manifest: Mapping[str, object]) -> tuple[str | None, bool | None]: + """The code pin a candidate manifest records, top level and under identity.""" + + top_level = { + key: manifest[key] for key in ("git_commit", "git_dirty") if key in manifest + } + identity = manifest.get("identity") + code = identity.get("code") if isinstance(identity, dict) else None + nested = ( + {key: code[key] for key in ("git_commit", "git_dirty") if key in code} + if isinstance(code, dict) + else {} + ) + _require( + all(top_level[key] == nested[key] for key in top_level.keys() & nested.keys()), + f"{MANIFEST_FILENAME} records two different code revisions", + ) + pin = {**nested, **top_level} + commit, dirty = pin.get("git_commit"), pin.get("git_dirty") + _require( + commit is None + or (type(commit) is str and GIT_COMMIT_PATTERN.fullmatch(commit) is not None), + f"{MANIFEST_FILENAME} records a malformed code revision", + ) + _require( + type(dirty) in (bool, type(None)), + f"{MANIFEST_FILENAME} records a malformed working-tree flag", + ) + return commit, dirty # type: ignore[return-value] + + +def _source_revision( + build: Path, recorded: tuple[str | None, bool | None], revision: str | None +) -> tuple[str, bool | None, str]: + """Resolve ``(git_commit, git_dirty, origin)`` from the build and the caller.""" + + if revision is not None: + _require( + GIT_COMMIT_PATTERN.fullmatch(revision) is not None, + "--revision must be a full lowercase git commit", + ) + recorded_commit, recorded_dirty = recorded + if recorded_commit is not None: + _require( + revision in (None, recorded_commit), + f"--revision {revision} does not match the recorded revision " + f"{recorded_commit}", + ) + return recorded_commit, recorded_dirty, "recorded" + _require( + revision is not None, + f"{build} records no code revision; pass --revision with the commit the " + "build ran from", + ) + assert revision is not None + return revision, None, "caller_asserted" + + +def _build_identity(*records: Mapping[str, object]) -> dict[str, str]: + """Identifiers the build recorded, from the first record that names each.""" + + identity = {} + for field in _BUILD_FIELDS: + value = next((record[field] for record in records if field in record), None) + if type(value) is str: + identity[field] = value + return identity + + +def _unused_evidence(build: Path) -> list[dict[str, str]]: + found = [] + for name in sorted(os.listdir(build)): + if name in _UNUSED_REASONS: + found.append({"name": name, "reason": _UNUSED_REASONS[name]}) + elif name.endswith(_RUN_MANIFEST_SUFFIX): + found.append({"name": name, "reason": _RUN_MANIFEST_REASON}) + return found + + +def _published_file(name: str, role: str, payload: bytes) -> dict[str, object]: + return { + "name": name, + "role": role, + "bytes": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + + +def _write_publication(output: Path, files: Mapping[str, bytes]) -> None: + """Write every file into a hidden sibling, then move it into place whole.""" + + output.parent.mkdir(parents=True, exist_ok=True) + staging = output.parent / f".{output.name}.{uuid.uuid4().hex}.partial" + staging.mkdir() + try: + for name, payload in files.items(): + with (staging / name).open("xb") as stream: + stream.write(payload) + _require(not os.path.lexists(output), f"output {output} already exists") + os.rename(staging, output) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + + +def reconstruct_uk_publication( + build: Path, output: Path, *, revision: str | None = None +) -> dict[str, object]: + """Write a declaration-only publication for ``build`` into a new ``output``. + + ``build`` is a UK build's output directory from before run bindings; it is + only read. ``revision`` must equal the code revision the build recorded, + or supplies one (marked as caller-asserted) when the build recorded none. + The returned summary report is also written as ``orrery.reconstruction.json`` + beside the publication files, outside the upload inventory. + """ + + build, output = Path(build), Path(output) + _require(build.is_dir(), f"build directory {build} does not exist") + _require(not os.path.lexists(output), f"output {output} already exists") + recorded_evidence = [ + name + for name in (SCHEMA_NAME, EVIDENCE_INDEX_NAME) + if os.path.lexists(build / name) + ] + _require( + not recorded_evidence, + f"{build} records Orrery evidence ({', '.join(recorded_evidence)}); export " + "it with python -m microcosm.graph.orrery instead", + ) + present = [name for name in sorted(_INPUT_ROLES) if os.path.lexists(build / name)] + _require( + DECLARATION_NAME in present, + f"{build} has no recorded graph declaration ({DECLARATION_NAME}); staged " + "bundles and builds from before the UK graph driver never wrote one. " + + (f"Found {', '.join(present)}" if present else "No build record found"), + ) + payloads = {name: _read_input(build, name) for name in present} + records = { + name: _json_object(payloads[name], name) + for name in (COMPLETION_NAME, MANIFEST_FILENAME) + if name in payloads + } + recorded = _recorded_digests(payloads, records) + verified = {} + for name, payload in payloads.items(): + for recorder, digest, size in recorded.get(name, []): + _require( + hashlib.sha256(payload).hexdigest() == digest + and size in (None, len(payload)), + f"{name} does not match the digest recorded in {recorder}", + ) + verified[name] = bool(recorded.get(name)) + manifest = records.get(MANIFEST_FILENAME, {}) + git_commit, git_dirty, origin = _source_revision( + build, _recorded_code(manifest), revision + ) + marker = reconstruction_marker( + git_commit=git_commit, + git_dirty=git_dirty, + revision_origin=origin, + inputs=[ + { + "name": name, + "role": _INPUT_ROLES[name], + "sha256": hashlib.sha256(payload).hexdigest(), + "bytes": len(payload), + "verification": "recorded_digest_matched" + if verified[name] + else "no_recorded_digest", + } + for name, payload in payloads.items() + ], + ) + schema = reconstructed_schema(payloads[DECLARATION_NAME], marker) + country = str(schema["country"]) + schema_bytes = canonical_json(schema) + graph_bytes = orrery_json_from_schema( + schema, + title=f"{country.upper()} · reconstructed declaration ({git_commit[:12]})", + ).encode("utf-8") + published = [ + _published_file(GRAPH_NAME, "graph", graph_bytes), + _published_file(SCHEMA_NAME, "schema", schema_bytes), + ] + # Derived from the published bytes, so reconstructing the same build twice + # yields the same publication rather than a second one. + publication_id = ( + PUBLICATION_ID_PREFIX + + hashlib.sha256(canonical_json(published)).hexdigest()[:32] + ) + inventory = {"version": 1, "publication_id": publication_id, "files": published} + warnings = [] + if origin == "caller_asserted": + warnings.append( + "The build records no code revision; the revision was supplied with " + "--revision and cannot be checked against the build." + ) + elif git_dirty is True: + warnings.append( + "The build ran from a working tree with uncommitted changes, so its " + "recorded revision does not fully identify the code that declared " + "the graph." + ) + elif git_dirty is None: + warnings.append("The build did not record whether its working tree was clean.") + if not verified[DECLARATION_NAME]: + warnings.append( + f"The build recorded no digest for {DECLARATION_NAME}; it is bound only " + "by the digest in the reconstruction marker." + ) + graph: Any = schema["graph"] + report: dict[str, object] = { + "protocol": REPORT_PROTOCOL, + "status": "reconstructed", + "basis": DECLARATION_ONLY, + "publication_id": publication_id, + "published_files": [item["name"] for item in published] + [INVENTORY_NAME], + "source_revision": marker["source_revision"], + "build": _build_identity(manifest, records.get(COMPLETION_NAME, {})), + "declaration": { + "name": DECLARATION_NAME, + "country": country, + "graph_sha256": schema["graph_sha256"], + "operations": len(graph["nodes"]), + "sources": len(graph["sources"]), + }, + "inputs": marker["inputs"], + "missing": [ + {"name": name, "reason": _ABSENT_REASONS[name]} + for name in sorted(_ABSENT_REASONS) + if name not in payloads + ], + "unused_evidence": _unused_evidence(build), + "warnings": warnings, + } + _write_publication( + output, + { + SCHEMA_NAME: schema_bytes, + GRAPH_NAME: graph_bytes, + INVENTORY_NAME: canonical_json(inventory), + REPORT_NAME: json_text(report).encode("utf-8"), + }, + ) + return report + + +def _count(number: int, noun: str) -> str: + return f"{number} {noun}{'' if number == 1 else 's'}" + + +def summary_text(report: Mapping[str, Any], output: Path) -> str: + """A short human-readable account of one reconstruction.""" + + revision, declaration = report["source_revision"], report["declaration"] + origin = ( + "recorded by the build" + if revision["origin"] == "recorded" + else "asserted with --revision" + ) + lines = [ + f"Wrote a reconstructed Orrery publication (declaration only) to {output}", + f" revision: {revision['git_commit']} ({origin})", + f" declaration: {declaration['name']}, " + f"{_count(declaration['operations'], 'operation')}, " + f"{_count(declaration['sources'], 'source')}", + " inputs: " + + ", ".join( + f"{item['name']} ({item['verification'].replace('_', ' ')})" + for item in report["inputs"] + ), + ] + for label, key in (("missing", "missing"), ("found, not bound", "unused_evidence")): + if report[key]: + names = ", ".join(item["name"] for item in report[key]) + lines.append(f" {label}: {names}") + lines.extend(f" warning: {warning}" for warning in report["warnings"]) + lines.append(f" report: {output / REPORT_NAME}") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + """Write a declaration-only Orrery publication for a UK build before #1144.""" + + parser = argparse.ArgumentParser( + prog="python -m microcosm.build.uk_runtime.orrery_reconstruction", + description=main.__doc__, + ) + parser.add_argument( + "--build", + type=Path, + required=True, + help="the past build's output directory, which is only read", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="new publication directory; it must not exist", + ) + parser.add_argument( + "--revision", + help=( + "full git commit the build ran from; it must match the revision the " + "build recorded, or supplies one when the build recorded none" + ), + ) + args = parser.parse_args(argv) + try: + report = reconstruct_uk_publication( + args.build, args.output, revision=args.revision + ) + except (OSError, ValueError, RecursionError) as error: + parser.error(str(error)) + print(summary_text(report, args.output)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py new file mode 100644 index 0000000000..bc283be490 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py @@ -0,0 +1,431 @@ +"""Declaration-only Orrery publications for UK builds from before run bindings. + +Fixtures reproduce the files a UK dense or national build wrote before +microcosm#1144: the declaration, its operation inventory and run manifests, +the candidate manifest carrying the code pin, the completion record binding +that manifest by digest and, for a staged build, ``sha256sums.txt``. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +from pathlib import Path + +import pytest +from microcosm_emitter.orrery.service.queue import publication_inventory + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime.graph import uk_spine_endpoint, uk_spine_graph +from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, uk_full_graph +from microcosm.build.uk_runtime.graph_calibration import UKGraphCalibrationConfig +from microcosm.build.uk_runtime.graph_evidence import add_uk_spine_gate_nodes +from microcosm.build.uk_runtime.graph_terminal import append_uk_full_gate_nodes +from microcosm.build.uk_runtime.orrery_publication import INVENTORY_NAME +from microcosm.build.uk_runtime.orrery_reconstruction import ( + GRAPH_NAME, + REPORT_NAME, + SCHEMA_NAME, + main, + reconstruct_uk_publication, +) +from microcosm.build.uk_runtime.rowwise_cli import MANIFEST_FILENAME, json_text +from microcosm.graph import Graph, Node, Owned, SourceRef, StructuralDelta +from microcosm.graph.canonical import canonical_json +from microcosm.graph.reconstruction import ( + RECONSTRUCTION_EXTENSION, + validate_reconstruction, +) +from microcosm.graph.schema import validate_graph_schema +from microcosm.graph.serialize import graph_to_json +from test_support.paths import paths_for + +_RECONSTRUCTION_VERIFY = ( + paths_for("microcosm-build").repository + / "tools" + / "orrery-contract" + / "verify-reconstruction.mjs" +) +RECORDED_COMMIT = "4bf7af406a1b2c3d4e5f60718293a4b5c6d7e8f9" +OTHER_COMMIT = "7f235941c0000000000000000000000000000000" + + +def sha256_hex(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def pre_1144_declaration() -> Graph: + return Graph( + "uk", + (SourceRef("uk_spine", "raw-bytes-v1", "Recorded FRS spine"),), + ( + Node( + "uk.full.spine_checkpoint", + "fixture.checkpoint@1", + sources=("uk_spine",), + structural=StructuralDelta.CREATE, + outputs=(Owned("household", "household_weight", "float64"),), + description="Load the recorded spine checkpoint", + ), + Node( + "uk.full.gates.calibrated", + "fixture.gates@1", + population="uk.full.spine_checkpoint", + description="Evaluate the calibrated gate battery", + ), + ), + ) + + +def write_pre_1144_build( + directory: Path, + *, + graph: Graph | None = None, + git_commit: str | None = RECORDED_COMMIT, + git_dirty: bool | None = False, + release_role: str = "dense", + staged: bool = True, +) -> Path: + """Write the files a UK build left in its output directory before #1144.""" + + directory.mkdir(parents=True) + declared = pre_1144_declaration() if graph is None else graph + (directory / "graph.json").write_bytes(graph_to_json(declared).encode()) + (directory / "operations.json").write_bytes( + canonical_json([{"id": node.id} for node in declared.nodes]) + ) + (directory / "build.graph.json").write_text('{"schema_version":3}') + (directory / "preflight.graph.json").write_text('{"schema_version":3}') + (directory / "evidence-index.json").write_text("{}") + (directory / ".graph-store").mkdir() + (directory / "candidate.h5").write_bytes(b"licensed population") + code = {"git_commit": git_commit, "git_dirty": git_dirty} + manifest = { + "schema_version": 4, + "build_kind": "uk_rowwise_calibrated_candidate", + "release_role": release_role, + "release_id": f"microcosm-uk-2025-{release_role}", + "created_at": "2026-10-01T12:00:00+00:00", + **code, + "identity": {"code": code}, + "outputs": {"dataset": {"path": "candidate.h5"}}, + } + manifest_bytes = json_text(manifest).encode() + (directory / MANIFEST_FILENAME).write_bytes(manifest_bytes) + (directory / "build.json").write_bytes( + canonical_json( + { + "kind": "uk_full_build_completion", + "release_role": release_role, + "rowwise_candidate_manifest": { + "filename": MANIFEST_FILENAME, + "sha256": sha256_hex(manifest_bytes), + "size_bytes": len(manifest_bytes), + "note": "staging receipts are appended after publication", + }, + } + ) + ) + if staged: + (directory / "sha256sums.txt").write_text( + f"{sha256_hex(b'licensed population')} candidate.h5\n" + f"{sha256_hex(manifest_bytes)} {MANIFEST_FILENAME}\n" + ) + return directory + + +def read_json(path: Path) -> dict: + return json.loads(path.read_bytes()) + + +def inputs_by_name(marker: dict) -> dict[str, dict]: + return {item["name"]: item for item in marker["inputs"]} + + +def test_dense_build_reconstructs_the_normal_publication_layout(tmp_path): + build = write_pre_1144_build(tmp_path / "build") + output = tmp_path / "publication" + + reconstruct_uk_publication(build, output) + + assert sorted(path.name for path in output.iterdir()) == sorted( + [GRAPH_NAME, SCHEMA_NAME, INVENTORY_NAME, REPORT_NAME] + ) + inventory = read_json(output / INVENTORY_NAME) + # The emitter recomputes the inventory before it queues a directory; the + # reconstructed one must be exactly what it would compute. + assert inventory == publication_inventory( + output, + {GRAPH_NAME: "graph", SCHEMA_NAME: "schema"}, + publication_id=inventory["publication_id"], + ) + schema = validate_graph_schema(read_json(output / SCHEMA_NAME)) + assert (output / SCHEMA_NAME).read_bytes() == canonical_json(schema) + document = read_json(output / GRAPH_NAME) + assert document["schemaVersion"] == "graph-explorer/v1" + assert document["metadata"]["microcosm"]["graph_sha256"] == sha256_hex( + (build / "graph.json").read_bytes() + ) + assert "reconstructed declaration" in document["title"] + assert not {"activities", "artifacts"} & document.keys() + assert all("statuses" not in node for node in document["nodes"]) + # Nothing from the build other than the declaration's projection is copied. + assert not (output / "candidate.h5").exists() + + +def test_marker_names_the_recorded_revision_and_every_input_digest(tmp_path): + build = write_pre_1144_build(tmp_path / "build") + output = tmp_path / "publication" + + reconstruct_uk_publication(build, output) + + schema = read_json(output / SCHEMA_NAME) + marker = validate_reconstruction(schema) + assert marker is not None + document = read_json(output / GRAPH_NAME) + assert ( + document["metadata"]["microcosm"]["extensions"][RECONSTRUCTION_EXTENSION] + == marker + ) + assert marker["source_revision"] == { + "git_commit": RECORDED_COMMIT, + "git_dirty": False, + "origin": "recorded", + } + inputs = inputs_by_name(marker) + assert set(inputs) == { + "build.json", + "graph.json", + MANIFEST_FILENAME, + "sha256sums.txt", + } + for name, item in inputs.items(): + payload = (build / name).read_bytes() + assert item["sha256"] == sha256_hex(payload) + assert item["bytes"] == len(payload) + assert inputs["graph.json"]["role"] == "declaration" + assert inputs["graph.json"]["verification"] == "no_recorded_digest" + assert inputs[MANIFEST_FILENAME]["role"] == "code_pin" + assert inputs[MANIFEST_FILENAME]["verification"] == "recorded_digest_matched" + + +def test_candidate_manifest_disagreeing_with_build_record_is_refused(tmp_path): + build = write_pre_1144_build(tmp_path / "build", staged=False) + completion = read_json(build / "build.json") + completion["rowwise_candidate_manifest"]["sha256"] = "0" * 64 + (build / "build.json").write_bytes(canonical_json(completion)) + + with pytest.raises(ValueError, match="does not match the digest recorded in"): + reconstruct_uk_publication(build, tmp_path / "publication") + assert not (tmp_path / "publication").exists() + + +def test_candidate_manifest_disagreeing_with_sha256sums_is_refused(tmp_path): + build = write_pre_1144_build(tmp_path / "build") + (build / "sha256sums.txt").write_text(f"{'0' * 64} {MANIFEST_FILENAME}\n") + + with pytest.raises(ValueError, match="sha256sums.txt"): + reconstruct_uk_publication(build, tmp_path / "publication") + assert not (tmp_path / "publication").exists() + + +@pytest.mark.parametrize( + "recorded_file", ["graph.schema.json", "execution.evidence.json"] +) +def test_build_with_recorded_orrery_evidence_is_refused(tmp_path, recorded_file): + build = write_pre_1144_build(tmp_path / "build") + (build / recorded_file).write_text("{}") + + with pytest.raises(ValueError, match="python -m microcosm.graph.orrery"): + reconstruct_uk_publication(build, tmp_path / "publication") + + +def test_staged_bundle_without_a_declaration_is_refused_naming_what_is_missing( + tmp_path, +): + # A staged bundle (staged//) never carried graph.json before #1144. + bundle = write_pre_1144_build(tmp_path / "bundle") + (bundle / "graph.json").unlink() + + with pytest.raises(ValueError, match="graph.json") as refusal: + reconstruct_uk_publication(bundle, tmp_path / "publication") + assert MANIFEST_FILENAME in str(refusal.value) + assert not (tmp_path / "publication").exists() + + +def test_caller_revision_must_match_the_recorded_revision(tmp_path): + build = write_pre_1144_build(tmp_path / "build") + + with pytest.raises(ValueError, match="does not match the recorded revision"): + reconstruct_uk_publication( + build, tmp_path / "publication", revision=OTHER_COMMIT + ) + reconstruct_uk_publication(build, tmp_path / "confirmed", revision=RECORDED_COMMIT) + marker = validate_reconstruction(read_json(tmp_path / "confirmed" / SCHEMA_NAME)) + assert marker["source_revision"]["origin"] == "recorded" + + +def test_caller_revision_supplies_an_unrecorded_revision_as_asserted(tmp_path): + build = write_pre_1144_build(tmp_path / "build", git_commit=None, git_dirty=None) + + report = reconstruct_uk_publication( + build, tmp_path / "publication", revision=OTHER_COMMIT + ) + + marker = validate_reconstruction(read_json(tmp_path / "publication" / SCHEMA_NAME)) + assert marker["source_revision"] == { + "git_commit": OTHER_COMMIT, + "git_dirty": None, + "origin": "caller_asserted", + } + assert any("--revision" in warning for warning in report["warnings"]) + + +@pytest.mark.parametrize("malformed_revision", ["4bf7af406", RECORDED_COMMIT.upper()]) +def test_malformed_caller_revision_is_refused(tmp_path, malformed_revision): + build = write_pre_1144_build(tmp_path / "build") + + with pytest.raises(ValueError, match="full lowercase git commit"): + reconstruct_uk_publication( + build, tmp_path / "publication", revision=malformed_revision + ) + + +def test_build_without_any_revision_is_refused(tmp_path): + build = write_pre_1144_build(tmp_path / "build", staged=False) + (build / MANIFEST_FILENAME).unlink() + (build / "build.json").unlink() + + with pytest.raises(ValueError, match="pass --revision"): + reconstruct_uk_publication(build, tmp_path / "publication") + + +def test_summary_reports_unused_evidence_missing_files_and_warnings(tmp_path): + build = write_pre_1144_build( + tmp_path / "build", git_dirty=True, release_role="national", staged=False + ) + + report = reconstruct_uk_publication(build, tmp_path / "publication") + + assert report == read_json(tmp_path / "publication" / REPORT_NAME) + assert report["status"] == "reconstructed" + assert report["basis"] == "declaration_only" + assert report["build"]["release_role"] == "national" + assert report["declaration"]["operations"] == 2 + assert report["declaration"]["sources"] == 1 + assert [item["name"] for item in report["missing"]] == ["sha256sums.txt"] + assert [item["name"] for item in report["unused_evidence"]] == [ + ".graph-store", + "build.graph.json", + "evidence-index.json", + "operations.json", + "preflight.graph.json", + ] + assert all(item["reason"] for item in report["missing"] + report["unused_evidence"]) + assert any("uncommitted changes" in warning for warning in report["warnings"]) + assert any("graph.json" in warning for warning in report["warnings"]) + published = { + item["name"] + for item in read_json(tmp_path / "publication" / INVENTORY_NAME)["files"] + } + assert REPORT_NAME not in published + + +def test_reconstruction_is_deterministic(tmp_path): + build = write_pre_1144_build(tmp_path / "build") + + reconstruct_uk_publication(build, tmp_path / "first") + reconstruct_uk_publication(build, tmp_path / "second") + + for name in (GRAPH_NAME, SCHEMA_NAME, INVENTORY_NAME, REPORT_NAME): + assert (tmp_path / "first" / name).read_bytes() == ( + tmp_path / "second" / name + ).read_bytes() + + +def test_existing_output_is_refused_and_failure_leaves_no_partial_output( + tmp_path, monkeypatch +): + build = write_pre_1144_build(tmp_path / "build") + existing = tmp_path / "existing" + existing.mkdir() + + with pytest.raises(ValueError, match="already exists"): + reconstruct_uk_publication(build, existing) + assert list(existing.iterdir()) == [] + + def interrupted_rename(source, destination): + raise OSError("interrupted before the publication was moved into place") + + # Every file is written, then the move into place fails: nothing remains. + monkeypatch.setattr(os, "rename", interrupted_rename) + with pytest.raises(OSError, match="interrupted"): + reconstruct_uk_publication(build, tmp_path / "publication") + assert sorted(path.name for path in tmp_path.iterdir()) == ["build", "existing"] + + +def test_declaration_behind_a_symbolic_link_is_refused(tmp_path): + build = write_pre_1144_build(tmp_path / "build") + elsewhere = tmp_path / "elsewhere.json" + (build / "graph.json").rename(elsewhere) + (build / "graph.json").symlink_to(elsewhere) + + with pytest.raises(ValueError, match="regular file"): + reconstruct_uk_publication(build, tmp_path / "publication") + + +def test_cli_writes_the_publication_and_prints_a_summary(tmp_path, capsys): + build = write_pre_1144_build(tmp_path / "build") + output = tmp_path / "publication" + + assert main(["--build", str(build), "--output", str(output)]) == 0 + + printed = capsys.readouterr().out + assert str(output) in printed + assert RECORDED_COMMIT in printed + assert "declaration only" in printed + assert (output / GRAPH_NAME).is_file() + with pytest.raises(SystemExit) as refusal: + main(["--build", str(build), "--output", str(output)]) + assert refusal.value.code == 2 + assert "already exists" in capsys.readouterr().err + + +def test_maintained_uk_dense_declaration_reconstructs_within_orrery_limits(tmp_path): + spine = add_uk_spine_gate_nodes( + uk_spine_graph(source_mode="split"), + spec=load_country_spec("uk"), + engine_identity="declaration-only", + ) + full = uk_full_graph( + UKFullBuildConfig( + calibration_year=2025, + geography_assignment="legacy", + calibration=UKGraphCalibrationConfig(), + ), + spine=spine, + ) + declared = append_uk_full_gate_nodes( + full.graph, + calibration=full.calibration, + spine_stage_names=uk_spine_endpoint(spine).stage_names, + engine_identity="declaration-only", + review_date="2026-10-08", + ) + build = write_pre_1144_build(tmp_path / "build", graph=declared) + output = tmp_path / "publication" + + report = reconstruct_uk_publication(build, output) + + assert report["declaration"]["operations"] == len(declared.nodes) + result = subprocess.run( + ["node", str(_RECONSTRUCTION_VERIFY)], + input=(output / GRAPH_NAME).read_text(encoding="utf-8"), + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout == "Orrery accepted the reconstructed declaration.\n" From 69bd5e012b6394c27cd98daeee73bb0032bd2bef Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:52:02 +0400 Subject: [PATCH 03/11] Thin US calibration epoch telemetry to the UK's rule The calibrator reports a calibration_epoch event for every epoch. The UK build forwards every tenth epoch and each solve's last to its staging bundle and its telemetry emitter, with a stride that grows so no run forwards more than about 2,400 rows. The US build passed its progress coordinator straight to the solver, so every epoch became a collector event and a full rewrite of a growing calibration_progress.json: 3,000 of each on a default L0-plus-refit run, against 300 under the UK rule. Define the rule once in microcosm.build.calibration_epoch_telemetry (stride 10, the 2,400-row cap, the stride function and the forwarding predicate) and use it in both countries: - The UK rowwise staging helpers now read it instead of their own constants, with no change in behavior. - The US _BuildProgress forwards only the rule's epochs to both sinks, at a stride computed from --epochs and the run's solves (one for --dense-default-dataset, otherwise a selection and its refit). Each solve's last epoch is always forwarded, so every solve's final loss and the latest-epoch summary in progress.json stay exact. Refs #1292 Co-Authored-By: Claude Opus 5.5 --- ...2-us-calibration-epoch-thinning.changed.md | 1 + docs/uk-full-build-graph.md | 2 +- .../build/calibration_epoch_telemetry.py | 58 ++++++++ .../build/uk_runtime/rowwise_staging.py | 44 +++--- .../test_calibration_epoch_telemetry.py | 65 +++++++++ .../uk/test_uk_rowwise_candidate.py | 4 +- .../us/test_us_fiscal_refresh_builder.py | 132 ++++++++++++++++++ tools/build_us_fiscal_refresh_release.py | 31 +++- tools/route_a/README.md | 12 +- 9 files changed, 314 insertions(+), 35 deletions(-) create mode 100644 changelog.d/1292-us-calibration-epoch-thinning.changed.md create mode 100644 packages/microcosm-build/src/microcosm/build/calibration_epoch_telemetry.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_calibration_epoch_telemetry.py diff --git a/changelog.d/1292-us-calibration-epoch-thinning.changed.md b/changelog.d/1292-us-calibration-epoch-thinning.changed.md new file mode 100644 index 0000000000..af006b111b --- /dev/null +++ b/changelog.d/1292-us-calibration-epoch-thinning.changed.md @@ -0,0 +1 @@ +US builds now forward calibration epochs to their staging bundle and hosted telemetry the way UK builds do: every tenth epoch and each solve's last epoch, with the stride growing so no run forwards more than about 2,400 rows. Previously every epoch was forwarded. The rule is defined once in `microcosm.build.calibration_epoch_telemetry` and both countries use it. diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index 36240e03d2..156e61e5de 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -110,7 +110,7 @@ The graph evaluates every declared UK gate once; `uk.full.gates.calibrated.gate_ The terminal graph node writes unsigned `certification.json` from those identified artifacts and any declared native or matched-size comparisons. `build.json` is the completion marker that binds both `candidate.json` and the certification artifact. Physical output bytes are checked against their declared artifacts. Bundle publication writes the completion marker last and rolls back handled failures or interrupts. A process kill or power loss can leave an absent completion marker; a directory without a valid bound marker is not a completed build. -A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a `uk-local-candidate` row spooled under `/logbook-spool` on every terminal outcome, chained through `--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`, with an error receipt on failure), version 3 staging telemetry with the run's sampling evidence (`sample: {"mode": "full"}` on the f100 rung, null below it, judged on the effective fraction), stage events around each graph phase and per-epoch `calibration_progress` rows from the dense solve and, with `--dataset-households`, the size search and refit (rows tagged with their `phase`), and the staged-dataset delivery of the published bundle under `staged//` in the private repository. `--staging-local-only`, `--no-staging`, `--staging-read-back` and `--no-staged-dataset` behave as in [UK staging operations](uk-staging-operations.md). Dry runs plan without solving or writing and record no Logbook row. +A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a `uk-local-candidate` row spooled under `/logbook-spool` on every terminal outcome, chained through `--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`, with an error receipt on failure), version 3 staging telemetry with the run's sampling evidence (`sample: {"mode": "full"}` on the f100 rung, null below it, judged on the effective fraction), stage events around each graph phase and `calibration_progress` rows for every tenth epoch and each solve's last (the rule every build shares, in `microcosm.build.calibration_epoch_telemetry`) from the dense solve and, with `--dataset-households`, the size search and refit (rows tagged with their `phase`), and the staged-dataset delivery of the published bundle under `staged//` in the private repository. `--staging-local-only`, `--no-staging`, `--staging-read-back` and `--no-staged-dataset` behave as in [UK staging operations](uk-staging-operations.md). Dry runs plan without solving or writing and record no Logbook row. Structural failures stop export. The maintained local statistical failure policy may still export an unreleasable diagnostic candidate with a nonzero process status. Missing evidence remains explicit. `--release-candidate` applies the maintained strictness and solve settings and requires the UK gate signing key and local targets, because it signs the local gate report; it does not publish or authorize a release. Fixture acceptance proves graph behavior. Native certification additionally requires measured incumbent comparison evidence supplied with `--native-scorecard`; exact-count promotion also requires the measured comparison at the requested k through `--matched-size-scorecard`. The certification node verifies their candidate and output identities before assessing readiness. diff --git a/packages/microcosm-build/src/microcosm/build/calibration_epoch_telemetry.py b/packages/microcosm-build/src/microcosm/build/calibration_epoch_telemetry.py new file mode 100644 index 0000000000..711ea677cf --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/calibration_epoch_telemetry.py @@ -0,0 +1,58 @@ +"""Which calibration epochs a build forwards to its telemetry. + +The calibrator reports one ``calibration_epoch`` event per epoch. Every build, +UK and US, forwards the same subset to its staging bundle and its telemetry +emitter (microcosm#1292): every ``CALIBRATION_EPOCH_EVERY``-th epoch and the +last epoch of each solve, so the loss curve and every solve's final loss +survive at a tenth of the rows. + +A UK size run at 2,000 epochs solves the dense pool, up to ten full-length L0 +probes and the refit: about 24,000 epochs. One forwarded epoch is one row in +the staging bundle's ``calibration_progress.json`` and one event in its +``events.ndjson``, both under the staging contract's 5 MiB remote cap, so the +stride also grows with the run: no run forwards more than +``CALIBRATION_MAX_EPOCH_ROWS`` strided epochs, whatever ``--epochs`` says. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +__all__ = [ + "CALIBRATION_EPOCH_EVERY", + "CALIBRATION_MAX_EPOCH_ROWS", + "calibration_epoch_stride", + "forwards_calibration_epoch", +] + +CALIBRATION_EPOCH_EVERY = 10 +CALIBRATION_MAX_EPOCH_ROWS = 2400 + + +def calibration_epoch_stride(total_epochs: int) -> int: + """The forwarding stride for a run whose solves total ``total_epochs``. + + At least ``CALIBRATION_EPOCH_EVERY``, and large enough that the strided + epochs stay within ``CALIBRATION_MAX_EPOCH_ROWS``. A caller that cannot + know its exact solve count (a budget search may stop early) passes the + most epochs the run can solve. + """ + + return max( + CALIBRATION_EPOCH_EVERY, + -(-int(total_epochs) // CALIBRATION_MAX_EPOCH_ROWS), + ) + + +def forwards_calibration_epoch(event: Mapping[str, object], *, every: int) -> bool: + """Whether a build forwards this calibrator report at stride ``every``. + + Only ``calibration_epoch`` reports are forwarded: every ``every``-th epoch + and each solve's last. A solve's epoch counter restarts at 1, so a + selection and its refit each keep their final epoch. + """ + + if event.get("kind") != "calibration_epoch": + return False + epoch = int(event["epoch"]) + return epoch % every == 0 or epoch == int(event["epochs"]) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py index 55c80a4565..acbdb3f053 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py @@ -22,6 +22,11 @@ from pathlib import Path from typing import Any +from microcosm.build.calibration_epoch_telemetry import ( + CALIBRATION_EPOCH_EVERY, + calibration_epoch_stride, + forwards_calibration_epoch, +) from microcosm.build.run_outcome import ( UNRECORDED_GATE_BLOCK, GateBlock, @@ -55,8 +60,6 @@ __all__ = [ "STAGED_DATASET_PHASES", - "STAGING_EPOCH_EVERY", - "STAGING_MAX_EPOCH_ROWS", "STAGING_UPLOAD_INTERVAL_SECONDS", "add_staging_artifact", "block_staging_run_bundle", @@ -85,16 +88,6 @@ # and loses uploads (the v20 national run did); five minutes keeps a # 1,500-epoch run well inside it. STAGING_UPLOAD_INTERVAL_SECONDS = 300.0 -# The staging bundle keeps one row per forwarded epoch in -# calibration_progress.json and one event in events.ndjson, both under the -# contract's 5 MiB remote cap. A size run at 2,000 epochs solves the dense -# pool, up to ten full-length L0 probes and the refit: about 24,000 epochs, -# which would breach the cap mid-run. Forwarding every tenth epoch and the -# last epoch of each phase keeps the loss curve and stays near 0.8 MB. -STAGING_EPOCH_EVERY = 10 -# ...and never more than this many forwarded epochs per run, whatever --epochs -# says: the stride grows with the run so the cap holds by construction. -STAGING_MAX_EPOCH_ROWS = 2400 STAGED_DATASET_PHASES = { "uploaded": "dataset_staged", "already_staged": "dataset_staged", @@ -102,8 +95,6 @@ "skipped": "dataset_stage_skipped", } _STAGING_UPLOAD_INTERVAL_SECONDS = STAGING_UPLOAD_INTERVAL_SECONDS -_STAGING_EPOCH_EVERY = STAGING_EPOCH_EVERY -_STAGING_MAX_EPOCH_ROWS = STAGING_MAX_EPOCH_ROWS _STAGED_DATASET_PHASES = STAGED_DATASET_PHASES _ACTIVE_EMITTER: LocalTelemetryEmitter | None = None @@ -354,27 +345,30 @@ def stage_sample( def staging_epoch_every(args: argparse.Namespace) -> int: - """The epoch stride that keeps the forwarded rows under the row budget. + """The shared calibration epoch stride over this run's solves. A dense run solves once; a size run solves the pool, up to ``budget_iters`` - full-length probes and the refit. The stride is at least - ``STAGING_EPOCH_EVERY`` and grows so at most ``STAGING_MAX_EPOCH_ROWS`` - epochs are forwarded, keeping ``calibration_progress.json`` and + full-length probes and the refit. The stride grows with that total, as + :func:`~microcosm.build.calibration_epoch_telemetry.calibration_epoch_stride` + defines for every build, keeping ``calibration_progress.json`` and ``events.ndjson`` under the contract's 5 MiB cap for any ``--epochs``. """ solves = 1 if args.dataset_households is None else 2 + BUDGET_ITERS - total = int(args.epochs) * solves - return max(STAGING_EPOCH_EVERY, -(-total // STAGING_MAX_EPOCH_ROWS)) + return calibration_epoch_stride(int(args.epochs) * solves) _staging_epoch_every = staging_epoch_every def thinned_epochs( - sink: Callable[[Mapping[str, Any]], None], *, every: int = STAGING_EPOCH_EVERY + sink: Callable[[Mapping[str, Any]], None], *, every: int = CALIBRATION_EPOCH_EVERY ) -> Callable[[dict[str, object]], None]: - """Forward every ``every``-th epoch and each phase's last epoch to ``sink``. + """Forward the shared rule's calibration epochs to ``sink``. + + :func:`~microcosm.build.calibration_epoch_telemetry.forwards_calibration_epoch` + selects every ``every``-th epoch and each phase's last epoch, as for every + build. The kernel flags probe epochs with ``budget_search: True``; the staging contract records that field as an integer or null, so the flag becomes 1 @@ -387,11 +381,7 @@ def thinned_epochs( def callback(event: dict[str, object]) -> None: nonlocal disabled - if disabled or event.get("kind") != "calibration_epoch": - return - epoch = int(event["epoch"]) - epochs = int(event["epochs"]) - if epoch % every != 0 and epoch != epochs: + if disabled or not forwards_calibration_epoch(event, every=every): return forwarded = dict(event) budget_search = forwarded.get("budget_search") diff --git a/packages/microcosm-build/tests/engine_free/shared/test_calibration_epoch_telemetry.py b/packages/microcosm-build/tests/engine_free/shared/test_calibration_epoch_telemetry.py new file mode 100644 index 0000000000..08628af1c7 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_calibration_epoch_telemetry.py @@ -0,0 +1,65 @@ +"""The calibration epochs every build forwards to its telemetry (microcosm#1292).""" + +import pytest + +from microcosm.build.calibration_epoch_telemetry import ( + CALIBRATION_EPOCH_EVERY, + CALIBRATION_MAX_EPOCH_ROWS, + calibration_epoch_stride, + forwards_calibration_epoch, +) + + +def _forwarded_epochs(epochs: int, *, every: int) -> list[int]: + return [ + epoch + for epoch in range(1, epochs + 1) + if forwards_calibration_epoch( + { + "kind": "calibration_epoch", + "epoch": epoch, + "epochs": epochs, + "loss": 1.0, + }, + every=every, + ) + ] + + +def test_calibration_epoch_rule_forwards_every_tenth_and_each_solve_last_epoch(): + assert CALIBRATION_EPOCH_EVERY == 10 + assert _forwarded_epochs(25, every=CALIBRATION_EPOCH_EVERY) == [10, 20, 25] + # A last epoch on the stride is forwarded once. + assert _forwarded_epochs(30, every=CALIBRATION_EPOCH_EVERY) == [10, 20, 30] + # A solve shorter than the stride still reports its final loss. + assert _forwarded_epochs(5, every=CALIBRATION_EPOCH_EVERY) == [5] + assert _forwarded_epochs(40, every=13) == [13, 26, 39, 40] + + +def test_calibration_epoch_rule_ignores_other_progress_events(): + # Budget-search reports share the progress callback but are not epochs; + # the closing report carries no epoch at all. + assert not forwards_calibration_epoch( + {"kind": "budget_probe", "epoch": 10, "epochs": 10}, every=10 + ) + assert not forwards_calibration_epoch({"kind": "budget_search_done"}, every=10) + + +@pytest.mark.parametrize( + ("total_epochs", "expected_stride"), + [ + (1, 10), + (3_000, 10), + (24_000, 10), + (24_001, 11), + (30_000, 13), + (120_000, 50), + ], +) +def test_calibration_epoch_stride_grows_to_keep_forwarded_rows_under_the_cap( + total_epochs, expected_stride +): + assert CALIBRATION_MAX_EPOCH_ROWS == 2_400 + stride = calibration_epoch_stride(total_epochs) + assert stride == expected_stride + assert total_epochs // stride <= CALIBRATION_MAX_EPOCH_ROWS diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py index ed5288635e..0b4770ad4d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py @@ -7,6 +7,7 @@ """ # ruff: noqa: F403, F405 +from microcosm.build.calibration_epoch_telemetry import CALIBRATION_MAX_EPOCH_ROWS from test_support.microcosm_build.uk_rowwise_candidate import * @@ -492,8 +493,7 @@ def stride(epochs, households): for epochs, households in ((2000, 55000), (10000, 55000), (100000, None)): solves = 1 if households is None else 2 + rowwise_cli.BUDGET_ITERS assert ( - epochs * solves / stride(epochs, households) - <= builder._STAGING_MAX_EPOCH_ROWS + epochs * solves / stride(epochs, households) <= CALIBRATION_MAX_EPOCH_ROWS ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index a1d494147c..f49443bb3a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -13755,6 +13755,8 @@ def namespace(no_staging: bool) -> SimpleNamespace: staging_run_id=None, staging_prefix=args.staging_prefix, staging_upload_interval_seconds=60.0, + epochs=module.DEFAULT_US_FISCAL_CALIBRATION_EPOCHS, + dense_default_dataset=False, ) progress = module._build_progress( @@ -13804,6 +13806,132 @@ def stage(self, *args, **kwargs): ] +def _solver_epoch_events(phase: str, epochs: int) -> list[dict[str, object]]: + """One solve's per-epoch reports, as the calibrator emits them.""" + + return [ + { + "phase": phase, + "kind": "calibration_epoch", + "epoch": epoch, + "epochs": epochs, + "loss": 100.0 / epoch, + } + for epoch in range(1, epochs + 1) + ] + + +def _forwarded_calibration_events(emitter: _TestEmitter) -> list[dict[str, object]]: + return [recorded[1] for recorded in emitter.events if recorded[0] == "calibration"] + + +def test_us_build_progress_forwards_every_tenth_and_each_solve_final_calibration_epoch( + tmp_path, +) -> None: + """The US forwards the UK's epochs: every tenth and each solve's last. + + The solver reports every epoch (microcosm#1292). An L0 selection and its + refit are two solves whose epoch counters each restart at 1, so each + keeps its own final epoch, and both the hosted emitter and the staging + bundle receive the same thinned rows. + """ + + module = _load_builder_module() + emitter = _TestEmitter() + run_dir = tmp_path / "stage" + progress = module._BuildProgress( + run_id="rel-1", + emitter=emitter, + staging_bundle=module.StagingRunBundleWriter( + run_id="rel-1", + candidate_release_id="rel-1", + run_dir=run_dir, + ), + ) + solver_reports = [ + *_solver_epoch_events("l0_selection", 25), + {"kind": "budget_probe", "budget_iteration": 1}, + *_solver_epoch_events("post_l0_refit", 25), + ] + + for report in solver_reports: + progress.calibration_progress(report) + + expected_rows = [ + (phase, epoch, 100.0 / epoch) + for phase in ("l0_selection", "post_l0_refit") + for epoch in (10, 20, 25) + ] + forwarded = _forwarded_calibration_events(emitter) + assert [ + (event.get("phase"), event.get("epoch"), event.get("loss")) + for event in forwarded + ] == expected_rows + staged_rows = json.loads((run_dir / "calibration_progress.json").read_text())[ + "events" + ] + assert [(row["epoch"], row["epochs"], row["loss"]) for row in staged_rows] == [ + (epoch, 25, loss) for _phase, epoch, loss in expected_rows + ] + latest = json.loads((run_dir / "progress.json").read_text())["calibration"] + assert (latest["epoch"], latest["epochs"], latest["loss"]) == (25, 25, 4.0) + + +@pytest.mark.parametrize( + ("epochs", "dense_default_dataset", "expected_stride", "expected_rows"), + [ + # The default L0 selection and refit: 2 x 1,500 epochs. + (1_500, False, 10, 300), + # 2 x 15,000 epochs would be 3,000 rows at the base stride. + (15_000, False, 13, 2_308), + # A single dense solve of the same total length. + (30_000, True, 13, 2_308), + ], +) +def test_us_build_progress_stride_keeps_long_runs_under_the_row_cap( + tmp_path, epochs, dense_default_dataset, expected_stride, expected_rows +) -> None: + """The stride grows with the run's solves, as the UK's does. + + No run forwards more than the UK's 2,400 rows plus each solve's last + epoch, whatever ``--epochs`` says (microcosm#1292). + """ + + module = _load_builder_module() + emitter = _TestEmitter() + args = SimpleNamespace( + no_staging=True, + staging_dir=None, + staging_repo_id=None, + staging_run_id=None, + staging_prefix=module.DEFAULT_STAGING_PREFIX, + staging_upload_interval_seconds=60.0, + epochs=epochs, + dense_default_dataset=dense_default_dataset, + ) + progress = module._build_progress( + args, + release_root=tmp_path, + release_id="rel-1", + emitter=emitter, + ) + phases = ("dense",) if dense_default_dataset else ("l0_selection", "post_l0_refit") + + for phase in phases: + for report in _solver_epoch_events(phase, epochs): + progress.calibration_progress(report) + + forwarded = _forwarded_calibration_events(emitter) + assert len(forwarded) == expected_rows + assert len(forwarded) <= 2_400 + len(phases) + for phase in phases: + assert [event["epoch"] for event in forwarded if event["phase"] == phase] == [ + epoch + for epoch in range(1, epochs + 1) + if epoch % expected_stride == 0 or epoch == epochs + ] + + def test_blank_staging_repo_id_is_refused_at_parse_time(monkeypatch, capsys) -> None: module = _load_builder_module() argv = [ @@ -13893,6 +14021,8 @@ def test_build_progress_replaces_any_previous_active_run(tmp_path) -> None: staging_run_id=None, staging_prefix=module.DEFAULT_STAGING_PREFIX, staging_upload_interval_seconds=60.0, + epochs=module.DEFAULT_US_FISCAL_CALIBRATION_EPOCHS, + dense_default_dataset=False, ) first = module._build_progress( args, @@ -13958,6 +14088,8 @@ def test_staging_bundle_refuses_a_destinationless_namespace(tmp_path) -> None: staging_run_id=None, staging_prefix=module.DEFAULT_STAGING_PREFIX, staging_upload_interval_seconds=60.0, + epochs=module.DEFAULT_US_FISCAL_CALIBRATION_EPOCHS, + dense_default_dataset=False, ) with pytest.raises(ValueError, match="no destination"): diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index be88543d84..94b1724143 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -54,6 +54,11 @@ import numpy as np import pandas as pd +from microcosm.build.calibration_epoch_telemetry import ( + CALIBRATION_EPOCH_EVERY, + calibration_epoch_stride, + forwards_calibration_epoch, +) from microcosm.build.gates import ( GateResult, TargetFitRequirement, @@ -11504,8 +11509,24 @@ def _assert_exact_k_original_pool_alignment( _ACTIVE_EMITTER: LocalTelemetryEmitter | None = None +def _calibration_epoch_every(args: argparse.Namespace) -> int: + """The shared calibration epoch stride over this build's solves. + + ``--dense-default-dataset`` solves once; the L0 and exact-k paths solve a + selection and then its refit, each for ``--epochs`` (an exact-k ladder + whose ``k`` is the whole pool solves once, so two is the most it solves). + """ + + solves = 1 if args.dense_default_dataset else 2 + return calibration_epoch_stride(int(args.epochs) * solves) + + class _BuildProgress: - """Report progress to an emitter and, independently, a staging bundle.""" + """Report progress to an emitter and, independently, a staging bundle. + + Both receive the same calibration epochs: every ``epoch_every``-th epoch + and each solve's last, the rule the UK build follows too (microcosm#1292). + """ def __init__( self, @@ -11514,11 +11535,13 @@ def __init__( emitter: LocalTelemetryEmitter, staging_bundle: StagingRunBundleWriter | None, staging_opt_out_reason: str | None = None, + epoch_every: int = CALIBRATION_EPOCH_EVERY, ) -> None: self.run_id = run_id self.emitter = emitter self.staging_bundle = staging_bundle self.staging_opt_out_reason = staging_opt_out_reason + self.epoch_every = epoch_every @property def repo_id(self) -> str | None: @@ -11557,6 +11580,8 @@ def stage( ) def calibration_progress(self, event: dict[str, object]) -> None: + if not forwards_calibration_epoch(event, every=self.epoch_every): + return self.emitter.transition_calibration_progress(event) if self.staging_bundle is not None: self.staging_bundle.calibration_progress(event) @@ -12249,12 +12274,14 @@ def _build_progress( # can never be marked failed in place of this build's. _ACTIVE_PROGRESS = None run_id = run_id or args.staging_run_id or release_id + epoch_every = _calibration_epoch_every(args) if args.no_staging: _ACTIVE_PROGRESS = _BuildProgress( run_id=run_id, emitter=emitter, staging_bundle=None, staging_opt_out_reason="--no-staging", + epoch_every=epoch_every, ) return _ACTIVE_PROGRESS if not args.staging_dir and not args.staging_repo_id: @@ -12279,6 +12306,7 @@ def _build_progress( run_id=run_id, emitter=emitter, staging_bundle=staging_bundle, + epoch_every=epoch_every, ) return _ACTIVE_PROGRESS @@ -12790,6 +12818,7 @@ def _main(argv: Sequence[str] | None = None) -> int | None: emitter=_ACTIVE_EMITTER, staging_bundle=None, staging_opt_out_reason="dry run", + epoch_every=_calibration_epoch_every(args), ) _ACTIVE_PROGRESS = telemetry if telemetry is not None: diff --git a/tools/route_a/README.md b/tools/route_a/README.md index 4d66455414..1abc36f0f1 100644 --- a/tools/route_a/README.md +++ b/tools/route_a/README.md @@ -212,10 +212,14 @@ uploads reach the dashboard's staging repository, the records for its Build progress tab include the release's run ID and candidate release ID, current status/stage, timestamps, messages and stage details from `progress.json`, the event history from `events.ndjson`, and calibration epoch/loss/budget-search information from -`calibration_progress.json`. The builder emits stages for loading the base, -source enrichments, target compilation, calibration, export/gates, diagnostics -and manifests, and uploads attached diagnostics as they are written. Completion -records status `passed`; reported failures record status `failed`. +`calibration_progress.json`. That file and the hosted telemetry carry every +tenth epoch of each solve and its last epoch, the rule the UK build follows +too (`microcosm.build.calibration_epoch_telemetry`); the stride grows past ten +only when the run's solves total more than 24,000 epochs. The builder emits +stages for loading the base, source enrichments, target compilation, +calibration, export/gates, diagnostics and manifests, and uploads attached +diagnostics as they are written. Completion records status `passed`; reported +failures record status `failed`. Telemetry is written locally under `/staging/runs//` and uploaded under From d0d2ef4d26f62db76b8ad1072f4e3077045a0a4f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:27:46 +0400 Subject: [PATCH 04/11] Forward each solve's first calibration epoch The calibration-diagnostics build monitor starts a new solver pass only when a calibration row's epoch is lower than the previous row's or its epoch total changes. With --epochs of 10 or fewer, the shared rule forwarded only each solve's last epoch, so an L0 run, or an exact-k run whose k is below the pool, forwarded (epoch N, epochs N) for both the selection and the refit. The monitor merged the two solves into one pass, with no start time it could back-date and no epoch rate. A UK size run, whose dense solve, probes and refit all run --epochs, merged the same way. Forward epoch 1 of every solve too, in the shared rule, so both countries get it. Each solve's forwarded rows now start at epoch 1, and the forwarded epochs fall at every solve boundary where the full per-epoch stream does. That adds one row per solve: 302 rows instead of 300 on a default US run, and 2,412 instead of 2,400 on a default UK size run. The stride still keeps the epochs on the stride within CALIBRATION_MAX_EPOCH_ROWS. Each solve's first and last epoch add at most two rows, so a run forwards at most CALIBRATION_MAX_EPOCH_ROWS + 2 rows per solve, where the bound was one per solve before. Folding the new rows into the 2,400 would move the default UK size run from stride 10 to 11 to save about 1% of the row budget, so the stride is unchanged. Refs #1292 Co-Authored-By: Claude Opus 5.5 --- ...2-us-calibration-epoch-thinning.changed.md | 2 +- docs/uk-full-build-graph.md | 2 +- .../build/calibration_epoch_telemetry.py | 36 +++++--- .../build/uk_runtime/rowwise_staging.py | 4 +- .../test_calibration_epoch_telemetry.py | 66 +++++++++++++-- .../uk/test_uk_rowwise_candidate.py | 57 +++++++++++++ .../us/test_us_fiscal_refresh_builder.py | 83 ++++++++++++++++--- tools/build_us_fiscal_refresh_release.py | 5 +- tools/route_a/README.md | 10 ++- 9 files changed, 224 insertions(+), 41 deletions(-) diff --git a/changelog.d/1292-us-calibration-epoch-thinning.changed.md b/changelog.d/1292-us-calibration-epoch-thinning.changed.md index af006b111b..0cd0c6255f 100644 --- a/changelog.d/1292-us-calibration-epoch-thinning.changed.md +++ b/changelog.d/1292-us-calibration-epoch-thinning.changed.md @@ -1 +1 @@ -US builds now forward calibration epochs to their staging bundle and hosted telemetry the way UK builds do: every tenth epoch and each solve's last epoch, with the stride growing so no run forwards more than about 2,400 rows. Previously every epoch was forwarded. The rule is defined once in `microcosm.build.calibration_epoch_telemetry` and both countries use it. +US builds now forward calibration epochs to their staging bundle and hosted telemetry the way UK builds do: each solve's first and last epoch and every tenth epoch, with the stride growing so no run forwards more than 2,400 epochs on the stride beside those first and last epochs. Previously every epoch was forwarded. UK builds now also forward each solve's first epoch, one more row per solve, so the forwarded epochs fall at every boundary between solves and a dashboard that starts a new solve when the epoch falls no longer merges solves of ten or fewer epochs. The rule is defined once in `microcosm.build.calibration_epoch_telemetry` and both countries use it. diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index 156e61e5de..2027e27bd0 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -110,7 +110,7 @@ The graph evaluates every declared UK gate once; `uk.full.gates.calibrated.gate_ The terminal graph node writes unsigned `certification.json` from those identified artifacts and any declared native or matched-size comparisons. `build.json` is the completion marker that binds both `candidate.json` and the certification artifact. Physical output bytes are checked against their declared artifacts. Bundle publication writes the completion marker last and rolls back handled failures or interrupts. A process kill or power loss can leave an absent completion marker; a directory without a valid bound marker is not a completed build. -A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a `uk-local-candidate` row spooled under `/logbook-spool` on every terminal outcome, chained through `--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`, with an error receipt on failure), version 3 staging telemetry with the run's sampling evidence (`sample: {"mode": "full"}` on the f100 rung, null below it, judged on the effective fraction), stage events around each graph phase and `calibration_progress` rows for every tenth epoch and each solve's last (the rule every build shares, in `microcosm.build.calibration_epoch_telemetry`) from the dense solve and, with `--dataset-households`, the size search and refit (rows tagged with their `phase`), and the staged-dataset delivery of the published bundle under `staged//` in the private repository. `--staging-local-only`, `--no-staging`, `--staging-read-back` and `--no-staged-dataset` behave as in [UK staging operations](uk-staging-operations.md). Dry runs plan without solving or writing and record no Logbook row. +A non-dry dense run is wrapped in the rowwise tool's operational envelope: the Logbook attempt (a `uk-local-candidate` row spooled under `/logbook-spool` on every terminal outcome, chained through `--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`, with an error receipt on failure), version 3 staging telemetry with the run's sampling evidence (`sample: {"mode": "full"}` on the f100 rung, null below it, judged on the effective fraction), stage events around each graph phase and `calibration_progress` rows for each solve's first and last epoch and every tenth epoch (the rule every build shares, in `microcosm.build.calibration_epoch_telemetry`) from the dense solve and, with `--dataset-households`, the size search and refit (rows tagged with their `phase`), and the staged-dataset delivery of the published bundle under `staged//` in the private repository. `--staging-local-only`, `--no-staging`, `--staging-read-back` and `--no-staged-dataset` behave as in [UK staging operations](uk-staging-operations.md). Dry runs plan without solving or writing and record no Logbook row. Structural failures stop export. The maintained local statistical failure policy may still export an unreleasable diagnostic candidate with a nonzero process status. Missing evidence remains explicit. `--release-candidate` applies the maintained strictness and solve settings and requires the UK gate signing key and local targets, because it signs the local gate report; it does not publish or authorize a release. Fixture acceptance proves graph behavior. Native certification additionally requires measured incumbent comparison evidence supplied with `--native-scorecard`; exact-count promotion also requires the measured comparison at the requested k through `--matched-size-scorecard`. The certification node verifies their candidate and output identities before assessing readiness. diff --git a/packages/microcosm-build/src/microcosm/build/calibration_epoch_telemetry.py b/packages/microcosm-build/src/microcosm/build/calibration_epoch_telemetry.py index 711ea677cf..286eabea38 100644 --- a/packages/microcosm-build/src/microcosm/build/calibration_epoch_telemetry.py +++ b/packages/microcosm-build/src/microcosm/build/calibration_epoch_telemetry.py @@ -2,16 +2,26 @@ The calibrator reports one ``calibration_epoch`` event per epoch. Every build, UK and US, forwards the same subset to its staging bundle and its telemetry -emitter (microcosm#1292): every ``CALIBRATION_EPOCH_EVERY``-th epoch and the -last epoch of each solve, so the loss curve and every solve's final loss -survive at a tenth of the rows. +emitter (microcosm#1292): the first and last epoch of each solve and every +``CALIBRATION_EPOCH_EVERY``-th epoch, so the loss curve and every solve's +starting and final loss survive at a tenth of the rows. + +Each solve's epoch counter restarts at 1, and forwarding that first epoch +keeps the restart visible: the forwarded epochs fall at every boundary +between solves, as the full per-epoch stream does. A reader that starts a +new solve when the epoch falls (the calibration-diagnostics build monitor +does) therefore sees every solve, even when ``--epochs`` is no longer than +the stride. A UK size run at 2,000 epochs solves the dense pool, up to ten full-length L0 probes and the refit: about 24,000 epochs. One forwarded epoch is one row in the staging bundle's ``calibration_progress.json`` and one event in its ``events.ndjson``, both under the staging contract's 5 MiB remote cap, so the stride also grows with the run: no run forwards more than -``CALIBRATION_MAX_EPOCH_ROWS`` strided epochs, whatever ``--epochs`` says. +``CALIBRATION_MAX_EPOCH_ROWS`` epochs on the stride, whatever ``--epochs`` +says. Each solve's first and last epoch can add one row each, so a run of +``solves`` solves forwards at most ``CALIBRATION_MAX_EPOCH_ROWS + 2 * solves`` +rows. """ from __future__ import annotations @@ -32,10 +42,11 @@ def calibration_epoch_stride(total_epochs: int) -> int: """The forwarding stride for a run whose solves total ``total_epochs``. - At least ``CALIBRATION_EPOCH_EVERY``, and large enough that the strided - epochs stay within ``CALIBRATION_MAX_EPOCH_ROWS``. A caller that cannot - know its exact solve count (a budget search may stop early) passes the - most epochs the run can solve. + At least ``CALIBRATION_EPOCH_EVERY``, and large enough that the epochs on + the stride stay within ``CALIBRATION_MAX_EPOCH_ROWS``; each solve's first + and last epoch are forwarded beside them. A caller that cannot know its + exact solve count (a budget search may stop early) passes the most epochs + the run can solve. """ return max( @@ -47,12 +58,13 @@ def calibration_epoch_stride(total_epochs: int) -> int: def forwards_calibration_epoch(event: Mapping[str, object], *, every: int) -> bool: """Whether a build forwards this calibrator report at stride ``every``. - Only ``calibration_epoch`` reports are forwarded: every ``every``-th epoch - and each solve's last. A solve's epoch counter restarts at 1, so a - selection and its refit each keep their final epoch. + Only ``calibration_epoch`` reports are forwarded: each solve's first and + last epoch and every ``every``-th epoch. A solve's epoch counter restarts + at 1, so a selection and its refit each keep their first and final epoch, + and the forwarded epochs fall where the refit begins. """ if event.get("kind") != "calibration_epoch": return False epoch = int(event["epoch"]) - return epoch % every == 0 or epoch == int(event["epochs"]) + return epoch == 1 or epoch % every == 0 or epoch == int(event["epochs"]) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py index acbdb3f053..6b3945dd8c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py @@ -367,8 +367,8 @@ def thinned_epochs( """Forward the shared rule's calibration epochs to ``sink``. :func:`~microcosm.build.calibration_epoch_telemetry.forwards_calibration_epoch` - selects every ``every``-th epoch and each phase's last epoch, as for every - build. + selects each phase's first and last epoch and every ``every``-th epoch, as + for every build. The kernel flags probe epochs with ``budget_search: True``; the staging contract records that field as an integer or null, so the flag becomes 1 diff --git a/packages/microcosm-build/tests/engine_free/shared/test_calibration_epoch_telemetry.py b/packages/microcosm-build/tests/engine_free/shared/test_calibration_epoch_telemetry.py index 08628af1c7..d0cfc0c525 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_calibration_epoch_telemetry.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_calibration_epoch_telemetry.py @@ -26,14 +26,39 @@ def _forwarded_epochs(epochs: int, *, every: int) -> list[int]: ] -def test_calibration_epoch_rule_forwards_every_tenth_and_each_solve_last_epoch(): +def test_calibration_epoch_rule_forwards_each_solve_first_last_and_every_tenth_epoch(): assert CALIBRATION_EPOCH_EVERY == 10 - assert _forwarded_epochs(25, every=CALIBRATION_EPOCH_EVERY) == [10, 20, 25] + assert _forwarded_epochs(25, every=CALIBRATION_EPOCH_EVERY) == [1, 10, 20, 25] # A last epoch on the stride is forwarded once. - assert _forwarded_epochs(30, every=CALIBRATION_EPOCH_EVERY) == [10, 20, 30] - # A solve shorter than the stride still reports its final loss. - assert _forwarded_epochs(5, every=CALIBRATION_EPOCH_EVERY) == [5] - assert _forwarded_epochs(40, every=13) == [13, 26, 39, 40] + assert _forwarded_epochs(30, every=CALIBRATION_EPOCH_EVERY) == [1, 10, 20, 30] + # A solve shorter than the stride still reports its first and final loss. + assert _forwarded_epochs(5, every=CALIBRATION_EPOCH_EVERY) == [1, 5] + # A one-epoch solve's first epoch is its last, forwarded once. + assert _forwarded_epochs(1, every=CALIBRATION_EPOCH_EVERY) == [1] + assert _forwarded_epochs(40, every=13) == [1, 13, 26, 39, 40] + + +def test_forwarded_calibration_epochs_fall_at_every_solve_boundary(): + """A reader can tell consecutive solves apart from the forwarded rows alone. + + A selection and its refit run the same number of epochs, and each solve's + epoch counter restarts at 1. The calibration-diagnostics build monitor + starts a new solver pass when a row's epoch is lower than the row before + it (or its epoch total differs), so the forwarded epochs must fall at each + boundary, as the full per-epoch stream does, even when both solves are + shorter than the stride. + """ + + boundaries_without_a_fall = [] + for every in (CALIBRATION_EPOCH_EVERY, 13): + for epochs in range(2, 41): + # Both solves forward the same epochs: the refit's first forwarded + # row directly follows the selection's last. + forwarded = _forwarded_epochs(epochs, every=every) + if forwarded[0] >= forwarded[-1]: + boundaries_without_a_fall.append((epochs, every)) + + assert boundaries_without_a_fall == [] def test_calibration_epoch_rule_ignores_other_progress_events(): @@ -56,10 +81,37 @@ def test_calibration_epoch_rule_ignores_other_progress_events(): (120_000, 50), ], ) -def test_calibration_epoch_stride_grows_to_keep_forwarded_rows_under_the_cap( +def test_calibration_epoch_stride_grows_to_keep_the_strided_epochs_under_the_cap( total_epochs, expected_stride ): assert CALIBRATION_MAX_EPOCH_ROWS == 2_400 stride = calibration_epoch_stride(total_epochs) assert stride == expected_stride assert total_epochs // stride <= CALIBRATION_MAX_EPOCH_ROWS + + +@pytest.mark.parametrize( + ("epochs_per_solve", "solves", "expected_rows"), + [ + # The default US run: an L0 selection and its refit. + (1_500, 2, 302), + # The default UK size run: the dense solve, ten budget-search probes + # and the refit, 2,400 epochs on the stride and each solve's first. + (2_000, 12, 2_412), + (15_000, 2, 2_310), + (30_000, 1, 2_309), + # Last epochs off the stride took even the rule without first epochs + # past a flat 2,400 rows (to 2,401 here). + (3_428, 7, 2_408), + ], +) +def test_forwarded_rows_stay_within_the_cap_plus_two_per_solve( + epochs_per_solve, solves, expected_rows +): + """The stride caps the strided epochs; each solve adds its first and last.""" + + every = calibration_epoch_stride(epochs_per_solve * solves) + forwarded_rows = solves * len(_forwarded_epochs(epochs_per_solve, every=every)) + + assert forwarded_rows == expected_rows + assert forwarded_rows <= CALIBRATION_MAX_EPOCH_ROWS + 2 * solves diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py index 0b4770ad4d..86358bfe41 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py @@ -487,6 +487,7 @@ def stride(epochs, households): assert stride(2, None) == 10 and stride(2, 300) == 10 assert stride(2000, None) == 10 # dense + ten probes + refit at 2,000 epochs: 24,000 epochs -> 2,400 rows + # on the stride, plus each of the twelve solves' first epoch assert stride(2000, 55000) == 10 assert stride(10000, 55000) == 50 assert stride(100000, None) == 42 @@ -497,6 +498,62 @@ def stride(epochs, households): ) +def test_uk_thinned_epochs_forward_each_solve_first_epoch(): + """A UK size run no longer than the stride keeps its solves apart. + + The dense solve, each budget-search probe and the refit all run + ``--epochs`` and restart the epoch counter at 1. Forwarding each solve's + first epoch beside its last makes the epoch fall at every boundary, which + is how the calibration-diagnostics build monitor starts a new solver pass + (microcosm#1292). + """ + + epochs = 5 + forwarded_rows: list[dict[str, object]] = [] + observer = rowwise_staging.thinned_epochs( + forwarded_rows.append, + every=rowwise_staging.staging_epoch_every( + SimpleNamespace(epochs=epochs, dataset_households=55000) + ), + ) + # The dense solve's reports carry no phase; probe epochs are flagged. + solve_contexts = [ + {}, + {"phase": "size_search", "budget_search": True}, + {"phase": "size_search", "budget_search": True}, + {"phase": "size_refit"}, + ] + + for solve_context in solve_contexts: + for epoch in range(1, epochs + 1): + observer( + { + **solve_context, + "kind": "calibration_epoch", + "epoch": epoch, + "epochs": epochs, + "loss": 1.0 / epoch, + } + ) + + assert [(row.get("phase"), row["epoch"]) for row in forwarded_rows] == [ + (solve_context.get("phase"), epoch) + for solve_context in solve_contexts + for epoch in (1, epochs) + ] + # The staging contract records the probe flag as 1 or null. + assert [row.get("budget_search") for row in forwarded_rows] == [ + None, + None, + 1, + 1, + 1, + 1, + None, + None, + ] + + def test_remote_dataset_staging_is_refused_up_front_without_credential_or_repo( monkeypatch, tmp_path, capsys ): diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index f49443bb3a..9373d815ed 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -13825,15 +13825,15 @@ def _forwarded_calibration_events(emitter: _TestEmitter) -> list[dict[str, objec return [recorded[1] for recorded in emitter.events if recorded[0] == "calibration"] -def test_us_build_progress_forwards_every_tenth_and_each_solve_final_calibration_epoch( +def test_us_build_progress_forwards_each_solve_first_final_and_every_tenth_epoch( tmp_path, ) -> None: - """The US forwards the UK's epochs: every tenth and each solve's last. + """The US forwards the UK's epochs: each solve's first and last, every tenth. The solver reports every epoch (microcosm#1292). An L0 selection and its refit are two solves whose epoch counters each restart at 1, so each - keeps its own final epoch, and both the hosted emitter and the staging - bundle receive the same thinned rows. + keeps its own first and final epoch, and both the hosted emitter and the + staging bundle receive the same thinned rows. """ module = _load_builder_module() @@ -13860,7 +13860,7 @@ def test_us_build_progress_forwards_every_tenth_and_each_solve_final_calibration expected_rows = [ (phase, epoch, 100.0 / epoch) for phase in ("l0_selection", "post_l0_refit") - for epoch in (10, 20, 25) + for epoch in (1, 10, 20, 25) ] forwarded = _forwarded_calibration_events(emitter) assert [ @@ -13877,15 +13877,74 @@ def test_us_build_progress_forwards_every_tenth_and_each_solve_final_calibration assert (latest["epoch"], latest["epochs"], latest["loss"]) == (25, 25, 4.0) +@pytest.mark.parametrize("epochs", [10, 5]) +@pytest.mark.parametrize( + "solve_phases", + [("l0_selection", "post_l0_refit"), ("l0_selection", "exact_k_refit")], + ids=["l0_refit", "exact_k_below_the_pool"], +) +def test_us_short_run_forwards_each_solve_first_epoch_so_the_epoch_falls_at_the_refit( + tmp_path, solve_phases, epochs +) -> None: + """A run no longer than the stride still shows both of its solves. + + With ``--epochs`` of 10 or fewer, no epoch but the last is a multiple of + the stride, so without each solve's first epoch an L0 run, or an exact-k + run whose k is below the pool, forwarded ``(epochs, epochs)`` twice. The + calibration-diagnostics build monitor starts a new solver pass only when + the epoch falls or the epoch total changes, so it merged the selection and + its refit into one pass with no start time it could back-date + (microcosm#1292). + """ + + module = _load_builder_module() + emitter = _TestEmitter() + run_dir = tmp_path / "stage" + args = SimpleNamespace( + no_staging=False, + staging_dir=run_dir, + staging_repo_id=None, + staging_run_id=None, + staging_prefix=module.DEFAULT_STAGING_PREFIX, + staging_upload_interval_seconds=60.0, + epochs=epochs, + dense_default_dataset=False, + ) + progress = module._build_progress( + args, + release_root=tmp_path, + release_id="rel-1", + emitter=emitter, + ) + + for phase in solve_phases: + for report in _solver_epoch_events(phase, epochs): + progress.calibration_progress(report) + + assert [ + (event["phase"], event["epoch"]) + for event in _forwarded_calibration_events(emitter) + ] == [(phase, epoch) for phase in solve_phases for epoch in (1, epochs)] + staged_epochs = [ + row["epoch"] + for row in json.loads((run_dir / "calibration_progress.json").read_text())[ + "events" + ] + ] + assert staged_epochs == [1, epochs, 1, epochs] + refit_first_row = 2 + assert staged_epochs[refit_first_row] < staged_epochs[refit_first_row - 1] + + @pytest.mark.parametrize( ("epochs", "dense_default_dataset", "expected_stride", "expected_rows"), [ # The default L0 selection and refit: 2 x 1,500 epochs. - (1_500, False, 10, 300), + (1_500, False, 10, 302), # 2 x 15,000 epochs would be 3,000 rows at the base stride. - (15_000, False, 13, 2_308), + (15_000, False, 13, 2_310), # A single dense solve of the same total length. - (30_000, True, 13, 2_308), + (30_000, True, 13, 2_309), ], ) def test_us_build_progress_stride_keeps_long_runs_under_the_row_cap( @@ -13893,8 +13952,8 @@ def test_us_build_progress_stride_keeps_long_runs_under_the_row_cap( ) -> None: """The stride grows with the run's solves, as the UK's does. - No run forwards more than the UK's 2,400 rows plus each solve's last - epoch, whatever ``--epochs`` says (microcosm#1292). + No run forwards more than the UK's 2,400 epochs on the stride plus each + solve's first and last epoch, whatever ``--epochs`` says (microcosm#1292). """ module = _load_builder_module() @@ -13923,12 +13982,12 @@ def test_us_build_progress_stride_keeps_long_runs_under_the_row_cap( forwarded = _forwarded_calibration_events(emitter) assert len(forwarded) == expected_rows - assert len(forwarded) <= 2_400 + len(phases) + assert len(forwarded) <= 2_400 + 2 * len(phases) for phase in phases: assert [event["epoch"] for event in forwarded if event["phase"] == phase] == [ epoch for epoch in range(1, epochs + 1) - if epoch % expected_stride == 0 or epoch == epochs + if epoch == 1 or epoch % expected_stride == 0 or epoch == epochs ] diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 94b1724143..44bebf56c4 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -11524,8 +11524,9 @@ def _calibration_epoch_every(args: argparse.Namespace) -> int: class _BuildProgress: """Report progress to an emitter and, independently, a staging bundle. - Both receive the same calibration epochs: every ``epoch_every``-th epoch - and each solve's last, the rule the UK build follows too (microcosm#1292). + Both receive the same calibration epochs: each solve's first and last and + every ``epoch_every``-th epoch, the rule the UK build follows too + (microcosm#1292). """ def __init__( diff --git a/tools/route_a/README.md b/tools/route_a/README.md index 1abc36f0f1..068be2c984 100644 --- a/tools/route_a/README.md +++ b/tools/route_a/README.md @@ -212,10 +212,12 @@ uploads reach the dashboard's staging repository, the records for its Build progress tab include the release's run ID and candidate release ID, current status/stage, timestamps, messages and stage details from `progress.json`, the event history from `events.ndjson`, and calibration epoch/loss/budget-search information from -`calibration_progress.json`. That file and the hosted telemetry carry every -tenth epoch of each solve and its last epoch, the rule the UK build follows -too (`microcosm.build.calibration_epoch_telemetry`); the stride grows past ten -only when the run's solves total more than 24,000 epochs. The builder emits +`calibration_progress.json`. That file and the hosted telemetry carry each +solve's first and last epoch and every tenth epoch between them, the rule the +UK build follows too (`microcosm.build.calibration_epoch_telemetry`); the +stride grows past ten only when the run's solves total more than 24,000 +epochs. Because each solve's rows start at epoch 1, the epoch falls wherever +the refit begins, even when `--epochs` is ten or fewer. The builder emits stages for loading the base, source enrichments, target compilation, calibration, export/gates, diagnostics and manifests, and uploads attached diagnostics as they are written. Completion records status `passed`; reported From 96abc1ec85060ed95a85d5686885017380f9b531 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:30:15 +0400 Subject: [PATCH 05/11] Check a stale completion digest against the digest list The dense driver merged with #901 (2026-09-29) recorded the candidate manifest's digest in build.json with the note "staging receipts are appended after publication". Its close step then appended staging_delivery and staged_dataset to the manifest and refreshed only sha256sums.txt. build.json was not reissued over the final bytes until _refresh_completion arrived with #1057 (cb0d39045, 2026-09-30). Treating every digest mismatch as fatal therefore refused every completed dense build from that window, even with --revision. A build.json digest carrying that exact note is now skipped when it no longer matches, and the manifest must match sha256sums.txt instead. The marker records this as fallback_digest_matched. With no digest list (staging disabled), nothing the build recorded checks the manifest, so the marker records only_stale_digest_recorded. The report lists each skipped digest with its note and what the file was checked against, and warns. Any other mismatch is still refused. The fixture now replays the drivers' write sequence instead of pairing the note with a digest of the final manifest, which no real build wrote. It has three layouts: the #901 window (note and stale digest), the refreshed dense layout, and the national completion record (uk_national_build_completion with build_record), which arrived in #1057 together with the refresh. Co-Authored-By: Claude Opus 5.5 --- .../1291-uk-orrery-reconstruction.added.md | 2 +- docs/orrery-adapter.md | 34 +- .../build/uk_runtime/orrery_reconstruction.py | 116 +++++-- .../uk/test_uk_orrery_reconstruction.py | 302 +++++++++++++++--- .../src/microcosm/graph/reconstruction.py | 21 +- 5 files changed, 407 insertions(+), 68 deletions(-) diff --git a/changelog.d/1291-uk-orrery-reconstruction.added.md b/changelog.d/1291-uk-orrery-reconstruction.added.md index 2499f6c656..399008a367 100644 --- a/changelog.d/1291-uk-orrery-reconstruction.added.md +++ b/changelog.d/1291-uk-orrery-reconstruction.added.md @@ -1 +1 @@ -Add `python -m microcosm.build.uk_runtime.orrery_reconstruction`, which turns the output directory of a UK dense or national build from before #1144 into a declaration-only Orrery publication. It writes a new directory with the normal publication layout (`graph.schema.json`, `graph.orrery.json`, `orrery.upload.json`) and a summary report. The declaration comes from the build's recorded `graph.json` and the revision from its recorded code pin. Every input is checked against the digests the build recorded, builds that already record Orrery evidence are refused, and nothing is uploaded. +Add `python -m microcosm.build.uk_runtime.orrery_reconstruction`, which turns the output directory of a UK dense or national build from before #1144 into a declaration-only Orrery publication. It writes a new directory with the normal publication layout (`graph.schema.json`, `graph.orrery.json`, `orrery.upload.json`) and a summary report. The declaration comes from the build's recorded `graph.json` and the revision from its recorded code pin. Every input is checked against the digests the build recorded. The one exception is the candidate manifest digest that dense builds closed by #901's driver left in `build.json` beside the note that staging receipts follow: it is skipped and reported, and the manifest is checked against `sha256sums.txt` instead. Builds that already record Orrery evidence are refused, and nothing is uploaded. diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index e04313f9e3..3d87366c8e 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -313,10 +313,18 @@ The marker is the producer extension `microcosm.reconstruction`: revision itself, and `caller_asserted` when the build recorded none and the caller supplied one. `git_dirty` is `null` when the build did not record it. - **`inputs`:** every file the reconstruction read, in name order, by digest - and size. `verification` is `recorded_digest_matched` when a digest the build - itself recorded for that file matched, and `no_recorded_digest` when the - build recorded none. Exactly one input is the declaration, and its digest - equals the schema's `graph_sha256`. + and size. Exactly one input is the declaration, and its digest equals the + schema's `graph_sha256`. `verification` compares the file with the digests + the build itself recorded for it: + + | Value | Meaning | + | --- | --- | + | `recorded_digest_matched` | Every recorded digest matched. | + | `fallback_digest_matched` | The build marked one recorded digest as taken before it rewrote the file. That digest was skipped, and another recorded digest matched. | + | `only_stale_digest_recorded` | Every recorded digest is marked as taken before the rewrite, so nothing the build recorded checks the file. | + | `no_recorded_digest` | The build recorded no digest for the file. | + + In the last two cases the file is bound only by the digest in the marker. - **Exact bytes:** `reconstructed_schema` refuses a declaration that the current serializer would not write byte for byte. The schema therefore describes the recorded file, not a reinterpretation of it. @@ -354,7 +362,17 @@ The build directory is only read. The command: devices and inputs over 32 MiB are refused. - **Checks recorded digests:** a file whose bytes disagree with a digest the build recorded for it is refused. `build.json` binds the candidate manifest, - and `sha256sums.txt` binds the files it lists. + and `sha256sums.txt` binds the files it lists. One exception covers the dense + driver merged with #901 (2026-09-29). It recorded the manifest's digest in + `build.json` with the note `staging receipts are appended after + publication`, then appended `staging_delivery` and `staged_dataset` to the + manifest and refreshed only `sha256sums.txt`. `build.json` was reissued over + the final bytes, without the note, only from #1057 (2026-09-30). A + `build.json` digest with that exact note that no longer matches is skipped + and reported, and the manifest must then match `sha256sums.txt` + (`fallback_digest_matched`). A build run with `--no-staging` or + `--no-staged-dataset` wrote no digest list, so its manifest is reported as + `only_stale_digest_recorded`, with a warning. - **Resolves the revision:** it takes the candidate manifest's `git_commit` and `git_dirty`, which must agree with the copy under `identity.code`. `--revision` must match that commit, or supplies one when the build recorded @@ -380,12 +398,14 @@ directory into place only after every file is written: The summary report, of which the command prints a short version, lists: - the inputs and how each was verified; +- the recorded digests skipped as stale, each with its note and the digest + lists the file was checked against instead; - the expected files that are missing; - evidence found but not bound to operations: run manifests, `evidence-index.json`, `operations.json` and the content store; - warnings, for a dirty or unrecorded working tree, a caller-asserted revision, - or a declaration the build recorded no digest for. Before #1144, the last one - applies to every build. + a skipped stale digest, or a declaration the build recorded no digest for. + Before #1144, the last one applies to every build. Output is deterministic: reconstructing the same build twice gives identical bytes and the same publication ID. The command uploads nothing and starts no diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py index aafa750a29..68ae6a36bb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py @@ -18,7 +18,8 @@ import shutil import stat import uuid -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from dataclasses import dataclass from pathlib import Path from typing import Any, NoReturn @@ -91,6 +92,26 @@ "declaration-only reconstruction" ) _BUILD_FIELDS = ("build_kind", "release_role", "release_id", "build_id", "created_at") +# The note the UK drivers wrote beside the candidate manifest's digest in +# build.json. The close step then appended the staging receipts +# (``staging_delivery``, ``staged_dataset``) to the manifest and refreshed only +# sha256sums.txt, so the digest beside the note is of the manifest as first +# written. ``_refresh_completion`` (cb0d39045, merged with #1057 on 2026-09-30) +# reissues build.json over the final bytes and removes the note; dense builds +# closed by the driver merged with #901 (2026-09-29) until then kept it. +STAGING_RECEIPTS_NOTE = "staging receipts are appended after publication" + + +@dataclass(frozen=True) +class _RecordedDigest: + """One digest a build file recorded for another file.""" + + recorder: str + sha256: str + size_bytes: int | None = None + # Set when the recorder says the digest was taken before the file was + # rewritten, so a mismatch is expected rather than evidence of tampering. + stale_note: str | None = None def _refuse(message: str) -> NoReturn: @@ -148,14 +169,16 @@ def _json_object(payload: bytes, name: str) -> dict[str, object]: def _recorded_digests( payloads: Mapping[str, bytes], records: Mapping[str, dict[str, object]] -) -> dict[str, list[tuple[str, str, int | None]]]: - """Digests the build itself recorded, as ``name -> [(recorder, sha256, size)]``.""" +) -> dict[str, list[_RecordedDigest]]: + """Digests the build itself recorded, by the name of the file they cover.""" - recorded: dict[str, list[tuple[str, str, int | None]]] = {} + recorded: dict[str, list[_RecordedDigest]] = {} if SHA256SUMS_FILENAME in payloads: listing = payloads[SHA256SUMS_FILENAME].decode("utf-8") for digest, name in parse_sha256sums(listing): - recorded.setdefault(name, []).append((SHA256SUMS_FILENAME, digest, None)) + recorded.setdefault(name, []).append( + _RecordedDigest(SHA256SUMS_FILENAME, digest) + ) completion = records.get(COMPLETION_NAME, {}) manifest_record = completion.get("rowwise_candidate_manifest") if isinstance(manifest_record, dict) and "sha256" in manifest_record: @@ -164,16 +187,57 @@ def _recorded_digests( f"{COMPLETION_NAME} records an unexpected candidate manifest name", ) size = manifest_record.get("size_bytes") + note = manifest_record.get("note") recorded.setdefault(MANIFEST_FILENAME, []).append( - ( + _RecordedDigest( COMPLETION_NAME, str(manifest_record["sha256"]), size if type(size) is int else None, + note if note == STAGING_RECEIPTS_NOTE else None, ) ) return recorded +def _verification( + name: str, payload: bytes, records: Sequence[_RecordedDigest] +) -> tuple[str, list[dict[str, object]]]: + """Check ``payload`` against its recorded digests: ``(state, stale records)``. + + A mismatch is refused, except against a record that says it was taken + before the file was rewritten. Such a record is skipped and reported, and + the file counts as verified only if another recorded digest matches it. + """ + + digest, size = hashlib.sha256(payload).hexdigest(), len(payload) + matched, skipped = [], [] + for record in records: + if record.sha256 == digest and record.size_bytes in (None, size): + matched.append(record.recorder) + elif record.stale_note is not None: + skipped.append(record) + else: + _refuse(f"{name} does not match the digest recorded in {record.recorder}") + stale = [ + { + "name": name, + "recorded_in": record.recorder, + "sha256": record.sha256, + "bytes": record.size_bytes, + "note": record.stale_note, + "checked_against": matched, + } + for record in skipped + ] + if not records: + return "no_recorded_digest", stale + if not skipped: + return "recorded_digest_matched", stale + return ( + "fallback_digest_matched" if matched else "only_stale_digest_recorded" + ), stale + + def _recorded_code(manifest: Mapping[str, object]) -> tuple[str | None, bool | None]: """The code pin a candidate manifest records, top level and under identity.""" @@ -205,6 +269,24 @@ def _recorded_code(manifest: Mapping[str, object]) -> tuple[str | None, bool | N return commit, dirty # type: ignore[return-value] +def _stale_digest_warning(stale: Mapping[str, Any]) -> str: + """Name a recorded digest skipped as stale, and what checked the file instead.""" + + skipped = ( + f"{stale['recorded_in']} records the digest {stale['name']} had before the " + "staging receipts were appended, as its note says" + ) + if stale["checked_against"]: + return ( + f"{skipped}, so {stale['name']} was checked against " + f"{', '.join(stale['checked_against'])} instead." + ) + return ( + f"{skipped}, and the build recorded no later digest, so nothing the build " + f"recorded checks {stale['name']}." + ) + + def _source_revision( build: Path, recorded: tuple[str | None, bool | None], revision: str | None ) -> tuple[str, bool | None, str]: @@ -318,15 +400,13 @@ def reconstruct_uk_publication( if name in payloads } recorded = _recorded_digests(payloads, records) - verified = {} + verification: dict[str, str] = {} + stale_digests: list[dict[str, object]] = [] for name, payload in payloads.items(): - for recorder, digest, size in recorded.get(name, []): - _require( - hashlib.sha256(payload).hexdigest() == digest - and size in (None, len(payload)), - f"{name} does not match the digest recorded in {recorder}", - ) - verified[name] = bool(recorded.get(name)) + verification[name], stale_records = _verification( + name, payload, recorded.get(name, []) + ) + stale_digests.extend(stale_records) manifest = records.get(MANIFEST_FILENAME, {}) git_commit, git_dirty, origin = _source_revision( build, _recorded_code(manifest), revision @@ -341,9 +421,7 @@ def reconstruct_uk_publication( "role": _INPUT_ROLES[name], "sha256": hashlib.sha256(payload).hexdigest(), "bytes": len(payload), - "verification": "recorded_digest_matched" - if verified[name] - else "no_recorded_digest", + "verification": verification[name], } for name, payload in payloads.items() ], @@ -380,7 +458,8 @@ def reconstruct_uk_publication( ) elif git_dirty is None: warnings.append("The build did not record whether its working tree was clean.") - if not verified[DECLARATION_NAME]: + warnings.extend(_stale_digest_warning(stale) for stale in stale_digests) + if verification[DECLARATION_NAME] == "no_recorded_digest": warnings.append( f"The build recorded no digest for {DECLARATION_NAME}; it is bound only " "by the digest in the reconstruction marker." @@ -402,6 +481,7 @@ def reconstruct_uk_publication( "sources": len(graph["sources"]), }, "inputs": marker["inputs"], + "stale_recorded_digests": stale_digests, "missing": [ {"name": name, "reason": _ABSENT_REASONS[name]} for name in sorted(_ABSENT_REASONS) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py index bc283be490..5680f91621 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py @@ -1,9 +1,11 @@ """Declaration-only Orrery publications for UK builds from before run bindings. -Fixtures reproduce the files a UK dense or national build wrote before -microcosm#1144: the declaration, its operation inventory and run manifests, -the candidate manifest carrying the code pin, the completion record binding -that manifest by digest and, for a staged build, ``sha256sums.txt``. +Fixtures replay the files a UK dense or national build wrote before +microcosm#1144, in the order that era's ``full_build_cli`` wrote them: the +declaration, its operation inventory and run manifests, the candidate manifest +carrying the code pin, the completion record binding that manifest by digest, +for a staged build ``sha256sums.txt``, and then the close step, which appends +the staging receipts to the manifest. """ from __future__ import annotations @@ -18,6 +20,7 @@ from microcosm_emitter.orrery.service.queue import publication_inventory from microcosm.build.country_spec import load_country_spec +from microcosm.build.staging_dataset import refresh_sha256sums_entry from microcosm.build.uk_runtime.graph import uk_spine_endpoint, uk_spine_graph from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, uk_full_graph from microcosm.build.uk_runtime.graph_calibration import UKGraphCalibrationConfig @@ -50,6 +53,22 @@ ) RECORDED_COMMIT = "4bf7af406a1b2c3d4e5f60718293a4b5c6d7e8f9" OTHER_COMMIT = "7f235941c0000000000000000000000000000000" +NATIONAL_BUILD_ID = "uk-national-20260930T120000Z-s42" +# The completion-record layouts UK builds wrote before #1144: +# - DENSE_STALE_COMPLETION: the dense driver merged in #901 (430187fbc, merged +# 2026-09-29). ``build.json`` binds the candidate manifest as first written, +# with STAGING_RECEIPTS_NOTE; the close step then rewrites the manifest and +# refreshes only ``sha256sums.txt``, so that digest is stale. +# - DENSE_REFRESHED_COMPLETION: from cb0d39045 (#1057, merged 2026-09-30), +# ``_refresh_completion`` reissues ``build.json`` over the final bytes and +# removes the note. +# - NATIONAL_COMPLETION: the national graph path, which arrived in #1057 with +# the refresh, so its ``build.json`` (``uk_national_build_completion``) is +# always refreshed, its ``build_record`` entry included. +DENSE_STALE_COMPLETION = "dense_stale_completion" +DENSE_REFRESHED_COMPLETION = "dense_refreshed_completion" +NATIONAL_COMPLETION = "national_completion" +STAGING_RECEIPTS_NOTE = "staging receipts are appended after publication" def sha256_hex(payload: bytes) -> str: @@ -79,17 +98,45 @@ def pre_1144_declaration() -> Graph: ) +def file_entry(path: Path) -> dict[str, object]: + """A completion-record entry, as ``artifact_files.file_artifact`` wrote it.""" + + payload = path.read_bytes() + return { + "filename": path.name, + "sha256": sha256_hex(payload), + "size_bytes": len(payload), + } + + +def output_entry(path: Path) -> dict[str, object]: + """A candidate manifest ``outputs`` entry, as both drivers wrote it.""" + + payload = path.read_bytes() + return {"path": str(path), "sha256": sha256_hex(payload), "bytes": len(payload)} + + def write_pre_1144_build( directory: Path, *, graph: Graph | None = None, git_commit: str | None = RECORDED_COMMIT, git_dirty: bool | None = False, - release_role: str = "dense", + layout: str = DENSE_REFRESHED_COMPLETION, staged: bool = True, ) -> Path: - """Write the files a UK build left in its output directory before #1144.""" - + """Replay the files a UK build left in its output directory before #1144. + + The order is the driver's: the candidate manifest, ``build.json`` over it + (noting that staging receipts follow), the staging sidecars when the + bundle was staged, then the close step, which appends ``staging_delivery`` + and ``staged_dataset`` to the manifest whether or not it staged. Only the + refreshed layouts then reissue ``build.json``. + """ + + national = layout == NATIONAL_COMPLETION + release_role = "national" if national else "dense" + release_id = f"microcosm-uk-2025-{release_role}" directory.mkdir(parents=True) declared = pre_1144_declaration() if graph is None else graph (directory / "graph.json").write_bytes(graph_to_json(declared).encode()) @@ -100,39 +147,95 @@ def write_pre_1144_build( (directory / "preflight.graph.json").write_text('{"schema_version":3}') (directory / "evidence-index.json").write_text("{}") (directory / ".graph-store").mkdir() - (directory / "candidate.h5").write_bytes(b"licensed population") + dataset = directory / ( + "microcosm_uk_2024_25.h5" if national else "microcosm_uk_2024_25_local.h5" + ) + dataset.write_bytes(b"licensed population") + outputs = {"dataset": output_entry(dataset)} code = {"git_commit": git_commit, "git_dirty": git_dirty} - manifest = { + manifest: dict[str, object] = { "schema_version": 4, - "build_kind": "uk_rowwise_calibrated_candidate", "release_role": release_role, - "release_id": f"microcosm-uk-2025-{release_role}", - "created_at": "2026-10-01T12:00:00+00:00", + "release_id": release_id, + "created_at": "2026-09-30T12:00:00+00:00", **code, "identity": {"code": code}, - "outputs": {"dataset": {"path": "candidate.h5"}}, + "outputs": outputs, } - manifest_bytes = json_text(manifest).encode() - (directory / MANIFEST_FILENAME).write_bytes(manifest_bytes) - (directory / "build.json").write_bytes( - canonical_json( - { - "kind": "uk_full_build_completion", - "release_role": release_role, - "rowwise_candidate_manifest": { - "filename": MANIFEST_FILENAME, - "sha256": sha256_hex(manifest_bytes), - "size_bytes": len(manifest_bytes), - "note": "staging receipts are appended after publication", - }, - } + if national: + build_record = directory / "build_record.json" + build_record.write_text(json_text({"build_id": NATIONAL_BUILD_ID})) + outputs["build_record"] = output_entry(build_record) + manifest.update( + build_kind="uk_national_calibrated_candidate", build_id=NATIONAL_BUILD_ID ) - ) + else: + manifest.update(build_kind="uk_rowwise_calibrated_candidate") + manifest_path = directory / MANIFEST_FILENAME + manifest_path.write_text(json_text(manifest)) + manifest_entry = {**file_entry(manifest_path), "note": STAGING_RECEIPTS_NOTE} + if national: + completion: dict[str, object] = { + "schema_version": 1, + "kind": "uk_national_build_completion", + "release_role": release_role, + "release_id": release_id, + "build_id": NATIONAL_BUILD_ID, + "dataset": {**file_entry(dataset), "path": str(dataset)}, + "build_record": {**file_entry(build_record), "path": str(build_record)}, + "rowwise_candidate_manifest": manifest_entry, + "graph": {"artifacts": {}, "readback": {"passed": True}}, + "release_authorized": False, + } + else: + candidate = directory / "candidate.json" + candidate.write_bytes(canonical_json({"readback_passed": True})) + completion = { + "readback_passed": True, + "kind": "uk_full_build_completion", + "release_role": release_role, + "candidate_manifest": file_entry(candidate), + "rowwise_candidate_manifest": manifest_entry, + } + (directory / "build.json").write_bytes(canonical_json(completion)) + sums = directory / "sha256sums.txt" if staged: - (directory / "sha256sums.txt").write_text( - f"{sha256_hex(b'licensed population')} candidate.h5\n" - f"{sha256_hex(manifest_bytes)} {MANIFEST_FILENAME}\n" + # ``write_sidecars``: every output, the manifest as staged and the staged + # manifest, sorted by name. + (directory / "staged_manifest.json").write_text('{"run_id":"fixture"}') + listed = sorted( + [Path(str(entry["path"])).name for entry in outputs.values()] + + [MANIFEST_FILENAME, "staged_manifest.json"] + ) + sums.write_text( + "".join( + f"{sha256_hex((directory / name).read_bytes())} {name}\n" + for name in listed + ) ) + # The close step (``_close_attempt`` / ``_close_national_attempt``). + manifest["staging_delivery"] = {"mode": "hub" if staged else "disabled"} + manifest["staged_dataset"] = {"status": "uploaded" if staged else "disabled"} + rewritten = {"rowwise_candidate_manifest": manifest_path} + if national: + build_record.write_text( + json_text( + {"build_id": NATIONAL_BUILD_ID, "staging_delivery": {"mode": "hub"}} + ) + ) + outputs["build_record"] = output_entry(build_record) + rewritten["build_record"] = build_record + manifest_path.write_text(json_text(manifest)) + if sums.is_file(): + for path in rewritten.values(): + refresh_sha256sums_entry(directory, path.name) + if layout != DENSE_STALE_COMPLETION: + # ``_refresh_completion``: drop the note and bind the final bytes. + for role, path in rewritten.items(): + entry = dict(completion[role]) # type: ignore[arg-type] + entry.pop("note", None) + completion[role] = {**entry, **file_entry(path)} + (directory / "build.json").write_bytes(canonical_json(completion)) return directory @@ -172,7 +275,9 @@ def test_dense_build_reconstructs_the_normal_publication_layout(tmp_path): assert not {"activities", "artifacts"} & document.keys() assert all("statuses" not in node for node in document["nodes"]) # Nothing from the build other than the declaration's projection is copied. - assert not (output / "candidate.h5").exists() + assert not {path.name for path in build.iterdir()} & { + path.name for path in output.iterdir() + } def test_marker_names_the_recorded_revision_and_every_input_digest(tmp_path): @@ -211,26 +316,132 @@ def test_marker_names_the_recorded_revision_and_every_input_digest(tmp_path): assert inputs[MANIFEST_FILENAME]["verification"] == "recorded_digest_matched" -def test_candidate_manifest_disagreeing_with_build_record_is_refused(tmp_path): - build = write_pre_1144_build(tmp_path / "build", staged=False) +@pytest.mark.parametrize("layout", [DENSE_REFRESHED_COMPLETION, NATIONAL_COMPLETION]) +def test_candidate_manifest_disagreeing_with_a_refreshed_build_record_is_refused( + tmp_path, layout +): + build = write_pre_1144_build(tmp_path / "build", layout=layout, staged=False) completion = read_json(build / "build.json") completion["rowwise_candidate_manifest"]["sha256"] = "0" * 64 (build / "build.json").write_bytes(canonical_json(completion)) - with pytest.raises(ValueError, match="does not match the digest recorded in"): + with pytest.raises( + ValueError, match="does not match the digest recorded in build.json" + ): reconstruct_uk_publication(build, tmp_path / "publication") assert not (tmp_path / "publication").exists() -def test_candidate_manifest_disagreeing_with_sha256sums_is_refused(tmp_path): - build = write_pre_1144_build(tmp_path / "build") +def test_only_the_staging_receipts_note_marks_a_completion_digest_as_stale(tmp_path): + build = write_pre_1144_build(tmp_path / "build", layout=DENSE_STALE_COMPLETION) + completion = read_json(build / "build.json") + completion["rowwise_candidate_manifest"]["note"] = "checked by hand" + (build / "build.json").write_bytes(canonical_json(completion)) + + with pytest.raises( + ValueError, match="does not match the digest recorded in build.json" + ): + reconstruct_uk_publication(build, tmp_path / "publication") + + +@pytest.mark.parametrize("layout", [DENSE_REFRESHED_COMPLETION, DENSE_STALE_COMPLETION]) +def test_candidate_manifest_disagreeing_with_sha256sums_is_refused(tmp_path, layout): + # A stale completion digest is skipped, but the digest list still binds. + build = write_pre_1144_build(tmp_path / "build", layout=layout) (build / "sha256sums.txt").write_text(f"{'0' * 64} {MANIFEST_FILENAME}\n") - with pytest.raises(ValueError, match="sha256sums.txt"): + with pytest.raises( + ValueError, match="does not match the digest recorded in sha256sums.txt" + ): reconstruct_uk_publication(build, tmp_path / "publication") assert not (tmp_path / "publication").exists() +def test_dense_build_closed_before_the_completion_refresh_is_checked_against_the_digest_list( + tmp_path, +): + build = write_pre_1144_build(tmp_path / "build", layout=DENSE_STALE_COMPLETION) + stale_entry = read_json(build / "build.json")["rowwise_candidate_manifest"] + # What #901's driver left: the note, beside the digest of the manifest as it + # was before the close step appended the staging receipts. + assert stale_entry["note"] == STAGING_RECEIPTS_NOTE + assert stale_entry["sha256"] != sha256_hex((build / MANIFEST_FILENAME).read_bytes()) + + report = reconstruct_uk_publication(build, tmp_path / "publication") + + marker = validate_reconstruction(read_json(tmp_path / "publication" / SCHEMA_NAME)) + assert marker["source_revision"]["git_commit"] == RECORDED_COMMIT + manifest_input = inputs_by_name(marker)[MANIFEST_FILENAME] + assert manifest_input["verification"] == "fallback_digest_matched" + assert report["inputs"] == marker["inputs"] + assert report["stale_recorded_digests"] == [ + { + "name": MANIFEST_FILENAME, + "recorded_in": "build.json", + "sha256": stale_entry["sha256"], + "bytes": stale_entry["size_bytes"], + "note": STAGING_RECEIPTS_NOTE, + "checked_against": ["sha256sums.txt"], + } + ] + assert any( + "build.json" in warning and "checked against sha256sums.txt" in warning + for warning in report["warnings"] + ) + + +def test_stale_completion_digest_without_a_digest_list_leaves_the_manifest_unchecked( + tmp_path, +): + # With staging disabled the driver wrote no sidecars, but the close step + # still appended the receipts, so no current digest covers the manifest. + build = write_pre_1144_build( + tmp_path / "build", layout=DENSE_STALE_COMPLETION, staged=False + ) + + report = reconstruct_uk_publication(build, tmp_path / "publication") + + marker = validate_reconstruction(read_json(tmp_path / "publication" / SCHEMA_NAME)) + manifest_input = inputs_by_name(marker)[MANIFEST_FILENAME] + assert manifest_input["verification"] == "only_stale_digest_recorded" + assert [item["checked_against"] for item in report["stale_recorded_digests"]] == [ + [] + ] + assert any( + "nothing the build recorded checks" in warning for warning in report["warnings"] + ) + + +def test_national_build_reconstructs_from_its_completion_record(tmp_path): + build = write_pre_1144_build(tmp_path / "build", layout=NATIONAL_COMPLETION) + completion = read_json(build / "build.json") + assert completion["kind"] == "uk_national_build_completion" + assert "note" not in completion["rowwise_candidate_manifest"] + assert completion["build_record"]["sha256"] == sha256_hex( + (build / "build_record.json").read_bytes() + ) + + report = reconstruct_uk_publication(build, tmp_path / "publication") + + marker = validate_reconstruction(read_json(tmp_path / "publication" / SCHEMA_NAME)) + inputs = inputs_by_name(marker) + assert set(inputs) == { + "build.json", + "graph.json", + MANIFEST_FILENAME, + "sha256sums.txt", + } + assert inputs[MANIFEST_FILENAME]["verification"] == "recorded_digest_matched" + assert report["stale_recorded_digests"] == [] + assert report["build"] == { + "build_kind": "uk_national_calibrated_candidate", + "release_role": "national", + "release_id": "microcosm-uk-2025-national", + "build_id": NATIONAL_BUILD_ID, + "created_at": "2026-09-30T12:00:00+00:00", + } + + @pytest.mark.parametrize( "recorded_file", ["graph.schema.json", "execution.evidence.json"] ) @@ -304,7 +515,7 @@ def test_build_without_any_revision_is_refused(tmp_path): def test_summary_reports_unused_evidence_missing_files_and_warnings(tmp_path): build = write_pre_1144_build( - tmp_path / "build", git_dirty=True, release_role="national", staged=False + tmp_path / "build", git_dirty=True, layout=NATIONAL_COMPLETION, staged=False ) report = reconstruct_uk_publication(build, tmp_path / "publication") @@ -393,6 +604,19 @@ def test_cli_writes_the_publication_and_prints_a_summary(tmp_path, capsys): assert "already exists" in capsys.readouterr().err +def test_cli_reconstructs_a_build_closed_before_the_completion_refresh( + tmp_path, capsys +): + build = write_pre_1144_build(tmp_path / "build", layout=DENSE_STALE_COMPLETION) + output = tmp_path / "publication" + + arguments = ["--build", str(build), "--output", str(output)] + assert main([*arguments, "--revision", RECORDED_COMMIT]) == 0 + + printed = capsys.readouterr().out + assert f"{MANIFEST_FILENAME} (fallback digest matched)" in printed + + def test_maintained_uk_dense_declaration_reconstructs_within_orrery_limits(tmp_path): spine = add_uk_spine_gate_nodes( uk_spine_graph(source_mode="split"), diff --git a/packages/microcosm-graph/src/microcosm/graph/reconstruction.py b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py index 3f8fe424d7..1b2a74bc90 100644 --- a/packages/microcosm-graph/src/microcosm/graph/reconstruction.py +++ b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py @@ -41,9 +41,24 @@ # ``recorded``: the build wrote the revision itself. ``caller_asserted``: the # build recorded none, and whoever ran the reconstruction supplied it. REVISION_ORIGINS = frozenset({"recorded", "caller_asserted"}) -# Whether a digest the build itself recorded for the file was matched, or the -# build recorded none and the file is bound only by the digest listed here. -INPUT_VERIFICATIONS = frozenset({"recorded_digest_matched", "no_recorded_digest"}) +# How an input's bytes compare with the digests the build itself recorded: +# - ``recorded_digest_matched``: every recorded digest matched. +# - ``fallback_digest_matched``: the build marked one recorded digest as taken +# before it rewrote the file. That digest was skipped, and another digest the +# build recorded matched. +# - ``only_stale_digest_recorded``: every digest the build recorded for the file +# is marked as taken before the rewrite, so nothing it recorded checks the +# current bytes. +# - ``no_recorded_digest``: the build recorded none. +# In the last two cases the file is bound only by the digest listed here. +INPUT_VERIFICATIONS = frozenset( + { + "recorded_digest_matched", + "fallback_digest_matched", + "only_stale_digest_recorded", + "no_recorded_digest", + } +) RECONSTRUCTION_SCOPE = ( "Rebuilt after the build from its recorded graph declaration. Operations " "carry no execution, cache or gate state; nothing here was observed running." From 5d44bc7e46cc8c58de46b2ab2b331a2aecf3a82b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:31:48 +0400 Subject: [PATCH 06/11] Refuse a recorded declaration that is not a graph The serializer reports JSON of the wrong shape as a TypeError: a top-level array or string, or a graph object whose node list is not an array. reconstructed_schema passed that error through, and the UK command's main turns only OSError, ValueError and RecursionError into a refusal. So a graph.json of [] reconstructed with --revision ended in an uncaught traceback and exit status 1, instead of a refusal with status 2. reconstructed_schema now converts the serializer's TypeError into a ValueError that says the recorded declaration is not a graph. A recorded file is data, so a malformed one is refused like any other input that fails a check. Co-Authored-By: Claude Opus 5.5 --- docs/orrery-adapter.md | 4 +++- .../uk/test_uk_orrery_reconstruction.py | 14 ++++++++++++++ .../src/microcosm/graph/reconstruction.py | 10 +++++++++- .../shared/test_graph_reconstruction.py | 15 +++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index 3d87366c8e..b09cc59d49 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -327,7 +327,9 @@ The marker is the producer extension `microcosm.reconstruction`: In the last two cases the file is bound only by the digest in the marker. - **Exact bytes:** `reconstructed_schema` refuses a declaration that the current serializer would not write byte for byte. The schema therefore - describes the recorded file, not a reinterpretation of it. + describes the recorded file, not a reinterpretation of it. A file that is not + a graph declaration at all, such as a JSON array, is also refused with a + `ValueError`. - **Where it travels:** the marker is saved in `graph.schema.json`, so any later projection of the saved schema keeps it. Orrery documents show it at `metadata.microcosm.extensions["microcosm.reconstruction"]`. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py index 5680f91621..54c5170949 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py @@ -604,6 +604,20 @@ def test_cli_writes_the_publication_and_prints_a_summary(tmp_path, capsys): assert "already exists" in capsys.readouterr().err +def test_cli_refuses_a_declaration_that_is_not_a_json_object(tmp_path, capsys): + build = write_pre_1144_build(tmp_path / "build", git_commit=None, git_dirty=None) + (build / "graph.json").write_bytes(b"[]") + output = tmp_path / "publication" + + arguments = ["--build", str(build), "--output", str(output)] + with pytest.raises(SystemExit) as refusal: + main([*arguments, "--revision", OTHER_COMMIT]) + + assert refusal.value.code == 2 + assert "recorded declaration is not a graph" in capsys.readouterr().err + assert not output.exists() + + def test_cli_reconstructs_a_build_closed_before_the_completion_refresh( tmp_path, capsys ): diff --git a/packages/microcosm-graph/src/microcosm/graph/reconstruction.py b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py index 1b2a74bc90..8bad2aee13 100644 --- a/packages/microcosm-graph/src/microcosm/graph/reconstruction.py +++ b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py @@ -230,8 +230,16 @@ def reconstructed_schema( raise ValueError( "Graph reconstruction: declaration must be UTF-8 JSON." ) from error + try: + declared = graph_from_json(text) + except TypeError as error: + # The serializer reports JSON of the wrong shape (an array where the + # graph object belongs, say) as a TypeError; recorded files are data. + raise ValueError( + f"Graph reconstruction: the recorded declaration is not a graph ({error})." + ) from error schema = graph_schema( - compile_graph(graph_from_json(text)), + compile_graph(declared), extensions={RECONSTRUCTION_EXTENSION: checked}, ) _require( diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py index 93bd1e95f6..3f38cf0563 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py @@ -124,6 +124,21 @@ def test_declaration_that_does_not_reproduce_its_recorded_bytes_is_refused(): reconstructed_schema(reformatted, marker_for(reformatted)) +@pytest.mark.parametrize( + "declaration", + [ + b"[]", + b'"graph"', + # A graph object whose node list has the wrong type. + b'{"country":"uk","nodes":{},"sources":[]}', + ], + ids=["array", "string", "node_list_of_wrong_type"], +) +def test_declaration_that_is_not_a_graph_is_refused(declaration): + with pytest.raises(ValueError, match="recorded declaration is not a graph"): + reconstructed_schema(declaration, marker_for(declaration)) + + def test_marker_must_name_the_declaration_it_was_built_from(): other_declaration = graph_to_json(compiled_default_population_graph().graph).encode( "utf-8" From fc3e2327e5a0e250727dd478dc7dc4db6b0e0ecd Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:33:17 +0400 Subject: [PATCH 07/11] Refuse an output directory inside the build directory The CLI help and the docs say the build directory is only read, but an --output inside it was accepted: the command created its hidden staging directory and the publication inside the build. Output equal to the build was refused only because it already exists, with a message that did not name the problem. The output path is now resolved, and each of its existing ancestors is compared with the build directory by os.path.samefile. An output equal to the build, or beneath it, is refused before anything is written, so "..", symbolic links and case-insensitive file systems cannot disguise the build directory. Co-Authored-By: Claude Opus 5.5 --- .../1291-uk-orrery-reconstruction.added.md | 2 +- docs/orrery-adapter.md | 4 +++- .../build/uk_runtime/orrery_reconstruction.py | 23 ++++++++++++++++++- .../uk/test_uk_orrery_reconstruction.py | 23 +++++++++++++++++++ 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/changelog.d/1291-uk-orrery-reconstruction.added.md b/changelog.d/1291-uk-orrery-reconstruction.added.md index 399008a367..7d3bd474eb 100644 --- a/changelog.d/1291-uk-orrery-reconstruction.added.md +++ b/changelog.d/1291-uk-orrery-reconstruction.added.md @@ -1 +1 @@ -Add `python -m microcosm.build.uk_runtime.orrery_reconstruction`, which turns the output directory of a UK dense or national build from before #1144 into a declaration-only Orrery publication. It writes a new directory with the normal publication layout (`graph.schema.json`, `graph.orrery.json`, `orrery.upload.json`) and a summary report. The declaration comes from the build's recorded `graph.json` and the revision from its recorded code pin. Every input is checked against the digests the build recorded. The one exception is the candidate manifest digest that dense builds closed by #901's driver left in `build.json` beside the note that staging receipts follow: it is skipped and reported, and the manifest is checked against `sha256sums.txt` instead. Builds that already record Orrery evidence are refused, and nothing is uploaded. +Add `python -m microcosm.build.uk_runtime.orrery_reconstruction`, which turns the output directory of a UK dense or national build from before #1144 into a declaration-only Orrery publication. It writes a new directory with the normal publication layout (`graph.schema.json`, `graph.orrery.json`, `orrery.upload.json`) and a summary report. The declaration comes from the build's recorded `graph.json` and the revision from its recorded code pin. Every input is checked against the digests the build recorded. The one exception is the candidate manifest digest that dense builds closed by #901's driver left in `build.json` beside the note that staging receipts follow: it is skipped and reported, and the manifest is checked against `sha256sums.txt` instead. Builds that already record Orrery evidence are refused, as is an output directory inside the build directory, which is only read. Nothing is uploaded. diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index b09cc59d49..0bcec652d3 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -388,7 +388,9 @@ The build directory is only read. The command: it. It writes a new directory with the layout of a normal publication, and moves the -directory into place only after every file is written: +directory into place only after every file is written. The output must not +exist, and must not be the build directory or lie inside it once `..` and links +are resolved: | File | Contents | | --- | --- | diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py index 68ae6a36bb..71dc6dbaf4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py @@ -325,6 +325,21 @@ def _build_identity(*records: Mapping[str, object]) -> dict[str, str]: return identity +def _within(directory: Path, candidate: Path) -> bool: + """Whether ``candidate`` is ``directory`` or lies beneath it, links resolved. + + Each existing ancestor of the resolved path is compared with ``directory`` + as a file, so ``..``, symbolic links and case-insensitive file systems + cannot disguise the build directory. + """ + + resolved = candidate.resolve() + return any( + ancestor.exists() and os.path.samefile(ancestor, directory) + for ancestor in (resolved, *resolved.parents) + ) + + def _unused_evidence(build: Path) -> list[dict[str, str]]: found = [] for name in sorted(os.listdir(build)): @@ -375,6 +390,10 @@ def reconstruct_uk_publication( build, output = Path(build), Path(output) _require(build.is_dir(), f"build directory {build} does not exist") + _require( + not _within(build, output), + f"output {output} is inside the build directory {build}, which is only read", + ) _require(not os.path.lexists(output), f"output {output} already exists") recorded_evidence = [ name @@ -553,7 +572,9 @@ def main(argv: list[str] | None = None) -> int: "--output", type=Path, required=True, - help="new publication directory; it must not exist", + help=( + "new publication directory, outside the build directory; it must not exist" + ), ) parser.add_argument( "--revision", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py index 54c5170949..579a864363 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py @@ -577,6 +577,29 @@ def interrupted_rename(source, destination): assert sorted(path.name for path in tmp_path.iterdir()) == ["build", "existing"] +@pytest.mark.parametrize( + "output_parts", + [ + ("build",), + ("build", "publication"), + ("elsewhere", "..", "build", "publication"), + ("link-to-build", "publication"), + ], + ids=["the_build_itself", "inside", "through_a_parent_reference", "through_a_link"], +) +def test_output_inside_the_build_directory_is_refused(tmp_path, output_parts): + build = write_pre_1144_build(tmp_path / "build") + (tmp_path / "elsewhere").mkdir() + (tmp_path / "link-to-build").symlink_to(build, target_is_directory=True) + listing = sorted(path.name for path in build.iterdir()) + + with pytest.raises(ValueError, match="inside the build directory"): + reconstruct_uk_publication(build, tmp_path.joinpath(*output_parts)) + + # The build is only read: nothing was written into it, not even staging. + assert sorted(path.name for path in build.iterdir()) == listing + + def test_declaration_behind_a_symbolic_link_is_refused(tmp_path): build = write_pre_1144_build(tmp_path / "build") elsewhere = tmp_path / "elsewhere.json" From 14445a65e3e3cfbc422f3a504154a957fcd9597c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:39:58 +0400 Subject: [PATCH 08/11] Record the code revision that performed the reconstruction The docs, the changelog and the PR said nothing in a reconstruction is re-derived from current code. Only the declaration's bytes are the build's own, though: the schema's compiled, fields and input_bindings sections, and the Orrery document projected from it, are computed by the code that runs the reconstruction. The marker named only the source revision, so a reader could not tell which code produced those parts, or that a reconstruction at another revision may differ in them. The marker now has a reconstruction_revision field next to source_revision, with git_commit and git_dirty. code_revision(path) reads them from the git checkout that tracks the reconstructing module's own file, not from the current directory, and returns nulls when no checkout tracks that file, so an installed copy inside some unrelated repository is never attributed to it. The UK command records it, and shows it in the report and the summary. It warns when the revision is unknown, or when the working tree is dirty or could not be checked. The scope text, docs and changelog now say which parts come from the build and which come from the reconstructing code. The protocol stays v1: it has not been released. Co-Authored-By: Claude Opus 5.5 --- ...1291-orrery-reconstruction-marker.added.md | 2 +- .../1291-uk-orrery-reconstruction.added.md | 2 +- docs/orrery-adapter.md | 72 +++++++++---- .../build/uk_runtime/orrery_reconstruction.py | 40 ++++++- .../uk/test_uk_orrery_reconstruction.py | 57 ++++++++++ .../src/microcosm/graph/reconstruction.py | 102 ++++++++++++++++-- .../shared/test_graph_reconstruction.py | 100 +++++++++++++++++ 7 files changed, 338 insertions(+), 37 deletions(-) diff --git a/changelog.d/1291-orrery-reconstruction-marker.added.md b/changelog.d/1291-orrery-reconstruction-marker.added.md index 52e26303e9..f19de06661 100644 --- a/changelog.d/1291-orrery-reconstruction-marker.added.md +++ b/changelog.d/1291-orrery-reconstruction-marker.added.md @@ -1 +1 @@ -Add `microcosm.graph.reconstruction`, which rebuilds a declaration-only graph schema from a build's recorded `graph.json` and marks it with the `microcosm.reconstruction` extension (protocol `microcosm.graph.reconstruction.v1`). The marker names the source revision and every input file by digest, and it travels in `graph.schema.json` into the Orrery document's metadata. A declaration that the current serializer would not reproduce byte for byte is refused, and no operation is presented as executed. +Add `microcosm.graph.reconstruction`, which rebuilds a declaration-only graph schema from a build's recorded `graph.json` and marks it with the `microcosm.reconstruction` extension (protocol `microcosm.graph.reconstruction.v1`). The marker names the revision that declared the graph (`source_revision`), the revision of the code that performed the reconstruction (`reconstruction_revision`, read by `code_revision` from the checkout that tracks that code), and every input file by digest. Only the declaration's bytes come from the build: the schema's `compiled`, `fields` and `input_bindings` sections and the Orrery projection come from the reconstructing code. The marker travels in `graph.schema.json` into the Orrery document's metadata. A declaration that the current serializer would not reproduce byte for byte is refused, and no operation is presented as executed. diff --git a/changelog.d/1291-uk-orrery-reconstruction.added.md b/changelog.d/1291-uk-orrery-reconstruction.added.md index 7d3bd474eb..258227a587 100644 --- a/changelog.d/1291-uk-orrery-reconstruction.added.md +++ b/changelog.d/1291-uk-orrery-reconstruction.added.md @@ -1 +1 @@ -Add `python -m microcosm.build.uk_runtime.orrery_reconstruction`, which turns the output directory of a UK dense or national build from before #1144 into a declaration-only Orrery publication. It writes a new directory with the normal publication layout (`graph.schema.json`, `graph.orrery.json`, `orrery.upload.json`) and a summary report. The declaration comes from the build's recorded `graph.json` and the revision from its recorded code pin. Every input is checked against the digests the build recorded. The one exception is the candidate manifest digest that dense builds closed by #901's driver left in `build.json` beside the note that staging receipts follow: it is skipped and reported, and the manifest is checked against `sha256sums.txt` instead. Builds that already record Orrery evidence are refused, as is an output directory inside the build directory, which is only read. Nothing is uploaded. +Add `python -m microcosm.build.uk_runtime.orrery_reconstruction`, which turns the output directory of a UK dense or national build from before #1144 into a declaration-only Orrery publication. It writes a new directory with the normal publication layout (`graph.schema.json`, `graph.orrery.json`, `orrery.upload.json`) and a summary report. The declaration comes from the build's recorded `graph.json` and the revision from its recorded code pin. The schema's compiled sections and the Orrery document are computed by the code running the command, whose revision the marker records beside the build's. Every input is checked against the digests the build recorded. The one exception is the candidate manifest digest that dense builds closed by #901's driver left in `build.json` beside the note that staging receipts follow: it is skipped and reported, and the manifest is checked against `sha256sums.txt` instead. Builds that already record Orrery evidence are refused, as is an output directory inside the build directory, which is only read. Nothing is uploaded. diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index 0bcec652d3..d598177b60 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -276,17 +276,29 @@ binding that execution export validates, so its execution cannot be shown. declaration alone and marks it as reconstructed: ```python -from microcosm.graph.reconstruction import reconstructed_schema, reconstruction_marker +from microcosm.graph.reconstruction import ( + code_revision, + reconstructed_schema, + reconstruction_marker, +) marker = reconstruction_marker( git_commit=recorded_commit, git_dirty=False, revision_origin="recorded", + reconstruction_revision=code_revision(__file__), inputs=inputs, # one {name, role, sha256, bytes, verification} per file read ) schema = reconstructed_schema(declaration_bytes, marker) ``` +Only the declaration comes from the build. The schema's `graph` section and +`graph_sha256` reproduce the recorded file byte for byte. Its `compiled`, +`fields` and `input_bindings` sections, and the Orrery document projected from +it, are computed by the code running the reconstruction. A reconstruction made +at another revision can therefore differ in those parts, which is why the +marker records that revision separately from the build's. + The marker is the producer extension `microcosm.reconstruction`: ```json @@ -299,19 +311,30 @@ The marker is the producer extension `microcosm.reconstruction`: "git_dirty": false, "origin": "recorded" }, + "reconstruction_revision": { + "git_commit": "", + "git_dirty": false + }, "inputs": [ {"name": "graph.json", "role": "declaration", "sha256": "", "bytes": 4096, "verification": "no_recorded_digest"}, {"name": "rowwise_candidate_manifest.json", "role": "code_pin", "sha256": "", "bytes": 2048, "verification": "recorded_digest_matched"} ], - "scope": "Rebuilt after the build from its recorded graph declaration. ..." + "scope": "Rebuilt after the build from its recorded graph declaration, the only part taken from the build: ..." } ``` -- **`source_revision`:** `origin` is `recorded` when the build wrote the - revision itself, and `caller_asserted` when the build recorded none and the - caller supplied one. `git_dirty` is `null` when the build did not record it. +- **`source_revision`:** the code that declared the graph. `origin` is + `recorded` when the build wrote the revision itself, and `caller_asserted` + when the build recorded none and the caller supplied one. `git_dirty` is + `null` when the build did not record it. +- **`reconstruction_revision`:** the code that performed the reconstruction, + and so computed every section except the declaration. `code_revision(path)` + reads it from the git checkout that tracks `path` (the reconstructing + module's own file), not from the current directory. Both fields are `null` + when no checkout tracks that file, as for an installed wheel. `git_dirty` is + `null` alone when git could not check the working tree. - **`inputs`:** every file the reconstruction read, in name order, by digest and size. Exactly one input is the declaration, and its digest equals the schema's `graph_sha256`. `verification` compares the file with the digests @@ -326,10 +349,10 @@ The marker is the producer extension `microcosm.reconstruction`: In the last two cases the file is bound only by the digest in the marker. - **Exact bytes:** `reconstructed_schema` refuses a declaration that the - current serializer would not write byte for byte. The schema therefore - describes the recorded file, not a reinterpretation of it. A file that is not - a graph declaration at all, such as a JSON array, is also refused with a - `ValueError`. + current serializer would not write byte for byte. The schema's `graph` + section is therefore the recorded file, not a reinterpretation of it. A file + that is not a graph declaration at all, such as a JSON array, is also refused + with a `ValueError`. - **Where it travels:** the marker is saved in `graph.schema.json`, so any later projection of the saved schema keeps it. Orrery documents show it at `metadata.microcosm.extensions["microcosm.reconstruction"]`. @@ -353,9 +376,13 @@ uv run python -m microcosm.build.uk_runtime.orrery_reconstruction \ --output path/to/reconstructed-publication ``` -The declaration is the build's own `graph.json`, never one re-derived from -current code. The declaration format has not changed since the UK graph driver, -so a recorded file reproduces byte for byte; one that does not is refused. +The declaration is the build's own `graph.json`. It is not rebuilt by running +the UK graph builders at the current revision. The declaration format has not +changed since the UK graph driver, so a recorded file reproduces byte for byte; +one that does not is refused. Everything derived from it (the schema's +compiled sections and the Orrery document) is computed by the code running the +command, and the marker records that code's revision as +`reconstruction_revision`. The build directory is only read. The command: @@ -402,18 +429,21 @@ are resolved: The summary report, of which the command prints a short version, lists: - the inputs and how each was verified; -- the recorded digests skipped as stale, each with its note and the digest - lists the file was checked against instead; +- the source revision and the reconstruction revision; +- the recorded digests skipped as stale, each with its note and the records the + file was checked against instead; - the expected files that are missing; - evidence found but not bound to operations: run manifests, `evidence-index.json`, `operations.json` and the content store; -- warnings, for a dirty or unrecorded working tree, a caller-asserted revision, - a skipped stale digest, or a declaration the build recorded no digest for. - Before #1144, the last one applies to every build. - -Output is deterministic: reconstructing the same build twice gives identical -bytes and the same publication ID. The command uploads nothing and starts no -emitter service, so publishing the directory remains a separate step. +- warnings, for a dirty or unrecorded working tree in the build or the + reconstruction, an unknown reconstruction revision, a caller-asserted + revision, a skipped stale digest, or a declaration the build recorded no + digest for. Before #1144, the last one applies to every build. + +Output is deterministic: reconstructing the same build twice from the same +code revision gives identical bytes and the same publication ID. The command +uploads nothing and starts no emitter service, so publishing the directory +remains a separate step. ## Orrery evidence panels diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py index 71dc6dbaf4..fcaeb97ded 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py @@ -4,9 +4,12 @@ declaration (``graph.json``) and their code pin, but no run binding, so their recorded execution cannot be validated. This module projects the recorded declaration alone into the normal publication layout, marked as reconstructed -by :mod:`microcosm.graph.reconstruction`. It reads only the build directory, -executes no kernel and uploads nothing: the publication directory is submitted -later, if at all, through the usual publication path. +by :mod:`microcosm.graph.reconstruction`. Only the declaration's bytes come +from the build: the schema's compiled sections and the Orrery document are +computed by the code running this module, whose revision the marker records +beside the build's. It reads only the build directory, executes no kernel and +uploads nothing: the publication directory is submitted later, if at all, +through the usual publication path. """ from __future__ import annotations @@ -30,6 +33,7 @@ DECLARATION_ONLY, DECLARATION_ROLE, GIT_COMMIT_PATTERN, + code_revision, reconstructed_schema, reconstruction_marker, ) @@ -430,10 +434,14 @@ def reconstruct_uk_publication( git_commit, git_dirty, origin = _source_revision( build, _recorded_code(manifest), revision ) + # The code computing the schema's sections and the Orrery projection, + # identified from this module's file rather than the working directory. + reconstructing = code_revision(Path(__file__)) marker = reconstruction_marker( git_commit=git_commit, git_dirty=git_dirty, revision_origin=origin, + reconstruction_revision=reconstructing, inputs=[ { "name": name, @@ -477,6 +485,22 @@ def reconstruct_uk_publication( ) elif git_dirty is None: warnings.append("The build did not record whether its working tree was clean.") + if reconstructing["git_commit"] is None: + warnings.append( + "The code running this reconstruction is not a file tracked by a git " + "checkout, so the marker records no reconstruction revision." + ) + elif reconstructing["git_dirty"] is None: + warnings.append( + "git could not say whether the working tree running this " + "reconstruction was clean." + ) + elif reconstructing["git_dirty"]: + warnings.append( + "This reconstruction ran from a working tree with uncommitted changes, " + "so its reconstruction revision does not fully identify the code that " + "compiled the schema and projected the Orrery document." + ) warnings.extend(_stale_digest_warning(stale) for stale in stale_digests) if verification[DECLARATION_NAME] == "no_recorded_digest": warnings.append( @@ -491,6 +515,7 @@ def reconstruct_uk_publication( "publication_id": publication_id, "published_files": [item["name"] for item in published] + [INVENTORY_NAME], "source_revision": marker["source_revision"], + "reconstruction_revision": marker["reconstruction_revision"], "build": _build_identity(manifest, records.get(COMPLETION_NAME, {})), "declaration": { "name": DECLARATION_NAME, @@ -534,9 +559,18 @@ def summary_text(report: Mapping[str, Any], output: Path) -> str: if revision["origin"] == "recorded" else "asserted with --revision" ) + reconstructing = report["reconstruction_revision"] + if reconstructing["git_commit"] is None: + reconstructed_by = "unknown revision" + else: + tree = {False: "clean", True: "uncommitted changes"}.get( + reconstructing["git_dirty"], "working tree not checked" + ) + reconstructed_by = f"{reconstructing['git_commit']} ({tree})" lines = [ f"Wrote a reconstructed Orrery publication (declaration only) to {output}", f" revision: {revision['git_commit']} ({origin})", + f" reconstructed by: {reconstructed_by}", f" declaration: {declaration['name']}, " f"{_count(declaration['operations'], 'operation')}, " f"{_count(declaration['sources'], 'source')}", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py index 579a864363..051269f7a5 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py @@ -21,6 +21,7 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.staging_dataset import refresh_sha256sums_entry +from microcosm.build.uk_runtime import orrery_reconstruction from microcosm.build.uk_runtime.graph import uk_spine_endpoint, uk_spine_graph from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, uk_full_graph from microcosm.build.uk_runtime.graph_calibration import UKGraphCalibrationConfig @@ -316,6 +317,61 @@ def test_marker_names_the_recorded_revision_and_every_input_digest(tmp_path): assert inputs[MANIFEST_FILENAME]["verification"] == "recorded_digest_matched" +def test_marker_records_the_code_that_ran_the_reconstruction_beside_the_source( + tmp_path, monkeypatch +): + looked_up = [] + + def reconstructing_checkout(path): + looked_up.append(Path(path)) + return {"git_commit": OTHER_COMMIT, "git_dirty": True} + + monkeypatch.setattr(orrery_reconstruction, "code_revision", reconstructing_checkout) + build = write_pre_1144_build(tmp_path / "build") + + report = reconstruct_uk_publication(build, tmp_path / "publication") + + marker = validate_reconstruction(read_json(tmp_path / "publication" / SCHEMA_NAME)) + # The revision is looked up from the reconstructing module's own file, not + # from whatever directory the command runs in. + assert looked_up == [Path(orrery_reconstruction.__file__)] + assert marker["source_revision"] == { + "git_commit": RECORDED_COMMIT, + "git_dirty": False, + "origin": "recorded", + } + assert marker["reconstruction_revision"] == { + "git_commit": OTHER_COMMIT, + "git_dirty": True, + } + assert report["reconstruction_revision"] == marker["reconstruction_revision"] + assert any( + "reconstruction ran from a working tree with uncommitted changes" in warning + for warning in report["warnings"] + ) + + +def test_unknown_reconstruction_revision_is_recorded_as_unknown( + tmp_path, monkeypatch, capsys +): + monkeypatch.setattr( + orrery_reconstruction, + "code_revision", + lambda path: {"git_commit": None, "git_dirty": None}, + ) + build = write_pre_1144_build(tmp_path / "build") + output = tmp_path / "publication" + + assert main(["--build", str(build), "--output", str(output)]) == 0 + + report = read_json(output / REPORT_NAME) + assert report["reconstruction_revision"] == {"git_commit": None, "git_dirty": None} + assert any( + "no reconstruction revision" in warning for warning in report["warnings"] + ) + assert "reconstructed by: unknown revision" in capsys.readouterr().out + + @pytest.mark.parametrize("layout", [DENSE_REFRESHED_COMPLETION, NATIONAL_COMPLETION]) def test_candidate_manifest_disagreeing_with_a_refreshed_build_record_is_refused( tmp_path, layout @@ -619,6 +675,7 @@ def test_cli_writes_the_publication_and_prints_a_summary(tmp_path, capsys): printed = capsys.readouterr().out assert str(output) in printed assert RECORDED_COMMIT in printed + assert "reconstructed by: " in printed assert "declaration only" in printed assert (output / GRAPH_NAME).is_file() with pytest.raises(SystemExit) as refusal: diff --git a/packages/microcosm-graph/src/microcosm/graph/reconstruction.py b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py index 8bad2aee13..fcce64b8ec 100644 --- a/packages/microcosm-graph/src/microcosm/graph/reconstruction.py +++ b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py @@ -3,10 +3,14 @@ A build that recorded its graph declaration but no run binding cannot have its execution verified, yet its declared graph is still exact. A reconstructed schema projects that recorded declaration alone and carries a marker in its -producer extensions: the marker says the schema was rebuilt afterwards, names -the code revision that declared the graph, and binds every input file by -digest. Because the marker lives in ``graph.schema.json``, any later -projection of the saved schema keeps it, and Orrery documents show it under +producer extensions: the marker says the schema was rebuilt afterwards and +binds every input file by digest. It names two code revisions: the one that +declared the graph (``source_revision``) and the one that performed the +reconstruction (``reconstruction_revision``). Only the declaration's bytes come +from the build. The schema's ``compiled``, ``fields`` and ``input_bindings`` +sections, and any Orrery projection of it, are computed by the reconstructing +code. Because the marker lives in ``graph.schema.json``, any later projection +of the saved schema keeps it, and Orrery documents show it under ``metadata.microcosm.extensions``. Nothing here presents an operation as executed; no receipt, cache or gate state is attached. """ @@ -15,7 +19,9 @@ import hashlib import re +import subprocess from collections.abc import Mapping, Sequence +from pathlib import Path from .decl import compile_graph from .schema import _plain_json, graph_schema @@ -29,6 +35,7 @@ "RECONSTRUCTION_EXTENSION", "RECONSTRUCTION_PROTOCOL", "REVISION_ORIGINS", + "code_revision", "reconstructed_schema", "reconstruction_marker", "validate_reconstruction", @@ -60,13 +67,24 @@ } ) RECONSTRUCTION_SCOPE = ( - "Rebuilt after the build from its recorded graph declaration. Operations " - "carry no execution, cache or gate state; nothing here was observed running." + "Rebuilt after the build from its recorded graph declaration, the only part " + "taken from the build: the compiled sections and this projection come from " + "the code at reconstruction_revision. Operations carry no execution, cache " + "or gate state; nothing here was observed running." ) _MARKER_FIELDS = frozenset( - {"protocol", "reconstructed", "basis", "source_revision", "inputs", "scope"} + { + "protocol", + "reconstructed", + "basis", + "source_revision", + "reconstruction_revision", + "inputs", + "scope", + } ) _REVISION_FIELDS = frozenset({"git_commit", "git_dirty", "origin"}) +_RECONSTRUCTION_REVISION_FIELDS = frozenset({"git_commit", "git_dirty"}) _INPUT_FIELDS = frozenset({"name", "role", "sha256", "bytes", "verification"}) # A full lowercase SHA-1 or SHA-256 git object name; abbreviations are refused. GIT_COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?") @@ -118,6 +136,24 @@ def _validated_marker(value: object) -> dict[str, object]: "source revision dirty flag must be true, false or null", ) _require(revision["origin"] in REVISION_ORIGINS, "source revision origin") + reconstruction = marker["reconstruction_revision"] + _require( + type(reconstruction) is dict + and set(reconstruction) == _RECONSTRUCTION_REVISION_FIELDS, + "reconstruction revision fields", + ) + commit = reconstruction["git_commit"] + _require( + commit is None + or (type(commit) is str and GIT_COMMIT_PATTERN.fullmatch(commit) is not None), + "reconstruction revision must be a full lowercase git commit or null", + ) + _require( + type(reconstruction["git_dirty"]) in (bool, type(None)) + and (commit is not None or reconstruction["git_dirty"] is None), + "reconstruction revision dirty flag must be true, false or, without a " + "commit, null", + ) inputs = marker["inputs"] _require(type(inputs) is list and bool(inputs), "marker inputs") names = [] @@ -152,18 +188,61 @@ def _declaration_input(marker: Mapping[str, object]) -> Mapping[str, object]: return next(item for item in inputs if item["role"] == DECLARATION_ROLE) +def code_revision(path: str | Path) -> dict[str, object]: + """The git commit and working-tree state of the checkout that tracks ``path``. + + Pass the file of the code performing the reconstruction. git runs in that + file's directory, not the current one, and the file must be tracked there: + an installed copy that happens to sit inside some checkout (a virtual + environment within a repository, say) identifies no revision. Both values + are ``None`` when no checkout tracks the file, so an unknown revision is + never recorded as a clean one. + """ + + location = Path(path).resolve() + unknown: dict[str, object] = {"git_commit": None, "git_dirty": None} + + def git(*arguments: str) -> str | None: + try: + result = subprocess.run( + ["git", "-C", str(location.parent), *arguments], + check=False, + capture_output=True, + text=True, + ) + except OSError: + return None + return result.stdout if result.returncode == 0 else None + + if git("ls-files", "--error-unmatch", "--", location.name) is None: + return unknown + head = git("rev-parse", "HEAD") + commit = None if head is None else head.strip() + if commit is None or GIT_COMMIT_PATTERN.fullmatch(commit) is None: + return unknown + changes = git("status", "--porcelain", "--untracked-files=no") + return { + "git_commit": commit, + "git_dirty": None if changes is None else bool(changes.strip()), + } + + def reconstruction_marker( *, git_commit: str, git_dirty: bool | None, revision_origin: str, + reconstruction_revision: Mapping[str, object], inputs: Sequence[Mapping[str, object]], ) -> dict[str, object]: - """Return a validated marker naming the revision and every input read. + """Return a validated marker naming both revisions and every input read. - Each input is a mapping with ``name`` (a plain file name), ``role``, - ``sha256``, ``bytes`` and ``verification``; exactly one has role - ``declaration``. Inputs are listed in name order. + ``git_commit``, ``git_dirty`` and ``revision_origin`` describe the code that + declared the graph. ``reconstruction_revision`` is :func:`code_revision` of + the code performing the reconstruction, which computes every section of the + schema except the declaration itself. Each input is a mapping with ``name`` + (a plain file name), ``role``, ``sha256``, ``bytes`` and ``verification``; + exactly one has role ``declaration``. Inputs are listed in name order. """ ordered = sorted( @@ -179,6 +258,7 @@ def reconstruction_marker( "git_dirty": git_dirty, "origin": revision_origin, }, + "reconstruction_revision": dict(reconstruction_revision), "inputs": ordered, "scope": RECONSTRUCTION_SCOPE, } diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py index 3f38cf0563..84a577045a 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py @@ -16,6 +16,7 @@ DECLARATION_ONLY, RECONSTRUCTION_EXTENSION, RECONSTRUCTION_PROTOCOL, + code_revision, reconstructed_schema, reconstruction_marker, validate_reconstruction, @@ -32,6 +33,11 @@ _TEST_PATHS.repository / "tools" / "orrery-contract" / "verify-reconstruction.mjs" ) RECORDED_COMMIT = "0123456789abcdef0123456789abcdef01234567" +# The revision of the code that ran the reconstruction, later than the build's. +RECONSTRUCTING_REVISION = { + "git_commit": "89abcdef0123456789abcdef0123456789abcdef", + "git_dirty": False, +} def recorded_declaration() -> bytes: @@ -55,6 +61,7 @@ def marker_for(declaration: bytes, **revision_overrides) -> dict: "git_commit": RECORDED_COMMIT, "git_dirty": False, "revision_origin": "recorded", + "reconstruction_revision": RECONSTRUCTING_REVISION, **revision_overrides, } return reconstruction_marker( @@ -100,6 +107,74 @@ def test_reconstructed_schema_carries_the_marker_into_orrery_metadata(): assert carried == marker +def test_marker_records_the_reconstructing_code_apart_from_the_source_revision(): + declaration = recorded_declaration() + + marker = marker_for(declaration) + unknown = marker_for( + declaration, + reconstruction_revision={"git_commit": None, "git_dirty": None}, + ) + + # The declaration's bytes come from the source revision; the compiled + # sections and the Orrery projection come from the reconstructing code. + assert marker["source_revision"]["git_commit"] == RECORDED_COMMIT + assert marker["reconstruction_revision"] == RECONSTRUCTING_REVISION + assert unknown["reconstruction_revision"] == {"git_commit": None, "git_dirty": None} + assert "reconstruction_revision" in marker["scope"] + schema = reconstructed_schema(declaration, marker) + assert validate_reconstruction(schema)["reconstruction_revision"] == ( + RECONSTRUCTING_REVISION + ) + + +def run_git(checkout, *arguments) -> str: + return subprocess.run( + [ + "git", + "-C", + str(checkout), + "-c", + "user.name=Reconstruction fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + *arguments, + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def test_code_revision_names_the_checkout_that_tracks_the_file(tmp_path): + checkout = tmp_path / "checkout" + checkout.mkdir() + run_git(checkout, "init", "--quiet") + reconstructing_code = checkout / "reconstruct.py" + reconstructing_code.write_text("VERSION = 1\n") + run_git(checkout, "add", "reconstruct.py") + run_git(checkout, "commit", "--quiet", "-m", "Add the reconstructing code") + head = run_git(checkout, "rev-parse", "HEAD") + + assert code_revision(reconstructing_code) == { + "git_commit": head, + "git_dirty": False, + } + reconstructing_code.write_text("VERSION = 2\n") + assert code_revision(reconstructing_code) == {"git_commit": head, "git_dirty": True} + # A file no checkout tracks (an installed copy that happens to sit inside + # a checkout, or one outside any) identifies no revision. + untracked_copy = checkout / "installed.py" + untracked_copy.write_text("VERSION = 2\n") + outside_any_checkout = tmp_path / "outside.py" + outside_any_checkout.write_text("VERSION = 2\n") + unknown = {"git_commit": None, "git_dirty": None} + assert code_revision(untracked_copy) == unknown + assert code_revision(outside_any_checkout) == unknown + + def test_reconstructed_document_claims_no_execution(): declaration = recorded_declaration() @@ -210,6 +285,26 @@ def _other_protocol(marker: dict) -> None: marker["protocol"] = "microcosm.graph.reconstruction.v0" +def _no_reconstruction_revision(marker: dict) -> None: + del marker["reconstruction_revision"] + + +def _short_reconstruction_commit(marker: dict) -> None: + marker["reconstruction_revision"]["git_commit"] = "89abcde" + + +def _textual_reconstruction_dirty_flag(marker: dict) -> None: + marker["reconstruction_revision"]["git_dirty"] = "no" + + +def _dirty_flag_without_a_reconstruction_commit(marker: dict) -> None: + marker["reconstruction_revision"] = {"git_commit": None, "git_dirty": False} + + +def _reconstruction_revision_with_an_origin(marker: dict) -> None: + marker["reconstruction_revision"]["origin"] = "recorded" + + @pytest.mark.parametrize( "corrupt", [ @@ -228,6 +323,11 @@ def _other_protocol(marker: dict) -> None: _recorded_basis, _not_reconstructed, _other_protocol, + _no_reconstruction_revision, + _short_reconstruction_commit, + _textual_reconstruction_dirty_flag, + _dirty_flag_without_a_reconstruction_commit, + _reconstruction_revision_with_an_origin, ], ids=lambda corrupt: corrupt.__name__.strip("_"), ) From 5b4da04e01942a1011fc795d62d611f02940ca1a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:15:39 +0400 Subject: [PATCH 09/11] Read the reconstructing code's revision without inherited git variables code_revision ran git in the reconstructing file's directory with this process's whole environment. An exported GIT_DIR, GIT_WORK_TREE, GIT_INDEX_FILE or GIT_COMMON_DIR, set by an outer git command, a hook or a shell, overrides git -C. git then reads that other checkout. With a second checkout that tracks a file of the same name, the marker recorded the other checkout's commit as the reconstruction revision, reported as clean. A file only the right checkout tracks was reported as an unknown revision. git now runs without the variables the installed git lists as local to one repository (git rev-parse --local-env-vars), as the Axiom checkout check already does. If git cannot list them, the revision is unknown. The adapter's helper is not imported, because it lives in a heavy module and this one stays light. Found by Codex's review of this pull request. Co-Authored-By: Claude Opus 5.5 --- ...1291-orrery-reconstruction-marker.added.md | 2 +- docs/orrery-adapter.md | 5 ++- .../src/microcosm/graph/reconstruction.py | 36 +++++++++++++++++- .../shared/test_graph_reconstruction.py | 37 +++++++++++++++++++ 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/changelog.d/1291-orrery-reconstruction-marker.added.md b/changelog.d/1291-orrery-reconstruction-marker.added.md index f19de06661..64c2f3bdd5 100644 --- a/changelog.d/1291-orrery-reconstruction-marker.added.md +++ b/changelog.d/1291-orrery-reconstruction-marker.added.md @@ -1 +1 @@ -Add `microcosm.graph.reconstruction`, which rebuilds a declaration-only graph schema from a build's recorded `graph.json` and marks it with the `microcosm.reconstruction` extension (protocol `microcosm.graph.reconstruction.v1`). The marker names the revision that declared the graph (`source_revision`), the revision of the code that performed the reconstruction (`reconstruction_revision`, read by `code_revision` from the checkout that tracks that code), and every input file by digest. Only the declaration's bytes come from the build: the schema's `compiled`, `fields` and `input_bindings` sections and the Orrery projection come from the reconstructing code. The marker travels in `graph.schema.json` into the Orrery document's metadata. A declaration that the current serializer would not reproduce byte for byte is refused, and no operation is presented as executed. +Add `microcosm.graph.reconstruction`, which rebuilds a declaration-only graph schema from a build's recorded `graph.json` and marks it with the `microcosm.reconstruction` extension (protocol `microcosm.graph.reconstruction.v1`). The marker names the revision that declared the graph (`source_revision`), the revision of the code that performed the reconstruction (`reconstruction_revision`, read by `code_revision` from the checkout that tracks that code, ignoring inherited repository-selecting git variables such as `GIT_DIR`), and every input file by digest. Only the declaration's bytes come from the build: the schema's `compiled`, `fields` and `input_bindings` sections and the Orrery projection come from the reconstructing code. The marker travels in `graph.schema.json` into the Orrery document's metadata. A declaration that the current serializer would not reproduce byte for byte is refused, and no operation is presented as executed. diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index a334f09eab..66bf80f8c5 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -378,7 +378,10 @@ The marker is the producer extension `microcosm.reconstruction`: - **`reconstruction_revision`:** the code that performed the reconstruction, and so computed every section except the declaration. `code_revision(path)` reads it from the git checkout that tracks `path` (the reconstructing - module's own file), not from the current directory. Both fields are `null` + module's own file), not from the current directory, and runs git without + the repository-selecting variables of `git rev-parse --local-env-vars` + (`GIT_DIR`, `GIT_WORK_TREE` and the rest), so an inherited one cannot point + it at another checkout. Both fields are `null` when no checkout tracks that file, as for an installed wheel. `git_dirty` is `null` alone when git could not check the working tree. - **`inputs`:** every file the reconstruction read, in name order, by digest diff --git a/packages/microcosm-graph/src/microcosm/graph/reconstruction.py b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py index fcce64b8ec..40db0f6f60 100644 --- a/packages/microcosm-graph/src/microcosm/graph/reconstruction.py +++ b/packages/microcosm-graph/src/microcosm/graph/reconstruction.py @@ -18,6 +18,7 @@ from __future__ import annotations import hashlib +import os import re import subprocess from collections.abc import Mapping, Sequence @@ -188,6 +189,33 @@ def _declaration_input(marker: Mapping[str, object]) -> Mapping[str, object]: return next(item for item in inputs if item["role"] == DECLARATION_ROLE) +def _git_environment() -> dict[str, str] | None: + """This process's environment without the variables that choose git's repository. + + The names removed are the installed git's own list of variables local to + one repository (``git rev-parse --local-env-vars``): ``GIT_DIR``, + ``GIT_WORK_TREE``, ``GIT_INDEX_FILE``, ``GIT_COMMON_DIR``, + ``GIT_OBJECT_DIRECTORY``, the ``git -c`` carriers ``GIT_CONFIG_PARAMETERS`` + and ``GIT_CONFIG_COUNT``, and the rest. ``None`` when git cannot list them. + ``microcosm.frame.adapters.axiom`` strips the same list for its checkout + check; this module does not import that adapter, so that it stays light. + """ + + try: + result = subprocess.run( + ["git", "rev-parse", "--local-env-vars"], + check=False, + capture_output=True, + text=True, + ) + except OSError: + return None + if result.returncode != 0: + return None + local = frozenset(result.stdout.split()) + return {name: value for name, value in os.environ.items() if name not in local} + + def code_revision(path: str | Path) -> dict[str, object]: """The git commit and working-tree state of the checkout that tracks ``path``. @@ -196,11 +224,16 @@ def code_revision(path: str | Path) -> dict[str, object]: an installed copy that happens to sit inside some checkout (a virtual environment within a repository, say) identifies no revision. Both values are ``None`` when no checkout tracks the file, so an unknown revision is - never recorded as a clean one. + never recorded as a clean one. git runs without the caller's + repository-selecting variables (``GIT_DIR``, ``GIT_WORK_TREE`` and the rest), + so an outer git command, hook or shell cannot point it at another checkout. """ location = Path(path).resolve() unknown: dict[str, object] = {"git_commit": None, "git_dirty": None} + environment = _git_environment() + if environment is None: + return unknown def git(*arguments: str) -> str | None: try: @@ -209,6 +242,7 @@ def git(*arguments: str) -> str | None: check=False, capture_output=True, text=True, + env=environment, ) except OSError: return None diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py index 84a577045a..7da4f87af8 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_reconstruction.py @@ -175,6 +175,43 @@ def test_code_revision_names_the_checkout_that_tracks_the_file(tmp_path): assert code_revision(outside_any_checkout) == unknown +def _checkout_tracking(checkout, files: dict[str, str]) -> str: + """Commit ``files`` to a new repository at ``checkout``; return its HEAD.""" + + checkout.mkdir() + run_git(checkout, "init", "--quiet") + for name, text in files.items(): + (checkout / name).write_text(text) + run_git(checkout, "add", *files) + run_git(checkout, "commit", "--quiet", "-m", "Add the fixture files") + return run_git(checkout, "rev-parse", "HEAD") + + +def test_code_revision_ignores_git_variables_that_select_another_checkout( + tmp_path, monkeypatch +): + reconstructing = tmp_path / "reconstructing" + reconstructing_head = _checkout_tracking( + reconstructing, + {"reconstruct.py": "VERSION = 1\n", "only_here.py": "VERSION = 1\n"}, + ) + other = tmp_path / "other" + other_head = _checkout_tracking(other, {"reconstruct.py": "VERSION = 2\n"}) + assert other_head != reconstructing_head + # An outer git command, a hook or a caller's shell can export these, and + # each one points git at the other checkout. + monkeypatch.setenv("GIT_DIR", str(other / ".git")) + monkeypatch.setenv("GIT_WORK_TREE", str(other)) + monkeypatch.setenv("GIT_INDEX_FILE", str(other / ".git" / "index")) + monkeypatch.setenv("GIT_COMMON_DIR", str(other / ".git")) + + expected = {"git_commit": reconstructing_head, "git_dirty": False} + # A file both checkouts track by the same name ... + assert code_revision(reconstructing / "reconstruct.py") == expected + # ... and one only the reconstructing checkout tracks. + assert code_revision(reconstructing / "only_here.py") == expected + + def test_reconstructed_document_claims_no_execution(): declaration = recorded_declaration() From 2b020992f4daf2a54624ea02d4de396e39e6abec Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:18:39 +0400 Subject: [PATCH 10/11] Check and write the same resolved output path reconstruct_uk_publication checked the resolved output against the build directory, but wrote to the path as given. build/missing/../../publication names a folder beside the build, so the check accepted it. Creating its parents then made build/missing inside the build, which the command must only read. With a read-only build, that mkdir failed, so such a build could not be reconstructed at all, even into a writable destination. The output is now resolved once, and that path is used both for the check and for every write. An output that already exists under the spelling the caller gave, such as a dangling link, is still refused rather than followed. A new test sends that path for a writable build and for a read-only one. The build's files and folders must be unchanged, and the publication must appear beside the build. Found by Codex's review of this pull request. Co-Authored-By: Claude Opus 5.5 --- docs/orrery-adapter.md | 4 +- .../build/uk_runtime/orrery_reconstruction.py | 14 +++++-- .../uk/test_uk_orrery_reconstruction.py | 38 +++++++++++++++++++ 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index 66bf80f8c5..1aaf27c1b3 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -466,7 +466,9 @@ The build directory is only read. The command: It writes a new directory with the layout of a normal publication, and moves the directory into place only after every file is written. The output must not exist, and must not be the build directory or lie inside it once `..` and links -are resolved: +are resolved. The command writes to that resolved path, so a path that passes +through a missing folder inside the build creates nothing there, and a +read-only build can be reconstructed: | File | Contents | | --- | --- | diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py index fcaeb97ded..62c4045b9b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py @@ -392,13 +392,21 @@ def reconstruct_uk_publication( beside the publication files, outside the upload inventory. """ - build, output = Path(build), Path(output) + build, requested_output = Path(build), Path(output) + # Check and write the same resolved path. Otherwise a path that enters a + # missing folder inside the build and leaves it again with ``..`` passes + # the check, and creating its parents makes that folder in the build. + output = requested_output.resolve() _require(build.is_dir(), f"build directory {build} does not exist") _require( not _within(build, output), - f"output {output} is inside the build directory {build}, which is only read", + f"output {requested_output} is inside the build directory {build}, which " + "is only read", + ) + _require( + not os.path.lexists(requested_output) and not os.path.lexists(output), + f"output {requested_output} already exists", ) - _require(not os.path.lexists(output), f"output {output} already exists") recorded_evidence = [ name for name in (SCHEMA_NAME, EVIDENCE_INDEX_NAME) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py index 051269f7a5..c4751955b1 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py @@ -656,6 +656,44 @@ def test_output_inside_the_build_directory_is_refused(tmp_path, output_parts): assert sorted(path.name for path in build.iterdir()) == listing +def _tree_snapshot(directory: Path) -> dict[str, bytes | None]: + """Every path under ``directory``, with each file's bytes (``None`` for a folder).""" + + return { + path.relative_to(directory).as_posix(): ( + None if path.is_dir() else path.read_bytes() + ) + for path in sorted(directory.rglob("*")) + } + + +@pytest.mark.parametrize( + "build_is_writable", [True, False], ids=["writable_build", "read_only_build"] +) +def test_output_leaving_a_missing_folder_inside_the_build_creates_nothing_there( + tmp_path, build_is_writable +): + # It names a folder beside the build, so it is accepted; the folders it + # passes through on the way must not be created inside the build. + build = write_pre_1144_build(tmp_path / "build") + output = build / "missing" / ".." / ".." / "publication" + before = _tree_snapshot(build) + original_mode = build.stat().st_mode + if not build_is_writable: + if os.geteuid() == 0: + pytest.skip("root ignores directory permissions") + build.chmod(0o555) + try: + reconstruct_uk_publication(build, output) + finally: + build.chmod(original_mode) + + assert _tree_snapshot(build) == before + assert sorted(path.name for path in (tmp_path / "publication").iterdir()) == ( + sorted([GRAPH_NAME, SCHEMA_NAME, INVENTORY_NAME, REPORT_NAME]) + ) + + def test_declaration_behind_a_symbolic_link_is_refused(tmp_path): build = write_pre_1144_build(tmp_path / "build") elsewhere = tmp_path / "elsewhere.json" From bb908fed6b64e6582096f1cf2679dc47ef39649c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:52:10 +0400 Subject: [PATCH 11/11] Print the resolved output path the reconstruction wrote main passed the --output spelling to summary_text, but the publication is written at the resolved path, which the spelling need not reach. For --output /missing/../../cli_publication the printed "Wrote ... to" and "report:" paths did not exist, because missing cannot be entered. main now resolves the output before the write, as reconstruct_uk_publication does, and prints that path. It still passes the given spelling, so the checks on it run. New tests: - the command prints the resolved path, for both lines, and never the given spelling; - an output whose given name exists as a symbolic link but whose resolved path does not (a dangling link leading outside the build) is refused as already existing, and nothing is written where the link points. The adapter doc says the output must not exist as given or once resolved, and that the command prints the resolved path. Co-Authored-By: Claude Opus 5.5 --- docs/orrery-adapter.md | 10 +++-- .../build/uk_runtime/orrery_reconstruction.py | 9 +++- .../uk/test_uk_orrery_reconstruction.py | 41 +++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index 1aaf27c1b3..e9422e6ae8 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -465,10 +465,12 @@ The build directory is only read. The command: It writes a new directory with the layout of a normal publication, and moves the directory into place only after every file is written. The output must not -exist, and must not be the build directory or lie inside it once `..` and links -are resolved. The command writes to that resolved path, so a path that passes -through a missing folder inside the build creates nothing there, and a -read-only build can be reconstructed: +exist, either as given (a symbolic link counts, even one whose target does not +exist) or once resolved. It must not be the build directory or lie inside it +once `..` and links are resolved. The command writes to that resolved path, so +a path that passes through a missing folder inside the build creates nothing +there, and a read-only build can be reconstructed. It prints the resolved path, +because the path as given need not reach it: | File | Contents | | --- | --- | diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py index 62c4045b9b..9b5dd8b628 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_reconstruction.py @@ -627,12 +627,19 @@ def main(argv: list[str] | None = None) -> int: ) args = parser.parse_args(argv) try: + # The publication is written at the resolved path, which the given + # spelling need not reach: ``/missing/../../publication`` + # resolves beside the build, but ``missing`` cannot be entered. + # Resolve before the write, as ``reconstruct_uk_publication`` does, + # and print that path. The given spelling is still what is passed, so + # that the checks on it run. + written_output = args.output.resolve() report = reconstruct_uk_publication( args.build, args.output, revision=args.revision ) except (OSError, ValueError, RecursionError) as error: parser.error(str(error)) - print(summary_text(report, args.output)) + print(summary_text(report, written_output)) return 0 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py index c4751955b1..ede92261ca 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_reconstruction.py @@ -694,6 +694,26 @@ def test_output_leaving_a_missing_folder_inside_the_build_creates_nothing_there( ) +def test_output_named_by_a_dangling_link_leading_outside_the_build_is_refused( + tmp_path, +): + # The given name exists, as a symbolic link, but the path it resolves to + # does not. Checking only the resolved path would accept it and write the + # publication wherever the link points. + build = write_pre_1144_build(tmp_path / "build") + (tmp_path / "elsewhere").mkdir() + link_target = tmp_path / "elsewhere" / "publication" + dangling_link = tmp_path / "publication-link" + dangling_link.symlink_to(link_target, target_is_directory=True) + + with pytest.raises(ValueError, match="already exists"): + reconstruct_uk_publication(build, dangling_link) + + assert dangling_link.is_symlink() + assert not os.path.lexists(link_target) + assert list((tmp_path / "elsewhere").iterdir()) == [] + + def test_declaration_behind_a_symbolic_link_is_refused(tmp_path): build = write_pre_1144_build(tmp_path / "build") elsewhere = tmp_path / "elsewhere.json" @@ -722,6 +742,27 @@ def test_cli_writes_the_publication_and_prints_a_summary(tmp_path, capsys): assert "already exists" in capsys.readouterr().err +def test_cli_prints_the_resolved_output_path_it_wrote(tmp_path, capsys): + # The given spelling enters a folder the build does not have and leaves + # it with "..", so it names no reachable path; the publication is written + # at its resolved path beside the build, which is the one to print. + build = write_pre_1144_build(tmp_path / "build") + given_output = build / "missing" / ".." / ".." / "cli_publication" + written_output = tmp_path.resolve() / "cli_publication" + + assert main(["--build", str(build), "--output", str(given_output)]) == 0 + + printed = capsys.readouterr().out + assert ( + f"Wrote a reconstructed Orrery publication (declaration only) to " + f"{written_output}\n" + ) in printed + assert f" report: {written_output / REPORT_NAME}" in printed.splitlines() + assert str(given_output) not in printed + assert (written_output / REPORT_NAME).is_file() + assert not os.path.lexists(given_output) + + def test_cli_refuses_a_declaration_that_is_not_a_json_object(tmp_path, capsys): build = write_pre_1144_build(tmp_path / "build", git_commit=None, git_dirty=None) (build / "graph.json").write_bytes(b"[]")