diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8715a74a..f00a619f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,7 @@ jobs: set -eu echo "authority: $AUTHORITY ($AUTHORITY_NA not applicable)" echo "templates: $TEMPLATES ($TEMPLATES_NA not applicable)" - test "$AUTHORITY" = "verified 24/24" + test "$AUTHORITY" = "verified 25/25" # G13 is N/A on SQLite, the action's default store: SQLite has no clock of its own # to diverge from; G15 is N/A because neither document declares `max_attempts`. # G16 is graded on both: verify brings its own precondition provider (SPEC-v0.7 §8.9). @@ -148,7 +148,11 @@ jobs: # grants name no task, which the templates example's do not. The authority example # binds one, so its count is unchanged and its passing total moved 19 to 20 instead. test "$AUTHORITY_NA" = "3" - test "$TEMPLATES" = "verified 11/11" + # + # SPEC-v0.11 §8 moved the templates count again: G31 needs only an action to + # build a chain from, so it is graded wherever any guarantee is, and both + # examples gained one passing row when item 4 landed. + test "$TEMPLATES" = "verified 12/12" test "$TEMPLATES_NA" = "16" test -s verify-badge.json test -s verify-report.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7580e518..1b77a8c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,28 @@ any change to one appears here. trusted. Found by review; the tests that missed it all tampered with a row's *content*, and `{}` and a float among the controls are both valid JSON. +- **One chain, five receipt schema versions, walked end to end** (`SPEC-v0.11.md` §6). A store + kept since v0.6 holds five: `v3` (0.6), `v4` (0.7), `v5` (0.8), `v6` (0.9), `v7` (0.10). + **No new field**: `schema` has existed since `SPEC-v0.3.md` §12.2. What is new is the proof + that `verify_chain` walks such a chain hash by hash, **each row hashed by the rule its own + version wrote**. v0.10's release pass proved the `v6`/`v7` boundary against the released 0.9.0 + and stopped there. + + `scripts/five_schema_chain.py` builds the chain from the **released wheels** rather than from + fixtures: five environments, `pip install ctrlrun==0.6.1`, `0.7.0`, `0.8.0`, `0.9.0`, `0.10.0`, + one store, then this build verifies across the whole thing. A fixture is this build's opinion + of what 0.6 wrote; the wheel is what it wrote. + + And a receipt whose schema label this binary does **not** know is named, not reported as a + break: `SPEC-v0.6.md` §3.2's distinction, and the difference between *this evidence is from a + future version* and *this evidence is tampered with*. Relabelling a stored row without + rehashing it is still `content_altered`, because that is somebody editing evidence. + +- **`G31`, five receipt schemas verify**, and `ctrlrun.guarantees/v6` becomes **`v7`**, moved + once. `G28` to `G30` and `G32` are not in the catalogue yet: `SPEC-v0.11.md` §8 assigns ids in + item order so that splitting the milestone renumbers nothing, and a row whose check does not + exist would report something before it could. + ### Changed - **`StateStore.receipts()` returns `tuple[Receipt | UnreadableReceipt, ...]`**, amending diff --git a/scripts/five_schema_chain.py b/scripts/five_schema_chain.py new file mode 100644 index 00000000..d48a4b0f --- /dev/null +++ b/scripts/five_schema_chain.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: 2026 The CTRLRun contributors +# SPDX-License-Identifier: Apache-2.0 +"""Build one receipt chain with five RELEASED ctrlrun wheels, then verify it with this build. + +`SPEC-v0.11.md` §6.1. A store kept since v0.6 holds five receipt schema versions: `v3` (0.6), +`v4` (0.7), `v5` (0.8), `v6` (0.9), `v7` (0.10). Nothing proved that `verify` walks such a chain +end to end, hash by hash, **each row hashed by the rule its own version wrote**. v0.10's release +pass proved the `v6`/`v7` boundary against the released 0.9.0 and stopped there. + +**The chain is written by the released wheels and not by fixtures this build produces**, because +a fixture is this build's opinion of what 0.6 wrote and the wheel is what it actually wrote. +v0.10's upgrade check did this and it is the reason it found anything. + +This needs a network and five virtual environments, so it is a script an operator or a release +pass runs rather than a test the ordinary suite runs. `tests/test_five_schema_versions.py` keeps +the invariant checkable without a network, and `G31` grades the walk. + + python scripts/five_schema_chain.py # build and verify, print the transcript + python scripts/five_schema_chain.py --keep DIR # leave the store behind to poke at + +The transcript it printed on 2026-09-14, against this build, is in that test file's docstring. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import tempfile +import venv +from pathlib import Path + +#: One release per receipt schema version a store kept since v0.6 can hold. 0.6.1 rather than +#: 0.6.0 because it is the last 0.6, and `v3` is what both wrote. +RELEASES: tuple[tuple[str, str], ...] = ( + ("0.6.1", "ctrlrun.receipt/v3"), + ("0.7.0", "ctrlrun.receipt/v4"), + ("0.8.0", "ctrlrun.receipt/v5"), + ("0.9.0", "ctrlrun.receipt/v6"), + ("0.10.0", "ctrlrun.receipt/v7"), +) + +#: `ctrlrun.policy/v1`, which every release from 0.6 on still reads: the rule since +#: `SPEC-v0.3.md` §12.2 is that every reader upgrades before any writer switches, and a document +#: only the newest binary parses would make this script test the policy loader instead of the +#: chain. +POLICY = "schema: ctrlrun.policy/v1\nactions:\n stripe.refund:\n decision: allow\n" + +#: Run by each wheel inside its own environment. It must use only what 0.6.1 already had, so +#: nothing here touches `Receipt.schema`: that attribute does not exist in 0.6.1, and an earlier +#: version of this script raised `AttributeError` out of its own reporting line **after** the +#: writes had landed, which reads as a failed write and is not one. +WRITER = """ +import sys +from ctrlrun import Control, Policy, SQLiteStateStore +from ctrlrun.action import Action, Principal + +database, tag, policy = sys.argv[1], sys.argv[2], sys.argv[3] +store = SQLiteStateStore(database) +control = Control(Policy.from_yaml(policy), store) +for index in range(2): + control.execute( + Action( + name="stripe.refund", + arguments={"payment_id": "%s-%d" % (tag, index), "amount": 2000}, + principal=Principal(agent="chain-agent"), + ), + lambda: {"ok": True}, + "refund:%s-%d" % (tag, index), + ) +print(len(store.receipts())) +store.close() +""" + + +#: The environment every subprocess gets: this one, with anything that could put **this** +#: build's source onto a released wheel's `sys.path` removed. +#: +#: **This is the whole methodology and it was wrong first.** Run as `PYTHONPATH=src python +#: scripts/five_schema_chain.py`, which is how anyone runs a script against an uninstalled +#: checkout, the variable is inherited by every child -- so each "released wheel" imported this +#: build's `src/ctrlrun` instead of the wheel just installed beside it. The script printed a +#: chain of ten receipts that verified perfectly and reported **one** schema version, because +#: all five writers were this binary. A run that checked nothing looked exactly like a pass. +def _clean_environment() -> dict[str, str]: + environment = dict(os.environ) + for leak in ("PYTHONPATH", "PYTHONHOME", "PYTHONSTARTUP"): + environment.pop(leak, None) + return environment + + +def _require_wrote_as(python: Path, version: str, schema: str) -> None: + """Fail loudly if the wheel that just ran was not the released one. + + The positive control for `_clean_environment`, and it is not optional: the only symptom of a + leaked `sys.path` is a schema count, which is the number this script exists to produce. A + check that can be fooled by the bug it checks for is not a check. + """ + seen = subprocess.run( + [ + str(python), + "-c", + "import ctrlrun, ctrlrun.receipt as r;" + "print(ctrlrun.__file__);print(getattr(r, 'RECEIPT_SCHEMA', 'none'))", + ], + check=True, + capture_output=True, + text=True, + env=_clean_environment(), + ).stdout.split() + if f"venv-{version}" not in seen[0]: + raise SystemExit( + f"{version} ran from {seen[0]}, which is not its own environment: something put " + "another ctrlrun on its sys.path, so this run proves nothing" + ) + if seen[1] != schema: + raise SystemExit( + f"{version} writes {seen[1]}, and this script says it writes {schema}. One of the " + "two is wrong, and RELEASES is the thing to fix" + ) + + +def _environment(root: Path, version: str) -> Path: + """A scratch environment with exactly one released ctrlrun in it.""" + target = root / f"venv-{version}" + venv.EnvBuilder(with_pip=True).create(target) + python = target / "bin" / "python" + subprocess.run( + [ + str(python), + "-m", + "pip", + "install", + "-q", + "--disable-pip-version-check", + f"ctrlrun=={version}", + ], + check=True, + env=_clean_environment(), + ) + return python + + +def build(root: Path) -> Path: + """Write two receipts with each release, oldest first, into one store.""" + database = root / "chain.db" + writer = root / "writer.py" + writer.write_text(WRITER, encoding="utf-8") + for version, schema in RELEASES: + python = _environment(root, version) + done = subprocess.run( + [str(python), str(writer), str(database), f"v{version}", POLICY], + check=True, + capture_output=True, + text=True, + env=_clean_environment(), + ) + _require_wrote_as(python, version, schema) + print( + f" {version:<7} wrote 2 receipts under {schema}; store now holds {done.stdout.strip()}" + ) + return database + + +def verify(database: Path) -> int: + """Verify the whole chain with THIS build, and report what it walked.""" + from ctrlrun.receipt import verify_chain + from ctrlrun.state import SQLiteStateStore + + store = SQLiteStateStore(str(database)) + try: + rows = store.receipts() + schemas = sorted({row.schema for row in rows}) + report = verify_chain(store) + finally: + store.close() + + print() + print(f" receipts: {len(rows)}") + print(f" schemas in ONE chain: {len(schemas)}") + for schema in schemas: + at = [row.seq for row in rows if row.schema == schema] + print(f" {schema:<22} at seq {at}") + print( + f" verify_chain: ok={report.ok} verified={report.verified} " + f"chained={report.chained} unchained={report.unchained}" + ) + print(f" breaks: {[(b.name, b.seq) for b in report.breaks] or 'none'}") + + expected = {schema for _, schema in RELEASES} + if set(schemas) != expected: + print(f"\nFAIL: expected {sorted(expected)}, walked {schemas}") + return 1 + if not report.ok or report.verified != len(rows): + print("\nFAIL: the chain did not verify end to end") + return 1 + print(f"\nOK: one chain, {len(schemas)} receipt schema versions, verified end to end") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--keep", type=Path, help="build in this directory and leave it behind") + arguments = parser.parse_args() + + print(f"Building one chain with {len(RELEASES)} released wheels from PyPI:") + if arguments.keep: + arguments.keep.mkdir(parents=True, exist_ok=True) + return verify(build(arguments.keep)) + with tempfile.TemporaryDirectory() as scratch: + return verify(build(Path(scratch))) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/ctrlrun/verify/guarantees.py b/src/ctrlrun/verify/guarantees.py index a8a787ba..eff1cae0 100644 --- a/src/ctrlrun/verify/guarantees.py +++ b/src/ctrlrun/verify/guarantees.py @@ -31,7 +31,11 @@ #: SPEC-v0.9 §8: `v5` was G1 to G24, and it moved once, with G24. G22 and G23 joined it with #: their items. No stub rows: a guarantee that reports anything before its check exists is a #: false green, which is what 0.6.1 had to fix and what G17 shipped as in v0.8. -CATALOGUE: Final = "ctrlrun.guarantees/v6" +#: SPEC-v0.11 §8: `v7` is G1 to G32, and it moves once, here, with item 4's G31. G28 (item 2), +#: G29, G30 and G32 (item 3) join it with their items, and the release item asserts every row +#: present before the release. No stub rows: a guarantee that reports anything before its check +#: exists is a false green. +CATALOGUE: Final = "ctrlrun.guarantees/v7" @dataclass(frozen=True) @@ -209,6 +213,15 @@ class Guarantee: "a swapped upstream is denied", ("v0.10 §4.3", "v0.10 §4.7 T490", "v0.10 §4.7 T493"), ), + Guarantee( + "G31", + # 27 characters against `report._TITLE_WIDTH`'s 32. It grades **the walk**, not the + # field: `schema` has existed since v0.3 and `SPEC-v0.11 §6` adds no field. What was + # never proved is that `verify_chain` walks a chain holding more than one of them, + # hash by hash, each row hashed by the rule its own version wrote. + "five receipt schemas verify", + ("v0.11 §6", "v0.11 §6.1 T521", "v0.11 §6.2 T523"), + ), ) #: By id, for `--only` and for the report. Insertion order is catalogue order. diff --git a/src/ctrlrun/verify/scenarios.py b/src/ctrlrun/verify/scenarios.py index 7bc4cbb8..92341e2b 100644 --- a/src/ctrlrun/verify/scenarios.py +++ b/src/ctrlrun/verify/scenarios.py @@ -35,7 +35,7 @@ import subprocess import sys import threading -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from contextlib import suppress from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta @@ -109,11 +109,15 @@ ) from ..receipt import ( BLOCKED_ATTEMPT_CEILING, + GENESIS_HASH, + KNOWN_RECEIPT_SCHEMAS, + RECEIPT_SCHEMA, Event, EventType, Receipt, ReceiptResult, UnreadableReceipt, + _document_hash, iso_timestamp, new_receipt_id, verify_chain, @@ -4579,6 +4583,183 @@ def body(detail: dict[str, Any]) -> None: _upstream.forget(f"{reg.SYNTHETIC_PREFIX}-upstream") store.close() + # --- G31: one chain, five receipt schema versions, walked end to end ------------------ + + def g31(self) -> GuaranteeResult: + """SPEC-v0.11 §6, §8. A chain spanning five receipt schema versions verifies end to end. + + **The walk, not the field.** `schema` has existed since `SPEC-v0.3.md` §12.2 and v0.11 + adds no field. What was never proved is that `verify_chain` walks a chain holding more + than one of them, hash by hash, **each row hashed by the rule its own version wrote**. + v0.10's release pass proved the `v6`/`v7` boundary against the released 0.9.0 and stopped + there. + + **Why this can be graded without installing five wheels**, which is the question to ask + of a scenario about released versions. `SPEC-v0.7.md` §6.11 made `to_dict` render under + *its own* schema's label and key set, and made a receipt hash the document it was read + from, through the same `_document_hash` every version has used. So a `Receipt` carrying + `ctrlrun.receipt/v3` serializes to v3's 26 keys and hashes to what 0.6 stored. + + **It cannot go through `put_receipt`, and that is correct.** Every backend's + `put_receipt` does `replace(receipt, schema=RECEIPT_SCHEMA, ...)`: a writer writes under + its own schema, so an older receipt cannot be forged through the public API. The first + version of this scenario tried exactly that and control 1 below caught it, reporting a + chain of one label where it had asked for five. So the chain is presented as a + `ChainSource` **view**, which is what `G11` already does to alter a row, and the links + are computed the way `put_receipt` computes them: `seq` and `prev_hash` into the + document, then `_document_hash` over it, then that digest is the next row's `prev_hash`. + + **The construction is self-checking.** If it were wrong, `verify_chain` would report + breaks and this scenario would fail rather than pass: there is no way for a badly built + chain to grade `PASS` here. What that does not cover is a walk that quietly skipped rows + whose label it did not recognise, so there are two controls, and neither is decoration: + + 1. The chain must really hold **five distinct schema labels** before the walk is graded. + A chain of five `v7` rows verifies perfectly and proves nothing, which is + `SPEC-v0.4.md` §2.2's guarantee that could not have failed. + 2. An **older** row is then altered and the break must be named at its `seq`. Without it + "verified" would be a count of the rows the walk bothered to read. + + `scripts/five_schema_chain.py` is the other half, and it is the one built from the + **released wheels**: five environments, five `pip install ctrlrun==`, one real store. + This guarantee is what an operator can run on their own host with no network; that + script is what proves the premise against what 0.6 actually wrote. + """ + selection = self.select(decisions=(Decision.ALLOW, Decision.APPROVE, Decision.DENY)) + if selection is None: + return self.na("G31", self.unselected(reg.NO_ACTIONS)) + control, store, recorder, _ = self._control_for("G31", selection) + + def body(detail: dict[str, Any]) -> None: + action = selection.build() + key = ( + None + if selection.effect_key is None + else f"{selection.effect_key!s}-{reg.SYNTHETIC_PREFIX}-schemas" + ) + # One real receipt first, written by this binary through the ordinary path, so every + # row below is a real receipt's fields rather than a shape this scenario invented. + with suppress(CTRLRunError): + self.execute( + control, + action, + _Executor(lambda: f"{APPROVER}-result"), + key, + self.approve(control, store, action, selection), + ) + seed = _written(store) + _expect_control( + bool(seed), + "the scenario wrote a receipt to build the chain from", + "no receipt reached the store", + ) + + labels = (*_OLDER_RECEIPT_SCHEMAS, RECEIPT_SCHEMA) + rows: list[Receipt] = [] + previous = GENESIS_HASH + for position, label in enumerate(labels, start=1): + # `seq` and `prev_hash` go into the document BEFORE it is hashed, because that + # is what makes them tamper-evident (SPEC-v0.6 §6.2). `hash` is a column and + # never a key, so it is set after. + row = replace( + seed[-1], + receipt_id=new_receipt_id(), + schema=label, + seq=position, + prev_hash=previous, + hash=None, + ) + previous = _document_hash(row.to_dict()) + rows.append(replace(row, hash=previous)) + + chain = _AlteredChain(tuple(rows), (len(rows), previous)) + present = sorted({item.schema for item in rows}) + detail["schemas"] = present + detail["receipts"] = len(rows) + # Control 1. Without it every assertion below passes on a chain of one version, and + # the first version of this scenario failed exactly here. + _expect_control( + len(present) == len(labels), + f"the chain holds {len(labels)} distinct receipt schemas", + f"it holds {present}", + ) + + report = verify_chain(chain) + _expect( + report.ok and report.verified == len(rows), + f"a chain of {len(present)} receipt schema versions verifies end to end", + f"it reported ok={report.ok} verified={report.verified} of {len(rows)}, " + f"{[(item.name, item.seq) for item in report.breaks]}", + ) + + # Control 2. Alter an OLDER row, never the newest: a walk that skipped labels it did + # not know would have passed everything above. + target = rows[0] + altered = replace(target, decision_reason=f"{target.decision_reason}-altered") + damaged = verify_chain( + _AlteredChain( + tuple(altered if item.seq == target.seq else item for item in rows), + (len(rows), previous), + ) + ) + detail["older_row_altered"] = {"schema": target.schema, "seq": target.seq} + _expect( + any( + item.name == "content_altered" and item.seq == target.seq + for item in damaged.breaks + ), + f"altering a {target.schema} row is named `content_altered` at seq {target.seq}", + f"it was reported as {[(b.name, b.seq) for b in damaged.breaks]}", + ) + + try: + return self.graded("G31", selection, store, recorder, body) + finally: + store.close() + + +#: SPEC-v0.11 §6. Every receipt schema version a **chain** can hold other than the current one. +#: +#: Derived from `receipt.py`'s constants and never listed, because §6 says the count is taken +#: from the only place it cannot be stale: `ROADMAP.md` has said "three shapes", then "four", and +#: both went wrong at the next release. G31 then grades whatever this binary can actually write. +#: +#: **From `v3`, not from `v1`.** The chain itself arrived in `v3` (`SPEC-v0.6.md` §6.2), so a +#: `v1` or `v2` receipt has no `seq` at all and is `unchained`, which is a different case, which +#: `G11` already covers, and which is never a pass. +#: +#: The version is parsed as a **number** rather than compared as a string: `"ctrlrun.receipt/v10"` +#: sorts below `"ctrlrun.receipt/v3"` lexically, so a string comparison here would quietly drop +#: every row from v0.13 onward and G31 would go on passing over a shorter chain. +def _schema_number(label: str) -> int: + return int(label.rsplit("/v", 1)[-1]) + + +_FIRST_CHAINED_SCHEMA: Final = 3 + + +def _chainable_schemas(known: Iterable[str], current: str) -> tuple[str, ...]: + """The chainable receipt schemas older than `current`, oldest first. + + **A function rather than a comprehension at module scope**, so a test can hand it a set + containing `ctrlrun.receipt/v10` and see what it does. The module constant is computed once + at import, so a test cannot reach the ordering by patching `KNOWN_RECEIPT_SCHEMAS`, and a + mutation replacing the numeric comparison with a string one survived the whole suite for + exactly that reason: every version that exists today is one digit, so the two agree. + """ + return tuple( + sorted( + ( + label + for label in known + if _schema_number(label) >= _FIRST_CHAINED_SCHEMA and label != current + ), + key=_schema_number, + ) + ) + + +_OLDER_RECEIPT_SCHEMAS: Final = _chainable_schemas(KNOWN_RECEIPT_SCHEMAS, RECEIPT_SCHEMA) #: SPEC-v0.8 §3.4, §11.7 — the claim verify's own approver principals carry their roles in. #: Named for what it is, and `SYNTHETIC_PREFIX`ed nowhere, because it is a claim **name** and a diff --git a/tests/test_approver.py b/tests/test_approver.py index fa215413..10f50442 100644 --- a/tests/test_approver.py +++ b/tests/test_approver.py @@ -846,7 +846,7 @@ def test_T294_the_catalogue_is_v4_and_carries_G18(): """§11.4: the version moves once, here, and G18 lands with it.""" from ctrlrun.verify.guarantees import CATALOGUE, GUARANTEES - assert CATALOGUE == "ctrlrun.guarantees/v6" + assert CATALOGUE == "ctrlrun.guarantees/v7" identifiers = [guarantee.id for guarantee in GUARANTEES] assert "G18" in identifiers assert identifiers == sorted(identifiers, key=lambda name: int(name[1:])) diff --git a/tests/test_attempt_cap.py b/tests/test_attempt_cap.py index 5c73653f..93a8e35e 100644 --- a/tests/test_attempt_cap.py +++ b/tests/test_attempt_cap.py @@ -1653,7 +1653,7 @@ def _verify(tmp_path, document, *, only): def test_T252_G15_is_in_the_catalogue(): from ctrlrun.verify import guarantees as reg - assert reg.CATALOGUE == "ctrlrun.guarantees/v6" + assert reg.CATALOGUE == "ctrlrun.guarantees/v7" assert "G15" in reg.BY_ID assert "v0.1 §5.4" in reg.BY_ID["G15"].descends_from diff --git a/tests/test_clock_skew.py b/tests/test_clock_skew.py index cc7206da..f2baf8dd 100644 --- a/tests/test_clock_skew.py +++ b/tests/test_clock_skew.py @@ -584,7 +584,7 @@ def test_T219_G13_is_not_applicable_on_sqlite_with_its_sentence(tmp_path): def test_T219_the_catalogue_is_v3_and_G13_is_in_it(): - assert reg.CATALOGUE == "ctrlrun.guarantees/v6" + assert reg.CATALOGUE == "ctrlrun.guarantees/v7" assert "G13" in reg.BY_ID assert "v0.1 §5.3 E3" in reg.BY_ID["G13"].descends_from diff --git a/tests/test_five_schema_versions.py b/tests/test_five_schema_versions.py new file mode 100644 index 00000000..549faa32 --- /dev/null +++ b/tests/test_five_schema_versions.py @@ -0,0 +1,644 @@ +# SPDX-FileCopyrightText: 2026 The CTRLRun contributors +# SPDX-License-Identifier: Apache-2.0 +"""One chain, five receipt schema versions, walked end to end. SPEC-v0.11 §6; T521-T525. + +A store kept since v0.6 holds **five** receipt schema versions: `v3` (0.6), `v4` (0.7), `v5` +(0.8), `v6` (0.9), `v7` (0.10). The count comes from `receipt.py`'s constants, which is the only +place it cannot be stale: `ROADMAP.md` has said "three shapes", then "four", and was wrong at the +next release both times. + +**No new field.** `schema` has existed since `SPEC-v0.3.md` §12.2 and the rule since then is that +every reader upgrades before any writer switches, so an older receipt on disk still parses. What +was never proved is that `verify_chain` walks such a chain **hash by hash, each row hashed by the +rule its own version wrote**. v0.10's release pass proved the `v6`/`v7` boundary against the +released 0.9.0 and stopped there. + +**The premise is proved against the released wheels, not against fixtures**, by +`scripts/five_schema_chain.py`, which needs a network and five virtual environments and is +therefore a script rather than a test. Its transcript on 2026-09-14, against this build:: + + 0.6.1 wrote 2 receipts under ctrlrun.receipt/v3; store now holds 2 + 0.7.0 wrote 2 receipts under ctrlrun.receipt/v4; store now holds 4 + 0.8.0 wrote 2 receipts under ctrlrun.receipt/v5; store now holds 6 + 0.9.0 wrote 2 receipts under ctrlrun.receipt/v6; store now holds 8 + 0.10.0 wrote 2 receipts under ctrlrun.receipt/v7; store now holds 10 + + receipts: 10 + schemas in ONE chain: 5 + ctrlrun.receipt/v3 at seq [1, 2] + ctrlrun.receipt/v4 at seq [3, 4] + ctrlrun.receipt/v5 at seq [5, 6] + ctrlrun.receipt/v6 at seq [7, 8] + ctrlrun.receipt/v7 at seq [9, 10] + verify_chain: ok=True verified=10 chained=10 unchained=0 + breaks: none + +**That run was wrong the first time and the way it was wrong is worth keeping.** Run as +`PYTHONPATH=src python scripts/five_schema_chain.py`, the variable is inherited by every child, +so each "released wheel" imported this build's `src/ctrlrun` instead of the wheel installed +beside it. It printed a chain of ten receipts that verified perfectly and reported **one** schema +version. The script now strips the variable and asserts, per release, that the interpreter it +just ran came from that release's own environment. + +What is kept here is the invariant, so that a later change breaks the ordinary suite rather than +only a release rehearsal. +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from ctrlrun import Control, Policy, SQLiteStateStore +from ctrlrun.action import Action, Principal +from ctrlrun.receipt import ( + GENESIS_HASH, + KNOWN_RECEIPT_SCHEMAS, + RECEIPT_SCHEMA, + Receipt, + _document_hash, + new_receipt_id, + verify_chain, +) +from ctrlrun.verify.report import Status + +T0 = datetime(2026, 1, 1, 12, 0, tzinfo=UTC) +LEASE = timedelta(minutes=5) +ALLOW = "schema: ctrlrun.policy/v1\nactions:\n stripe.refund:\n decision: allow\n" + +#: The five a store kept since v0.6 can hold, oldest first. Written out **here** deliberately, +#: where `scenarios.py` derives its list from `KNOWN_RECEIPT_SCHEMAS`: a derived list and a +#: derived assertion about it agree with each other no matter what either says, and T522 is what +#: makes the derivation answerable to something. +CHAINED_SCHEMAS = ( + "ctrlrun.receipt/v3", + "ctrlrun.receipt/v4", + "ctrlrun.receipt/v5", + "ctrlrun.receipt/v6", + "ctrlrun.receipt/v7", +) + + +#: `scripts/five_schema_chain.py` belongs to the **repository** and not to the package: it builds +#: five virtual environments and needs a network, and `MANIFEST.in` prunes `scripts/` for the same +#: reason it prunes `.github`, because a downstream packager builds a library. So the two tests +#: about it skip where the file is absent, exactly as `test_verify_action.py` does for `action.yml` +#: and the CI workflow, and run with everything asserted in a checkout and in CI's `check` job, +#: which is where a change to the script is actually made. +#: +#: Found by the `package` job: the first version read the path unconditionally and both tests +#: failed with `FileNotFoundError` inside the sdist. +SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "five_schema_chain.py" + + +def _repository_script() -> str: + if not SCRIPT.exists(): # pragma: no cover - only outside a checkout + pytest.skip("scripts/five_schema_chain.py is not in the sdist; this asserts a repo file") + return SCRIPT.read_text(encoding="utf-8") + + +def an_action(payment_id: str) -> Action: + return Action( + name="stripe.refund", + arguments={"payment_id": payment_id, "amount": 2000}, + principal=Principal(agent="chain-agent"), + ) + + +def a_chain_of_every_schema(seed: Receipt) -> tuple[tuple[Receipt, ...], str]: + """One chained receipt per schema version, linked the way `put_receipt` links them. + + `seq` and `prev_hash` go **into** the document before it is hashed, which is what makes them + tamper-evident (`SPEC-v0.6.md` §6.2); `hash` is a column and never a key, so it is attached + afterwards. The document renders under each row's own schema's key set, which is + `SPEC-v0.7.md` §6.11 and the whole reason one chain can hold five shapes. + """ + rows: list[Receipt] = [] + previous = GENESIS_HASH + for position, label in enumerate(CHAINED_SCHEMAS, start=1): + row = replace( + seed, + receipt_id=new_receipt_id(), + schema=label, + seq=position, + prev_hash=previous, + hash=None, + ) + previous = _document_hash(row.to_dict()) + rows.append(replace(row, hash=previous)) + return tuple(rows), previous + + +class _Chain: + """A `ChainSource` over receipts this test built (`SPEC-v0.6.md` §6.5's protocol).""" + + def __init__(self, rows: tuple[Receipt, ...], head: tuple[int, str] | None) -> None: + self._rows = rows + self._head = head + + def receipts(self) -> tuple[Receipt, ...]: + return self._rows + + def chain_head(self) -> tuple[int, str] | None: + return self._head + + +@pytest.fixture +def seed(tmp_path) -> Receipt: + """A real receipt, written by this binary through the ordinary path.""" + store = SQLiteStateStore(tmp_path / "seed.db", clock=lambda: T0) + control = Control(Policy.from_yaml(ALLOW), store, clock=lambda: T0) + control.execute(an_action("p0"), lambda: {"ok": True}, "refund:p0", lease=LEASE) + written = store.receipts()[0] + store.close() + assert isinstance(written, Receipt) + return written + + +# --- T521: the walk ---------------------------------------------------------------------------- + + +def test_T521_a_chain_of_five_receipt_schema_versions_verifies_end_to_end(seed) -> None: + """SPEC-v0.11 §6. **The deliverable.** + + The positive control comes first and is not decoration: a chain of five rows that all carry + the *current* schema verifies perfectly and proves nothing, so the distinct-label count is + asserted before the walk is. That is `SPEC-v0.4.md` §2.2's guarantee that could not have + failed, in the place it would be easiest to write by accident. + """ + rows, head = a_chain_of_every_schema(seed) + labels = sorted({row.schema for row in rows}) + + assert len(labels) == 5, f"the chain does not span five schemas: {labels}" + assert labels == sorted(CHAINED_SCHEMAS) + + report = verify_chain(_Chain(rows, (len(rows), head))) + + assert report.ok, [(item.name, item.seq) for item in report.breaks] + assert report.verified == 5 + assert report.chained == 5 + assert report.unchained == 0 + assert report.breaks == [] + + +@pytest.mark.parametrize("position", range(5), ids=[s.rsplit("/", 1)[-1] for s in CHAINED_SCHEMAS]) +def test_T521b_altering_the_row_of_any_one_version_is_named_at_its_seq(seed, position) -> None: + """Every row is load-bearing, not just the newest. + + Without this, a walk that skipped rows whose label it did not recognise would pass `T521` + and "verified" would be a count of the rows it bothered to read. Parametrized over all five, + because a walk could plausibly read the two it knows best and skip the rest. + """ + rows, head = a_chain_of_every_schema(seed) + target = rows[position] + altered = replace(target, decision_reason=f"{target.decision_reason}-altered") + damaged = tuple(altered if row.seq == target.seq else row for row in rows) + + report = verify_chain(_Chain(damaged, (len(rows), head))) + + assert not report.ok, f"altering the {target.schema} row was not detected" + named = [(item.name, item.seq) for item in report.breaks] + assert ("content_altered", target.seq) in named, ( + f"altering a {target.schema} row was reported as {named}" + ) + + +# --- T522: the count comes from the constants, not from a document ----------------------------- + + +def test_T522_the_schemas_a_chain_can_hold_are_the_ones_receipt_py_declares() -> None: + """SPEC-v0.11 §6. `ROADMAP.md` said "three shapes", then "four", and was wrong at the next + release both times, which is the argument for reading `receipt.py`'s constants instead. + + This is what makes `scenarios.py`'s **derived** list answerable to something. A derived list + and a derived assertion about it agree no matter what either says; this states the five + independently, so adding `v8` without adding it here goes red and somebody decides whether + the new version belongs in the chain G31 grades. + """ + from ctrlrun.verify.scenarios import _OLDER_RECEIPT_SCHEMAS + + assert RECEIPT_SCHEMA == "ctrlrun.receipt/v7", ( + "the current receipt schema moved; CHAINED_SCHEMAS and SPEC-v0.11 §6's count of five " + "both need a decision, and G31 grades whatever this list says" + ) + assert set(CHAINED_SCHEMAS) <= KNOWN_RECEIPT_SCHEMAS + assert CHAINED_SCHEMAS[-1] == RECEIPT_SCHEMA + assert tuple(_OLDER_RECEIPT_SCHEMAS) == CHAINED_SCHEMAS[:-1], ( + f"verify grades {_OLDER_RECEIPT_SCHEMAS} and this file says {CHAINED_SCHEMAS[:-1]}" + ) + # `v1` and `v2` are known and are NOT in the chain's set: the chain arrived in `v3` + # (SPEC-v0.6 §6.2), so those rows carry no `seq` and are `unchained`, which G11 covers and + # which is never a pass. + assert {"ctrlrun.receipt/v1", "ctrlrun.receipt/v2"} <= KNOWN_RECEIPT_SCHEMAS + assert "ctrlrun.receipt/v1" not in CHAINED_SCHEMAS + assert "ctrlrun.receipt/v2" not in CHAINED_SCHEMAS + + +# --- T523: a version this binary does not know ------------------------------------------------- + + +def test_T523_a_receipt_from_a_future_version_is_named_and_is_not_a_break(seed) -> None: + """SPEC-v0.11 §6.2. `SPEC-v0.6.md` §3.2 draws the same line for a `schema_version` row the + binary does not know, and the difference matters to the only person who reads the output: + *this evidence is from a future version* and *this evidence is tampered with* are different + sentences that call for different actions. + + **Constructed honestly**, which is the whole difficulty: relabelling a stored row is a + *tamper*, and `content_altered` is the right answer to that. A receipt a future writer + actually wrote carries a hash computed over its own document, so that is what this builds. + """ + future = "ctrlrun.receipt/v9" + assert future not in KNOWN_RECEIPT_SCHEMAS, "pick a version this binary really does not know" + + rows, _ = a_chain_of_every_schema(seed) + tail = replace( + rows[-1], + receipt_id=new_receipt_id(), + schema=future, + seq=len(rows) + 1, + prev_hash=rows[-1].hash, + hash=None, + ) + digest = _document_hash(tail.to_dict()) + rows = (*rows, replace(tail, hash=digest)) + + report = verify_chain(_Chain(rows, (len(rows), digest))) + + assert report.ok, ( + "a receipt written by a version this binary does not know was reported as a break: " + f"{[(item.name, item.seq) for item in report.breaks]}" + ) + assert report.verified == 6 + assert report.breaks == [] + # And it is **named**: a reader can tell which row it could not fully interpret, rather than + # the row passing silently as though this binary had read every field in it. + unknown = [row.seq for row in rows if row.schema not in KNOWN_RECEIPT_SCHEMAS] + assert unknown == [6], unknown + + +def test_T523b_a_relabelled_row_is_a_tamper_and_is_reported_as_one(seed) -> None: + """The other side of §6.2, and the reason it needs stating. + + Taking a stored `v7` row and writing `v9` on it **without** rehashing is somebody editing + evidence, not a future version. It must be `content_altered`, and a reader that treated any + unknown label as "from the future, nothing to see" would have made relabelling a way to + launder a tamper. + """ + rows, head = a_chain_of_every_schema(seed) + target = rows[-1] + relabelled = replace(target, schema="ctrlrun.receipt/v9") + damaged = tuple(relabelled if row.seq == target.seq else row for row in rows) + + report = verify_chain(_Chain(damaged, (len(rows), head))) + + assert not report.ok, "relabelling a stored row to an unknown version was not detected" + assert ("content_altered", target.seq) in [(item.name, item.seq) for item in report.breaks] + + +# --- T524: G31 grades it, and agrees with itself ----------------------------------------------- + + +def test_T524_G31_is_in_the_catalogue_and_the_catalogue_moved_once() -> None: + """SPEC-v0.11 §8. The catalogue moves to `v7` once, with whichever item lands first.""" + from ctrlrun.verify import guarantees as reg + + assert reg.CATALOGUE == "ctrlrun.guarantees/v7" + assert reg.BY_ID["G31"].title == "five receipt schemas verify" + assert reg.BY_ID["G31"].descends_from, "G31 names no acceptance test" + # §8 assigns ids in item order, so G31 lands before G28 to G30 exist. A stub row for them + # would report something before its check existed, which is the false green §8 forbids. + assert {"G28", "G29", "G30", "G32"}.isdisjoint({g.id for g in reg.GUARANTEES}) + + +# --- T525: the script that proves the premise -------------------------------------------------- + + +def test_T525_the_released_wheel_script_strips_what_would_make_it_lie() -> None: + """The methodology, asserted rather than trusted. + + `scripts/five_schema_chain.py` is the half of §6.1 that uses the **released** wheels, and its + first run was wrong in a way that looked exactly like a pass: `PYTHONPATH` is inherited by + every child, so every "released wheel" imported this build's source and the script reported + one schema version across a chain of ten receipts. + + The script is not run here: it needs a network and builds five virtual environments. What is + checked is that the two guards which make its answer mean anything are still in it, because + a script whose methodology quietly regressed would go on printing a convincing transcript. + """ + source = _repository_script() + + assert "PYTHONPATH" in source, "the script no longer strips PYTHONPATH from its children" + assert "_clean_environment" in source + assert source.count("env=_clean_environment()") >= 3, ( + "a subprocess in the script runs with this process's environment, so it may import this " + "build instead of the wheel it just installed" + ) + assert "_require_wrote_as" in source, ( + "the script no longer checks that each release ran from its own environment, which is " + "the positive control for the stripping above" + ) + for version, _schema in ( + ("0.6.1", ""), + ("0.7.0", ""), + ("0.8.0", ""), + ("0.9.0", ""), + ("0.10.0", ""), + ): + assert version in source, f"the script no longer builds the chain with {version}" + for schema in CHAINED_SCHEMAS: + assert schema in source, f"the script no longer expects {schema}" + + +# --- T521c: the same chain, read back off a real store ---------------------------------------- + + +def test_T521c_five_schemas_on_disk_rehash_to_their_stored_hashes(tmp_path, seed) -> None: + """The mechanism, not the arithmetic: a receipt read from a store is hashed as **the document + it was read from** (`SPEC-v0.7.md` §6.11), and that is the only reason one chain can hold + five shapes at all. + + **`T521` does not cover this and a mutation proved it.** `T521` builds its receipts with + `replace()`, which drops the stored document, so `chain_hash()` there falls back to rendering + with `to_dict()` and the stored-document branch is never taken. Deleting that branch left + `T521` green. This test puts the five documents **on disk** and reads them back through the + store, which is the path an operator's chain actually takes. + + The rows go in with SQL because `put_receipt` does `replace(receipt, schema=RECEIPT_SCHEMA)`: + a writer writes under its own schema, so an older receipt cannot be forged through the public + API. That is correct, and it is why this test writes the table the way 0.6 left it. + """ + database = tmp_path / "five.db" + store = SQLiteStateStore(database, clock=lambda: T0) + store.close() + + rows, head = a_chain_of_every_schema(seed) + connection = sqlite3.connect(database) + for row in rows: + connection.execute( + "INSERT INTO receipts (receipt_id, action_id, ts, json, seq, prev_hash, hash) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + row.receipt_id, + row.action_id, + row.finished_at.isoformat(), + json.dumps(row.to_dict(), ensure_ascii=False, separators=(",", ":")), + row.seq, + row.prev_hash, + row.hash, + ), + ) + connection.execute( + "INSERT INTO receipt_chain (id, seq, hash) VALUES (1, ?, ?) " + "ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, hash = excluded.hash", + (len(rows), head), + ) + connection.commit() + connection.close() + + reopened = SQLiteStateStore(database, clock=lambda: T0) + read_back = reopened.receipts() + report = verify_chain(reopened) + reopened.close() + + assert len(read_back) == 5 + assert sorted({row.schema for row in read_back}) == sorted(CHAINED_SCHEMAS) + # The stored document really is what came back, so the fallback is not what is being tested. + for row in read_back: + assert isinstance(row, Receipt) + assert row.chain_hash() == row.hash, ( + f"the {row.schema} row at seq {row.seq} does not rehash to its stored hash, so it is " + "being rendered by this binary rather than read as it was written" + ) + assert report.ok, [(item.name, item.seq) for item in report.breaks] + assert report.verified == 5 + + +def test_T521d_a_stored_document_this_binary_would_not_render_still_rehashes(tmp_path, seed): + """The stored-document branch, on the only input that can distinguish it. + + **`T521c` does not reach it either, and a mutation proved that too.** `T521c` writes + `json.dumps(row.to_dict())` to disk, so re-rendering with `to_dict()` produces the same bytes + and `chain_hash()` gives the same answer whichever branch it takes. The branch only matters + when the document on disk is something this binary would **not** produce: a key it has never + heard of, written by a version that came later. + + That is the case `SPEC-v0.7.md` §6.11 exists for, and the one an operator hits when a newer + writer has touched their store. Deleting the stored-document branch makes this row's hash + unreproducible and the chain reports `content_altered` about evidence nobody altered. + """ + database = tmp_path / "future.db" + store = SQLiteStateStore(database, clock=lambda: T0) + store.close() + + document = dict(seed.to_dict()) + document["schema"] = "ctrlrun.receipt/v9" + document["seq"] = 1 + document["prev_hash"] = GENESIS_HASH + # The field that makes the document unrenderable by this binary: `to_dict` projects a fixed + # key set per schema, so nothing here can put this key back. + document["settled_at"] = "2026-09-14T00:00:00Z" + digest = _document_hash(document) + + connection = sqlite3.connect(database) + connection.execute( + "INSERT INTO receipts (receipt_id, action_id, ts, json, seq, prev_hash, hash) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + str(document["receipt_id"]), + str(document["action_id"]), + seed.finished_at.isoformat(), + json.dumps(document, ensure_ascii=False, separators=(",", ":")), + 1, + GENESIS_HASH, + digest, + ), + ) + connection.execute( + "INSERT INTO receipt_chain (id, seq, hash) VALUES (1, 1, ?) " + "ON CONFLICT(id) DO UPDATE SET seq = excluded.seq, hash = excluded.hash", + (digest,), + ) + connection.commit() + connection.close() + + reopened = SQLiteStateStore(database, clock=lambda: T0) + rows = reopened.receipts() + report = verify_chain(reopened) + reopened.close() + + assert len(rows) == 1 + row = rows[0] + assert isinstance(row, Receipt) + assert row.schema == "ctrlrun.receipt/v9" + assert "settled_at" not in row.to_dict(), ( + "this binary rendered a key it does not know, so the document is NOT one it would " + "refuse to reproduce and this test proves nothing" + ) + assert row.chain_hash() == digest, ( + "a receipt carrying a key this binary has never heard of did not rehash to its stored " + "hash, so it is being re-rendered rather than read as it was written (SPEC-v0.7 §6.11)" + ) + assert report.ok, ( + "a receipt a later version wrote was reported as a break: " + f"{[(item.name, item.seq) for item in report.breaks]}" + ) + assert report.verified == 1 + + +# --- T522b: the version is a number, and two digits is where a string comparison breaks -------- + + +def test_T522b_schema_versions_are_ordered_as_numbers_and_not_as_strings() -> None: + """`"ctrlrun.receipt/v10"` sorts **below** `"ctrlrun.receipt/v3"` lexically. + + A string comparison in `_OLDER_RECEIPT_SCHEMAS` gives the right answer for every version that + exists today and the wrong one from v0.13 on: it would drop `v10` and later out of the set + G31 grades, and G31 would go on passing over a shorter chain with nothing to say about it. + A mutation replacing the parse with `label >= "ctrlrun.receipt/v3"` survived the rest of this + file, because nothing here reaches two digits yet. This is what makes the guard load-bearing + now rather than in three milestones. + """ + from ctrlrun.verify.scenarios import _schema_number + + assert _schema_number("ctrlrun.receipt/v3") == 3 + assert _schema_number("ctrlrun.receipt/v10") == 10 + assert "ctrlrun.receipt/v10" < "ctrlrun.receipt/v3", ( + "this test is about a lexical ordering that no longer holds; if that changed, the parse " + "may no longer be needed" + ) + ordered = sorted( + ["ctrlrun.receipt/v10", "ctrlrun.receipt/v3", "ctrlrun.receipt/v9"], key=_schema_number + ) + assert ordered == [ + "ctrlrun.receipt/v3", + "ctrlrun.receipt/v9", + "ctrlrun.receipt/v10", + ], ordered + + +# --- T525b: the script's guard, run rather than grepped --------------------------------------- + + +def test_T525b_the_scripts_environment_guard_actually_strips_the_variables() -> None: + """**`T525` greps and a mutation walked through it.** Replacing the body of + `_clean_environment` with `pass` left every string `T525` looks for in place, so it passed + over a script that hands its own `sys.path` to the wheels it is testing. + + That is the project's own rule about auditing by grep, applied to a script: searching for the + identifier finds the identifier. This imports the function and runs it. + """ + import importlib.util + + _repository_script() # skips outside a checkout, for SCRIPT's reason + spec = importlib.util.spec_from_file_location("five_schema_chain", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + leaked = dict(os.environ) + leaked["PYTHONPATH"] = "/somewhere/that/would/shadow/the/wheel/src" + leaked["PYTHONHOME"] = "/somewhere/else" + original = os.environ.copy() + try: + os.environ.update(leaked) + cleaned = module._clean_environment() + finally: + os.environ.clear() + os.environ.update(original) + + assert "PYTHONPATH" not in cleaned, ( + "the script would hand its own sys.path to each released wheel, so every wheel would " + "import THIS build and the run would report one schema version while looking like a pass" + ) + assert "PYTHONHOME" not in cleaned + assert "PATH" in cleaned, "the child needs an environment it can still run in" + assert module.RELEASES[0][0] == "0.6.1" + assert tuple(schema for _version, schema in module.RELEASES) == CHAINED_SCHEMAS + + +# --- T524b: G31's own control, forced ---------------------------------------------------------- + + +def test_T524b_G31_fails_its_control_when_the_chain_spans_one_schema(tmp_path, monkeypatch): + """**The guarantee that could not have failed, checked by making it fail.** + + `G31`'s first control requires the chain it built to hold five distinct labels, because a + chain of five `v7` rows verifies perfectly and proves nothing. A mutation replacing that + control with `True` survived the whole suite: nothing ever drove `G31` with a one-version + chain, so the control was green and not load-bearing. + + Forcing it is the only way to know the control works, and `control failed` is the right + status: it means the kernel, not the operator's document, is wrong + (`SPEC-v0.4.md` §1.3). + """ + from ctrlrun.verify import scenarios + + # Duplicates of the CURRENT schema, which is exactly what the failure looked like when it + # really happened: the first version of this scenario built its rows with `put_receipt`, + # which does `replace(receipt, schema=RECEIPT_SCHEMA, ...)`, so five rows came back carrying + # one label. Emptying the tuple instead would make the control trivially *satisfied* -- one + # expected label, one present -- which is a test of nothing. + monkeypatch.setattr( + scenarios, "_OLDER_RECEIPT_SCHEMAS", (RECEIPT_SCHEMA, RECEIPT_SCHEMA, RECEIPT_SCHEMA) + ) + + policy = tmp_path / "ctrlrun.yaml" + policy.write_text( + "schema: ctrlrun.policy/v2\n" + "actions:\n" + " stripe.refund:\n" + " decision: allow\n" + ' effect: "refund:{payment_id}"\n', + encoding="utf-8", + ) + from ctrlrun.verify import run + + report = run(policy, only=("G31",)) + result = next(item for item in report.guarantees if item.id == "G31") + + assert result.status is not Status.PASS, ( + "G31 graded PASS over a chain holding one schema version, which is exactly the claim it " + "is supposed to refuse to make" + ) + assert result.reason == "control failed", result.reason + + +def test_T522c_the_derivation_orders_a_two_digit_version_correctly() -> None: + """The ordering, on a set that contains the case a string comparison gets wrong. + + `_OLDER_RECEIPT_SCHEMAS` is computed once at import, so patching `KNOWN_RECEIPT_SCHEMAS` + cannot reach it, and a mutation replacing the numeric comparison with `label >= + "ctrlrun.receipt/v3"` survived every other test in this file: every version that exists + today is one digit and the two comparisons agree. `_chainable_schemas` takes its inputs so + that this can hand it `v10` and `v11`, which is where they stop agreeing. + """ + from ctrlrun.verify.scenarios import _chainable_schemas + + known = { + "ctrlrun.receipt/v1", + "ctrlrun.receipt/v2", + "ctrlrun.receipt/v3", + "ctrlrun.receipt/v9", + "ctrlrun.receipt/v10", + "ctrlrun.receipt/v11", + } + + assert _chainable_schemas(known, "ctrlrun.receipt/v11") == ( + "ctrlrun.receipt/v3", + "ctrlrun.receipt/v9", + "ctrlrun.receipt/v10", + ), ( + "a two-digit receipt schema was dropped or misordered, which is what a lexical " + "comparison does: 'ctrlrun.receipt/v10' sorts below 'ctrlrun.receipt/v3'" + ) + # And the real inputs still give the real answer, so the function is the constant's producer + # and not a second implementation beside it. + from ctrlrun.verify.scenarios import _OLDER_RECEIPT_SCHEMAS + + assert _chainable_schemas(KNOWN_RECEIPT_SCHEMAS, RECEIPT_SCHEMA) == _OLDER_RECEIPT_SCHEMAS diff --git a/tests/test_idempotency.py b/tests/test_idempotency.py index 97b321a4..b09b7b47 100644 --- a/tests/test_idempotency.py +++ b/tests/test_idempotency.py @@ -679,7 +679,7 @@ def _g14(tmp_path, document: str = WITH_EFFECTS): def test_T239_G14_is_in_the_catalogue(): - assert reg.CATALOGUE == "ctrlrun.guarantees/v6" + assert reg.CATALOGUE == "ctrlrun.guarantees/v7" assert "G14" in reg.BY_ID assert reg.BY_ID["G14"].descends_from, "a guarantee names the tests it is the deployed form of" diff --git a/tests/test_preconditions.py b/tests/test_preconditions.py index 818212a0..9022674a 100644 --- a/tests/test_preconditions.py +++ b/tests/test_preconditions.py @@ -2096,7 +2096,7 @@ def _g16(path, **kwargs): def test_T269_G16_is_in_the_catalogue(): from ctrlrun.verify import guarantees as reg - assert reg.CATALOGUE == "ctrlrun.guarantees/v6" + assert reg.CATALOGUE == "ctrlrun.guarantees/v7" assert "G16" in reg.BY_ID assert reg.BY_ID["G16"].descends_from diff --git a/tests/test_schema_completeness.py b/tests/test_schema_completeness.py index a9cb1955..219e6f8f 100644 --- a/tests/test_schema_completeness.py +++ b/tests/test_schema_completeness.py @@ -14,7 +14,7 @@ from __future__ import annotations from datetime import UTC, datetime, timedelta -from typing import Any +from typing import Any, Final import pytest @@ -100,15 +100,48 @@ def test_every_field_v6_froze_is_written_by_something() -> None: assert document[field], f"{field} is empty in the serialized receipt" -def test_the_guarantee_catalogue_is_contiguous_from_G1() -> None: - """`ctrlrun.guarantees/v6`. A catalogue with a stub row reads as a shipped guarantee, which is - why §10.1 has item 1 move the schema once rather than three branches racing it.""" - assert CATALOGUE == "ctrlrun.guarantees/v6", CATALOGUE +#: SPEC-v0.11 §8's table, which assigns ids **in item order** so that splitting the milestone +#: renumbers nothing. The items land one at a time, so between them the catalogue legitimately +#: has holes: item 4 lands `G31` while `G28` (item 2), `G29`, `G30` and `G32` (item 3) are still +#: unbuilt. A hole is allowed **only** if it is one of these, and the release item asserts the +#: set is complete before 0.11.0 ships. +_V0_11_GUARANTEES: Final = { + "G28": "item 2, the anchor", + "G29": "item 3, the prune", + "G30": "item 3, the hold", + "G31": "item 4, five receipt schemas", + "G32": "item 3, an honestly pruned chain leaves a clean anchor report", +} + + +def test_the_guarantee_catalogue_has_no_hole_it_cannot_account_for() -> None: + """`ctrlrun.guarantees/v7`. A catalogue with a stub row reads as a shipped guarantee, which is + why the schema moves once per milestone rather than several branches racing it. + + **This asserted contiguity from `G1` until v0.11**, and contiguity is not what the check was + ever for: it was for *a number nobody built*. `SPEC-v0.11.md` §8 assigns ids in **item** + order rather than landing order, deliberately, so that cutting the milestone in two + renumbers nothing, and item 4's `G31` therefore lands while `G28` to `G30` do not exist yet. + Relaxing to "increasing and unique" would have given up the check entirely, so instead a gap + must be a number §8 named and nothing else, and every id present must be one this project + assigned. + """ + assert CATALOGUE == "ctrlrun.guarantees/v7", CATALOGUE ids = [entry.id for entry in GUARANTEES] - # Contiguous from G1, derived from the catalogue's own length: a milestone that adds an id - # should not have to edit the number here, and one that leaves a GAP should go red. - assert ids == [f"G{number}" for number in range(1, len(GUARANTEES) + 1)], ids + numbers = [int(entry.id[1:]) for entry in GUARANTEES] + assert numbers == sorted(set(numbers)), f"the catalogue is out of order or repeats: {ids}" + + missing = [f"G{number}" for number in range(1, max(numbers) + 1) if f"G{number}" not in ids] + unexplained = [name for name in missing if name not in _V0_11_GUARANTEES] + assert not unexplained, ( + f"the catalogue skips {unexplained}, and SPEC-v0.11 §8 does not assign those numbers to " + "an item that has not landed. A gap here is a guarantee nobody built" + ) + if missing: + # Not an error, but it must be a number §8 named, and it says which item owes it. The + # release item is where this list is required to be empty. + assert all(name in _V0_11_GUARANTEES for name in missing), missing for entry in GUARANTEES: # A row whose title is a placeholder reads as a shipped guarantee in every report that # prints the catalogue, which is the failure this assertion is actually for. diff --git a/tests/test_verify.py b/tests/test_verify.py index b22976a6..2a9ecf83 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -178,7 +178,8 @@ def test_T101_a_policy_with_no_approve_rule_makes_G1_and_G2_not_applicable(tmp_p # section, G13, which is N/A on every SQLite run: SQLite has no clock of its own, and G15, # because this document names no `max_attempts` (SPEC-v0.7 §8.9). The rest are applicable, # G14 among them, and the count is over those. - assert report.applicable == 11 + # 12 since v0.11 item 4's `G31`, which needs only an action to build a chain from. + assert report.applicable == 12 # Derived: every guarantee is applicable or not, exactly once. The literal moved with every # milestone that added an id (G19, then G25), and the invariant never did. assert report.applicable + report.not_applicable == len(reg.GUARANTEES) @@ -777,22 +778,26 @@ def test_G11_is_applicable_even_where_every_action_is_denied(tmp_path): def test_the_catalogue_is_closed_and_ordered(): - """SPEC-v0.10 §7: `v6` is G1 to G27, and each id lands with its item. Ordered by number, so + """SPEC-v0.11 §8: `v7` is G1 to G32, and each id lands with its item. Ordered by number, so an id that arrives before a lower one still sits where a reader looks for it, and unreleased - `main` carries a partial `v6` until the release item asserts all three. - - **No stub rows.** The upper bound is what v0.10 may reach; what is asserted about the middle - is that every id present is one of them and that none is missing from the order. A catalogue - holding an id whose check does not exist yet would report something before it could, which - is a false green (`v0.7 §9.4`'s D27). G26 and G27 are absent here on purpose: items 2 and 3 - bring them, and an assertion that they are present would be the stub row this forbids.""" - assert reg.CATALOGUE == "ctrlrun.guarantees/v6" + `main` carries a partial `v7` until the release item asserts every row. + + **No stub rows.** The upper bound is what v0.11 may reach; what is asserted about the middle + is that every id present is one of them and that none is out of order. A catalogue holding an + id whose check does not exist yet would report something before it could, which is a false + green (`v0.7 §9.4`'s D27). + + **G28 to G30 and G32 are absent here on purpose**, and G31 is present without them: §8 + assigns ids in **item** order rather than landing order, so that splitting the milestone + renumbers nothing, and item 4 lands before items 2 and 3. An assertion that G28 is present + would be the stub row this forbids.""" + assert reg.CATALOGUE == "ctrlrun.guarantees/v7" ids = [guarantee.id for guarantee in reg.GUARANTEES] assert ids[:11] == [f"G{n}" for n in range(1, 12)] assert "G13" in ids and "G16" in ids and "G18" in ids assert ids == sorted(ids, key=lambda gid: int(gid[1:])), ids assert len(ids) == len(set(ids)) - assert set(ids) <= {f"G{n}" for n in range(1, 28)}, ids + assert set(ids) <= {f"G{n}" for n in range(1, 33)}, ids for guarantee in reg.GUARANTEES: assert guarantee.descends_from, f"{guarantee.id} names no acceptance test" @@ -860,7 +865,7 @@ def test_observe_mode_is_refused_before_any_scenario_runs(tmp_path): assert "observe" in str(refused.value) -def test_the_v1_payments_template_reports_eleven_over_eleven(): +def test_the_v1_payments_template_reports_twelve_over_twelve(): """The definition of done, dogfooded rather than described (SPEC-v0.4 §4.1). Ten and not nine since v0.8 item 6, and nine and not eight since item 2. G18 is graded @@ -871,10 +876,12 @@ def test_the_v1_payments_template_reports_eleven_over_eleven(): report = run(V1_PAYMENTS) assert report.exit_code == 0 - assert (report.passed, report.applicable) == (11, 11) + # 12 since v0.11 item 4 added `G31`, which needs only an action to build a chain from + # and is therefore applicable wherever this template's other eleven are. + assert (report.passed, report.applicable) == (12, 12) assert report.applicable + report.not_applicable == len(reg.GUARANTEES) text = report.to_text() - assert "11/11 declared guarantees pass." in text + assert "12/12 declared guarantees pass." in text # G13 is N/A on SQLite, which has no clock of its own; G14 and G15 join G3, G4 and G5 where # the effect template lives in the @protect decorator verify does not read, and where the # document names no `max_attempts`. G16 and G18 are graded: verify brings its own provider diff --git a/tests/test_verify_action.py b/tests/test_verify_action.py index 4e25bc8c..8d0f1cd3 100644 --- a/tests/test_verify_action.py +++ b/tests/test_verify_action.py @@ -139,8 +139,8 @@ def test_T118_ci_asserts_the_two_shapes_the_specification_names(): steps = _workflow()["jobs"]["verify"]["steps"] script = "\n".join(step.get("run", "") for step in steps) - assert 'test "$AUTHORITY" = "verified 24/24"' in script - assert 'test "$TEMPLATES" = "verified 11/11"' in script + assert 'test "$AUTHORITY" = "verified 25/25"' in script + assert 'test "$TEMPLATES" = "verified 12/12"' in script assert 'test "$AUTHORITY_NA" = "3"' in script assert 'test "$TEMPLATES_NA" = "16"' in script @@ -158,7 +158,7 @@ def test_T118_the_two_configurations_really_do_report_those_shapes(): # `examples/authority/payments.yaml` is the one with a delegable grant to hop from. This # pin stays a literal on purpose (it is a CI pin on a shipped example, which exists to fail # when a shape changes) while the N/A counts above are derived. - assert authority.badge["message"] == "verified 24/24" + assert authority.badge["message"] == "verified 25/25" # G13 and G15: SQLite has no clock of its own to diverge from, and the document declares # no `max_attempts` (SPEC-v0.7 §8.9). **And G27**, because no action entry in this document # pins an upstream: SPEC-v0.10 §7.3's exit criterion wants a shipped example that does, and @@ -166,7 +166,7 @@ def test_T118_the_two_configurations_really_do_report_those_shapes(): # it demonstrates the refusal rather than a working call. That example is the release item's. assert authority.not_applicable == 3 assert templates.badge is not None - assert templates.badge["message"] == "verified 11/11" + assert templates.badge["message"] == "verified 12/12" assert templates.applicable + templates.not_applicable == len(reg.GUARANTEES) @@ -209,7 +209,11 @@ def test_T119_the_denominator_is_applicable_and_never_the_catalogue_size(): assert badge is not None assert badge["message"] == f"verified {report.passed}/{report.applicable}" - assert report.applicable == 11 + # 12 since v0.11 item 4: `G31` needs only an action to build a chain from, so it is + # applicable wherever this configuration's other eleven are. The number is pinned rather + # than derived because the claim under test is that the denominator moves with what was + # *graded* and not with the catalogue's size, and a derived number could not fail. + assert report.applicable == 12 assert report.applicable < len(reg.GUARANTEES) assert f"/{len(reg.GUARANTEES)}" not in badge["message"] @@ -295,7 +299,7 @@ def test_T120_a_configuration_with_not_applicable_guarantees_still_writes_a_badg assert report.exit_code == 0 assert report.badge is not None - assert report.badge["message"] == "verified 11/11" + assert report.badge["message"] == "verified 12/12" def test_T120_a_failing_run_writes_a_red_badge_and_a_non_zero_exit(tmp_path, monkeypatch): diff --git a/tests/test_verify_report.py b/tests/test_verify_report.py index 9ab64033..b1a569b6 100644 --- a/tests/test_verify_report.py +++ b/tests/test_verify_report.py @@ -149,8 +149,8 @@ def test_T113_a_failing_report_names_the_subject_and_prints_the_counterexample( # where nothing does. So the first two gain a pass and the third gains an N/A. # The passes are literal because they are the point; the N/A count is derived, for the # reason above. `len(reg.GUARANTEES)` moves with the catalogue and these fixtures do not. - (ALL_APPLICABLE, "16/16 declared guarantees pass."), - (WITH_NOT_APPLICABLE, "11/11 declared guarantees pass."), + (ALL_APPLICABLE, "17/17 declared guarantees pass."), + (WITH_NOT_APPLICABLE, "12/12 declared guarantees pass."), (EMPTY, "0/0 declared guarantees pass."), ], ids=["passing", "some-na", "all-na"], @@ -208,7 +208,7 @@ def test_T114_the_document_matches_the_schema_field_for_field(tmp_path): assert set(document) == TOP_LEVEL assert document["schema"] == REPORT_SCHEMA == "ctrlrun.verify/v1" - assert document["catalogue"] == reg.CATALOGUE == "ctrlrun.guarantees/v6" + assert document["catalogue"] == reg.CATALOGUE == "ctrlrun.guarantees/v7" assert set(document["policy"]) == {"path", "sha256", "schema", "mode", "actions"} assert document["authority"] is None assert document["store"] == {"backend": "sqlite", "scratch": True}