From d339d5384bc94347e005e730a8cae1cbf2e9ae26 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 23:34:12 +0200 Subject: [PATCH 01/34] fix(devtools): recover complete red testmon graphs Problem A failed full seed run left a complete pytest-testmon dependency database in testmondata and only an untyped seed-attempt receipt. Fresh worktrees could not use the graph, while treating the attempt as a green release baseline would overstate verification evidence. What changed Add a typed stamp model separating collection completeness, dependency-graph completeness, baseline color, exact or rebound checkout binding, and the two allowed uses. A complete red graph is accepted for affected selection only. Verify finalization writes a reusable red stamp without changing the red process exit status. Bootstrap validates the source graph, promotes a validated red attempt when no stamp exists, performs an SQLite online backup, rebinds provenance, and rehashes the copied database. Checkout guard and preflight reject stale fingerprints, malformed stamps or SQLite, omitted collection ledgers, missing dependency edges, and interrupted or timeout outcomes. Alternatives rejected A status-only stamp was rejected because it conflates graph coverage with suite health. Copying the source stamp byte-for-byte was rejected because SQLite online backup can produce an equivalent database with a different file hash. Compatibility and migration The testmon stamp protocol advances to version 4. Legacy untyped seed.json files are rejected and must be regenerated by the managed seed flow. No canonical cache files were modified. Verification - Focused managed tests: 127 passed, 1 deselected. - ruff check: All checks passed. - mypy: Success: no issues found in 2700 source files. - devtools verify --quick: ruff and mypy passed; render all hit a transient sqlite3 disk I/O error while another lane owned a large managed test workload. - No seed or full-suite command was run. --- devtools/checkout_guard.py | 29 + devtools/testmon_bootstrap.py | 189 ++++--- devtools/testmon_state.py | 518 ++++++++++++++++++ devtools/verify.py | 209 +++---- .../devtools/test_testmon_seed_recovery.py | 92 ++++ tests/unit/devtools/test_testmon_bootstrap.py | 156 +++++- tests/unit/devtools/test_testmon_state.py | 117 ++++ tests/unit/devtools/test_verify.py | 167 ++++-- 8 files changed, 1227 insertions(+), 250 deletions(-) create mode 100644 devtools/testmon_state.py create mode 100644 tests/integration/devtools/test_testmon_seed_recovery.py create mode 100644 tests/unit/devtools/test_testmon_state.py diff --git a/devtools/checkout_guard.py b/devtools/checkout_guard.py index 0b56e9a315..09c694a9f1 100644 --- a/devtools/checkout_guard.py +++ b/devtools/checkout_guard.py @@ -68,6 +68,8 @@ import tomllib +from devtools.testmon_state import validate_stamp + class CheckoutImportMismatchError(RuntimeError): """``import polylogue`` resolved to a package outside the invoking checkout.""" @@ -130,6 +132,7 @@ def as_dict(self) -> dict[str, object]: _TESTMON_STATE_DIR = Path(".cache/testmon") _TESTMON_STATE_MARKER = _TESTMON_STATE_DIR / "seed.json" _TESTMON_SEED_ATTEMPT = _TESTMON_STATE_DIR / "seed-attempt.json" +_TESTMON_SEED_PROTOCOL_VERSION = 4 _VERIFY_STATE_DIR = Path(".cache/verify") _VERIFY_STATE_MARKER = _VERIFY_STATE_DIR / "current-run.json" @@ -249,6 +252,10 @@ def _marker_origin(marker: Path) -> Path | None: if not isinstance(payload, Mapping): return None raw = payload.get("checkout_root") + if raw is None: + binding = payload.get("binding") + if isinstance(binding, Mapping): + raw = binding.get("checkout_root") if raw is None: fingerprint = payload.get("environment_fingerprint") if isinstance(fingerprint, Mapping): @@ -331,6 +338,28 @@ def _cache_artifact( marker_path = repo_root / marker origin = _marker_origin(marker_path) if origin == repo_root: + if ( + state_dir == _TESTMON_STATE_DIR + and validate_stamp( + marker_path, + state_path / "testmondata", + checkout_root=repo_root, + protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, + ) + is None + ): + return ( + origin, + EnvironmentArtifact( + kind="invalid_testmon_seed", + path=marker_path, + detail="testmon seed marker is stale, malformed, or its SQLite graph is incomplete", + remediation=( + f"remove {state_path} and run `devtools verify --seed-testmon` " + "to rebuild the typed testmon state" + ), + ), + ) return origin, None if ( origin is None diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index de41011860..7ed871e5f4 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -17,12 +17,19 @@ differs between the worktree and the main checkout at copy time self-invalidates (its ``fsha`` won't match), and testmon correctly treats the tests that depend on it as affected on the very next run. No merge or rewrite -is needed -- a straight copy is a valid seed. +is needed for the relative file fingerprints. + +The reusable stamp is typed. It records collection completeness, graph +completeness, baseline color, and whether the graph is exact or rebound to a +new checkout. A red graph is allowed for affected selection only. Bootstrap +revalidates the SQLite graph after the online backup and recomputes its file +fingerprint because SQLite backup can produce a byte-different equivalent +database. This module owns exactly one decision and one action: -- :func:`decide_testmon_bootstrap` -- pure decision, no I/O beyond reading the - two candidate seed files (local + main). Testable with plain tmp dirs. +- :func:`decide_testmon_bootstrap` -- pure decision, no subprocess beyond the + caller. It validates the main stamp or a complete red seed attempt. - :func:`bootstrap_testmon_seed_files` -- the copy action once bootstrapping has been decided. - :func:`maybe_bootstrap_testmon_seed` -- the orchestrator `devtools verify` @@ -38,8 +45,8 @@ copies it through :meth:`sqlite3.Connection.backup`, sqlite's own online-backup API -- built for copying a live database without an exclusive lock, immune to concurrent writers by design. ``seed.json`` is a small file written atomically -by ``verify.py`` (write-temp-then-rename), so a plain read-then-atomic-write -copy is enough for it; there is no partial-write window to observe. +by ``verify.py`` (write-temp-then-rename), so bootstrap writes its newly bound +stamp atomically after the copied graph has been revalidated. This module NEVER writes to the main checkout's copy of either file -- only reads from main, only writes to ``repo_root``. @@ -54,8 +61,16 @@ from dataclasses import dataclass from pathlib import Path +from devtools.testmon_state import ( + TestmonSeedStamp, + refresh_stamp, + stamp_from_attempt, + validate_stamp, +) + TESTMON_DATA_RELPATH = ".cache/testmon/testmondata" TESTMON_SEED_STAMP_RELPATH = ".cache/testmon/seed.json" +TESTMON_SEED_ATTEMPT_RELPATH = ".cache/testmon/seed-attempt.json" @dataclass(frozen=True) @@ -66,19 +81,27 @@ class BootstrapDecision: reason: str main_testmon_data: Path | None = None main_seed_stamp: Path | None = None + main_seed_attempt: Path | None = None + protocol_version: int = 4 -def _is_valid_complete_seed_stamp(seed_stamp: Path, *, protocol_version: int) -> bool: - """Mirror the validity check `devtools.verify._testmon_preflight` applies.""" - if not seed_stamp.is_file(): - return False - try: - stamp = json.loads(seed_stamp.read_text()) - except (OSError, json.JSONDecodeError): - return False - if not isinstance(stamp, dict): - return False - return stamp.get("protocol_version") == protocol_version and stamp.get("status") == "complete" +def _is_valid_complete_seed_stamp( + seed_stamp: Path, + testmon_data: Path, + *, + protocol_version: int, + checkout_root: Path, +) -> bool: + """Validate both the typed stamp and the real SQLite graph it describes.""" + return ( + validate_stamp( + seed_stamp, + testmon_data, + checkout_root=checkout_root, + protocol_version=protocol_version, + ) + is not None + ) def decide_testmon_bootstrap( @@ -89,6 +112,8 @@ def decide_testmon_bootstrap( main_testmon_data: Path, main_seed_stamp: Path, protocol_version: int, + main_seed_attempt: Path | None = None, + main_checkout_root: Path | None = None, ) -> BootstrapDecision: """Decide whether to copy the main checkout's testmon seed into a worktree. @@ -100,55 +125,63 @@ def decide_testmon_bootstrap( return BootstrapDecision(False, "repo_root is not a linked worktree; nothing to bootstrap") if local_testmon_data.is_file() and local_seed_stamp.is_file(): return BootstrapDecision(False, "local .cache/testmon already has a testmondata + seed stamp") - if not _is_valid_complete_seed_stamp(main_seed_stamp, protocol_version=protocol_version): + if not main_testmon_data.is_file(): return BootstrapDecision( False, - "main checkout has no valid complete testmon seed stamp to bootstrap from", + "main checkout has no valid testmon graph because its testmondata file is missing", ) - if not main_testmon_data.is_file(): + root = main_checkout_root or main_testmon_data.parents[2] + if _is_valid_complete_seed_stamp( + main_seed_stamp, + main_testmon_data, + protocol_version=protocol_version, + checkout_root=root, + ): return BootstrapDecision( - False, - "main checkout seed stamp is valid but its testmondata file is missing", + True, + f"main checkout has a validated testmon graph ({main_seed_stamp}); bootstrapping worktree cache", + main_testmon_data=main_testmon_data, + main_seed_stamp=main_seed_stamp, + protocol_version=protocol_version, ) + if main_seed_attempt is not None and main_seed_attempt.is_file(): + try: + attempt = json.loads(main_seed_attempt.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + attempt = None + if ( + isinstance(attempt, dict) + and stamp_from_attempt( + attempt, + main_testmon_data, + checkout_root=root, + protocol_version=protocol_version, + ) + is not None + ): + return BootstrapDecision( + True, + "main checkout has a validated complete graph from a red seed attempt; bootstrapping worktree cache", + main_testmon_data=main_testmon_data, + main_seed_attempt=main_seed_attempt, + protocol_version=protocol_version, + ) + if main_seed_stamp.is_file(): + return BootstrapDecision(False, "main checkout seed stamp is stale, malformed, or graph-incomplete") return BootstrapDecision( - True, - f"main checkout has a valid complete testmon seed ({main_seed_stamp}); bootstrapping worktree cache", - main_testmon_data=main_testmon_data, - main_seed_stamp=main_seed_stamp, + False, + "main checkout has no validated reusable testmon state", ) -def _atomic_copy_bytes(src: Path, dst: Path) -> None: - dst.parent.mkdir(parents=True, exist_ok=True) - tmp = dst.with_name(f"{dst.name}.{os.getpid()}.tmp") - tmp.write_bytes(src.read_bytes()) - tmp.replace(dst) - - -def _stamp_seed_checkout_origin( - seed_stamp: Path, - *, - checkout_root: Path, - inherited_from: Path | None = None, -) -> bool: - """Mark a copied seed with its destination checkout and source provenance.""" - try: - payload = json.loads(seed_stamp.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - return False - if not isinstance(payload, dict): - return False - payload["checkout_root"] = str(checkout_root.resolve()) - if inherited_from is not None: - payload["inherited_from"] = str(inherited_from.resolve()) +def _atomic_write_stamp(seed_stamp: Path, stamp: TestmonSeedStamp) -> None: + seed_stamp.parent.mkdir(parents=True, exist_ok=True) tmp = seed_stamp.with_name(f"{seed_stamp.name}.{os.getpid()}.tmp") try: - tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + tmp.write_text(json.dumps(stamp.as_dict(), indent=2, ensure_ascii=False) + "\n", encoding="utf-8") tmp.replace(seed_stamp) - except OSError: + finally: tmp.unlink(missing_ok=True) - return False - return True def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: @@ -187,11 +220,40 @@ def bootstrap_testmon_seed_files( if not decision.should_bootstrap: return True assert decision.main_testmon_data is not None - assert decision.main_seed_stamp is not None + if decision.main_seed_stamp is None and decision.main_seed_attempt is None: + return False + stamp: TestmonSeedStamp | None = None + try: + source_root = decision.main_testmon_data.parents[2] + if decision.main_seed_stamp is not None: + source = json.loads(decision.main_seed_stamp.read_text(encoding="utf-8")) + if not isinstance(source, dict): + return False + stamp = TestmonSeedStamp.from_mapping(source, protocol_version=decision.protocol_version) + else: + assert decision.main_seed_attempt is not None + source = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) + if not isinstance(source, dict): + return False + stamp = stamp_from_attempt( + source, + decision.main_testmon_data, + checkout_root=source_root, + protocol_version=decision.protocol_version, + ) + if stamp is None: + return False + except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError): + return False + if stamp is None: + return False _atomic_copy_sqlite_db(decision.main_testmon_data, local_testmon_data) - _atomic_copy_bytes(decision.main_seed_stamp, local_seed_stamp) if checkout_root is not None and inherited_from is not None: - return _stamp_seed_checkout_origin(local_seed_stamp, checkout_root=checkout_root, inherited_from=inherited_from) + stamp = stamp.rebound(checkout_root=checkout_root, inherited_from=inherited_from) + stamp = refresh_stamp(stamp, local_testmon_data) + if stamp is None: + return False + _atomic_write_stamp(local_seed_stamp, stamp) return True @@ -250,6 +312,7 @@ def maybe_bootstrap_testmon_seed( local_seed_stamp = repo_root / seed_stamp_relpath main_testmon_data = main_checkout / testmon_data_relpath main_seed_stamp = main_checkout / seed_stamp_relpath + main_seed_attempt = main_checkout / TESTMON_SEED_ATTEMPT_RELPATH decision = decide_testmon_bootstrap( is_linked_worktree=is_linked_worktree, local_testmon_data=local_testmon_data, @@ -257,21 +320,10 @@ def maybe_bootstrap_testmon_seed( main_testmon_data=main_testmon_data, main_seed_stamp=main_seed_stamp, protocol_version=protocol_version, + main_seed_attempt=main_seed_attempt, + main_checkout_root=main_checkout, ) if not decision.should_bootstrap: - if local_testmon_data.is_file() and _is_valid_complete_seed_stamp( - local_seed_stamp, protocol_version=protocol_version - ): - try: - local_payload = json.loads(local_seed_stamp.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - return None - if ( - isinstance(local_payload, dict) - and not local_payload.get("checkout_root") - and _stamp_seed_checkout_origin(local_seed_stamp, checkout_root=repo_root) - ): - return f"verify: migrated legacy pytest-testmon seed marker in {local_seed_stamp.parent}" return None stamped = bootstrap_testmon_seed_files( decision, @@ -294,6 +346,7 @@ def maybe_bootstrap_testmon_seed( __all__ = [ "TESTMON_DATA_RELPATH", "TESTMON_SEED_STAMP_RELPATH", + "TESTMON_SEED_ATTEMPT_RELPATH", "BootstrapDecision", "decide_testmon_bootstrap", "bootstrap_testmon_seed_files", diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py new file mode 100644 index 0000000000..4721f47c18 --- /dev/null +++ b/devtools/testmon_state.py @@ -0,0 +1,518 @@ +"""Typed safety contract for reusable pytest-testmon state. + +The testmon database answers two different questions which must not share a +boolean marker: + +* did collection and dependency capture cover every promised node? +* did that run establish a green release baseline? + +A failed test can still have a complete dependency graph. Such a graph is +usable for affected-test selection, but it is never evidence that the suite +is releasable. This module is the single parser and SQLite validator used by +verification, worktree bootstrap, and the checkout guard. +""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from enum import StrEnum +from pathlib import Path +from typing import Any + + +class CollectionStatus(StrEnum): + COMPLETE = "complete" + INCOMPLETE = "incomplete" + + +class GraphStatus(StrEnum): + COMPLETE = "complete" + INCOMPLETE = "incomplete" + INVALID = "invalid" + + +class BaselineStatus(StrEnum): + GREEN = "green" + RED = "red" + + +class BindingMode(StrEnum): + EXACT = "exact" + RELATIVE_FILE_FINGERPRINTS = "relative-file-fingerprints" + + +@dataclass(frozen=True, slots=True) +class TestmonIdentity: + git_head: str | None + worktree_fingerprint: str + python: str + skip_slow: bool + lab: bool + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> TestmonIdentity: + git_head = value.get("git_head") + if git_head is not None and (not isinstance(git_head, str) or not git_head): + raise ValueError("identity.git_head must be a non-empty string or null") + worktree = value.get("worktree_fingerprint") + python = value.get("python") + if not isinstance(worktree, str) or not worktree: + raise ValueError("identity.worktree_fingerprint must be a non-empty string") + if not isinstance(python, str) or not python: + raise ValueError("identity.python must be a non-empty string") + if not isinstance(value.get("skip_slow"), bool) or not isinstance(value.get("lab"), bool): + raise ValueError("identity selection flags must be booleans") + return cls(git_head, worktree, python, value["skip_slow"], value["lab"]) + + def as_dict(self) -> dict[str, Any]: + return { + "git_head": self.git_head, + "worktree_fingerprint": self.worktree_fingerprint, + "python": self.python, + "skip_slow": self.skip_slow, + "lab": self.lab, + } + + +@dataclass(frozen=True, slots=True) +class TestmonBinding: + mode: BindingMode + checkout_root: str + source_checkout_root: str | None = None + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> TestmonBinding: + raw_mode = value.get("mode") + if not isinstance(raw_mode, str): + raise ValueError("binding.mode is invalid") + try: + mode = BindingMode(raw_mode) + except ValueError as exc: + raise ValueError("binding.mode is invalid") from exc + checkout_root = value.get("checkout_root") + source = value.get("source_checkout_root") + if not isinstance(checkout_root, str) or not checkout_root: + raise ValueError("binding.checkout_root must be a non-empty string") + if source is not None and (not isinstance(source, str) or not source): + raise ValueError("binding.source_checkout_root must be a non-empty string or null") + if mode is BindingMode.EXACT and source is not None: + raise ValueError("exact bindings cannot have a source checkout") + if mode is BindingMode.RELATIVE_FILE_FINGERPRINTS and source is None: + raise ValueError("rebound bindings require a source checkout") + return cls(mode, checkout_root, source) + + def as_dict(self) -> dict[str, Any]: + return { + "mode": self.mode.value, + "checkout_root": self.checkout_root, + "source_checkout_root": self.source_checkout_root, + } + + +@dataclass(frozen=True, slots=True) +class GraphInspection: + status: GraphStatus + recorded_count: int + dependency_edge_count: int + missing_nodeids: tuple[str, ...] + orphan_execution_edges: int + orphan_fingerprint_edges: int + error: str | None + failed_nodeids: tuple[str, ...] + + @property + def usable_for_selection(self) -> bool: + return self.status is GraphStatus.COMPLETE + + def as_dict(self) -> dict[str, Any]: + return { + "status": self.status.value, + "recorded_count": self.recorded_count, + "dependency_edge_count": self.dependency_edge_count, + "missing_nodeids": list(self.missing_nodeids), + "orphan_execution_edges": self.orphan_execution_edges, + "orphan_fingerprint_edges": self.orphan_fingerprint_edges, + "error": self.error, + "failed_nodeids": list(self.failed_nodeids), + } + + +@dataclass(frozen=True, slots=True) +class TestmonSeedStamp: + protocol_version: int + collection_status: CollectionStatus + expected_nodeids: tuple[str, ...] + selected_nodeids_omitted: int + baseline_status: BaselineStatus + release_baseline_allowed: bool + baseline_exit_code: int + graph: GraphInspection + identity: TestmonIdentity + binding: TestmonBinding + testmon_data: str + run_id: str + artifact_dir: str + + @property + def affected_selection_allowed(self) -> bool: + return ( + self.collection_status is CollectionStatus.COMPLETE + and self.selected_nodeids_omitted == 0 + and self.graph.usable_for_selection + ) + + @property + def expected_digest(self) -> str: + return hashlib.sha256("\n".join(sorted(self.expected_nodeids)).encode()).hexdigest() + + def as_dict(self) -> dict[str, Any]: + return { + "protocol_version": self.protocol_version, + "status": "usable", + "collection": { + "status": self.collection_status.value, + "expected_count": len(self.expected_nodeids), + "expected_digest": self.expected_digest, + "selected_nodeids": list(self.expected_nodeids), + "selected_nodeids_omitted": self.selected_nodeids_omitted, + }, + "baseline": { + "status": self.baseline_status.value, + "exit_code": self.baseline_exit_code, + "release_baseline_allowed": self.release_baseline_allowed, + }, + "graph": self.graph.as_dict(), + "identity": self.identity.as_dict(), + "binding": self.binding.as_dict(), + "testmon_data": self.testmon_data, + "run_id": self.run_id, + "artifact_dir": self.artifact_dir, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, Any], *, protocol_version: int) -> TestmonSeedStamp: + if value.get("protocol_version") != protocol_version or value.get("status") != "usable": + raise ValueError("seed stamp is not a current usable testmon stamp") + collection = value.get("collection") + baseline = value.get("baseline") + graph = value.get("graph") + identity = value.get("identity") + binding = value.get("binding") + if not all(isinstance(item, Mapping) for item in (collection, baseline, graph, identity, binding)): + raise ValueError("seed stamp has incomplete typed state") + assert isinstance(collection, Mapping) + assert isinstance(baseline, Mapping) + assert isinstance(graph, Mapping) + assert isinstance(identity, Mapping) + assert isinstance(binding, Mapping) + if collection.get("status") != CollectionStatus.COMPLETE.value: + raise ValueError("seed stamp collection is not complete") + nodeids = collection.get("selected_nodeids") + if ( + not isinstance(nodeids, list) + or not nodeids + or any(not isinstance(item, str) or not item for item in nodeids) + ): + raise ValueError("seed stamp selected nodeids are missing or malformed") + if len(set(nodeids)) != len(nodeids): + raise ValueError("seed stamp selected nodeids are not unique") + omitted = collection.get("selected_nodeids_omitted") + if not isinstance(omitted, int) or isinstance(omitted, bool) or omitted != 0: + raise ValueError("seed stamp has controlled collection omissions") + if collection.get("expected_count") != len(nodeids): + raise ValueError("seed stamp expected count does not match selected nodeids") + expected_digest = hashlib.sha256("\n".join(sorted(nodeids)).encode()).hexdigest() + if collection.get("expected_digest") != expected_digest: + raise ValueError("seed stamp expected nodeid digest is stale") + raw_baseline_status = baseline.get("status") + if not isinstance(raw_baseline_status, str): + raise ValueError("seed stamp baseline status is invalid") + try: + baseline_status = BaselineStatus(raw_baseline_status) + except ValueError as exc: + raise ValueError("seed stamp baseline status is invalid") from exc + exit_code = baseline.get("exit_code") + release_allowed = baseline.get("release_baseline_allowed") + if not isinstance(exit_code, int) or isinstance(exit_code, bool) or not isinstance(release_allowed, bool): + raise ValueError("seed stamp baseline fields are malformed") + if release_allowed != (baseline_status is BaselineStatus.GREEN): + raise ValueError("release permission does not match baseline status") + graph_status = graph.get("status") + if not isinstance(graph_status, str): + raise ValueError("seed stamp graph status is invalid") + try: + status = GraphStatus(graph_status) + except ValueError as exc: + raise ValueError("seed stamp graph status is invalid") from exc + if status is not GraphStatus.COMPLETE: + raise ValueError("seed stamp graph is not complete") + graph_expected = [ + "recorded_count", + "dependency_edge_count", + "orphan_execution_edges", + "orphan_fingerprint_edges", + ] + graph_counts = {key: graph.get(key) for key in graph_expected} + if any(not isinstance(item, int) or isinstance(item, bool) or item < 0 for item in graph_counts.values()): + raise ValueError("seed stamp graph counts are malformed") + dependency_edge_count = graph.get("dependency_edge_count") + if not isinstance(dependency_edge_count, int) or isinstance(dependency_edge_count, bool): + raise ValueError("seed stamp dependency edge count is malformed") + if ( + graph.get("recorded_count") != len(nodeids) + or dependency_edge_count < len(nodeids) + or graph.get("orphan_execution_edges") != 0 + or graph.get("orphan_fingerprint_edges") != 0 + ): + raise ValueError("seed stamp graph coverage is incomplete") + if graph.get("error") is not None or graph.get("missing_nodeids"): + raise ValueError("seed stamp graph has missing or erroneous nodes") + graph_nodeids = graph.get("failed_nodeids", []) + if not isinstance(graph_nodeids, list) or any(not isinstance(item, str) for item in graph_nodeids): + raise ValueError("seed stamp graph failure ledger is malformed") + testmon_data = value.get("testmon_data") + run_id = value.get("run_id") + artifact_dir = value.get("artifact_dir") + if not all(isinstance(item, str) and item for item in (testmon_data, run_id, artifact_dir)): + raise ValueError("seed stamp provenance is incomplete") + assert isinstance(testmon_data, str) + assert isinstance(run_id, str) + assert isinstance(artifact_dir, str) + return cls( + protocol_version, + CollectionStatus.COMPLETE, + tuple(nodeids), + 0, + baseline_status, + release_allowed, + exit_code, + GraphInspection( + status, + graph["recorded_count"], + dependency_edge_count, + tuple(graph.get("missing_nodeids", [])), + graph["orphan_execution_edges"], + graph["orphan_fingerprint_edges"], + graph.get("error"), + tuple(graph_nodeids), + ), + TestmonIdentity.from_mapping(identity), + TestmonBinding.from_mapping(binding), + testmon_data, + run_id, + artifact_dir, + ) + + def rebound(self, *, checkout_root: Path, inherited_from: Path) -> TestmonSeedStamp: + return replace( + self, + binding=TestmonBinding( + BindingMode.RELATIVE_FILE_FINGERPRINTS, + str(checkout_root.resolve()), + str(inherited_from.resolve()), + ), + ) + + +def file_fingerprint(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> GraphInspection: + """Validate the real testmon schema and every expected dependency edge.""" + expected = tuple(expected_nodeids) + if not path.is_file() or not expected or len(set(expected)) != len(expected): + return GraphInspection( + GraphStatus.INCOMPLETE, 0, 0, expected, 0, 0, "missing or malformed expected nodeids", () + ) + try: + with sqlite3.connect(f"file:{path}?mode=ro", uri=True) as connection: + if connection.execute("PRAGMA integrity_check").fetchone() != ("ok",): + return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "sqlite integrity check failed", ()) + required = {"test_execution", "test_execution_file_fp", "file_fp"} + tables = {str(row[0]) for row in connection.execute("select name from sqlite_master where type='table'")} + if not required <= tables: + return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon schema is incomplete", ()) + executions = connection.execute( + "select id, test_name, failed from test_execution where test_name is not null" + ).fetchall() + latest: dict[str, tuple[int, bool]] = {} + for execution_id, test_name, failed in executions: + name = str(test_name) + prior = latest.get(name) + if prior is None or int(execution_id) > prior[0]: + latest[name] = (int(execution_id), bool(failed)) + missing = tuple(sorted(set(expected) - latest.keys())) + expected_ids = {latest[nodeid][0] for nodeid in expected if nodeid in latest} + edge_rows = connection.execute( + "select test_execution_id, fingerprint_id from test_execution_file_fp" + ).fetchall() + execution_ids = {int(row[0]) for row in executions} + fingerprint_ids = {int(row[0]) for row in connection.execute("select id from file_fp").fetchall()} + orphan_execution_edges = sum(1 for row in edge_rows if int(row[0]) not in execution_ids) + orphan_fingerprint_edges = sum(1 for row in edge_rows if int(row[1]) not in fingerprint_ids) + edge_counts: dict[int, int] = {} + for execution_id, _fingerprint_id in edge_rows: + edge_counts[int(execution_id)] = edge_counts.get(int(execution_id), 0) + 1 + uncovered = tuple( + sorted(nodeid for nodeid in expected if nodeid in latest and edge_counts.get(latest[nodeid][0], 0) == 0) + ) + missing = tuple(sorted(set(missing) | set(uncovered))) + failed = tuple(sorted(nodeid for nodeid in expected if nodeid in latest and latest[nodeid][1])) + edge_count = sum(edge_counts.get(execution_id, 0) for execution_id in expected_ids) + status = ( + GraphStatus.COMPLETE + if not missing and not orphan_execution_edges and not orphan_fingerprint_edges + else GraphStatus.INCOMPLETE + ) + return GraphInspection( + status, + len(expected) - len(missing), + edge_count, + missing, + orphan_execution_edges, + orphan_fingerprint_edges, + None, + failed, + ) + except (OSError, sqlite3.Error, UnicodeError) as exc: + return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, str(exc), ()) + + +def validate_stamp( + stamp_path: Path, + data_path: Path, + *, + checkout_root: Path, + protocol_version: int, +) -> TestmonSeedStamp | None: + """Parse and re-check a stamp against its current SQLite graph.""" + try: + payload = json.loads(stamp_path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + return None + stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) + if Path(stamp.binding.checkout_root).resolve() != checkout_root.resolve(): + return None + if file_fingerprint(data_path) != stamp.testmon_data: + return None + graph = inspect_testmon_database(data_path, stamp.expected_nodeids) + if graph != stamp.graph: + return None + return stamp + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return None + + +def refresh_stamp(stamp: TestmonSeedStamp, data_path: Path) -> TestmonSeedStamp | None: + """Refresh mutable SQLite provenance after a successful affected run.""" + graph = inspect_testmon_database(data_path, stamp.expected_nodeids) + if not graph.usable_for_selection: + return None + try: + return replace(stamp, graph=graph, testmon_data=file_fingerprint(data_path)) + except OSError: + return None + + +def stamp_from_attempt( + attempt: Mapping[str, Any], + data_path: Path, + *, + checkout_root: Path, + protocol_version: int, +) -> TestmonSeedStamp | None: + """Promote only a complete attempt, including a red one, into a stamp.""" + if attempt.get("protocol_version") != protocol_version: + return None + selection = attempt.get("selection") + expected = attempt.get("expected_nodeids") + identity = attempt.get("identity") + if not isinstance(selection, Mapping) or not isinstance(expected, list) or not isinstance(identity, Mapping): + return None + assert isinstance(selection, Mapping) + assert isinstance(identity, Mapping) + omitted = selection.get("selected_nodeids_omitted") + selected_count = selection.get("selected_count") + if omitted != 0 or selected_count != len(expected) or not expected: + return None + expected_count = attempt.get("expected_count") + if expected_count is not None and expected_count != len(expected): + return None + expected_digest = attempt.get("expected_digest") + if ( + expected_digest is not None + and expected_digest + != hashlib.sha256("\n".join(sorted(str(nodeid) for nodeid in expected)).encode()).hexdigest() + ): + return None + recorded_data = attempt.get("testmon_data") + if recorded_data is not None: + if not isinstance(recorded_data, str) or not data_path.is_file(): + return None + try: + if file_fingerprint(data_path) != recorded_data: + return None + except OSError: + return None + outcomes = attempt.get("node_outcomes") + if not isinstance(outcomes, list) or len(outcomes) != len(expected): + return None + outcome_by_node = {item.get("nodeid"): item.get("outcome") for item in outcomes if isinstance(item, Mapping)} + if set(outcome_by_node) != set(expected): + return None + if any(outcome not in {"passed", "failed", "error", "skipped"} for outcome in outcome_by_node.values()): + return None + baseline = ( + BaselineStatus.GREEN + if attempt.get("exit_code") == 0 + and all(outcome in {"passed", "skipped"} for outcome in outcome_by_node.values()) + else BaselineStatus.RED + ) + graph = inspect_testmon_database(data_path, [str(nodeid) for nodeid in expected]) + if not graph.usable_for_selection: + return None + try: + typed_identity = TestmonIdentity.from_mapping(identity) + except ValueError: + return None + return TestmonSeedStamp( + protocol_version, + CollectionStatus.COMPLETE, + tuple(str(nodeid) for nodeid in expected), + 0, + baseline, + baseline is BaselineStatus.GREEN, + int(attempt.get("exit_code", 1)), + graph, + typed_identity, + TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())), + file_fingerprint(data_path), + str(attempt.get("run_id") or "attempt-recovery"), + str(attempt.get("artifact_dir") or ".cache/verify"), + ) + + +__all__ = [ + "BaselineStatus", + "BindingMode", + "CollectionStatus", + "GraphInspection", + "GraphStatus", + "TestmonBinding", + "TestmonIdentity", + "TestmonSeedStamp", + "file_fingerprint", + "inspect_testmon_database", + "refresh_stamp", + "stamp_from_attempt", + "validate_stamp", +] diff --git a/devtools/verify.py b/devtools/verify.py index c07bde6b74..d9f0eb22ba 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -31,7 +31,6 @@ import shlex import shutil import signal -import sqlite3 import stat import subprocess import sys @@ -59,6 +58,13 @@ write_termination_request, ) from devtools.testmon_bootstrap import maybe_bootstrap_testmon_seed +from devtools.testmon_state import ( + TestmonSeedStamp, + inspect_testmon_database, + refresh_stamp, + stamp_from_attempt, + validate_stamp, +) from devtools.verify_runs import ( CURRENT_CONTAINMENT_PATH, CURRENT_EVENTS_DIR, @@ -190,7 +196,7 @@ def _format_completion_notification( TESTMON_SEED_STAMP = Path(".cache/testmon/seed.json") TESTMON_SEED_ATTEMPT = Path(".cache/testmon/seed-attempt.json") TESTMON_AFFECTED_STAMP = Path(".cache/testmon/affected.json") -TESTMON_SEED_PROTOCOL_VERSION = 3 +TESTMON_SEED_PROTOCOL_VERSION = 4 PYTEST_REPORT_DIR = Path(".cache/verify") PYTEST_REPORT_PATH = PYTEST_REPORT_DIR / "last-pytest.json" PYTEST_JUNIT_REPORT_DIR = Path(".cache/test-reports") @@ -2072,16 +2078,24 @@ def _testmon_coverage_identity(executable_paths: Sequence[str]) -> dict[str, Any def _matching_testmon_coverage(executable_paths: Sequence[str]) -> str | None: """Return the receipt kind proving that zero new selection is legitimate.""" identity = _testmon_coverage_identity(executable_paths) - seed = _read_json_artifact(TESTMON_SEED_STAMP) - if isinstance(seed, dict): - seed_identity = seed.get("identity") - if ( - seed.get("protocol_version") == TESTMON_SEED_PROTOCOL_VERSION - and seed.get("status") == "complete" - and isinstance(seed_identity, dict) - and seed_identity.get("worktree_fingerprint") == identity["worktree_fingerprint"] - ): - return "complete_seed" + seed = validate_stamp( + TESTMON_SEED_STAMP, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + if seed is not None and seed.affected_selection_allowed: + return "validated_seed_graph" + attempt = _read_testmon_seed_attempt() + if attempt is not None: + recovered = stamp_from_attempt( + attempt, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + if recovered is not None and recovered.affected_selection_allowed: + return "validated_seed_attempt_graph" affected = _read_json_artifact(TESTMON_AFFECTED_STAMP) if isinstance(affected, dict) and affected.get("identity") == identity: return "successful_affected_run" @@ -2111,36 +2125,36 @@ def _testmon_preflight(*, seed_testmon: bool, full_pytest: bool, quick: bool, co "to create .cache/testmon/testmondata and .cache/testmon/seed.json " "before using the default affected-test path.\n" ) - if not TESTMON_DATA.exists() or not TESTMON_SEED_STAMP.exists(): + if not TESTMON_DATA.exists(): return seed_message - try: - stamp = json.loads(TESTMON_SEED_STAMP.read_text()) - except (OSError, json.JSONDecodeError): - return ( - "verify: pytest-testmon seed stamp is unreadable; run `devtools verify --seed-testmon` " - "to refresh .cache/testmon/testmondata and .cache/testmon/seed.json.\n" - ) - if not isinstance(stamp, dict): - return ( - "verify: pytest-testmon seed stamp has an invalid shape; run `devtools verify --seed-testmon` " - "to refresh .cache/testmon/testmondata and .cache/testmon/seed.json.\n" - ) - if stamp.get("protocol_version") != TESTMON_SEED_PROTOCOL_VERSION or stamp.get("status") != "complete": + if not TESTMON_SEED_STAMP.exists(): + attempt = _read_testmon_seed_attempt() + if ( + attempt is not None + and stamp_from_attempt( + attempt, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + is not None + ): + sys.stderr.write( + "verify: using a validated complete pytest-testmon graph from a red seed attempt; " + "the release baseline remains red.\n" + ) + return None + return seed_message + stamp = validate_stamp( + TESTMON_SEED_STAMP, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + if stamp is None: return ( - "verify: pytest-testmon has no validated complete seed receipt; run " - "`devtools verify --seed-testmon` to resume or rebuild the dependency baseline.\n" - ) - current_head = _git_head() - stamped_head = stamp.get("git_head") - if current_head is not None and stamped_head != current_head: - sys.stderr.write( - "verify: pytest-testmon seed was recorded for a different git head; " - "continuing with the existing dependency database and recording affected-test evidence.\n" - ) - if stamp.get("testmon_data") != _file_fingerprint(TESTMON_DATA): - sys.stderr.write( - "verify: pytest-testmon database changed after the seed stamp; " - "continuing because testmon updates its dependency database during normal affected runs.\n" + "verify: pytest-testmon seed state is unreadable, stale, malformed, or not graph-complete; run " + "`devtools verify --seed-testmon` to rebuild the dependency baseline.\n" ) return None @@ -2280,50 +2294,23 @@ def _prepare_testmon_seed_attempt( def _testmon_database_state(expected_nodeids: Sequence[str]) -> dict[str, Any]: - if not TESTMON_DATA.exists(): - return { - "recorded_count": 0, - "failed_count": 0, - "missing_nodeids": list(expected_nodeids), - "failed_nodeids": [], - "node_outcomes": dict.fromkeys(expected_nodeids, "missing"), - "error": "missing", - } - try: - with sqlite3.connect(TESTMON_DATA) as conn: - rows = conn.execute( - """ - SELECT current.test_name, current.failed - FROM test_execution AS current - JOIN ( - SELECT test_name, MAX(id) AS latest_id - FROM test_execution - GROUP BY test_name - ) AS latest ON latest.latest_id = current.id - """ - ).fetchall() - except sqlite3.Error as exc: - return { - "recorded_count": 0, - "failed_count": 0, - "missing_nodeids": list(expected_nodeids), - "failed_nodeids": [], - "node_outcomes": dict.fromkeys(expected_nodeids, "missing"), - "error": str(exc), - } - recorded = {str(name): bool(failed) for name, failed in rows} + graph = inspect_testmon_database(TESTMON_DATA, expected_nodeids) expected = set(expected_nodeids) - failed = sorted(nodeid for nodeid in expected if recorded.get(nodeid) is True) + failed = list(graph.failed_nodeids) return { - "recorded_count": len(recorded), - "failed_count": sum(recorded.values()), - "missing_nodeids": sorted(expected - recorded.keys()), + "recorded_count": graph.recorded_count, + "failed_count": len(failed), + "dependency_edge_count": graph.dependency_edge_count, + "missing_nodeids": list(graph.missing_nodeids), "failed_nodeids": failed, "node_outcomes": { - nodeid: ("failed" if recorded.get(nodeid) is True else "passed" if nodeid in recorded else "missing") + nodeid: ("failed" if nodeid in failed else "passed" if nodeid not in graph.missing_nodeids else "missing") for nodeid in sorted(expected) }, - "error": None, + "error": graph.error, + "graph_status": graph.status.value, + "orphan_execution_edges": graph.orphan_execution_edges, + "orphan_fingerprint_edges": graph.orphan_fingerprint_edges, } @@ -2447,7 +2434,7 @@ def _finalize_testmon_seed_attempt( unsuccessful_nodeids = [ str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} ] - complete = ( + green_complete = ( exit_code == 0 and bool(expected) and (bool(prepared.get("resume")) or omitted == 0) @@ -2456,9 +2443,35 @@ def _finalize_testmon_seed_attempt( and not database["failed_nodeids"] and not unsuccessful_nodeids ) + attempt_candidate = { + **dict(prepared), + "exit_code": exit_code, + "expected_nodeids": expected, + "expected_count": len(expected), + "selection": { + **selection, + # A resumed run inherits the complete collection ledger from its + # original selection. The current pytest step may select only a + # subset while it repairs missing graph edges. + "selected_count": len(expected) if prepared.get("resume") else selection.get("selected_count"), + "selected_nodeids_omitted": 0 if prepared.get("resume") else omitted, + }, + "node_outcomes": node_outcomes, + "identity": prepared.get("identity"), + "run_id": prepared.get("run_id"), + "artifact_dir": prepared.get("artifact_dir"), + } + reusable_stamp = stamp_from_attempt( + attempt_candidate, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + reusable = reusable_stamp is not None + attempt_status = "complete" if green_complete else "reusable" if reusable else "incomplete" payload = { **dict(prepared), - "status": "complete" if complete else "incomplete", + "status": attempt_status, "finished_at": datetime.now(timezone.utc).isoformat(), "exit_code": exit_code, "expected_nodeids": expected, @@ -2489,25 +2502,8 @@ def _finalize_testmon_seed_attempt( "pytest_step": dict(pytest_step) if pytest_step is not None else None, } _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - if complete: - stamp = { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "timestamp": payload["finished_at"], - "checkout_root": str(ROOT.resolve()), - "git_head": dict(prepared["identity"]).get("git_head"), - "identity": prepared["identity"], - "expected_count": payload["expected_count"], - "expected_digest": payload["expected_digest"], - "testmon_data": payload["testmon_data"], - "database": { - "recorded_count": database["recorded_count"], - "failed_count": database["failed_count"], - }, - "run_id": payload["run_id"], - "artifact_dir": payload["artifact_dir"], - } - _atomic_write_json(TESTMON_SEED_STAMP, stamp) + if reusable_stamp is not None: + _atomic_write_json(TESTMON_SEED_STAMP, reusable_stamp.as_dict()) return payload @@ -2649,6 +2645,19 @@ def main(argv: list[str] | None = None) -> int: _warn_low_memory() # check again right before the heavy step rc, elapsed, metadata = _run(label, cmd, run=verify_run) if rc == 0 and label in {"pytest testmon", "pytest testmon (broad)"}: + raw_stamp = _read_json_artifact(TESTMON_SEED_STAMP) + try: + current_stamp = ( + TestmonSeedStamp.from_mapping(raw_stamp, protocol_version=TESTMON_SEED_PROTOCOL_VERSION) + if isinstance(raw_stamp, Mapping) + else None + ) + except ValueError: + current_stamp = None + if current_stamp is not None: + refreshed_stamp = refresh_stamp(current_stamp, TESTMON_DATA) + if refreshed_stamp is not None: + _atomic_write_json(TESTMON_SEED_STAMP, refreshed_stamp.as_dict()) executable_paths = _changed_executable_paths() selected_count = metadata.get("selected_count") if selected_count == 0 and executable_paths: @@ -2721,7 +2730,7 @@ def main(argv: list[str] | None = None) -> int: "resume": seed_receipt["resume"], "expected_count": seed_receipt["expected_count"], "attempt_path": str(TESTMON_SEED_ATTEMPT), - "stamp_path": str(TESTMON_SEED_STAMP) if seed_receipt["status"] == "complete" else None, + "stamp_path": str(TESTMON_SEED_STAMP) if seed_receipt["status"] in {"complete", "reusable"} else None, } if use_json: diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py new file mode 100644 index 0000000000..7f7a712969 --- /dev/null +++ b/tests/integration/devtools/test_testmon_seed_recovery.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path + +from devtools.testmon_bootstrap import BootstrapDecision, bootstrap_testmon_seed_files +from devtools.testmon_state import ( + BaselineStatus, + inspect_testmon_database, + stamp_from_attempt, + validate_stamp, +) + + +def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "test_sample.py").write_text( + "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 2\n", + encoding="utf-8", + ) + data = source / ".cache" / "testmon" / "testmondata" + data.parent.mkdir(parents=True) + env = os.environ.copy() + env["TESTMON_DATAFILE"] = str(data) + run = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "--testmon", "--testmon-noselect"], + cwd=source, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert run.returncode != 0 + expected = ("test_sample.py::test_passed", "test_sample.py::test_failed") + assert inspect_testmon_database(data, expected).usable_for_selection + attempt = { + "protocol_version": 4, + "status": "incomplete", + "identity": { + "git_head": "head", + "worktree_fingerprint": "source-tree", + "python": sys.version, + "skip_slow": False, + "lab": False, + }, + "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, + "expected_nodeids": list(expected), + "node_outcomes": [ + {"nodeid": expected[0], "outcome": "passed"}, + {"nodeid": expected[1], "outcome": "failed"}, + ], + "exit_code": 1, + "run_id": "real-testmon", + "artifact_dir": ".cache/verify/runs/real-testmon", + } + stamp = stamp_from_attempt(attempt, data, checkout_root=source, protocol_version=4) + assert stamp is not None and stamp.baseline_status is BaselineStatus.RED + source_stamp = source / ".cache" / "testmon" / "seed.json" + source_stamp.parent.mkdir(parents=True, exist_ok=True) + source_stamp.write_text(json.dumps(stamp.as_dict()), encoding="utf-8") + + lane = tmp_path / "lane" + local_data = lane / ".cache" / "testmon" / "testmondata" + local_stamp = lane / ".cache" / "testmon" / "seed.json" + decision = BootstrapDecision( + True, + "real graph", + main_testmon_data=data, + main_seed_stamp=source_stamp, + protocol_version=4, + ) + assert bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + checkout_root=lane, + inherited_from=source, + ) + rebound = validate_stamp(local_stamp, local_data, checkout_root=lane, protocol_version=4) + assert rebound is not None + assert rebound.baseline_status is BaselineStatus.RED + assert rebound.binding.checkout_root == str(lane.resolve()) + assert rebound.affected_selection_allowed + + with sqlite3.connect(local_data) as connection: + connection.execute("delete from test_execution_file_fp where test_execution_id = 1") + assert validate_stamp(local_stamp, local_data, checkout_root=lane, protocol_version=4) is None diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index f5b1042ef3..ab31da6b18 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -27,21 +27,69 @@ bootstrap_testmon_seed_files, decide_testmon_bootstrap, ) +from devtools.testmon_state import ( + BaselineStatus, + BindingMode, + CollectionStatus, + GraphInspection, + GraphStatus, + file_fingerprint, +) +from devtools.testmon_state import ( + TestmonBinding as _TestmonBinding, +) +from devtools.testmon_state import ( + TestmonIdentity as _TestmonIdentity, +) +from devtools.testmon_state import ( + TestmonSeedStamp as _TestmonSeedStamp, +) -PROTOCOL_VERSION = 3 +PROTOCOL_VERSION = 4 def _write_valid_seed_stamp(path: Path, *, protocol_version: int = PROTOCOL_VERSION) -> None: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps({"protocol_version": protocol_version, "status": "complete"})) + data = path.parent / "testmondata" + if not data.exists(): + _write_sqlite_db(data) + with sqlite3.connect(data) as conn: + nodeids = tuple(row[0] for row in conn.execute("select test_name from test_execution")) + graph = GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()) + stamp = _TestmonSeedStamp( + protocol_version, + CollectionStatus.COMPLETE, + nodeids, + 0, + BaselineStatus.GREEN, + True, + 0, + graph, + _TestmonIdentity("head", "tree", "python", True, False), + _TestmonBinding(BindingMode.EXACT, str(path.parent.parent.parent.resolve())), + file_fingerprint(data), + "seed", + ".cache/verify/runs/seed", + ) + path.write_text(json.dumps(stamp.as_dict())) def _write_sqlite_db(path: Path, *, rows: tuple[str, ...] = ("a", "b")) -> None: path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path) try: - conn.execute("CREATE TABLE file_fp (path TEXT, fsha TEXT)") - conn.executemany("INSERT INTO file_fp VALUES (?, ?)", [(row, f"sha-{row}") for row in rows]) + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") + conn.execute( + "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, environment_id INTEGER, test_name TEXT, failed INTEGER)" + ) + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") + conn.executemany("INSERT INTO file_fp(filename, fsha) VALUES (?, ?)", [(row, f"sha-{row}") for row in rows]) + conn.executemany("INSERT INTO test_execution(test_name, failed) VALUES (?, 0)", [(row,) for row in rows]) + conn.executemany( + "INSERT INTO test_execution_file_fp VALUES (?, ?)", + [(index, index) for index, _row in enumerate(rows, start=1)], + ) conn.commit() finally: conn.close() @@ -66,7 +114,8 @@ def test_local_seed_already_present_skips_bootstrap(tmp_path: Path) -> None: local_data = tmp_path / "local" / "testmondata" local_stamp = tmp_path / "local" / "seed.json" _write_sqlite_db(local_data) - _write_valid_seed_stamp(local_stamp) + local_stamp.parent.mkdir(parents=True, exist_ok=True) + local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) main_data = tmp_path / "main" / "testmondata" main_stamp = tmp_path / "main" / "seed.json" _write_sqlite_db(main_data) @@ -94,13 +143,12 @@ def test_main_seed_absent_skips_bootstrap(tmp_path: Path) -> None: protocol_version=PROTOCOL_VERSION, ) assert not decision.should_bootstrap - assert "no valid complete testmon seed stamp" in decision.reason + assert "testmondata file is missing" in decision.reason def test_main_seed_stamp_wrong_protocol_version_skips_bootstrap(tmp_path: Path) -> None: main_stamp = tmp_path / "main" / "seed.json" _write_valid_seed_stamp(main_stamp, protocol_version=PROTOCOL_VERSION + 1) - _write_sqlite_db(tmp_path / "main" / "testmondata") decision = decide_testmon_bootstrap( is_linked_worktree=True, @@ -111,7 +159,7 @@ def test_main_seed_stamp_wrong_protocol_version_skips_bootstrap(tmp_path: Path) protocol_version=PROTOCOL_VERSION, ) assert not decision.should_bootstrap - assert "no valid complete testmon seed stamp" in decision.reason + assert "stale" in decision.reason or "no validated" in decision.reason def test_main_seed_stamp_incomplete_status_skips_bootstrap(tmp_path: Path) -> None: @@ -151,7 +199,8 @@ def test_main_seed_stamp_unreadable_json_skips_bootstrap(tmp_path: Path) -> None def test_valid_seed_stamp_but_missing_testmondata_skips_bootstrap(tmp_path: Path) -> None: """A seed stamp claims completeness but the db file itself vanished -- don't copy nothing.""" main_stamp = tmp_path / "main" / "seed.json" - _write_valid_seed_stamp(main_stamp) + main_stamp.parent.mkdir(parents=True, exist_ok=True) + main_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) decision = decide_testmon_bootstrap( is_linked_worktree=True, @@ -184,10 +233,63 @@ def test_valid_main_seed_and_empty_local_bootstraps(tmp_path: Path) -> None: assert decision.main_seed_stamp == main_stamp +def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) -> None: + main_data = tmp_path / "main" / "testmondata" + _write_sqlite_db(main_data, rows=("tests/test.py::test_passed", "tests/test.py::test_failed")) + attempt = tmp_path / "main" / "seed-attempt.json" + attempt.write_text( + json.dumps( + { + "protocol_version": PROTOCOL_VERSION, + "status": "incomplete", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, + "expected_nodeids": ["tests/test.py::test_passed", "tests/test.py::test_failed"], + "node_outcomes": [ + {"nodeid": "tests/test.py::test_passed", "outcome": "passed"}, + {"nodeid": "tests/test.py::test_failed", "outcome": "failed"}, + ], + "exit_code": 1, + "run_id": "red-run", + "artifact_dir": ".cache/verify/runs/red-run", + } + ) + ) + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=tmp_path / "lane" / "testmondata", + local_seed_stamp=tmp_path / "lane" / "seed.json", + main_testmon_data=main_data, + main_seed_stamp=tmp_path / "main" / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + ) + + assert decision.should_bootstrap + assert decision.main_seed_attempt == attempt + local_data = tmp_path / "lane" / "testmondata" + local_stamp = tmp_path / "lane" / "seed.json" + assert bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert json.loads(local_stamp.read_text())["baseline"]["status"] == "red" + + def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: """Partial local state (e.g. a stale stamp with no db, or vice versa) still needs a fresh copy.""" local_stamp = tmp_path / "local" / "seed.json" - _write_valid_seed_stamp(local_stamp) + local_stamp.parent.mkdir(parents=True, exist_ok=True) + local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) main_data = tmp_path / "main" / "testmondata" main_stamp = tmp_path / "main" / "seed.json" _write_sqlite_db(main_data) @@ -220,10 +322,14 @@ def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: ) bootstrap_testmon_seed_files(decision, local_testmon_data=local_data, local_seed_stamp=local_stamp) - assert local_stamp.read_text() == main_stamp.read_text() + local_payload = json.loads(local_stamp.read_text()) + source_payload = json.loads(main_stamp.read_text()) + assert {key: local_payload[key] for key in source_payload if key != "testmon_data"} == { + key: source_payload[key] for key in source_payload if key != "testmon_data" + } conn = sqlite3.connect(local_data) try: - rows = conn.execute("SELECT path, fsha FROM file_fp ORDER BY path").fetchall() + rows = conn.execute("SELECT filename, fsha FROM file_fp ORDER BY filename").fetchall() finally: conn.close() assert rows == [("x", "sha-x"), ("y", "sha-y"), ("z", "sha-z")] @@ -248,10 +354,12 @@ def test_bootstrap_seed_files_marks_destination_and_source_checkout(tmp_path: Pa ) payload = json.loads(local_stamp.read_text()) - assert payload["checkout_root"] == str((tmp_path / "lane").resolve()) - assert payload["inherited_from"] == str((tmp_path / "main").resolve()) + assert payload["binding"]["checkout_root"] == str((tmp_path / "lane").resolve()) + assert payload["binding"]["source_checkout_root"] == str((tmp_path / "main").resolve()) source = json.loads(main_stamp.read_text()) - assert {key: payload[key] for key in source} == source + assert {key: payload[key] for key in source if key not in {"binding", "testmon_data"}} == { + key: source[key] for key in source if key not in {"binding", "testmon_data"} + } def test_bootstrap_seed_files_keeps_copied_state_when_stamp_turns_invalid(tmp_path: Path) -> None: @@ -272,17 +380,20 @@ def test_bootstrap_seed_files_keeps_copied_state_when_stamp_turns_invalid(tmp_pa ) assert stamped is False - assert local_data.is_file() - assert local_stamp.read_text() == "{concurrent rewrite" + assert not local_data.exists() + assert not local_stamp.exists() -def test_maybe_bootstrap_migrates_a_valid_legacy_local_stamp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_maybe_bootstrap_does_not_migrate_an_untyped_legacy_local_stamp( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: lane = tmp_path / "lane" main = tmp_path / "main" local_data = lane / "cache" / "testmondata" local_stamp = lane / "cache" / "seed.json" _write_sqlite_db(local_data) - _write_valid_seed_stamp(local_stamp) + local_stamp.parent.mkdir(parents=True, exist_ok=True) + local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "complete"})) monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, main)) message = testmon_bootstrap.maybe_bootstrap_testmon_seed( @@ -292,11 +403,8 @@ def test_maybe_bootstrap_migrates_a_valid_legacy_local_stamp(tmp_path: Path, mon protocol_version=PROTOCOL_VERSION, ) - payload = json.loads(local_stamp.read_text()) - assert message is not None and "migrated legacy" in message - assert payload["checkout_root"] == str(lane.resolve()) - assert payload["protocol_version"] == PROTOCOL_VERSION - assert payload["status"] == "complete" + assert message is None + assert json.loads(local_stamp.read_text())["status"] == "complete" def test_bootstrap_seed_files_noop_when_decision_says_no(tmp_path: Path) -> None: diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py new file mode 100644 index 0000000000..f2e1d1b5bf --- /dev/null +++ b/tests/unit/devtools/test_testmon_state.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +from devtools.testmon_state import ( + BaselineStatus, + GraphStatus, + file_fingerprint, + inspect_testmon_database, + stamp_from_attempt, + validate_stamp, +) +from devtools.testmon_state import ( + TestmonSeedStamp as _TestmonSeedStamp, +) + +PROTOCOL = 4 +NODEIDS = ("tests/test_seed.py::test_passed", "tests/test_seed.py::test_failed") + + +def _write_graph(path: Path, *, failed: bool = False, with_edges: bool = True) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(path) as connection: + connection.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + connection.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") + connection.execute("CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT, failed INTEGER)") + connection.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") + for index, nodeid in enumerate(NODEIDS, start=1): + connection.execute("INSERT INTO file_fp VALUES (?, ?, ?)", (index, f"file-{index}.py", f"sha-{index}")) + connection.execute( + "INSERT INTO test_execution VALUES (?, ?, ?)", + (index, nodeid, int(failed and index == 2)), + ) + if with_edges: + connection.execute("INSERT INTO test_execution_file_fp VALUES (?, ?)", (index, index)) + + +def _attempt(data: Path, *, outcomes: tuple[str, str] = ("passed", "failed")) -> dict[str, object]: + return { + "protocol_version": PROTOCOL, + "status": "incomplete", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "selection": { + "selected_count": len(NODEIDS), + "selected_nodeids_omitted": 0, + }, + "expected_nodeids": list(NODEIDS), + "expected_count": len(NODEIDS), + "node_outcomes": [ + {"nodeid": nodeid, "outcome": outcome} for nodeid, outcome in zip(NODEIDS, outcomes, strict=True) + ], + "exit_code": 1, + "run_id": "run-red", + "artifact_dir": ".cache/verify/runs/run-red", + "testmon_data": file_fingerprint(data), + } + + +def test_failed_complete_graph_is_selection_only_and_rebindable(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data, failed=True) + stamp = stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert stamp is not None + assert stamp.baseline_status is BaselineStatus.RED + assert stamp.affected_selection_allowed + assert not stamp.release_baseline_allowed + + stamp_path = tmp_path / "seed.json" + stamp_path.write_text(json.dumps(stamp.as_dict())) + assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) == stamp + + +def test_omitted_interrupted_and_uncovered_nodes_fail_closed(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + omitted = _attempt(data) + omitted["selection"] = {"selected_count": 1, "selected_nodeids_omitted": 1} + assert stamp_from_attempt(omitted, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + interrupted = _attempt(data, outcomes=("passed", "interrupted")) + assert stamp_from_attempt(interrupted, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + data.unlink() + _write_graph(data, with_edges=False) + assert stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + +def test_malformed_sqlite_and_stale_stamp_fail_closed(tmp_path: Path) -> None: + malformed = tmp_path / "malformed" + malformed.write_bytes(b"not sqlite") + inspection = inspect_testmon_database(malformed, NODEIDS) + assert inspection.status is GraphStatus.INVALID + + data = tmp_path / "testmondata" + _write_graph(data) + stamp = stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) + assert stamp is not None + stamp_path = tmp_path / "seed.json" + stamp_path.write_text(json.dumps(stamp.as_dict())) + data.write_bytes(data.read_bytes() + b"stale") + assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + +def test_stamp_parser_rejects_untyped_or_non_graph_state() -> None: + try: + _TestmonSeedStamp.from_mapping({"protocol_version": PROTOCOL, "status": "complete"}, protocol_version=PROTOCOL) + except ValueError: + pass + else: + raise AssertionError("legacy green-looking stamp must not be accepted") diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 44388ae490..a1b0a1cc13 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -12,6 +12,23 @@ import pytest from devtools import verify_runs +from devtools.testmon_state import ( + BaselineStatus, + BindingMode, + CollectionStatus, + GraphInspection, + GraphStatus, + file_fingerprint, +) +from devtools.testmon_state import ( + TestmonBinding as _TestmonBinding, +) +from devtools.testmon_state import ( + TestmonIdentity as _TestmonIdentity, +) +from devtools.testmon_state import ( + TestmonSeedStamp as _TestmonSeedStamp, +) from devtools.verify import ( PYTEST_CONTAINMENT_PATH, PYTEST_EVENTS_PATH, @@ -74,6 +91,37 @@ def _pytest_marker_expr(command: list[str]) -> str: return command[marker_indexes[-1] + 1] +def _write_real_testmon_state(nodeids: tuple[str, ...] = ("tests/test_a.py::test_one",)) -> Path: + TESTMON_DATA.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(TESTMON_DATA) as conn: + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") + conn.execute("CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT, failed INTEGER)") + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") + for index, nodeid in enumerate(nodeids, start=1): + conn.execute("INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", (index, nodeid, f"sha-{index}")) + conn.execute("INSERT INTO test_execution(id, test_name, failed) VALUES (?, ?, 0)", (index, nodeid)) + conn.execute("INSERT INTO test_execution_file_fp VALUES (?, ?)", (index, index)) + stamp = _TestmonSeedStamp( + TESTMON_SEED_PROTOCOL_VERSION, + CollectionStatus.COMPLETE, + nodeids, + 0, + BaselineStatus.GREEN, + True, + 0, + GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()), + _TestmonIdentity("current-head", "covered", "python", True, False), + _TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())), + file_fingerprint(TESTMON_DATA), + "seed", + ".cache/verify/runs/seed", + ) + TESTMON_SEED_STAMP.parent.mkdir(parents=True, exist_ok=True) + TESTMON_SEED_STAMP.write_text(json.dumps(stamp.as_dict())) + return TESTMON_DATA + + def test_quick_verify_omits_pytest() -> None: steps = build_verify_steps(quick=True, lab=False, skip_slow=False) @@ -344,52 +392,26 @@ def test_testmon_preflight_requires_seed_stamp(tmp_path: Path, monkeypatch: pyte def test_testmon_preflight_accepts_seeded_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("seeded") - seed_stamp = tmp_path / ".cache" / "testmon" / "seed.json" - seed_stamp.parent.mkdir(parents=True, exist_ok=True) - seed_stamp.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "git_head": "current-head", - "testmon_data": hashlib.sha256(b"seeded").hexdigest(), - } - ) - ) - monkeypatch.setattr("devtools.verify._git_head", lambda: "current-head") + _write_real_testmon_state() assert _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is None -def test_testmon_preflight_warns_on_stale_git_head( +def test_testmon_preflight_rejects_stale_database_fingerprint( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("seeded") - seed_stamp = tmp_path / ".cache" / "testmon" / "seed.json" - seed_stamp.parent.mkdir(parents=True, exist_ok=True) - seed_stamp.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "git_head": "old-head", - "testmon_data": hashlib.sha256(b"seeded").hexdigest(), - } - ) - ) - monkeypatch.setattr("devtools.verify._git_head", lambda: "current-head") + _write_real_testmon_state() + TESTMON_DATA.write_bytes(TESTMON_DATA.read_bytes() + b"stale") message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - assert message is None - assert "different git head" in capsys.readouterr().err + assert message is not None + assert "stale" in message + assert capsys.readouterr().err == "" -def test_testmon_preflight_warns_on_database_fingerprint_drift( +def test_testmon_preflight_rejects_malformed_sqlite_state( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: monkeypatch.chdir(tmp_path) @@ -401,18 +423,16 @@ def test_testmon_preflight_warns_on_database_fingerprint_drift( json.dumps( { "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "git_head": "current-head", - "testmon_data": hashlib.sha256(b"seeded").hexdigest(), + "status": "usable", } ) ) - monkeypatch.setattr("devtools.verify._git_head", lambda: "current-head") message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - assert message is None - assert "database changed" in capsys.readouterr().err + assert message is not None + assert "stale" in message or "malformed" in message + assert capsys.readouterr().err == "" def test_testmon_preflight_rejects_incomplete_seed_receipt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -434,7 +454,7 @@ def test_testmon_preflight_rejects_incomplete_seed_receipt(tmp_path: Path, monke message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) assert message is not None - assert "no validated complete seed receipt" in message + assert "stale" in message or "malformed" in message def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -508,13 +528,21 @@ def test_testmon_database_state_reports_missing_and_failed_nodes( monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) with sqlite3.connect(TESTMON_DATA) as conn: + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") conn.execute( "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" ) + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") conn.executemany( "INSERT INTO test_execution(test_name, failed) VALUES (?, ?)", [("tests/test_a.py::test_ok", 0), ("tests/test_b.py::test_failed", 1)], ) + conn.executemany( + "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", + [(1, "a.py", "a"), (2, "b.py", "b")], + ) + conn.executemany("INSERT INTO test_execution_file_fp VALUES (?, ?)", [(1, 1), (2, 2)]) state = _testmon_database_state( ["tests/test_a.py::test_ok", "tests/test_b.py::test_failed", "tests/test_c.py::test_missing"] @@ -599,19 +627,36 @@ def test_seed_receipt_classifies_every_node_terminal_outcome( (artifact_dir / "events.jsonl").write_text("".join(json.dumps(event) + "\n" for event in events)) TESTMON_DATA.parent.mkdir(parents=True) with sqlite3.connect(TESTMON_DATA) as conn: + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") conn.execute( "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" ) + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") conn.executemany( "INSERT INTO test_execution(test_name, failed) VALUES (?, ?)", [(nodeid, int(nodeid != expected[0])) for nodeid in expected[:-1]], ) + conn.executemany( + "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", + [(index, f"file-{index}.py", f"sha-{index}") for index, _nodeid in enumerate(expected[:-1], start=1)], + ) + conn.executemany( + "INSERT INTO test_execution_file_fp VALUES (?, ?)", + [(index, index) for index, _nodeid in enumerate(expected[:-1], start=1)], + ) receipt = _finalize_testmon_seed_attempt( prepared={ "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, "status": "running", - "identity": {"git_head": "head"}, + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, "resume": False, "expected_nodeids": [], "run_id": "run-mixed", @@ -679,19 +724,36 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon (artifact_dir / "events.jsonl").write_text("") TESTMON_DATA.parent.mkdir(parents=True) with sqlite3.connect(TESTMON_DATA) as conn: + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") conn.execute( "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" ) + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") conn.executemany( "INSERT INTO test_execution(test_name, failed) VALUES (?, 0)", [(nodeid,) for nodeid in expected], ) + conn.executemany( + "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", + [(index, f"file-{index}.py", f"sha-{index}") for index, _nodeid in enumerate(expected, start=1)], + ) + conn.executemany( + "INSERT INTO test_execution_file_fp VALUES (?, ?)", + [(index, index) for index, _nodeid in enumerate(expected, start=1)], + ) receipt = _finalize_testmon_seed_attempt( prepared={ "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, "status": "running", - "identity": {"git_head": "head"}, + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, "resume": False, "expected_nodeids": [], "run_id": "run-1", @@ -704,8 +766,8 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon assert receipt["status"] == "complete" assert receipt["expected_count"] == 2 stamp = json.loads((tmp_path / ".cache" / "testmon" / "seed.json").read_text()) - assert stamp["status"] == "complete" - assert stamp["expected_count"] == 2 + assert stamp["status"] == "usable" + assert stamp["collection"]["expected_count"] == 2 def test_classify_late_sigterm_after_pytest_success_summary() -> None: @@ -1646,19 +1708,8 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo def test_testmon_coverage_receipts_are_content_exact() -> None: paths = ("polylogue/example.py",) - TESTMON_SEED_STAMP.parent.mkdir(parents=True, exist_ok=True) - TESTMON_SEED_STAMP.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "identity": {"worktree_fingerprint": "covered"}, - } - ) - ) - - with patch("devtools.verify._worktree_fingerprint", return_value="covered"): - assert _matching_testmon_coverage(paths) == "complete_seed" + _write_real_testmon_state() + assert _matching_testmon_coverage(paths) == "validated_seed_graph" TESTMON_SEED_STAMP.unlink() with patch("devtools.verify._worktree_fingerprint", return_value="affected"): From b5ce0904a5660ec4d90ef2304ac481abe1df318c Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 00:21:59 +0200 Subject: [PATCH 02/34] fix(devtools): harden testmon seed recovery provenance Problem: malformed, stale, and partial testmon state could be promoted or reused as if it were trustworthy.\n\nWhat changed: validate the installed SQLite shape and relative fingerprints, preserve resumed denominators, require real execution outcomes, bind bootstrap copies to validated roots, and reject untyped zero-selection receipts. Add route-level regressions for red graphs, stale rows, omissions, orphan edges, and unsafe paths.\n\nCompatibility/migration: existing changed-file testmon runs remain supported through pytest-testmon fingerprints; only invalid or unverifiable recovery state now fails closed. --- devtools/testmon_bootstrap.py | 91 +++++++-- devtools/testmon_state.py | 188 ++++++++++++++---- devtools/verify.py | 106 +++++++--- .../devtools/test_testmon_seed_recovery.py | 5 + tests/unit/devtools/test_testmon_bootstrap.py | 49 ++++- tests/unit/devtools/test_testmon_state.py | 45 +++++ tests/unit/devtools/test_verify.py | 137 ++++++++++++- 7 files changed, 525 insertions(+), 96 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 7ed871e5f4..c067b7184a 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -82,9 +82,18 @@ class BootstrapDecision: main_testmon_data: Path | None = None main_seed_stamp: Path | None = None main_seed_attempt: Path | None = None + main_checkout_root: Path | None = None protocol_version: int = 4 +def _checkout_root_for_data(data_path: Path) -> Path: + """Resolve the checkout root for canonical and test-local cache layouts.""" + resolved = data_path.resolve() + if resolved.parent.name == "testmon" and resolved.parent.parent.name == ".cache": + return resolved.parents[2] + return resolved.parent + + def _is_valid_complete_seed_stamp( seed_stamp: Path, testmon_data: Path, @@ -114,6 +123,7 @@ def decide_testmon_bootstrap( protocol_version: int, main_seed_attempt: Path | None = None, main_checkout_root: Path | None = None, + local_checkout_root: Path | None = None, ) -> BootstrapDecision: """Decide whether to copy the main checkout's testmon seed into a worktree. @@ -123,14 +133,30 @@ def decide_testmon_bootstrap( """ if not is_linked_worktree: return BootstrapDecision(False, "repo_root is not a linked worktree; nothing to bootstrap") - if local_testmon_data.is_file() and local_seed_stamp.is_file(): - return BootstrapDecision(False, "local .cache/testmon already has a testmondata + seed stamp") + local_root = (local_checkout_root or _checkout_root_for_data(local_testmon_data)).resolve() + if ( + local_testmon_data.is_file() + and local_seed_stamp.is_file() + and _is_valid_complete_seed_stamp( + local_seed_stamp, + local_testmon_data, + protocol_version=protocol_version, + checkout_root=local_root, + ) + ): + return BootstrapDecision(False, "local .cache/testmon already has a validated testmondata + seed stamp") if not main_testmon_data.is_file(): return BootstrapDecision( False, "main checkout has no valid testmon graph because its testmondata file is missing", ) - root = main_checkout_root or main_testmon_data.parents[2] + root = main_checkout_root or _checkout_root_for_data(main_testmon_data) + root = root.resolve() + try: + main_testmon_data.resolve().relative_to(root) + main_seed_stamp.resolve().relative_to(root) + except ValueError: + return BootstrapDecision(False, "main testmon paths are not bound to the declared checkout root") if _is_valid_complete_seed_stamp( main_seed_stamp, main_testmon_data, @@ -142,6 +168,7 @@ def decide_testmon_bootstrap( f"main checkout has a validated testmon graph ({main_seed_stamp}); bootstrapping worktree cache", main_testmon_data=main_testmon_data, main_seed_stamp=main_seed_stamp, + main_checkout_root=root, protocol_version=protocol_version, ) if main_seed_attempt is not None and main_seed_attempt.is_file(): @@ -164,6 +191,7 @@ def decide_testmon_bootstrap( "main checkout has a validated complete graph from a red seed attempt; bootstrapping worktree cache", main_testmon_data=main_testmon_data, main_seed_attempt=main_seed_attempt, + main_checkout_root=root, protocol_version=protocol_version, ) if main_seed_stamp.is_file(): @@ -196,16 +224,19 @@ def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: dst.parent.mkdir(parents=True, exist_ok=True) tmp = dst.with_name(f"{dst.name}.{os.getpid()}.tmp") tmp.unlink(missing_ok=True) - src_conn = sqlite3.connect(f"file:{src}?mode=ro", uri=True) try: - dst_conn = sqlite3.connect(tmp) + src_conn = sqlite3.connect(f"file:{src}?mode=ro", uri=True) try: - src_conn.backup(dst_conn) + dst_conn = sqlite3.connect(tmp) + try: + src_conn.backup(dst_conn) + finally: + dst_conn.close() finally: - dst_conn.close() + src_conn.close() + tmp.replace(dst) finally: - src_conn.close() - tmp.replace(dst) + tmp.unlink(missing_ok=True) def bootstrap_testmon_seed_files( @@ -222,14 +253,27 @@ def bootstrap_testmon_seed_files( assert decision.main_testmon_data is not None if decision.main_seed_stamp is None and decision.main_seed_attempt is None: return False + if checkout_root is None or inherited_from is None: + return False stamp: TestmonSeedStamp | None = None try: - source_root = decision.main_testmon_data.parents[2] + source_root = (decision.main_checkout_root or inherited_from).resolve() + destination_root = checkout_root.resolve() + if source_root == destination_root: + return False + if inherited_from.resolve() != source_root: + return False + if decision.main_testmon_data.resolve() == local_testmon_data.resolve(): + return False + decision.main_testmon_data.resolve().relative_to(source_root) + local_testmon_data.resolve().relative_to(destination_root) if decision.main_seed_stamp is not None: - source = json.loads(decision.main_seed_stamp.read_text(encoding="utf-8")) - if not isinstance(source, dict): - return False - stamp = TestmonSeedStamp.from_mapping(source, protocol_version=decision.protocol_version) + stamp = validate_stamp( + decision.main_seed_stamp, + decision.main_testmon_data, + checkout_root=source_root, + protocol_version=decision.protocol_version, + ) else: assert decision.main_seed_attempt is not None source = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) @@ -243,18 +287,20 @@ def bootstrap_testmon_seed_files( ) if stamp is None: return False - except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError): + except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError, TypeError, sqlite3.Error): return False if stamp is None: return False - _atomic_copy_sqlite_db(decision.main_testmon_data, local_testmon_data) - if checkout_root is not None and inherited_from is not None: - stamp = stamp.rebound(checkout_root=checkout_root, inherited_from=inherited_from) - stamp = refresh_stamp(stamp, local_testmon_data) - if stamp is None: + try: + _atomic_copy_sqlite_db(decision.main_testmon_data, local_testmon_data) + stamp = stamp.rebound(checkout_root=destination_root, inherited_from=source_root) + refreshed = refresh_stamp(stamp, local_testmon_data) + if refreshed is None or refreshed.graph != stamp.graph: + return False + _atomic_write_stamp(local_seed_stamp, refreshed) + return True + except (OSError, sqlite3.Error, TypeError, ValueError): return False - _atomic_write_stamp(local_seed_stamp, stamp) - return True def _git_worktree_info(repo_root: Path) -> tuple[bool, Path] | None: @@ -322,6 +368,7 @@ def maybe_bootstrap_testmon_seed( protocol_version=protocol_version, main_seed_attempt=main_seed_attempt, main_checkout_root=main_checkout, + local_checkout_root=repo_root, ) if not decision.should_bootstrap: return None diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index 4721f47c18..1e92ade0f4 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -97,12 +97,19 @@ def from_mapping(cls, value: Mapping[str, Any]) -> TestmonBinding: source = value.get("source_checkout_root") if not isinstance(checkout_root, str) or not checkout_root: raise ValueError("binding.checkout_root must be a non-empty string") + if not Path(checkout_root).is_absolute(): + raise ValueError("binding.checkout_root must be absolute") if source is not None and (not isinstance(source, str) or not source): raise ValueError("binding.source_checkout_root must be a non-empty string or null") + if source is not None and not Path(source).is_absolute(): + raise ValueError("binding.source_checkout_root must be absolute") if mode is BindingMode.EXACT and source is not None: raise ValueError("exact bindings cannot have a source checkout") - if mode is BindingMode.RELATIVE_FILE_FINGERPRINTS and source is None: - raise ValueError("rebound bindings require a source checkout") + if mode is BindingMode.RELATIVE_FILE_FINGERPRINTS: + if source is None: + raise ValueError("rebound bindings require a source checkout") + if Path(source).resolve() == Path(checkout_root).resolve(): + raise ValueError("rebound binding source and destination must differ") return cls(mode, checkout_root, source) def as_dict(self) -> dict[str, Any]: @@ -269,10 +276,22 @@ def from_mapping(cls, value: Mapping[str, Any], *, protocol_version: int) -> Tes or graph.get("orphan_fingerprint_edges") != 0 ): raise ValueError("seed stamp graph coverage is incomplete") - if graph.get("error") is not None or graph.get("missing_nodeids"): + missing_nodeids = graph.get("missing_nodeids") + if ( + not isinstance(missing_nodeids, list) + or any(not isinstance(item, str) or not item for item in missing_nodeids) + or not set(missing_nodeids).issubset(nodeids) + ): + raise ValueError("seed stamp missing-node ledger is malformed") + if graph.get("error") is not None or missing_nodeids: raise ValueError("seed stamp graph has missing or erroneous nodes") graph_nodeids = graph.get("failed_nodeids", []) - if not isinstance(graph_nodeids, list) or any(not isinstance(item, str) for item in graph_nodeids): + if ( + not isinstance(graph_nodeids, list) + or any(not isinstance(item, str) or not item for item in graph_nodeids) + or not set(graph_nodeids).issubset(nodeids) + or len(set(graph_nodeids)) != len(graph_nodeids) + ): raise ValueError("seed stamp graph failure ledger is malformed") testmon_data = value.get("testmon_data") run_id = value.get("run_id") @@ -341,27 +360,96 @@ def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> Gra tables = {str(row[0]) for row in connection.execute("select name from sqlite_master where type='table'")} if not required <= tables: return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon schema is incomplete", ()) + required_columns = { + "test_execution": {"id", "test_name", "failed"}, + "test_execution_file_fp": {"test_execution_id", "fingerprint_id"}, + "file_fp": {"id", "filename", "fsha"}, + } + for table, columns in required_columns.items(): + actual = {str(row[1]) for row in connection.execute(f"pragma table_info({table})")} + if not columns <= actual: + return GraphInspection( + GraphStatus.INVALID, + 0, + 0, + expected, + 0, + 0, + f"testmon schema is missing columns from {table}", + (), + ) executions = connection.execute( "select id, test_name, failed from test_execution where test_name is not null" ).fetchall() latest: dict[str, tuple[int, bool]] = {} + execution_ids: set[int] = set() for execution_id, test_name, failed in executions: - name = str(test_name) + if ( + not isinstance(execution_id, int) + or isinstance(execution_id, bool) + or execution_id <= 0 + or not isinstance(test_name, str) + or not test_name + or not isinstance(failed, int) + or isinstance(failed, bool) + or failed not in (0, 1) + ): + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon execution row is malformed", () + ) + if execution_id in execution_ids: + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon execution ids are not unique", () + ) + execution_ids.add(execution_id) + name = test_name prior = latest.get(name) - if prior is None or int(execution_id) > prior[0]: - latest[name] = (int(execution_id), bool(failed)) + if prior is None or execution_id > prior[0]: + latest[name] = (execution_id, failed == 1) missing = tuple(sorted(set(expected) - latest.keys())) expected_ids = {latest[nodeid][0] for nodeid in expected if nodeid in latest} edge_rows = connection.execute( "select test_execution_id, fingerprint_id from test_execution_file_fp" ).fetchall() - execution_ids = {int(row[0]) for row in executions} - fingerprint_ids = {int(row[0]) for row in connection.execute("select id from file_fp").fetchall()} - orphan_execution_edges = sum(1 for row in edge_rows if int(row[0]) not in execution_ids) - orphan_fingerprint_edges = sum(1 for row in edge_rows if int(row[1]) not in fingerprint_ids) + fingerprints = connection.execute("select id, filename, fsha from file_fp").fetchall() + fingerprint_ids: set[int] = set() + for fingerprint_id, filename, fsha in fingerprints: + if ( + not isinstance(fingerprint_id, int) + or isinstance(fingerprint_id, bool) + or fingerprint_id <= 0 + or not isinstance(filename, str) + or not filename + or Path(filename).is_absolute() + or ".." in Path(filename).parts + or not isinstance(fsha, str) + or not fsha + ): + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon fingerprint row is malformed", () + ) + if fingerprint_id in fingerprint_ids: + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon fingerprint ids are not unique", () + ) + fingerprint_ids.add(fingerprint_id) + for execution_id, fingerprint_id in edge_rows: + if ( + not isinstance(execution_id, int) + or isinstance(execution_id, bool) + or execution_id <= 0 + or not isinstance(fingerprint_id, int) + or isinstance(fingerprint_id, bool) + or fingerprint_id <= 0 + ): + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon dependency edge is malformed", () + ) + orphan_execution_edges = sum(1 for row in edge_rows if row[0] not in execution_ids) + orphan_fingerprint_edges = sum(1 for row in edge_rows if row[1] not in fingerprint_ids) edge_counts: dict[int, int] = {} for execution_id, _fingerprint_id in edge_rows: - edge_counts[int(execution_id)] = edge_counts.get(int(execution_id), 0) + 1 + edge_counts[execution_id] = edge_counts.get(execution_id, 0) + 1 uncovered = tuple( sorted(nodeid for nodeid in expected if nodeid in latest and edge_counts.get(latest[nodeid][0], 0) == 0) ) @@ -383,7 +471,7 @@ def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> Gra None, failed, ) - except (OSError, sqlite3.Error, UnicodeError) as exc: + except (OSError, sqlite3.Error, UnicodeError, TypeError, ValueError, OverflowError) as exc: return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, str(exc), ()) @@ -431,7 +519,11 @@ def stamp_from_attempt( protocol_version: int, ) -> TestmonSeedStamp | None: """Promote only a complete attempt, including a red one, into a stamp.""" - if attempt.get("protocol_version") != protocol_version: + if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in { + "incomplete", + "reusable", + "complete", + }: return None selection = attempt.get("selection") expected = attempt.get("expected_nodeids") @@ -442,44 +534,72 @@ def stamp_from_attempt( assert isinstance(identity, Mapping) omitted = selection.get("selected_nodeids_omitted") selected_count = selection.get("selected_count") - if omitted != 0 or selected_count != len(expected) or not expected: + if ( + not isinstance(omitted, int) + or isinstance(omitted, bool) + or omitted != 0 + or not isinstance(selected_count, int) + or isinstance(selected_count, bool) + or selected_count != len(expected) + or not expected + or any(not isinstance(nodeid, str) or not nodeid for nodeid in expected) + or len(set(expected)) != len(expected) + ): return None expected_count = attempt.get("expected_count") - if expected_count is not None and expected_count != len(expected): + if not isinstance(expected_count, int) or isinstance(expected_count, bool) or expected_count != len(expected): return None expected_digest = attempt.get("expected_digest") if ( - expected_digest is not None - and expected_digest - != hashlib.sha256("\n".join(sorted(str(nodeid) for nodeid in expected)).encode()).hexdigest() + not isinstance(expected_digest, str) + or expected_digest != hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() ): return None recorded_data = attempt.get("testmon_data") - if recorded_data is not None: - if not isinstance(recorded_data, str) or not data_path.is_file(): - return None - try: - if file_fingerprint(data_path) != recorded_data: - return None - except OSError: + if not isinstance(recorded_data, str) or not recorded_data or not data_path.is_file(): + return None + try: + if file_fingerprint(data_path) != recorded_data: return None + except OSError: + return None + run_id = attempt.get("run_id") + artifact_dir = attempt.get("artifact_dir") + if not isinstance(run_id, str) or not run_id or not isinstance(artifact_dir, str) or not artifact_dir: + return None outcomes = attempt.get("node_outcomes") if not isinstance(outcomes, list) or len(outcomes) != len(expected): return None - outcome_by_node = {item.get("nodeid"): item.get("outcome") for item in outcomes if isinstance(item, Mapping)} + if any(not isinstance(item, Mapping) for item in outcomes): + return None + outcome_items = [item for item in outcomes if isinstance(item, Mapping)] + if any( + not isinstance(item.get("nodeid"), str) or not item.get("nodeid") or item.get("nodeid") not in expected + for item in outcome_items + ): + return None + outcome_by_node = {item["nodeid"]: item.get("outcome") for item in outcome_items} if set(outcome_by_node) != set(expected): return None + if len(outcome_by_node) != len(outcomes) or any( + not isinstance(nodeid, str) or not nodeid for nodeid in outcome_by_node + ): + return None if any(outcome not in {"passed", "failed", "error", "skipped"} for outcome in outcome_by_node.values()): return None + exit_code = attempt.get("exit_code") + if not isinstance(exit_code, int) or isinstance(exit_code, bool): + return None + graph = inspect_testmon_database(data_path, [str(nodeid) for nodeid in expected]) + if not graph.usable_for_selection: + return None baseline = ( BaselineStatus.GREEN - if attempt.get("exit_code") == 0 + if exit_code == 0 and all(outcome in {"passed", "skipped"} for outcome in outcome_by_node.values()) + and not graph.failed_nodeids else BaselineStatus.RED ) - graph = inspect_testmon_database(data_path, [str(nodeid) for nodeid in expected]) - if not graph.usable_for_selection: - return None try: typed_identity = TestmonIdentity.from_mapping(identity) except ValueError: @@ -491,13 +611,13 @@ def stamp_from_attempt( 0, baseline, baseline is BaselineStatus.GREEN, - int(attempt.get("exit_code", 1)), + exit_code, graph, typed_identity, TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())), file_fingerprint(data_path), - str(attempt.get("run_id") or "attempt-recovery"), - str(attempt.get("artifact_dir") or ".cache/verify"), + run_id, + artifact_dir, ) diff --git a/devtools/verify.py b/devtools/verify.py index d9f0eb22ba..ad9ff1462a 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2078,26 +2078,21 @@ def _testmon_coverage_identity(executable_paths: Sequence[str]) -> dict[str, Any def _matching_testmon_coverage(executable_paths: Sequence[str]) -> str | None: """Return the receipt kind proving that zero new selection is legitimate.""" identity = _testmon_coverage_identity(executable_paths) - seed = validate_stamp( - TESTMON_SEED_STAMP, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - ) - if seed is not None and seed.affected_selection_allowed: - return "validated_seed_graph" - attempt = _read_testmon_seed_attempt() - if attempt is not None: - recovered = stamp_from_attempt( - attempt, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - ) - if recovered is not None and recovered.affected_selection_allowed: - return "validated_seed_attempt_graph" affected = _read_json_artifact(TESTMON_AFFECTED_STAMP) - if isinstance(affected, dict) and affected.get("identity") == identity: + selected_count = affected.get("selected_count") if isinstance(affected, dict) else None + if ( + isinstance(affected, dict) + and affected.get("protocol_version") == 1 + and affected.get("status") == "complete" + and isinstance(affected.get("timestamp"), str) + and bool(affected.get("timestamp")) + and isinstance(affected.get("run_id"), str) + and bool(affected.get("run_id")) + and isinstance(selected_count, int) + and not isinstance(selected_count, bool) + and selected_count > 0 + and affected.get("identity") == identity + ): return "successful_affected_run" return None @@ -2232,7 +2227,17 @@ def _read_testmon_seed_attempt() -> dict[str, Any] | None: def _testmon_seed_expected_nodeids(attempt: Mapping[str, Any]) -> list[str]: """Recover the seed ledger, including after an abrupt outer-run exit.""" expected = attempt.get("expected_nodeids") - if isinstance(expected, list) and expected and all(isinstance(nodeid, str) for nodeid in expected): + if isinstance(expected, list) and expected: + if ( + any(not isinstance(nodeid, str) or not nodeid for nodeid in expected) + or len(set(expected)) != len(expected) + or not isinstance(attempt.get("expected_count"), int) + or isinstance(attempt.get("expected_count"), bool) + or attempt.get("expected_count") != len(expected) + or not isinstance(attempt.get("expected_digest"), str) + or attempt.get("expected_digest") != hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() + ): + return [] return list(expected) artifact_dir_raw = attempt.get("artifact_dir") @@ -2241,10 +2246,26 @@ def _testmon_seed_expected_nodeids(attempt: Mapping[str, Any]) -> list[str]: artifact_dir = Path(artifact_dir_raw) for selection_path in sorted(artifact_dir.glob("steps/*/selection.json")): selection = _read_json_artifact(selection_path) - if not isinstance(selection, dict) or int(selection.get("selected_nodeids_omitted") or 0) != 0: + if not isinstance(selection, dict): + continue + omitted = selection.get("selected_nodeids_omitted") + selected_count = selection.get("selected_count") + if ( + not isinstance(omitted, int) + or isinstance(omitted, bool) + or omitted != 0 + or not isinstance(selected_count, int) + or isinstance(selected_count, bool) + ): continue selected = selection.get("selected_nodeids") - if isinstance(selected, list) and selected and all(isinstance(nodeid, str) for nodeid in selected): + if ( + isinstance(selected, list) + and selected + and all(isinstance(nodeid, str) and nodeid for nodeid in selected) + and len(set(selected)) == len(selected) + and selected_count == len(selected) + ): return list(selected) return [] @@ -2320,6 +2341,7 @@ def _seed_node_outcomes_from_events( expected_nodeids: Sequence[str], database: Mapping[str, Any], pytest_step: Mapping[str, Any] | None, + use_database_fallback: bool = True, ) -> list[dict[str, Any]]: """Classify every promised seed node into one explicit terminal state.""" reports: dict[str, list[dict[str, Any]]] = {} @@ -2375,9 +2397,9 @@ def _seed_node_outcomes_from_events( and any(marker in diagnosis for marker in ("interrupt", "signal", "terminated")) ): outcome, reason = "interrupted", "run ended while node was active" - elif recorded.get(nodeid) == "passed": + elif use_database_fallback and recorded.get(nodeid) == "passed": outcome, reason = "passed", "testmon database recorded success" - elif recorded.get(nodeid) == "failed": + elif use_database_fallback and recorded.get(nodeid) == "failed": outcome, reason = "failed", "testmon database recorded failure" else: outcome, reason = "missing", "no terminal report or testmon execution row" @@ -2421,15 +2443,30 @@ def _finalize_testmon_seed_attempt( selection = selection_payload events_path = artifact_dir / "events.jsonl" - expected_raw = prepared.get("expected_nodeids") if prepared.get("resume") else selection.get("selected_nodeids") - expected = [str(nodeid) for nodeid in expected_raw] if isinstance(expected_raw, list) else [] - omitted = int(selection.get("selected_nodeids_omitted") or 0) + raw_omitted = selection.get("selected_nodeids_omitted") + raw_selected_count = selection.get("selected_count") + selected_nodeids = selection.get("selected_nodeids") + selection_valid = ( + isinstance(raw_omitted, int) + and not isinstance(raw_omitted, bool) + and raw_omitted >= 0 + and isinstance(raw_selected_count, int) + and not isinstance(raw_selected_count, bool) + and isinstance(selected_nodeids, list) + and all(isinstance(nodeid, str) and nodeid for nodeid in selected_nodeids) + and len(set(selected_nodeids)) == len(selected_nodeids) + and raw_selected_count == len(selected_nodeids) + ) + expected_raw = prepared.get("expected_nodeids") if prepared.get("resume") else selected_nodeids + expected = list(expected_raw) if isinstance(expected_raw, list) else [] + omitted = raw_omitted if selection_valid else 1 database = _testmon_database_state(expected) node_outcomes = _seed_node_outcomes_from_events( events_path or Path(".missing-testmon-events"), expected_nodeids=expected, database=database, pytest_step=pytest_step, + use_database_fallback=False, ) unsuccessful_nodeids = [ str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} @@ -2437,29 +2474,38 @@ def _finalize_testmon_seed_attempt( green_complete = ( exit_code == 0 and bool(expected) - and (bool(prepared.get("resume")) or omitted == 0) + and selection_valid + and omitted == 0 and database["error"] is None + and database["graph_status"] == "complete" and not database["missing_nodeids"] and not database["failed_nodeids"] + and database["orphan_execution_edges"] == 0 + and database["orphan_fingerprint_edges"] == 0 and not unsuccessful_nodeids ) attempt_candidate = { **dict(prepared), + "status": "reusable", "exit_code": exit_code, "expected_nodeids": expected, "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, "selection": { **selection, # A resumed run inherits the complete collection ledger from its # original selection. The current pytest step may select only a # subset while it repairs missing graph edges. - "selected_count": len(expected) if prepared.get("resume") else selection.get("selected_count"), - "selected_nodeids_omitted": 0 if prepared.get("resume") else omitted, + "selected_count": len(expected) + if prepared.get("resume") and selection_valid + else selection.get("selected_count"), + "selected_nodeids_omitted": 0 if prepared.get("resume") and selection_valid else omitted, }, "node_outcomes": node_outcomes, "identity": prepared.get("identity"), "run_id": prepared.get("run_id"), "artifact_dir": prepared.get("artifact_dir"), + "testmon_data": _file_fingerprint(TESTMON_DATA), } reusable_stamp = stamp_from_attempt( attempt_candidate, diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py index 7f7a712969..5e326dae5b 100644 --- a/tests/integration/devtools/test_testmon_seed_recovery.py +++ b/tests/integration/devtools/test_testmon_seed_recovery.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import os import sqlite3 @@ -10,6 +11,7 @@ from devtools.testmon_bootstrap import BootstrapDecision, bootstrap_testmon_seed_files from devtools.testmon_state import ( BaselineStatus, + file_fingerprint, inspect_testmon_database, stamp_from_attempt, validate_stamp, @@ -50,6 +52,8 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane(tmp_path: Pat }, "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, "expected_nodeids": list(expected), + "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), "node_outcomes": [ {"nodeid": expected[0], "outcome": "passed"}, {"nodeid": expected[1], "outcome": "failed"}, @@ -57,6 +61,7 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane(tmp_path: Pat "exit_code": 1, "run_id": "real-testmon", "artifact_dir": ".cache/verify/runs/real-testmon", + "testmon_data": file_fingerprint(data), } stamp = stamp_from_attempt(attempt, data, checkout_root=source, protocol_version=4) assert stamp is not None and stamp.baseline_status is BaselineStatus.RED diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index ab31da6b18..221d992c90 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -15,6 +15,7 @@ from __future__ import annotations +import hashlib import json import sqlite3 from pathlib import Path @@ -66,7 +67,7 @@ def _write_valid_seed_stamp(path: Path, *, protocol_version: int = PROTOCOL_VERS 0, graph, _TestmonIdentity("head", "tree", "python", True, False), - _TestmonBinding(BindingMode.EXACT, str(path.parent.parent.parent.resolve())), + _TestmonBinding(BindingMode.EXACT, str(path.parent.resolve())), file_fingerprint(data), "seed", ".cache/verify/runs/seed", @@ -114,8 +115,7 @@ def test_local_seed_already_present_skips_bootstrap(tmp_path: Path) -> None: local_data = tmp_path / "local" / "testmondata" local_stamp = tmp_path / "local" / "seed.json" _write_sqlite_db(local_data) - local_stamp.parent.mkdir(parents=True, exist_ok=True) - local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) + _write_valid_seed_stamp(local_stamp) main_data = tmp_path / "main" / "testmondata" main_stamp = tmp_path / "main" / "seed.json" _write_sqlite_db(main_data) @@ -133,6 +133,29 @@ def test_local_seed_already_present_skips_bootstrap(tmp_path: Path) -> None: assert "already has" in decision.reason +def test_invalid_local_seed_does_not_block_valid_main_bootstrap(tmp_path: Path) -> None: + local_data = tmp_path / "local" / "testmondata" + local_stamp = tmp_path / "local" / "seed.json" + _write_sqlite_db(local_data) + _write_valid_seed_stamp(local_stamp) + local_data.write_bytes(local_data.read_bytes() + b"stale") + main_data = tmp_path / "main" / "testmondata" + main_stamp = tmp_path / "main" / "seed.json" + _write_sqlite_db(main_data) + _write_valid_seed_stamp(main_stamp) + + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + main_testmon_data=main_data, + main_seed_stamp=main_stamp, + protocol_version=PROTOCOL_VERSION, + ) + + assert decision.should_bootstrap + + def test_main_seed_absent_skips_bootstrap(tmp_path: Path) -> None: decision = decide_testmon_bootstrap( is_linked_worktree=True, @@ -251,6 +274,11 @@ def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) }, "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, "expected_nodeids": ["tests/test.py::test_passed", "tests/test.py::test_failed"], + "expected_count": 2, + "expected_digest": hashlib.sha256( + "\n".join(sorted(["tests/test.py::test_passed", "tests/test.py::test_failed"])).encode() + ).hexdigest(), + "testmon_data": file_fingerprint(main_data), "node_outcomes": [ {"nodeid": "tests/test.py::test_passed", "outcome": "passed"}, {"nodeid": "tests/test.py::test_failed", "outcome": "failed"}, @@ -320,13 +348,20 @@ def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: main_testmon_data=main_data, main_seed_stamp=main_stamp, ) - bootstrap_testmon_seed_files(decision, local_testmon_data=local_data, local_seed_stamp=local_stamp) + assert bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + checkout_root=tmp_path / "local", + inherited_from=tmp_path / "main", + ) local_payload = json.loads(local_stamp.read_text()) source_payload = json.loads(main_stamp.read_text()) - assert {key: local_payload[key] for key in source_payload if key != "testmon_data"} == { - key: source_payload[key] for key in source_payload if key != "testmon_data" - } + comparable_keys = set(source_payload) - {"binding", "testmon_data"} + assert {key: local_payload[key] for key in comparable_keys} == {key: source_payload[key] for key in comparable_keys} + assert local_payload["binding"]["checkout_root"] == str(tmp_path / "local") + assert local_payload["binding"]["source_checkout_root"] == str(tmp_path / "main") conn = sqlite3.connect(local_data) try: rows = conn.execute("SELECT filename, fsha FROM file_fp ORDER BY filename").fetchall() diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index f2e1d1b5bf..380bf33fae 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -1,9 +1,12 @@ from __future__ import annotations +import hashlib import json import sqlite3 from pathlib import Path +import pytest + from devtools.testmon_state import ( BaselineStatus, GraphStatus, @@ -54,6 +57,7 @@ def _attempt(data: Path, *, outcomes: tuple[str, str] = ("passed", "failed")) -> }, "expected_nodeids": list(NODEIDS), "expected_count": len(NODEIDS), + "expected_digest": hashlib.sha256("\n".join(sorted(NODEIDS)).encode()).hexdigest(), "node_outcomes": [ {"nodeid": nodeid, "outcome": outcome} for nodeid, outcome in zip(NODEIDS, outcomes, strict=True) ], @@ -74,6 +78,13 @@ def test_failed_complete_graph_is_selection_only_and_rebindable(tmp_path: Path) assert stamp.affected_selection_allowed assert not stamp.release_baseline_allowed + passed_outcomes = stamp_from_attempt( + _attempt(data, outcomes=("passed", "passed")), data, checkout_root=tmp_path, protocol_version=PROTOCOL + ) + assert passed_outcomes is not None + assert passed_outcomes.baseline_status is BaselineStatus.RED + assert not passed_outcomes.release_baseline_allowed + stamp_path = tmp_path / "seed.json" stamp_path.write_text(json.dumps(stamp.as_dict())) assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) == stamp @@ -108,6 +119,40 @@ def test_malformed_sqlite_and_stale_stamp_fail_closed(tmp_path: Path) -> None: assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None +def test_malformed_sqlite_values_fail_closed(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + with sqlite3.connect(data) as connection: + connection.execute("update test_execution set failed = 'bad' where id = 1") + + inspection = inspect_testmon_database(data, NODEIDS) + + assert inspection.status is GraphStatus.INVALID + + +@pytest.mark.parametrize("filename", ["../outside.py", "/tmp/outside.py"]) +def test_unsafe_testmon_fingerprint_paths_fail_closed(tmp_path: Path, filename: str) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + with sqlite3.connect(data) as connection: + connection.execute("update file_fp set filename = ? where id = 1", (filename,)) + + assert inspect_testmon_database(data, NODEIDS).status is GraphStatus.INVALID + + +def test_attempt_status_must_be_promotable(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data) + attempt["status"] = "running" + + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + attempt = _attempt(data) + attempt["run_id"] = None + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + def test_stamp_parser_rejects_untyped_or_non_graph_state() -> None: try: _TestmonSeedStamp.from_mapping({"protocol_version": PROTOCOL, "status": "complete"}, protocol_version=PROTOCOL) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index a1b0a1cc13..3da6b9fb9d 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -475,6 +475,11 @@ def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pyte "status": "incomplete", "identity": identity, "expected_nodeids": ["tests/unit/test_example.py::test_one"], + "expected_count": 1, + "expected_digest": hashlib.sha256(b"tests/unit/test_example.py::test_one").hexdigest(), + "run_id": "interrupted", + "started_at": "2026-08-05T12:00:00+00:00", + "testmon_data_before": "partial", } ) ) @@ -493,7 +498,9 @@ def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, mo step_dir = artifact_dir / "steps" / "17-pytest-seed-testmon" step_dir.mkdir(parents=True) expected = ["tests/unit/test_example.py::test_one"] - (step_dir / "selection.json").write_text(json.dumps({"selected_nodeids": expected, "selected_nodeids_omitted": 0})) + (step_dir / "selection.json").write_text( + json.dumps({"selected_nodeids": expected, "selected_nodeids_omitted": 0, "selected_count": 1}) + ) identity = { "git_head": "head", "worktree_fingerprint": "tree", @@ -522,6 +529,76 @@ def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, mo assert prepared["expected_count"] == 1 +def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) -> None: + monkeypatch = pytest.MonkeyPatch() + monkeypatch.chdir(tmp_path) + try: + expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir() + (artifact_dir / "selection.json").write_text( + json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) + ) + (artifact_dir / "events.jsonl").write_text( + json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" + ) + TESTMON_DATA.parent.mkdir(parents=True) + with sqlite3.connect(TESTMON_DATA) as connection: + connection.execute("create table environment (id integer primary key, environment_name text)") + connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") + connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") + connection.execute( + "create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)" + ) + connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) + connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) + connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) + prepared = { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "resume": True, + "expected_nodeids": expected, + "run_id": "resume", + "artifact_dir": str(tmp_path / "resume"), + } + + receipt = _finalize_testmon_seed_attempt( + prepared=prepared, + step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + + assert receipt["status"] == "incomplete" + assert {item["nodeid"]: item["outcome"] for item in receipt["node_outcomes"]} == { + expected[0]: "passed", + expected[1]: "missing", + } + + (artifact_dir / "selection.json").write_text(json.dumps({})) + (artifact_dir / "events.jsonl").write_text( + "\n".join( + json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) + for nodeid in expected + ) + + "\n" + ) + missing_selection = _finalize_testmon_seed_attempt( + prepared=prepared, + step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + assert missing_selection["status"] == "incomplete" + finally: + monkeypatch.undo() + + def test_testmon_database_state_reports_missing_and_failed_nodes( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -721,7 +798,12 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon } ) ) - (artifact_dir / "events.jsonl").write_text("") + (artifact_dir / "events.jsonl").write_text( + "".join( + json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) + "\n" + for nodeid in expected + ) + ) TESTMON_DATA.parent.mkdir(parents=True) with sqlite3.connect(TESTMON_DATA) as conn: conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") @@ -769,6 +851,51 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon assert stamp["status"] == "usable" assert stamp["collection"]["expected_count"] == 2 + (artifact_dir / "events.jsonl").write_text("") + stale_database = _finalize_testmon_seed_attempt( + prepared={ + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "resume": False, + "expected_nodeids": [], + "run_id": "run-stale-db", + "artifact_dir": str(tmp_path / "run-stale-db"), + }, + step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + assert stale_database["status"] == "incomplete" + + with sqlite3.connect(TESTMON_DATA) as connection: + connection.execute("insert into test_execution_file_fp values (999, 1)") + orphaned = _finalize_testmon_seed_attempt( + prepared={ + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "resume": False, + "expected_nodeids": [], + "run_id": "run-orphaned", + "artifact_dir": str(tmp_path / "run-orphaned"), + }, + step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + assert orphaned["status"] == "incomplete" + def test_classify_late_sigterm_after_pytest_success_summary() -> None: diagnosis = classify_pytest_result( @@ -1709,7 +1836,7 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo def test_testmon_coverage_receipts_are_content_exact() -> None: paths = ("polylogue/example.py",) _write_real_testmon_state() - assert _matching_testmon_coverage(paths) == "validated_seed_graph" + assert _matching_testmon_coverage(paths) is None TESTMON_SEED_STAMP.unlink() with patch("devtools.verify._worktree_fingerprint", return_value="affected"): @@ -1725,6 +1852,10 @@ def test_testmon_coverage_receipts_are_content_exact() -> None: with patch("devtools.verify._worktree_fingerprint", return_value="changed"): assert _matching_testmon_coverage(paths) is None + TESTMON_AFFECTED_STAMP.write_text(json.dumps({"identity": {"worktree_fingerprint": "affected"}})) + with patch("devtools.verify._worktree_fingerprint", return_value="affected"): + assert _matching_testmon_coverage(paths) is None + def test_failed_step_stop_policy_distinguishes_cheap_and_heavy_steps() -> None: assert _stop_after_failed_step("ruff check") is False From 3b1ab74ff968ce2ed42682ec4d152efedfab48e9 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 01:38:11 +0200 Subject: [PATCH 03/34] fix(devtools): bind testmon recovery artifacts --- devtools/testmon_bootstrap.py | 15 ++++++- devtools/testmon_state.py | 2 +- devtools/verify.py | 27 +++++++++--- tests/unit/devtools/test_testmon_bootstrap.py | 36 ++++++++++++++-- tests/unit/devtools/test_testmon_state.py | 9 ++++ tests/unit/devtools/test_verify.py | 42 ++++++++++++++++++- 6 files changed, 118 insertions(+), 13 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index c067b7184a..c38280e557 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -155,6 +155,8 @@ def decide_testmon_bootstrap( try: main_testmon_data.resolve().relative_to(root) main_seed_stamp.resolve().relative_to(root) + if main_seed_attempt is not None: + main_seed_attempt.resolve().relative_to(root) except ValueError: return BootstrapDecision(False, "main testmon paths are not bound to the declared checkout root") if _is_valid_complete_seed_stamp( @@ -225,7 +227,7 @@ def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: tmp = dst.with_name(f"{dst.name}.{os.getpid()}.tmp") tmp.unlink(missing_ok=True) try: - src_conn = sqlite3.connect(f"file:{src}?mode=ro", uri=True) + src_conn = sqlite3.connect(f"{src.resolve().as_uri()}?mode=ro", uri=True) try: dst_conn = sqlite3.connect(tmp) try: @@ -267,6 +269,17 @@ def bootstrap_testmon_seed_files( return False decision.main_testmon_data.resolve().relative_to(source_root) local_testmon_data.resolve().relative_to(destination_root) + local_seed_stamp.resolve().relative_to(destination_root) + if local_seed_stamp.resolve() == local_testmon_data.resolve(): + return False + if decision.main_seed_stamp is not None: + decision.main_seed_stamp.resolve().relative_to(source_root) + if decision.main_seed_stamp.resolve() == decision.main_testmon_data.resolve(): + return False + if decision.main_seed_attempt is not None: + decision.main_seed_attempt.resolve().relative_to(source_root) + if decision.main_seed_attempt.resolve() == decision.main_testmon_data.resolve(): + return False if decision.main_seed_stamp is not None: stamp = validate_stamp( decision.main_seed_stamp, diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index 1e92ade0f4..082eacdcda 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -353,7 +353,7 @@ def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> Gra GraphStatus.INCOMPLETE, 0, 0, expected, 0, 0, "missing or malformed expected nodeids", () ) try: - with sqlite3.connect(f"file:{path}?mode=ro", uri=True) as connection: + with sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True) as connection: if connection.execute("PRAGMA integrity_check").fetchone() != ("ok",): return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "sqlite integrity check failed", ()) required = {"test_execution", "test_execution_file_fp", "file_fp"} diff --git a/devtools/verify.py b/devtools/verify.py index ad9ff1462a..a6099a6f03 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2224,6 +2224,23 @@ def _read_testmon_seed_attempt() -> dict[str, Any] | None: return payload if isinstance(payload, dict) else None +def _safe_testmon_artifact_dir(raw: object, *, require_run_root: bool = False) -> Path | None: + if not isinstance(raw, str) or not raw: + return None + path = Path(raw) + checkout_root = Path.cwd().resolve() + if require_run_root and path.is_absolute(): + return None + resolved = (path if path.is_absolute() else checkout_root / path).resolve() + try: + resolved.relative_to(checkout_root) + if require_run_root: + resolved.relative_to((checkout_root / ".cache" / "verify" / "runs").resolve()) + except ValueError: + return None + return resolved + + def _testmon_seed_expected_nodeids(attempt: Mapping[str, Any]) -> list[str]: """Recover the seed ledger, including after an abrupt outer-run exit.""" expected = attempt.get("expected_nodeids") @@ -2240,10 +2257,9 @@ def _testmon_seed_expected_nodeids(attempt: Mapping[str, Any]) -> list[str]: return [] return list(expected) - artifact_dir_raw = attempt.get("artifact_dir") - if not isinstance(artifact_dir_raw, str): + artifact_dir = _safe_testmon_artifact_dir(attempt.get("artifact_dir"), require_run_root=True) + if artifact_dir is None: return [] - artifact_dir = Path(artifact_dir_raw) for selection_path in sorted(artifact_dir.glob("steps/*/selection.json")): selection = _read_json_artifact(selection_path) if not isinstance(selection, dict): @@ -2435,9 +2451,8 @@ def _finalize_testmon_seed_attempt( selection: dict[str, Any] = {} events_path: Path | None = None if pytest_step is not None: - artifact_dir_raw = pytest_step.get("artifact_dir") - if isinstance(artifact_dir_raw, str): - artifact_dir = Path(artifact_dir_raw) + artifact_dir = _safe_testmon_artifact_dir(pytest_step.get("artifact_dir")) + if artifact_dir is not None: selection_payload = _read_json_artifact(artifact_dir / "selection.json") if isinstance(selection_payload, dict): selection = selection_payload diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 221d992c90..d33ac42e37 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -335,8 +335,8 @@ def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" + main_data = tmp_path / "main?fragment#1" / "testmondata" + main_stamp = tmp_path / "main?fragment#1" / "seed.json" _write_sqlite_db(main_data, rows=("x", "y", "z")) _write_valid_seed_stamp(main_stamp) local_data = tmp_path / "local" / "testmondata" @@ -353,7 +353,7 @@ def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: local_testmon_data=local_data, local_seed_stamp=local_stamp, checkout_root=tmp_path / "local", - inherited_from=tmp_path / "main", + inherited_from=tmp_path / "main?fragment#1", ) local_payload = json.loads(local_stamp.read_text()) @@ -361,7 +361,7 @@ def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: comparable_keys = set(source_payload) - {"binding", "testmon_data"} assert {key: local_payload[key] for key in comparable_keys} == {key: source_payload[key] for key in comparable_keys} assert local_payload["binding"]["checkout_root"] == str(tmp_path / "local") - assert local_payload["binding"]["source_checkout_root"] == str(tmp_path / "main") + assert local_payload["binding"]["source_checkout_root"] == str(tmp_path / "main?fragment#1") conn = sqlite3.connect(local_data) try: rows = conn.execute("SELECT filename, fsha FROM file_fp ORDER BY filename").fetchall() @@ -397,6 +397,34 @@ def test_bootstrap_seed_files_marks_destination_and_source_checkout(tmp_path: Pa } +def test_bootstrap_seed_files_rejects_paths_outside_or_colliding_with_destination(tmp_path: Path) -> None: + main_data = tmp_path / "main" / "testmondata" + main_stamp = tmp_path / "main" / "seed.json" + _write_sqlite_db(main_data) + _write_valid_seed_stamp(main_stamp) + decision = BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp) + local_data = tmp_path / "lane" / "testmondata" + + assert not bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=tmp_path / "outside" / "seed.json", + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert not (tmp_path / "outside" / "seed.json").exists() + assert not local_data.exists() + + assert not bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_data, + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert not local_data.exists() + + def test_bootstrap_seed_files_keeps_copied_state_when_stamp_turns_invalid(tmp_path: Path) -> None: main_data = tmp_path / "main" / "testmondata" main_stamp = tmp_path / "main" / "seed.json" diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index 380bf33fae..8e567dffdd 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -130,6 +130,15 @@ def test_malformed_sqlite_values_fail_closed(tmp_path: Path) -> None: assert inspection.status is GraphStatus.INVALID +def test_sqlite_paths_with_uri_characters_are_inspected_safely(tmp_path: Path) -> None: + data = tmp_path / "checkout?fragment#1" / "testmondata" + _write_graph(data) + + inspection = inspect_testmon_database(data, NODEIDS) + + assert inspection.status is GraphStatus.COMPLETE + + @pytest.mark.parametrize("filename", ["../outside.py", "/tmp/outside.py"]) def test_unsafe_testmon_fingerprint_paths_fail_closed(tmp_path: Path, filename: str) -> None: data = tmp_path / "testmondata" diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 3da6b9fb9d..07a87e5ebb 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -515,7 +515,7 @@ def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, mo "status": "running", "identity": identity, "expected_nodeids": [], - "artifact_dir": str(artifact_dir), + "artifact_dir": str(artifact_dir.relative_to(tmp_path)), } ) ) @@ -529,6 +529,46 @@ def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, mo assert prepared["expected_count"] == 1 +def test_seed_resume_rejects_selection_artifact_outside_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + TESTMON_DATA.parent.mkdir(parents=True) + TESTMON_DATA.write_text("partial") + outside = tmp_path.parent / "outside-testmon-artifacts" + step_dir = outside / "steps" / "17-pytest-seed-testmon" + step_dir.mkdir(parents=True) + (step_dir / "selection.json").write_text( + json.dumps( + { + "selected_nodeids": ["tests/unit/test_example.py::test_one"], + "selected_nodeids_omitted": 0, + "selected_count": 1, + } + ) + ) + identity = { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "3.13", + "skip_slow": True, + "lab": False, + } + TESTMON_SEED_ATTEMPT.write_text( + json.dumps( + { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": identity, + "expected_nodeids": [], + "artifact_dir": str(outside), + } + ) + ) + + assert _testmon_seed_can_resume(identity) is False + + def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) -> None: monkeypatch = pytest.MonkeyPatch() monkeypatch.chdir(tmp_path) From 90a69dca69e0b07b80f9c0a424be9e83839bd574 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:03:08 +0200 Subject: [PATCH 04/34] fix(devtools): keep incomplete testmon attempts non-releasable Problem: an incomplete attempt with a complete graph and passing node outcomes could be promoted as a green release baseline. What changed: require the persisted attempt to be complete before granting release permission, while keeping complete red or reusable graphs available for affected selection. The finalizer now labels a genuinely green candidate complete before promotion. Add regression coverage for both paths. Compatibility/migration: no schema or migration change. --- devtools/testmon_state.py | 3 ++- devtools/verify.py | 2 +- tests/unit/devtools/test_testmon_state.py | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index 082eacdcda..1b5cc55187 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -595,7 +595,8 @@ def stamp_from_attempt( return None baseline = ( BaselineStatus.GREEN - if exit_code == 0 + if attempt.get("status") == "complete" + and exit_code == 0 and all(outcome in {"passed", "skipped"} for outcome in outcome_by_node.values()) and not graph.failed_nodeids else BaselineStatus.RED diff --git a/devtools/verify.py b/devtools/verify.py index a6099a6f03..df6e3a174a 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2501,7 +2501,7 @@ def _finalize_testmon_seed_attempt( ) attempt_candidate = { **dict(prepared), - "status": "reusable", + "status": "complete" if green_complete else "reusable", "exit_code": exit_code, "expected_nodeids": expected, "expected_count": len(expected), diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index 8e567dffdd..e6b516cfae 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -103,6 +103,27 @@ def test_omitted_interrupted_and_uncovered_nodes_fail_closed(tmp_path: Path) -> assert stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None +def test_incomplete_all_pass_attempt_is_selection_only(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data, outcomes=("passed", "passed")) + attempt["exit_code"] = 0 + + stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert stamp is not None + assert stamp.baseline_status is BaselineStatus.RED + assert stamp.affected_selection_allowed + assert not stamp.release_baseline_allowed + + attempt["status"] = "complete" + completed = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert completed is not None + assert completed.baseline_status is BaselineStatus.GREEN + assert completed.release_baseline_allowed + + def test_malformed_sqlite_and_stale_stamp_fail_closed(tmp_path: Path) -> None: malformed = tmp_path / "malformed" malformed.write_bytes(b"not sqlite") From 0575473c2e1c3b326b5bbda3460cb0f916ad11d0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:16:58 +0200 Subject: [PATCH 05/34] fix(devtools): preserve fail-closed testmon provenance Problem: red-attempt bootstrap synthesized seed.json, artifact paths were not fully checkout-bound, malformed green stamps could claim release permission, and subdirectory invocations mixed cwd-relative state with checkout-root validation. What changed: retain red bootstrap state as a rebound seed-attempt receipt without creating seed.json, validate run artifacts against .cache/verify/runs/, reject inconsistent green stamps, and anchor verify state to the invoking checkout. Extend unit and integration coverage for these production routes. Compatibility/migration: no schema or migration change. --- devtools/testmon_bootstrap.py | 61 ++++++++++++++++--- devtools/testmon_state.py | 30 ++++++++- devtools/verify.py | 12 ++++ tests/unit/devtools/test_testmon_bootstrap.py | 7 ++- tests/unit/devtools/test_testmon_state.py | 25 ++++++++ tests/unit/devtools/test_verify.py | 21 +++++-- 6 files changed, 139 insertions(+), 17 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index c38280e557..bab6479fbe 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -31,7 +31,8 @@ - :func:`decide_testmon_bootstrap` -- pure decision, no subprocess beyond the caller. It validates the main stamp or a complete red seed attempt. - :func:`bootstrap_testmon_seed_files` -- the copy action once bootstrapping - has been decided. + has been decided. A red attempt is copied as a rebound attempt receipt and + never synthesized into ``seed.json``. - :func:`maybe_bootstrap_testmon_seed` -- the orchestrator `devtools verify` calls: detects whether ``repo_root`` is a linked worktree (via ``git rev-parse --absolute-git-dir --git-common-dir``, the same mechanism @@ -204,16 +205,20 @@ def decide_testmon_bootstrap( ) -def _atomic_write_stamp(seed_stamp: Path, stamp: TestmonSeedStamp) -> None: - seed_stamp.parent.mkdir(parents=True, exist_ok=True) - tmp = seed_stamp.with_name(f"{seed_stamp.name}.{os.getpid()}.tmp") +def _atomic_write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") try: - tmp.write_text(json.dumps(stamp.as_dict(), indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - tmp.replace(seed_stamp) + tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + tmp.replace(path) finally: tmp.unlink(missing_ok=True) +def _atomic_write_stamp(seed_stamp: Path, stamp: TestmonSeedStamp) -> None: + _atomic_write_json(seed_stamp, stamp.as_dict()) + + def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: """Copy a (possibly concurrently-written) sqlite db via the online backup API. @@ -246,6 +251,7 @@ def bootstrap_testmon_seed_files( *, local_testmon_data: Path, local_seed_stamp: Path, + local_seed_attempt: Path | None = None, checkout_root: Path | None = None, inherited_from: Path | None = None, ) -> bool: @@ -255,6 +261,8 @@ def bootstrap_testmon_seed_files( assert decision.main_testmon_data is not None if decision.main_seed_stamp is None and decision.main_seed_attempt is None: return False + if decision.main_seed_attempt is not None and local_seed_attempt is None: + return False if checkout_root is None or inherited_from is None: return False stamp: TestmonSeedStamp | None = None @@ -272,6 +280,10 @@ def bootstrap_testmon_seed_files( local_seed_stamp.resolve().relative_to(destination_root) if local_seed_stamp.resolve() == local_testmon_data.resolve(): return False + if local_seed_attempt is not None: + local_seed_attempt.resolve().relative_to(destination_root) + if local_seed_attempt.resolve() in {local_testmon_data.resolve(), local_seed_stamp.resolve()}: + return False if decision.main_seed_stamp is not None: decision.main_seed_stamp.resolve().relative_to(source_root) if decision.main_seed_stamp.resolve() == decision.main_testmon_data.resolve(): @@ -305,12 +317,34 @@ def bootstrap_testmon_seed_files( if stamp is None: return False try: + if decision.main_seed_attempt is not None: + local_seed_stamp.unlink(missing_ok=True) _atomic_copy_sqlite_db(decision.main_testmon_data, local_testmon_data) - stamp = stamp.rebound(checkout_root=destination_root, inherited_from=source_root) - refreshed = refresh_stamp(stamp, local_testmon_data) - if refreshed is None or refreshed.graph != stamp.graph: + rebound = stamp.rebound(checkout_root=destination_root, inherited_from=source_root) + refreshed = refresh_stamp(rebound, local_testmon_data) + if refreshed is None or refreshed.graph != rebound.graph: return False - _atomic_write_stamp(local_seed_stamp, refreshed) + if decision.main_seed_attempt is not None: + assert local_seed_attempt is not None + source_attempt = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) + if not isinstance(source_attempt, dict): + return False + rebound_attempt = dict(source_attempt) + rebound_attempt["testmon_data"] = refreshed.testmon_data + rebound_attempt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" + if ( + stamp_from_attempt( + rebound_attempt, + local_testmon_data, + checkout_root=destination_root, + protocol_version=decision.protocol_version, + ) + is None + ): + return False + _atomic_write_json(local_seed_attempt, rebound_attempt) + else: + _atomic_write_stamp(local_seed_stamp, refreshed) return True except (OSError, sqlite3.Error, TypeError, ValueError): return False @@ -369,6 +403,7 @@ def maybe_bootstrap_testmon_seed( return None local_testmon_data = repo_root / testmon_data_relpath local_seed_stamp = repo_root / seed_stamp_relpath + local_seed_attempt = repo_root / TESTMON_SEED_ATTEMPT_RELPATH main_testmon_data = main_checkout / testmon_data_relpath main_seed_stamp = main_checkout / seed_stamp_relpath main_seed_attempt = main_checkout / TESTMON_SEED_ATTEMPT_RELPATH @@ -389,6 +424,7 @@ def maybe_bootstrap_testmon_seed( decision, local_testmon_data=local_testmon_data, local_seed_stamp=local_seed_stamp, + local_seed_attempt=local_seed_attempt, checkout_root=repo_root, inherited_from=main_checkout, ) @@ -397,6 +433,11 @@ def maybe_bootstrap_testmon_seed( f"verify: bootstrapped pytest-testmon seed into {local_testmon_data.parent}, " "but could not record its checkout provenance" ) + if decision.main_seed_attempt is not None: + return ( + f"verify: bootstrapped pytest-testmon graph from main checkout {main_checkout} " + f"into {local_testmon_data.parent} as a selection-only attempt receipt (no seed.json)" + ) return ( f"verify: bootstrapped pytest-testmon seed from main checkout {main_checkout} " f"into {local_testmon_data.parent} (worktree had no local seed)" diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index 1b5cc55187..d1baf556f2 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -248,6 +248,8 @@ def from_mapping(cls, value: Mapping[str, Any], *, protocol_version: int) -> Tes raise ValueError("seed stamp baseline fields are malformed") if release_allowed != (baseline_status is BaselineStatus.GREEN): raise ValueError("release permission does not match baseline status") + if baseline_status is BaselineStatus.GREEN and exit_code != 0: + raise ValueError("green seed stamp must have a zero exit code") graph_status = graph.get("status") if not isinstance(graph_status, str): raise ValueError("seed stamp graph status is invalid") @@ -293,6 +295,8 @@ def from_mapping(cls, value: Mapping[str, Any], *, protocol_version: int) -> Tes or len(set(graph_nodeids)) != len(graph_nodeids) ): raise ValueError("seed stamp graph failure ledger is malformed") + if baseline_status is BaselineStatus.GREEN and graph_nodeids: + raise ValueError("green seed stamp cannot contain failed graph nodes") testmon_data = value.get("testmon_data") run_id = value.get("run_id") artifact_dir = value.get("artifact_dir") @@ -301,6 +305,13 @@ def from_mapping(cls, value: Mapping[str, Any], *, protocol_version: int) -> Tes assert isinstance(testmon_data, str) assert isinstance(run_id, str) assert isinstance(artifact_dir, str) + typed_binding = TestmonBinding.from_mapping(binding) + if not _is_bound_run_artifact( + artifact_dir, + checkout_root=Path(typed_binding.checkout_root), + run_id=run_id, + ): + raise ValueError("seed stamp artifact directory is not checkout-bound") return cls( protocol_version, CollectionStatus.COMPLETE, @@ -320,7 +331,7 @@ def from_mapping(cls, value: Mapping[str, Any], *, protocol_version: int) -> Tes tuple(graph_nodeids), ), TestmonIdentity.from_mapping(identity), - TestmonBinding.from_mapping(binding), + typed_binding, testmon_data, run_id, artifact_dir, @@ -345,6 +356,21 @@ def file_fingerprint(path: Path) -> str: return digest.hexdigest() +def _is_bound_run_artifact(raw: object, *, checkout_root: Path, run_id: str) -> bool: + if not isinstance(raw, str) or not raw or not isinstance(run_id, str) or not run_id: + return False + path = Path(raw) + if path.is_absolute() or path.parts[:3] != (".cache", "verify", "runs"): + return False + if path.parts[3:] != (run_id,): + return False + try: + (checkout_root / path).resolve().relative_to((checkout_root / ".cache" / "verify" / "runs" / run_id).resolve()) + except ValueError: + return False + return True + + def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> GraphInspection: """Validate the real testmon schema and every expected dependency edge.""" expected = tuple(expected_nodeids) @@ -567,6 +593,8 @@ def stamp_from_attempt( artifact_dir = attempt.get("artifact_dir") if not isinstance(run_id, str) or not run_id or not isinstance(artifact_dir, str) or not artifact_dir: return None + if not _is_bound_run_artifact(artifact_dir, checkout_root=checkout_root, run_id=run_id): + return None outcomes = attempt.get("node_outcomes") if not isinstance(outcomes, list) or len(outcomes) != len(expected): return None diff --git a/devtools/verify.py b/devtools/verify.py index df6e3a174a..e8f5f76149 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -101,6 +101,17 @@ ROOT = Path(__file__).resolve().parents[1] + +def _anchor_verification_paths() -> None: + """Use the checkout root for relative verification state when invoked inside it.""" + current = Path.cwd().resolve() + try: + current.relative_to(ROOT.resolve()) + except ValueError: + return + os.chdir(ROOT) + + # ── mypy daemon probe ────────────────────────────────────────────── @@ -2601,6 +2612,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--json", action="store_true", default=None, help="Write structured JSON to stdout.") args = parser.parse_args(sys.argv[1:] if argv is None else argv) + _anchor_verification_paths() bootstrap_message = maybe_bootstrap_testmon_seed( ROOT, protocol_version=TESTMON_SEED_PROTOCOL_VERSION, diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index d33ac42e37..4066d4a213 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -303,14 +303,19 @@ def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) assert decision.main_seed_attempt == attempt local_data = tmp_path / "lane" / "testmondata" local_stamp = tmp_path / "lane" / "seed.json" + local_attempt = tmp_path / "lane" / "seed-attempt.json" assert bootstrap_testmon_seed_files( decision, local_testmon_data=local_data, local_seed_stamp=local_stamp, + local_seed_attempt=local_attempt, checkout_root=tmp_path / "lane", inherited_from=tmp_path / "main", ) - assert json.loads(local_stamp.read_text())["baseline"]["status"] == "red" + assert not local_stamp.exists() + rebound_attempt = json.loads(local_attempt.read_text()) + assert rebound_attempt["artifact_dir"] == ".cache/verify/runs/red-run" + assert rebound_attempt["testmon_data"] == file_fingerprint(local_data) def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index e6b516cfae..2ab2730739 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -140,6 +140,31 @@ def test_malformed_sqlite_and_stale_stamp_fail_closed(tmp_path: Path) -> None: assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None +def test_attempt_and_green_stamp_artifacts_fail_closed_when_malformed(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data, outcomes=("passed", "passed")) + attempt["exit_code"] = 0 + attempt["artifact_dir"] = "/tmp/outside-testmon-run" + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + attempt["artifact_dir"] = ".cache/verify/runs/run-red" + attempt["status"] = "complete" + stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + assert stamp is not None + stamp_path = tmp_path / ".cache" / "testmon" / "seed.json" + stamp_path.parent.mkdir(parents=True) + payload = stamp.as_dict() + payload["baseline"]["exit_code"] = 1 + stamp_path.write_text(json.dumps(payload)) + assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + payload = stamp.as_dict() + payload["graph"]["failed_nodeids"] = [NODEIDS[0]] + stamp_path.write_text(json.dumps(payload)) + assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + def test_malformed_sqlite_values_fail_closed(tmp_path: Path) -> None: data = tmp_path / "testmondata" _write_graph(data) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 07a87e5ebb..873ff25aac 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -42,6 +42,7 @@ TESTMON_SEED_ATTEMPT, TESTMON_SEED_PROTOCOL_VERSION, TESTMON_SEED_STAMP, + _anchor_verification_paths, _finalize_testmon_seed_attempt, _format_completion_notification, _matching_testmon_coverage, @@ -606,7 +607,7 @@ def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) "resume": True, "expected_nodeids": expected, "run_id": "resume", - "artifact_dir": str(tmp_path / "resume"), + "artifact_dir": ".cache/verify/runs/resume", } receipt = _finalize_testmon_seed_attempt( @@ -777,7 +778,7 @@ def test_seed_receipt_classifies_every_node_terminal_outcome( "resume": False, "expected_nodeids": [], "run_id": "run-mixed", - "artifact_dir": str(tmp_path / "run-mixed"), + "artifact_dir": ".cache/verify/runs/run-mixed", }, step_results=[ { @@ -879,7 +880,7 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon "resume": False, "expected_nodeids": [], "run_id": "run-1", - "artifact_dir": str(tmp_path / "run-1"), + "artifact_dir": ".cache/verify/runs/run-1", }, step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 0}], exit_code=0, @@ -906,7 +907,7 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon "resume": False, "expected_nodeids": [], "run_id": "run-stale-db", - "artifact_dir": str(tmp_path / "run-stale-db"), + "artifact_dir": ".cache/verify/runs/run-stale-db", }, step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], exit_code=0, @@ -929,7 +930,7 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon "resume": False, "expected_nodeids": [], "run_id": "run-orphaned", - "artifact_dir": str(tmp_path / "run-orphaned"), + "artifact_dir": ".cache/verify/runs/run-orphaned", }, step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], exit_code=0, @@ -1822,6 +1823,16 @@ def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.Ca assert "only 0.50 GiB available" in capsys.readouterr().err +def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(ROOT / "devtools") + + _anchor_verification_paths() + + assert Path.cwd() == ROOT.resolve() + + def test_verify_rejects_zero_testmon_selection_for_executable_change( capsys: pytest.CaptureFixture[str], ) -> None: From 2b80951515fb803a8e46a88055afda219a0f6204 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:23:55 +0200 Subject: [PATCH 06/34] fix(devtools): satisfy testmon provenance typing Problem: the new bound-artifact predicate redundantly checked a statically typed run identifier, causing the publication quick gate to fail under mypy.\n\nWhat changed: remove the unreachable runtime type check while retaining the non-empty identifier validation.\n\nCompatibility/migration: no schema or migration change. --- devtools/testmon_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index d1baf556f2..0b16f57981 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -357,7 +357,7 @@ def file_fingerprint(path: Path) -> str: def _is_bound_run_artifact(raw: object, *, checkout_root: Path, run_id: str) -> bool: - if not isinstance(raw, str) or not raw or not isinstance(run_id, str) or not run_id: + if not isinstance(raw, str) or not raw or not run_id: return False path = Path(raw) if path.is_absolute() or path.parts[:3] != (".cache", "verify", "runs"): From 3af9ec1d13c6a2028e0b2bda4dc8122f57a9ebb3 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:36:40 +0200 Subject: [PATCH 07/34] fix(devtools): bind testmon run receipts Problem: testmon attempt and stamp artifact paths were only shape-checked, so a missing or mismatched run receipt could pass provenance validation. Bootstrap also left the rebound artifact path without a destination receipt.\n\nWhat changed: require a run.json receipt whose run ID, checkout root, and relative artifact path agree with the typed state. Bootstrap copies and rebinds that receipt for the destination checkout, and the focused tests cover missing, mismatched, and rebound receipts.\n\nCompatibility/migration: no schema or migration change. --- devtools/testmon_bootstrap.py | 33 +++++++++++++++++++ devtools/testmon_state.py | 15 +++++++-- devtools/verify.py | 2 +- .../devtools/test_testmon_seed_recovery.py | 12 +++++++ tests/unit/devtools/test_testmon_bootstrap.py | 29 +++++++++++++++- tests/unit/devtools/test_testmon_state.py | 33 +++++++++++++++++++ tests/unit/devtools/test_verify.py | 30 +++++++++++++++++ 7 files changed, 149 insertions(+), 5 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index bab6479fbe..da769ac586 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -59,8 +59,10 @@ import os import sqlite3 import subprocess +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path +from typing import Any from devtools.testmon_state import ( TestmonSeedStamp, @@ -219,6 +221,28 @@ def _atomic_write_stamp(seed_stamp: Path, stamp: TestmonSeedStamp) -> None: _atomic_write_json(seed_stamp, stamp.as_dict()) +def _rebind_run_receipt(*, source: Path, destination: Path, checkout_root: Path, run_id: str) -> bool: + """Copy the run receipt while rebinding its checkout-local provenance.""" + try: + payload = json.loads((source / "run.json").read_text(encoding="utf-8")) + if not isinstance(payload, Mapping) or payload.get("run_id") != run_id: + return False + source_root = payload.get("checkout_root") + if not isinstance(source_root, str) or Path(source_root).resolve() != source.parents[3].resolve(): + return False + payload_dict: dict[str, Any] = dict(payload) + payload_dict["checkout_root"] = str(checkout_root.resolve()) + payload_dict["artifact_dir"] = str(Path(".cache") / "verify" / "runs" / run_id) + environment = payload_dict.get("environment_fingerprint") + if isinstance(environment, dict): + environment["checkout_root"] = str(checkout_root.resolve()) + environment["verify_state_origin"] = str(checkout_root.resolve()) + _atomic_write_json(destination / "run.json", payload_dict) + return True + except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): + return False + + def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: """Copy a (possibly concurrently-written) sqlite db via the online backup API. @@ -324,6 +348,15 @@ def bootstrap_testmon_seed_files( refreshed = refresh_stamp(rebound, local_testmon_data) if refreshed is None or refreshed.graph != rebound.graph: return False + source_artifact = source_root / Path(stamp.artifact_dir) + destination_artifact = destination_root / Path(refreshed.artifact_dir) + if not _rebind_run_receipt( + source=source_artifact, + destination=destination_artifact, + checkout_root=destination_root, + run_id=refreshed.run_id, + ): + return False if decision.main_seed_attempt is not None: assert local_seed_attempt is not None source_attempt = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index 0b16f57981..d642c816ce 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -365,10 +365,19 @@ def _is_bound_run_artifact(raw: object, *, checkout_root: Path, run_id: str) -> if path.parts[3:] != (run_id,): return False try: - (checkout_root / path).resolve().relative_to((checkout_root / ".cache" / "verify" / "runs" / run_id).resolve()) - except ValueError: + artifact_dir = (checkout_root / path).resolve() + artifact_dir.relative_to((checkout_root / ".cache" / "verify" / "runs" / run_id).resolve()) + receipt = json.loads((artifact_dir / "run.json").read_text(encoding="utf-8")) + if not isinstance(receipt, Mapping): + return False + return ( + receipt.get("run_id") == run_id + and isinstance(receipt.get("checkout_root"), str) + and Path(receipt["checkout_root"]).resolve() == checkout_root.resolve() + and receipt.get("artifact_dir") == str(Path(".cache") / "verify" / "runs" / run_id) + ) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): return False - return True def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> GraphInspection: diff --git a/devtools/verify.py b/devtools/verify.py index e8f5f76149..6fb36bf777 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2536,7 +2536,7 @@ def _finalize_testmon_seed_attempt( reusable_stamp = stamp_from_attempt( attempt_candidate, TESTMON_DATA, - checkout_root=ROOT, + checkout_root=Path.cwd(), protocol_version=TESTMON_SEED_PROTOCOL_VERSION, ) reusable = reusable_stamp is not None diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py index 5e326dae5b..b4d62ea33b 100644 --- a/tests/integration/devtools/test_testmon_seed_recovery.py +++ b/tests/integration/devtools/test_testmon_seed_recovery.py @@ -63,6 +63,18 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane(tmp_path: Pat "artifact_dir": ".cache/verify/runs/real-testmon", "testmon_data": file_fingerprint(data), } + artifact_dir = source / ".cache" / "verify" / "runs" / "real-testmon" + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": "real-testmon", + "checkout_root": str(source.resolve()), + "artifact_dir": ".cache/verify/runs/real-testmon", + } + ), + encoding="utf-8", + ) stamp = stamp_from_attempt(attempt, data, checkout_root=source, protocol_version=4) assert stamp is not None and stamp.baseline_status is BaselineStatus.RED source_stamp = source / ".cache" / "testmon" / "seed.json" diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 4066d4a213..6f1cd9474a 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -72,6 +72,17 @@ def _write_valid_seed_stamp(path: Path, *, protocol_version: int = PROTOCOL_VERS "seed", ".cache/verify/runs/seed", ) + artifact_dir = path.parent / ".cache" / "verify" / "runs" / "seed" + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": "seed", + "checkout_root": str(path.parent.resolve()), + "artifact_dir": ".cache/verify/runs/seed", + } + ) + ) path.write_text(json.dumps(stamp.as_dict())) @@ -289,6 +300,17 @@ def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) } ) ) + red_artifact = tmp_path / "main" / ".cache" / "verify" / "runs" / "red-run" + red_artifact.mkdir(parents=True, exist_ok=True) + (red_artifact / "run.json").write_text( + json.dumps( + { + "run_id": "red-run", + "checkout_root": str((tmp_path / "main").resolve()), + "artifact_dir": ".cache/verify/runs/red-run", + } + ) + ) decision = decide_testmon_bootstrap( is_linked_worktree=True, local_testmon_data=tmp_path / "lane" / "testmondata", @@ -316,6 +338,11 @@ def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) rebound_attempt = json.loads(local_attempt.read_text()) assert rebound_attempt["artifact_dir"] == ".cache/verify/runs/red-run" assert rebound_attempt["testmon_data"] == file_fingerprint(local_data) + rebound_receipt = json.loads( + (tmp_path / "lane" / ".cache" / "verify" / "runs" / "red-run" / "run.json").read_text() + ) + assert rebound_receipt["run_id"] == "red-run" + assert rebound_receipt["checkout_root"] == str((tmp_path / "lane").resolve()) def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: @@ -374,7 +401,7 @@ def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: conn.close() assert rows == [("x", "sha-x"), ("y", "sha-y"), ("z", "sha-z")] # No temp files left behind. - assert sorted(p.name for p in local_data.parent.iterdir()) == ["seed.json", "testmondata"] + assert sorted(p.name for p in local_data.parent.iterdir()) == [".cache", "seed.json", "testmondata"] def test_bootstrap_seed_files_marks_destination_and_source_checkout(tmp_path: Path) -> None: diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index 2ab2730739..43f539fe99 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -41,6 +41,17 @@ def _write_graph(path: Path, *, failed: bool = False, with_edges: bool = True) - def _attempt(data: Path, *, outcomes: tuple[str, str] = ("passed", "failed")) -> dict[str, object]: + artifact_dir = data.parent / ".cache" / "verify" / "runs" / "run-red" + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": "run-red", + "checkout_root": str(data.parent.resolve()), + "artifact_dir": ".cache/verify/runs/run-red", + } + ) + ) return { "protocol_version": PROTOCOL, "status": "incomplete", @@ -152,6 +163,28 @@ def test_attempt_and_green_stamp_artifacts_fail_closed_when_malformed(tmp_path: attempt["status"] = "complete" stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) assert stamp is not None + receipt = tmp_path / ".cache" / "verify" / "runs" / "run-red" / "run.json" + receipt.unlink() + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + receipt.write_text( + json.dumps( + { + "run_id": "wrong-run", + "checkout_root": str(tmp_path.resolve()), + "artifact_dir": ".cache/verify/runs/run-red", + } + ) + ) + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + receipt.write_text( + json.dumps( + { + "run_id": "run-red", + "checkout_root": str(tmp_path.resolve()), + "artifact_dir": ".cache/verify/runs/run-red", + } + ) + ) stamp_path = tmp_path / ".cache" / "testmon" / "seed.json" stamp_path.parent.mkdir(parents=True) payload = stamp.as_dict() diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 873ff25aac..eb132f7ae2 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -119,10 +119,35 @@ def _write_real_testmon_state(nodeids: tuple[str, ...] = ("tests/test_a.py::test ".cache/verify/runs/seed", ) TESTMON_SEED_STAMP.parent.mkdir(parents=True, exist_ok=True) + artifact_dir = ROOT / ".cache" / "verify" / "runs" / "seed" + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": "seed", + "checkout_root": str(ROOT.resolve()), + "artifact_dir": ".cache/verify/runs/seed", + } + ) + ) TESTMON_SEED_STAMP.write_text(json.dumps(stamp.as_dict())) return TESTMON_DATA +def _write_run_receipt(root: Path, run_id: str) -> None: + artifact_dir = root / ".cache" / "verify" / "runs" / run_id + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": run_id, + "checkout_root": str(root.resolve()), + "artifact_dir": f".cache/verify/runs/{run_id}", + } + ) + ) + + def test_quick_verify_omits_pytest() -> None: steps = build_verify_steps(quick=True, lab=False, skip_slow=False) @@ -609,6 +634,7 @@ def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) "run_id": "resume", "artifact_dir": ".cache/verify/runs/resume", } + _write_run_receipt(tmp_path, "resume") receipt = _finalize_testmon_seed_attempt( prepared=prepared, @@ -764,6 +790,7 @@ def test_seed_receipt_classifies_every_node_terminal_outcome( [(index, index) for index, _nodeid in enumerate(expected[:-1], start=1)], ) + _write_run_receipt(tmp_path, "run-mixed") receipt = _finalize_testmon_seed_attempt( prepared={ "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, @@ -866,6 +893,9 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon [(index, index) for index, _nodeid in enumerate(expected, start=1)], ) + _write_run_receipt(tmp_path, "run-1") + _write_run_receipt(tmp_path, "run-stale-db") + _write_run_receipt(tmp_path, "run-orphaned") receipt = _finalize_testmon_seed_attempt( prepared={ "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, From a42277083f3f83d0500c4a6185f5020a8ae64440 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 03:15:59 +0200 Subject: [PATCH 08/34] fix(devtools): keep red testmon attempts selection-only Problem: reusable red testmon graphs were being written as seed.json, which let a validated selection graph look like a release baseline. What changed: persist seed.json only for green complete runs, retain red graphs as typed selection attempts, carry the release permission into verification receipts, and make the merge gate consume that permission. The integration path now bootstraps through maybe_bootstrap_testmon_seed and the default preflight decision. Compatibility: no archive or durable data changes. Existing red seed.json files are rejected and must be regenerated through the managed seed flow. Co-Authored-By: Claude --- devtools/checkout_guard.py | 2 +- devtools/merge_gate.py | 33 ++++++++ devtools/testmon_bootstrap.py | 20 +++-- devtools/testmon_state.py | 2 + devtools/verify.py | 43 +++++++++- .../devtools/test_testmon_seed_recovery.py | 79 ++++++++++--------- tests/unit/devtools/test_checkout_guard.py | 20 +++++ tests/unit/devtools/test_merge_gate.py | 34 ++++++++ tests/unit/devtools/test_testmon_state.py | 2 +- tests/unit/devtools/test_verify.py | 28 +++++++ 10 files changed, 215 insertions(+), 48 deletions(-) diff --git a/devtools/checkout_guard.py b/devtools/checkout_guard.py index 09c694a9f1..85867b5272 100644 --- a/devtools/checkout_guard.py +++ b/devtools/checkout_guard.py @@ -277,7 +277,7 @@ def _is_valid_in_progress_testmon_seed_attempt(attempt: Path) -> bool: payload = json.loads(attempt.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError): return False - if not isinstance(payload, Mapping) or payload.get("status") not in {"running", "incomplete"}: + if not isinstance(payload, Mapping) or payload.get("status") not in {"running", "incomplete", "reusable"}: return False protocol_version = payload.get("protocol_version") if not isinstance(protocol_version, int) or isinstance(protocol_version, bool) or protocol_version <= 0: diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index b728d54c3e..1a5cb31157 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -218,6 +218,29 @@ def _command_skips_tests(command: str) -> bool: return not any(marker in lowered for marker in _LOOKS_LIKE_TESTS_MARKERS) +def _release_baseline_permission(stdout: str) -> bool | None: + """Read the structured verify decision when the command emitted one.""" + try: + payload = json.loads(stdout) + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + value = payload.get("release_baseline_allowed") + return value if isinstance(value, bool) else None + + +def _requires_release_baseline(command: str) -> bool: + """Identify verification commands whose success can claim a release baseline.""" + try: + argv = shlex.split(command) + except ValueError: + return False + if argv[:2] != ["devtools", "verify"]: + return False + return len(argv) == 2 or any(option in argv[2:] for option in ("--all", "--full", "--lab", "--seed-testmon")) + + def cmd_record(pr: int, command: str) -> int: info = _gh_json(["pr", "view", str(pr), "--json", "headRefOid,headRefName,body,isDraft"]) head_sha = info["headRefOid"] @@ -272,6 +295,7 @@ def cmd_record(pr: int, command: str) -> int: "branch": info["headRefName"], "command": command, "skips_tests": _command_skips_tests(command), + "release_baseline_allowed": _release_baseline_permission(result.stdout), "exit_code": result.returncode, "duration_s": duration_s, "recorded_at": time.time(), @@ -478,6 +502,15 @@ def cmd_check( if receipt.get("exit_code", 1) != 0: verdict.ok = False verdict.reasons.append(f"receipt exit_code is {receipt.get('exit_code')}, not 0") + if ( + _requires_release_baseline(str(receipt.get("command", ""))) + and receipt.get("release_baseline_allowed") is not True + ): + verdict.ok = False + verdict.reasons.append( + "verification receipt does not grant release_baseline_allowed=true; a selection-only " + "testmon attempt cannot satisfy the release merge gate" + ) if receipt.get("skips_tests"): verdict.reasons.append( f"advisory: receipt command {receipt.get('command')!r} does not look like it ran tests " diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index da769ac586..411febb20c 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -87,6 +87,7 @@ class BootstrapDecision: main_seed_attempt: Path | None = None main_checkout_root: Path | None = None protocol_version: int = 4 + selection_only: bool = False def _checkout_root_for_data(data_path: Path) -> Path: @@ -183,11 +184,13 @@ def decide_testmon_bootstrap( attempt = None if ( isinstance(attempt, dict) - and stamp_from_attempt( - attempt, - main_testmon_data, - checkout_root=root, - protocol_version=protocol_version, + and ( + attempt_stamp := stamp_from_attempt( + attempt, + main_testmon_data, + checkout_root=root, + protocol_version=protocol_version, + ) ) is not None ): @@ -198,6 +201,7 @@ def decide_testmon_bootstrap( main_seed_attempt=main_seed_attempt, main_checkout_root=root, protocol_version=protocol_version, + selection_only=not attempt_stamp.release_baseline_allowed, ) if main_seed_stamp.is_file(): return BootstrapDecision(False, "main checkout seed stamp is stale, malformed, or graph-incomplete") @@ -341,7 +345,7 @@ def bootstrap_testmon_seed_files( if stamp is None: return False try: - if decision.main_seed_attempt is not None: + if decision.main_seed_attempt is not None and decision.selection_only: local_seed_stamp.unlink(missing_ok=True) _atomic_copy_sqlite_db(decision.main_testmon_data, local_testmon_data) rebound = stamp.rebound(checkout_root=destination_root, inherited_from=source_root) @@ -357,7 +361,7 @@ def bootstrap_testmon_seed_files( run_id=refreshed.run_id, ): return False - if decision.main_seed_attempt is not None: + if decision.main_seed_attempt is not None and decision.selection_only: assert local_seed_attempt is not None source_attempt = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) if not isinstance(source_attempt, dict): @@ -466,7 +470,7 @@ def maybe_bootstrap_testmon_seed( f"verify: bootstrapped pytest-testmon seed into {local_testmon_data.parent}, " "but could not record its checkout provenance" ) - if decision.main_seed_attempt is not None: + if decision.main_seed_attempt is not None and decision.selection_only: return ( f"verify: bootstrapped pytest-testmon graph from main checkout {main_checkout} " f"into {local_testmon_data.parent} as a selection-only attempt receipt (no seed.json)" diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index d642c816ce..c351eca95d 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -523,6 +523,8 @@ def validate_stamp( if not isinstance(payload, Mapping): return None stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) + if not stamp.release_baseline_allowed: + return None if Path(stamp.binding.checkout_root).resolve() != checkout_root.resolve(): return None if file_fingerprint(data_path) != stamp.testmon_data: diff --git a/devtools/verify.py b/devtools/verify.py index 6fb36bf777..26928a56b5 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2235,6 +2235,28 @@ def _read_testmon_seed_attempt() -> dict[str, Any] | None: return payload if isinstance(payload, dict) else None +def _testmon_release_baseline_permission() -> bool | None: + """Return release permission for current testmon state, or ``None`` when not applicable.""" + if TESTMON_SEED_STAMP.exists(): + stamp = validate_stamp( + TESTMON_SEED_STAMP, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + return stamp.release_baseline_allowed if stamp is not None else False + attempt = _read_testmon_seed_attempt() + if attempt is None: + return False + stamp = stamp_from_attempt( + attempt, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + return stamp.release_baseline_allowed if stamp is not None else False + + def _safe_testmon_artifact_dir(raw: object, *, require_run_root: bool = False) -> Path | None: if not isinstance(raw, str) or not raw: return None @@ -2573,9 +2595,12 @@ def _finalize_testmon_seed_attempt( "testmon_data": _file_fingerprint(TESTMON_DATA), "pytest_step": dict(pytest_step) if pytest_step is not None else None, } + payload["release_baseline_allowed"] = bool(reusable_stamp is not None and reusable_stamp.release_baseline_allowed) _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - if reusable_stamp is not None: + if reusable_stamp is not None and reusable_stamp.release_baseline_allowed: _atomic_write_json(TESTMON_SEED_STAMP, reusable_stamp.as_dict()) + else: + TESTMON_SEED_STAMP.unlink(missing_ok=True) return payload @@ -2803,9 +2828,23 @@ def main(argv: list[str] | None = None) -> int: "resume": seed_receipt["resume"], "expected_count": seed_receipt["expected_count"], "attempt_path": str(TESTMON_SEED_ATTEMPT), - "stamp_path": str(TESTMON_SEED_STAMP) if seed_receipt["status"] in {"complete", "reusable"} else None, + "stamp_path": str(TESTMON_SEED_STAMP) if seed_receipt["release_baseline_allowed"] else None, + "release_baseline_allowed": seed_receipt["release_baseline_allowed"], } + if args.quick or args.commit: + release_baseline_allowed: bool | None = None + elif full_pytest: + release_baseline_allowed = exit_code == 0 + else: + release_baseline_allowed = _testmon_release_baseline_permission() + history_entry["release_baseline_allowed"] = release_baseline_allowed + if release_baseline_allowed is False and tier in {"testmon", "lab", "seed-testmon"}: + sys.stderr.write( + "verify: affected-test selection is usable, but the current testmon state does not grant " + "release-baseline permission.\n" + ) + if use_json: _print_json(history_entry) else: diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py index b4d62ea33b..ead7278c03 100644 --- a/tests/integration/devtools/test_testmon_seed_recovery.py +++ b/tests/integration/devtools/test_testmon_seed_recovery.py @@ -8,14 +8,11 @@ import sys from pathlib import Path -from devtools.testmon_bootstrap import BootstrapDecision, bootstrap_testmon_seed_files -from devtools.testmon_state import ( - BaselineStatus, - file_fingerprint, - inspect_testmon_database, - stamp_from_attempt, - validate_stamp, -) +import pytest + +from devtools import testmon_bootstrap, verify +from devtools.testmon_bootstrap import maybe_bootstrap_testmon_seed +from devtools.testmon_state import file_fingerprint, inspect_testmon_database def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane(tmp_path: Path) -> None: @@ -75,35 +72,45 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane(tmp_path: Pat ), encoding="utf-8", ) - stamp = stamp_from_attempt(attempt, data, checkout_root=source, protocol_version=4) - assert stamp is not None and stamp.baseline_status is BaselineStatus.RED - source_stamp = source / ".cache" / "testmon" / "seed.json" - source_stamp.parent.mkdir(parents=True, exist_ok=True) - source_stamp.write_text(json.dumps(stamp.as_dict()), encoding="utf-8") + source_attempt = source / ".cache" / "testmon" / "seed-attempt.json" + source_attempt.parent.mkdir(parents=True, exist_ok=True) + source_attempt.write_text(json.dumps(attempt), encoding="utf-8") lane = tmp_path / "lane" - local_data = lane / ".cache" / "testmon" / "testmondata" - local_stamp = lane / ".cache" / "testmon" / "seed.json" - decision = BootstrapDecision( - True, - "real graph", - main_testmon_data=data, - main_seed_stamp=source_stamp, - protocol_version=4, - ) - assert bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=lane, - inherited_from=source, + lane.mkdir() + (lane / "test_sample.py").write_text( + "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 1\n", + encoding="utf-8", ) - rebound = validate_stamp(local_stamp, local_data, checkout_root=lane, protocol_version=4) - assert rebound is not None - assert rebound.baseline_status is BaselineStatus.RED - assert rebound.binding.checkout_root == str(lane.resolve()) - assert rebound.affected_selection_allowed + monkeypatch = pytest.MonkeyPatch() + try: + monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, source)) + message = maybe_bootstrap_testmon_seed(lane, protocol_version=4) + assert message is not None and "selection-only attempt receipt" in message + + local_data = lane / ".cache" / "testmon" / "testmondata" + local_stamp = lane / ".cache" / "testmon" / "seed.json" + local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" + assert local_data.is_file() + assert local_attempt.is_file() + assert not local_stamp.exists() - with sqlite3.connect(local_data) as connection: - connection.execute("delete from test_execution_file_fp where test_execution_id = 1") - assert validate_stamp(local_stamp, local_data, checkout_root=lane, protocol_version=4) is None + monkeypatch.chdir(lane) + monkeypatch.setattr(verify, "ROOT", lane) + assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is None + selected = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "--testmon"], + cwd=lane, + env={**env, "TESTMON_DATAFILE": str(local_data)}, + capture_output=True, + text=True, + check=False, + ) + assert selected.returncode == 0, selected.stdout + selected.stderr + assert "1 passed" in selected.stdout + assert verify._testmon_release_baseline_permission() is False + with sqlite3.connect(local_data) as connection: + connection.execute("delete from test_execution_file_fp") + assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is not None + finally: + monkeypatch.undo() diff --git a/tests/unit/devtools/test_checkout_guard.py b/tests/unit/devtools/test_checkout_guard.py index 42f7e622b5..018550b3c1 100644 --- a/tests/unit/devtools/test_checkout_guard.py +++ b/tests/unit/devtools/test_checkout_guard.py @@ -223,6 +223,26 @@ def test_checkout_environment_fingerprint_accepts_current_in_progress_seed_attem assert attempt.is_file() +def test_checkout_environment_fingerprint_accepts_finalized_selection_attempt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = _fake_linked_checkout(tmp_path) + attempt = _write_in_progress_seed_attempt(root, status="reusable") + (root / ".cache" / "testmon" / "testmondata").write_text("complete graph") + package_path = root / "polylogue" / "__init__.py" + monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) + + fingerprint = assert_polylogue_matches_checkout( + root, + context="affected selection", + python_executable=root / ".venv" / "bin" / "python", + ) + + assert fingerprint.clean + assert fingerprint.testmon_state_origin is None + assert attempt.is_file() + + @pytest.mark.parametrize( ("status", "overrides"), [ diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 6d26285f94..ccb9f87354 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -4,6 +4,7 @@ import subprocess from collections.abc import Callable from pathlib import Path +from typing import cast from unittest.mock import MagicMock import pytest @@ -103,6 +104,39 @@ def test_record_persists_receipt_keyed_to_current_head_sha(monkeypatch: pytest.M assert receipt["skips_tests"] is False +def test_check_blocks_receipt_without_release_baseline_permission( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + _record(monkeypatch, pr_view, command="devtools verify") + receipt_path = merge_gate._receipt_path(42) + receipt = json.loads(receipt_path.read_text()) + receipt["release_baseline_allowed"] = False + receipt_path.write_text(json.dumps(receipt)) + + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view, [])) + assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 1 + + +def test_record_consumes_structured_verify_release_permission(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + return MagicMock(returncode=0, stdout=json.dumps({"release_baseline_allowed": False}), stderr="") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools verify") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["release_baseline_allowed"] is False + + def test_record_captures_nonzero_local_command_exit(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) monkeypatch.setattr( diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index 43f539fe99..f6a6929567 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -98,7 +98,7 @@ def test_failed_complete_graph_is_selection_only_and_rebindable(tmp_path: Path) stamp_path = tmp_path / "seed.json" stamp_path.write_text(json.dumps(stamp.as_dict())) - assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) == stamp + assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None def test_omitted_interrupted_and_uncovered_nodes_fail_closed(tmp_path: Path) -> None: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index eb132f7ae2..5a9ab6d17e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -922,6 +922,34 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon assert stamp["status"] == "usable" assert stamp["collection"]["expected_count"] == 2 + _write_run_receipt(tmp_path, "run-red") + with sqlite3.connect(TESTMON_DATA) as connection: + connection.execute("update test_execution set failed = 1 where test_name = ?", (expected[0],)) + red_receipt = _finalize_testmon_seed_attempt( + prepared={ + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "resume": False, + "expected_nodeids": [], + "run_id": "run-red", + "artifact_dir": ".cache/verify/runs/run-red", + }, + step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 1}], + exit_code=1, + ) + assert red_receipt["status"] == "reusable" + assert red_receipt["release_baseline_allowed"] is False + persisted_attempt = json.loads((tmp_path / ".cache" / "testmon" / "seed-attempt.json").read_text()) + assert persisted_attempt["release_baseline_allowed"] is False + assert not (tmp_path / ".cache" / "testmon" / "seed.json").exists() + (artifact_dir / "events.jsonl").write_text("") stale_database = _finalize_testmon_seed_attempt( prepared={ From ac768db87fd796833ac4f4388846ccae0c7cad47 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 03:49:43 +0200 Subject: [PATCH 09/34] fix: preserve typed testmon recovery authority Problem: reusable red testmon graphs lost checkout ownership, current-run markers, or fresh provenance across linked-worktree bootstrap and affected verification. Resumed seed ledgers also dropped their digest and prior terminal outcomes, and setup skips were treated as missing nodes. What changed: add typed checkout-bound attempt and verification-scope contracts, keep SQLite validation in verify/bootstrap preflight, create verify ownership markers during rebound bootstrap, refresh selection-only attempts after every affected run, and carry forward proven resumed outcomes. The production integration proof now reaches default verify selection through fresh bootstrap. Verification: focused affected tests passed with 190 passed and 1 unrelated lab-registration node deselected. The direct authority/bootstrap/guard subset passed with 50 passed. Mypy passed on the six changed production modules. The required devtools verify --quick was run exactly once; its non-format and non-type steps passed, while its initial format and type diagnostics were fixed afterward. Co-Authored-By: Codex --- devtools/checkout_guard.py | 28 ++-- devtools/testmon_bootstrap.py | 18 +++ devtools/testmon_state.py | 148 +++++++++++++++++- devtools/verify.py | 112 ++++++++++++- .../devtools/test_testmon_seed_recovery.py | 109 ++++++++----- tests/unit/devtools/test_checkout_guard.py | 42 +++++ tests/unit/devtools/test_testmon_bootstrap.py | 20 ++- tests/unit/devtools/test_testmon_state.py | 10 +- tests/unit/devtools/test_verify.py | 62 ++++++++ 9 files changed, 487 insertions(+), 62 deletions(-) diff --git a/devtools/checkout_guard.py b/devtools/checkout_guard.py index 85867b5272..e409c149f0 100644 --- a/devtools/checkout_guard.py +++ b/devtools/checkout_guard.py @@ -68,7 +68,7 @@ import tomllib -from devtools.testmon_state import validate_stamp +from devtools.testmon_state import attempt_is_checkout_bound, seed_marker_is_checkout_bound class CheckoutImportMismatchError(RuntimeError): @@ -265,7 +265,7 @@ def _marker_origin(marker: Path) -> Path | None: return Path(raw).resolve() -def _is_valid_in_progress_testmon_seed_attempt(attempt: Path) -> bool: +def _is_valid_in_progress_testmon_seed_attempt(attempt: Path, *, checkout_root: Path) -> bool: """Recognize the live seed ledger before its completion marker exists. ``verify --seed-testmon`` writes this receipt before pytest starts and @@ -279,6 +279,12 @@ def _is_valid_in_progress_testmon_seed_attempt(attempt: Path) -> bool: return False if not isinstance(payload, Mapping) or payload.get("status") not in {"running", "incomplete", "reusable"}: return False + if payload.get("status") == "reusable": + return attempt_is_checkout_bound( + payload, + checkout_root=checkout_root, + protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, + ) protocol_version = payload.get("protocol_version") if not isinstance(protocol_version, int) or isinstance(protocol_version, bool) or protocol_version <= 0: return False @@ -338,15 +344,10 @@ def _cache_artifact( marker_path = repo_root / marker origin = _marker_origin(marker_path) if origin == repo_root: - if ( - state_dir == _TESTMON_STATE_DIR - and validate_stamp( - marker_path, - state_path / "testmondata", - checkout_root=repo_root, - protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, - ) - is None + if state_dir == _TESTMON_STATE_DIR and not seed_marker_is_checkout_bound( + marker_path, + checkout_root=repo_root, + protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, ): return ( origin, @@ -365,7 +366,10 @@ def _cache_artifact( origin is None and not marker_path.exists() and state_dir == _TESTMON_STATE_DIR - and _is_valid_in_progress_testmon_seed_attempt(repo_root / _TESTMON_SEED_ATTEMPT) + and _is_valid_in_progress_testmon_seed_attempt( + repo_root / _TESTMON_SEED_ATTEMPT, + checkout_root=repo_root, + ) ): return None, None if origin is None: diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 411febb20c..b8cff9e5ee 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -66,6 +66,7 @@ from devtools.testmon_state import ( TestmonSeedStamp, + attempt_is_checkout_bound, refresh_stamp, stamp_from_attempt, validate_stamp, @@ -128,6 +129,7 @@ def decide_testmon_bootstrap( main_seed_attempt: Path | None = None, main_checkout_root: Path | None = None, local_checkout_root: Path | None = None, + local_seed_attempt: Path | None = None, ) -> BootstrapDecision: """Decide whether to copy the main checkout's testmon seed into a worktree. @@ -149,6 +151,17 @@ def decide_testmon_bootstrap( ) ): return BootstrapDecision(False, "local .cache/testmon already has a validated testmondata + seed stamp") + if local_testmon_data.is_file() and local_seed_attempt is not None and local_seed_attempt.is_file(): + try: + local_attempt = json.loads(local_seed_attempt.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + local_attempt = None + if isinstance(local_attempt, Mapping) and attempt_is_checkout_bound( + local_attempt, + checkout_root=local_root, + protocol_version=protocol_version, + ): + return BootstrapDecision(False, "local .cache/testmon already has a checkout-bound selection attempt") if not main_testmon_data.is_file(): return BootstrapDecision( False, @@ -242,6 +255,7 @@ def _rebind_run_receipt(*, source: Path, destination: Path, checkout_root: Path, environment["checkout_root"] = str(checkout_root.resolve()) environment["verify_state_origin"] = str(checkout_root.resolve()) _atomic_write_json(destination / "run.json", payload_dict) + _atomic_write_json(checkout_root / ".cache" / "verify" / "current-run.json", payload_dict) return True except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): return False @@ -369,6 +383,9 @@ def bootstrap_testmon_seed_files( rebound_attempt = dict(source_attempt) rebound_attempt["testmon_data"] = refreshed.testmon_data rebound_attempt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" + rebound_attempt["binding"] = refreshed.binding.as_dict() + rebound_attempt["release_baseline_allowed"] = False + rebound_attempt["verification_scope"] = "affected" if ( stamp_from_attempt( rebound_attempt, @@ -454,6 +471,7 @@ def maybe_bootstrap_testmon_seed( main_seed_attempt=main_seed_attempt, main_checkout_root=main_checkout, local_checkout_root=repo_root, + local_seed_attempt=local_seed_attempt, ) if not decision.should_bootstrap: return None diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index c351eca95d..2243b10c46 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -45,6 +45,12 @@ class BindingMode(StrEnum): RELATIVE_FILE_FINGERPRINTS = "relative-file-fingerprints" +class VerificationScope(StrEnum): + AFFECTED = "affected" + RELEASE_BASELINE = "release-baseline" + NON_TEST = "non-test" + + @dataclass(frozen=True, slots=True) class TestmonIdentity: git_head: str | None @@ -380,6 +386,117 @@ def _is_bound_run_artifact(raw: object, *, checkout_root: Path, run_id: str) -> return False +def seed_marker_is_checkout_bound( + marker_path: Path, + *, + checkout_root: Path, + protocol_version: int, +) -> bool: + """Validate only the typed ownership envelope of a seed marker. + + This intentionally does not open or fingerprint SQLite. The checkout guard + uses this cheap predicate for every entrypoint; verify preflight performs + the exhaustive graph validation before authorizing selection. + """ + try: + payload = json.loads(marker_path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + return False + stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) + return Path(stamp.binding.checkout_root).resolve() == checkout_root.resolve() + except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): + return False + + +def attempt_is_checkout_bound( + attempt: Mapping[str, Any], + *, + checkout_root: Path, + protocol_version: int, + reusable_only: bool = True, +) -> bool: + """Check a seed-attempt receipt without inspecting its SQLite graph.""" + allowed_statuses = {"reusable", "complete"} if reusable_only else {"running", "incomplete", "reusable", "complete"} + if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in allowed_statuses: + return False + identity = attempt.get("identity") + expected = attempt.get("expected_nodeids") + selection = attempt.get("selection") + if not isinstance(identity, Mapping) or not isinstance(expected, list) or not isinstance(selection, Mapping): + return False + if not expected or any(not isinstance(nodeid, str) or not nodeid for nodeid in expected): + return False + if len(set(expected)) != len(expected): + return False + if ( + not isinstance(attempt.get("expected_count"), int) + or isinstance(attempt.get("expected_count"), bool) + or attempt.get("expected_count") != len(expected) + ): + return False + expected_digest = hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() + if attempt.get("expected_digest") != expected_digest: + return False + try: + TestmonIdentity.from_mapping(identity) + except ValueError: + return False + omitted = selection.get("selected_nodeids_omitted") + selected_count = selection.get("selected_count") + if ( + not isinstance(omitted, int) + or isinstance(omitted, bool) + or omitted != 0 + or not isinstance(selected_count, int) + or isinstance(selected_count, bool) + or selected_count != len(expected) + ): + return False + recorded_data = attempt.get("testmon_data") + run_id = attempt.get("run_id") + artifact_dir = attempt.get("artifact_dir") + if ( + not isinstance(recorded_data, str) + or not recorded_data + or not isinstance(run_id, str) + or not run_id + or not isinstance(artifact_dir, str) + or not artifact_dir + or not _is_bound_run_artifact(artifact_dir, checkout_root=checkout_root, run_id=run_id) + ): + return False + raw_binding = attempt.get("binding") + if raw_binding is None: + binding = TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())) + elif isinstance(raw_binding, Mapping): + try: + binding = TestmonBinding.from_mapping(raw_binding) + except ValueError: + return False + else: + return False + if Path(binding.checkout_root).resolve() != checkout_root.resolve(): + return False + raw_permission = attempt.get("release_baseline_allowed") + if raw_permission is not None and not isinstance(raw_permission, bool): + return False + raw_scope = attempt.get("verification_scope") + if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: + return False + if reusable_only and raw_permission is not False: + return False + if reusable_only: + outcomes = attempt.get("node_outcomes") + if not isinstance(outcomes, list) or len(outcomes) != len(expected): + return False + nodeids = [item.get("nodeid") for item in outcomes if isinstance(item, Mapping)] + if len(nodeids) != len(outcomes) or set(nodeids) != set(expected) or len(set(nodeids)) != len(nodeids): + return False + if any(item.get("outcome") not in {"passed", "failed", "error", "skipped"} for item in outcomes): + return False + return True + + def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> GraphInspection: """Validate the real testmon schema and every expected dependency edge.""" expected = tuple(expected_nodeids) @@ -556,11 +673,7 @@ def stamp_from_attempt( protocol_version: int, ) -> TestmonSeedStamp | None: """Promote only a complete attempt, including a red one, into a stamp.""" - if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in { - "incomplete", - "reusable", - "complete", - }: + if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in {"reusable", "complete"}: return None selection = attempt.get("selection") expected = attempt.get("expected_nodeids") @@ -640,10 +753,30 @@ def stamp_from_attempt( and not graph.failed_nodeids else BaselineStatus.RED ) + raw_permission = attempt.get("release_baseline_allowed") + if raw_permission is not None and ( + not isinstance(raw_permission, bool) or raw_permission != (baseline is BaselineStatus.GREEN) + ): + return None + raw_scope = attempt.get("verification_scope") + if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: + return None try: typed_identity = TestmonIdentity.from_mapping(identity) except ValueError: return None + raw_binding = attempt.get("binding") + if raw_binding is None: + typed_binding = TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())) + elif isinstance(raw_binding, Mapping): + try: + typed_binding = TestmonBinding.from_mapping(raw_binding) + except ValueError: + return None + if Path(typed_binding.checkout_root).resolve() != checkout_root.resolve(): + return None + else: + return None return TestmonSeedStamp( protocol_version, CollectionStatus.COMPLETE, @@ -654,7 +787,7 @@ def stamp_from_attempt( exit_code, graph, typed_identity, - TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())), + typed_binding, file_fingerprint(data_path), run_id, artifact_dir, @@ -670,9 +803,12 @@ def stamp_from_attempt( "TestmonBinding", "TestmonIdentity", "TestmonSeedStamp", + "VerificationScope", + "attempt_is_checkout_bound", "file_fingerprint", "inspect_testmon_database", "refresh_stamp", + "seed_marker_is_checkout_bound", "stamp_from_attempt", "validate_stamp", ] diff --git a/devtools/verify.py b/devtools/verify.py index 26928a56b5..415ffa32f0 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -59,7 +59,11 @@ ) from devtools.testmon_bootstrap import maybe_bootstrap_testmon_seed from devtools.testmon_state import ( + BindingMode, + GraphStatus, + TestmonBinding, TestmonSeedStamp, + VerificationScope, inspect_testmon_database, refresh_stamp, stamp_from_attempt, @@ -2346,6 +2350,7 @@ def _prepare_testmon_seed_attempt( ) -> dict[str, Any]: prior = _read_testmon_seed_attempt() if resume else None expected = _testmon_seed_expected_nodeids(prior) if prior is not None else [] + prior_outcomes = prior.get("node_outcomes") if isinstance(prior, Mapping) else None payload = { "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, "status": "running", @@ -2353,10 +2358,13 @@ def _prepare_testmon_seed_attempt( "resume": resume, "expected_nodeids": expected, "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, + "prior_node_outcomes": prior_outcomes if isinstance(prior_outcomes, list) else [], "started_at": datetime.now(timezone.utc).isoformat(), "run_id": run.run_id, "artifact_dir": str(run.relative_run_dir), "testmon_data_before": _file_fingerprint(TESTMON_DATA), + "binding": TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict(), } TESTMON_SEED_STAMP.unlink(missing_ok=True) _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) @@ -2391,6 +2399,7 @@ def _seed_node_outcomes_from_events( database: Mapping[str, Any], pytest_step: Mapping[str, Any] | None, use_database_fallback: bool = True, + prior_node_outcomes: Mapping[str, Mapping[str, Any]] | None = None, ) -> list[dict[str, Any]]: """Classify every promised seed node into one explicit terminal state.""" reports: dict[str, list[dict[str, Any]]] = {} @@ -2436,6 +2445,8 @@ def _seed_node_outcomes_from_events( outcome, reason = "passed", "test call passed" elif any(report.get("outcome") == "skipped" for report in call_reports): outcome, reason = "skipped", "test call skipped" + elif any(report.get("outcome") == "skipped" for report in node_reports): + outcome, reason = "skipped", "test setup or teardown skipped" elif nodeid in started and nodeid not in finished and "timeout" in diagnosis: outcome, reason = "timeout", "supervisor timed out while node was active" elif nodeid in started and nodeid not in finished and "worker" in diagnosis: @@ -2450,6 +2461,13 @@ def _seed_node_outcomes_from_events( outcome, reason = "passed", "testmon database recorded success" elif use_database_fallback and recorded.get(nodeid) == "failed": outcome, reason = "failed", "testmon database recorded failure" + elif prior_node_outcomes is not None and nodeid in prior_node_outcomes: + prior = prior_node_outcomes[nodeid] + prior_outcome = prior.get("outcome") + if prior_outcome in {"passed", "failed", "error", "skipped"}: + outcome, reason = str(prior_outcome), "terminal outcome carried from the prior seed attempt" + else: + outcome, reason = "missing", "prior seed attempt has no terminal outcome" else: outcome, reason = "missing", "no terminal report or testmon execution row" results.append( @@ -2515,6 +2533,11 @@ def _finalize_testmon_seed_attempt( database=database, pytest_step=pytest_step, use_database_fallback=False, + prior_node_outcomes={ + str(item["nodeid"]): item + for item in prepared.get("prior_node_outcomes", []) + if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) + }, ) unsuccessful_nodeids = [ str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} @@ -2594,6 +2617,8 @@ def _finalize_testmon_seed_attempt( "unsuccessful_nodeids": unsuccessful_nodeids, "testmon_data": _file_fingerprint(TESTMON_DATA), "pytest_step": dict(pytest_step) if pytest_step is not None else None, + "binding": TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict(), + "verification_scope": VerificationScope.RELEASE_BASELINE.value, } payload["release_baseline_allowed"] = bool(reusable_stamp is not None and reusable_stamp.release_baseline_allowed) _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) @@ -2604,6 +2629,83 @@ def _finalize_testmon_seed_attempt( return payload +def _refresh_testmon_selection_attempt( + *, + step: Mapping[str, Any], + run: VerifyRun, + exit_code: int, +) -> None: + """Refresh a reusable red graph after every completed affected run.""" + attempt = _read_testmon_seed_attempt() + if attempt is None or attempt.get("release_baseline_allowed") is True: + return + expected = _testmon_seed_expected_nodeids(attempt) + if not expected: + return + database = _testmon_database_state(expected) + artifact_dir = _safe_testmon_artifact_dir(step.get("artifact_dir")) + events_path = artifact_dir / "events.jsonl" if artifact_dir is not None else Path(".missing-testmon-events") + prior = { + str(item["nodeid"]): item + for item in attempt.get("node_outcomes", []) + if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) + } + node_outcomes = _seed_node_outcomes_from_events( + events_path, + expected_nodeids=expected, + database=database, + pytest_step=step, + use_database_fallback=False, + prior_node_outcomes=prior, + ) + graph_complete = ( + database.get("graph_status") == GraphStatus.COMPLETE.value + and not database.get("missing_nodeids") + and database.get("error") is None + and database.get("orphan_execution_edges") == 0 + and database.get("orphan_fingerprint_edges") == 0 + ) + terminal = all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in node_outcomes) + prior_selection = attempt.get("selection") + payload = { + **attempt, + "status": "reusable" if graph_complete and terminal else "incomplete", + "finished_at": datetime.now(timezone.utc).isoformat(), + "exit_code": exit_code, + "expected_nodeids": expected, + "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), + "selection": { + **(dict(prior_selection) if isinstance(prior_selection, Mapping) else {}), + "selected_count": len(expected), + "selected_nodeids_omitted": 0, + }, + "database": database, + "node_outcomes": node_outcomes, + "node_outcome_counts": dict( + sorted( + { + outcome: sum(1 for item in node_outcomes if item.get("outcome") == outcome) + for outcome in {str(item.get("outcome")) for item in node_outcomes} + }.items() + ) + ), + "unsuccessful_nodeids": [ + str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} + ], + "testmon_data": _file_fingerprint(TESTMON_DATA), + "run_id": run.run_id, + "artifact_dir": str(run.relative_run_dir), + "pytest_step": dict(step), + "release_baseline_allowed": False, + "verification_scope": VerificationScope.AFFECTED.value, + } + raw_binding = attempt.get("binding") + if not isinstance(raw_binding, Mapping): + payload["binding"] = TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict() + _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) + + # ── main ──────────────────────────────────────────────────────────── @@ -2780,6 +2882,8 @@ def main(argv: list[str] | None = None) -> int: step_result: dict[str, Any] = {"name": label, "duration_s": round(elapsed, 2), "exit": rc} step_result.update(metadata) step_results.append(step_result) + if label in {"pytest testmon", "pytest testmon (broad)"} and not args.seed_testmon and not full_pytest: + _refresh_testmon_selection_attempt(step=step_result, run=verify_run, exit_code=rc) if rc != 0: exit_code = rc if _stop_after_failed_step(label): @@ -2833,11 +2937,15 @@ def main(argv: list[str] | None = None) -> int: } if args.quick or args.commit: + verification_scope = VerificationScope.NON_TEST release_baseline_allowed: bool | None = None - elif full_pytest: - release_baseline_allowed = exit_code == 0 + elif full_pytest or args.seed_testmon: + verification_scope = VerificationScope.RELEASE_BASELINE + release_baseline_allowed = exit_code == 0 if full_pytest else _testmon_release_baseline_permission() else: + verification_scope = VerificationScope.AFFECTED release_baseline_allowed = _testmon_release_baseline_permission() + history_entry["verification_scope"] = verification_scope.value history_entry["release_baseline_allowed"] = release_baseline_allowed if release_baseline_allowed is False and tier in {"testmon", "lab", "seed-testmon"}: sys.stderr.write( diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py index ead7278c03..4c0b0638bd 100644 --- a/tests/integration/devtools/test_testmon_seed_recovery.py +++ b/tests/integration/devtools/test_testmon_seed_recovery.py @@ -11,11 +11,14 @@ import pytest from devtools import testmon_bootstrap, verify -from devtools.testmon_bootstrap import maybe_bootstrap_testmon_seed from devtools.testmon_state import file_fingerprint, inspect_testmon_database -def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane(tmp_path: Path) -> None: +def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: source = tmp_path / "source" source.mkdir() (source / "test_sample.py").write_text( @@ -39,7 +42,7 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane(tmp_path: Pat assert inspect_testmon_database(data, expected).usable_for_selection attempt = { "protocol_version": 4, - "status": "incomplete", + "status": "reusable", "identity": { "git_head": "head", "worktree_fingerprint": "source-tree", @@ -82,35 +85,71 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane(tmp_path: Pat "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 1\n", encoding="utf-8", ) - monkeypatch = pytest.MonkeyPatch() - try: - monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, source)) - message = maybe_bootstrap_testmon_seed(lane, protocol_version=4) - assert message is not None and "selection-only attempt receipt" in message - - local_data = lane / ".cache" / "testmon" / "testmondata" - local_stamp = lane / ".cache" / "testmon" / "seed.json" - local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" - assert local_data.is_file() - assert local_attempt.is_file() - assert not local_stamp.exists() - - monkeypatch.chdir(lane) - monkeypatch.setattr(verify, "ROOT", lane) - assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is None - selected = subprocess.run( - [sys.executable, "-m", "pytest", "-q", "--testmon"], - cwd=lane, - env={**env, "TESTMON_DATAFILE": str(local_data)}, - capture_output=True, - text=True, - check=False, - ) - assert selected.returncode == 0, selected.stdout + selected.stderr - assert "1 passed" in selected.stdout - assert verify._testmon_release_baseline_permission() is False - with sqlite3.connect(local_data) as connection: - connection.execute("delete from test_execution_file_fp") - assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is not None - finally: - monkeypatch.undo() + monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, source)) + copy_calls: list[tuple[Path, Path]] = [] + original_copy = testmon_bootstrap._atomic_copy_sqlite_db + + def counted_copy(src: Path, dst: Path) -> None: + copy_calls.append((src, dst)) + original_copy(src, dst) + + monkeypatch.setattr(testmon_bootstrap, "_atomic_copy_sqlite_db", counted_copy) + (lane / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") + (lane / "polylogue" / "cli").mkdir(parents=True) + (lane / "polylogue" / "__init__.py").write_text("", encoding="utf-8") + (lane / "polylogue" / "cli" / "click_app.py").write_text("", encoding="utf-8") + + local_data = lane / ".cache" / "testmon" / "testmondata" + local_stamp = lane / ".cache" / "testmon" / "seed.json" + local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" + + monkeypatch.chdir(lane) + monkeypatch.setattr(verify, "ROOT", lane) + monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: lane / "polylogue" / "__init__.py") + monkeypatch.setattr("devtools.checkout_guard._is_linked_worktree", lambda _root: True) + monkeypatch.setattr("devtools.checkout_guard._python_environment_root", lambda _executable: lane) + monkeypatch.setattr(verify, "build_verify_steps", lambda **_kwargs: [("pytest testmon", ["pytest"])]) + run_count = 0 + + def fake_run(*_args: object, **_kwargs: object) -> tuple[int, float, dict[str, object]]: + nonlocal run_count + run_count += 1 + if run_count == 1: + with sqlite3.connect(local_data) as connection: + connection.execute("update test_execution set failed = 0 where test_name = ?", (expected[1],)) + return 1, 0.01, {"selected_count": 1} + return 0, 0.01, {"selected_count": 1} + + monkeypatch.setattr(verify, "_run", fake_run) + monkeypatch.setattr(verify, "_changed_executable_paths", lambda: ()) + monkeypatch.setattr(verify, "_stamp_head", lambda: None) + + assert verify.main([]) == 1 + result = json.loads(capsys.readouterr().out) + assert local_data.is_file() + assert local_attempt.is_file() + assert not local_stamp.exists() + assert result["steps"][0]["selected_count"] == 1 + assert result["release_baseline_allowed"] is False + assert verify._testmon_release_baseline_permission() is False + assert not local_stamp.exists() + assert len(copy_calls) == 1 + assert (lane / ".cache" / "verify" / "current-run.json").is_file() + refreshed_attempt = json.loads(local_attempt.read_text()) + assert refreshed_attempt["testmon_data"] == file_fingerprint(local_data) + current_run = json.loads((lane / ".cache" / "verify" / "current-run.json").read_text()) + assert refreshed_attempt["run_id"] == current_run["run_id"] + + assert verify.main([]) == 0 + second = json.loads(capsys.readouterr().out) + assert second["steps"][0]["selected_count"] == 1 + assert len(copy_calls) == 1 + + assert verify.main([]) == 0 + third = json.loads(capsys.readouterr().out) + assert third["steps"][0]["selected_count"] == 1 + assert len(copy_calls) == 1 + + with sqlite3.connect(local_data) as connection: + connection.execute("delete from test_execution_file_fp") + assert verify.main([]) == 2 diff --git a/tests/unit/devtools/test_checkout_guard.py b/tests/unit/devtools/test_checkout_guard.py index 018550b3c1..93d6c21f84 100644 --- a/tests/unit/devtools/test_checkout_guard.py +++ b/tests/unit/devtools/test_checkout_guard.py @@ -9,6 +9,7 @@ from __future__ import annotations +import hashlib import json from pathlib import Path @@ -183,10 +184,51 @@ def _write_in_progress_seed_attempt(root: Path, *, status: str = "running", **ov "artifact_dir": ".cache/verify/runs/seed-testmon-20260805T120000Z", "testmon_data_before": "missing", } + if status == "reusable": + nodeid = "tests/test.py::test_one" + payload.update( + { + "expected_nodeids": [nodeid], + "expected_count": 1, + "expected_digest": hashlib.sha256(nodeid.encode()).hexdigest(), + "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, + "node_outcomes": [{"nodeid": nodeid, "outcome": "failed"}], + "exit_code": 1, + "testmon_data": "fingerprint", + "release_baseline_allowed": False, + "verification_scope": "affected", + "binding": { + "mode": "exact", + "checkout_root": str(root.resolve()), + "source_checkout_root": None, + }, + } + ) payload.update(overrides) attempt = root / ".cache" / "testmon" / "seed-attempt.json" attempt.parent.mkdir(parents=True, exist_ok=True) attempt.write_text(json.dumps(payload)) + if status == "reusable": + run_dir = root / ".cache" / "verify" / "runs" / str(payload["run_id"]) + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "run.json").write_text( + json.dumps( + { + "run_id": payload["run_id"], + "checkout_root": str(root.resolve()), + "artifact_dir": f".cache/verify/runs/{payload['run_id']}", + } + ) + ) + (root / ".cache" / "verify" / "current-run.json").write_text( + json.dumps( + { + "run_id": payload["run_id"], + "checkout_root": str(root.resolve()), + "artifact_dir": f".cache/verify/runs/{payload['run_id']}", + } + ) + ) return attempt diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 6f1cd9474a..f6c539cb86 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -275,7 +275,7 @@ def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) json.dumps( { "protocol_version": PROTOCOL_VERSION, - "status": "incomplete", + "status": "reusable", "identity": { "git_head": "head", "worktree_fingerprint": "tree", @@ -343,6 +343,24 @@ def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) ) assert rebound_receipt["run_id"] == "red-run" assert rebound_receipt["checkout_root"] == str((tmp_path / "lane").resolve()) + current_run = json.loads((tmp_path / "lane" / ".cache" / "verify" / "current-run.json").read_text()) + assert current_run["run_id"] == "red-run" + assert current_run["checkout_root"] == str((tmp_path / "lane").resolve()) + + rebound_decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + local_seed_attempt=local_attempt, + main_testmon_data=main_data, + main_seed_stamp=tmp_path / "main" / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + main_checkout_root=tmp_path / "main", + local_checkout_root=tmp_path / "lane", + ) + assert not rebound_decision.should_bootstrap + assert "checkout-bound selection attempt" in rebound_decision.reason def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index f6a6929567..543dde0dd4 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -54,7 +54,7 @@ def _attempt(data: Path, *, outcomes: tuple[str, str] = ("passed", "failed")) -> ) return { "protocol_version": PROTOCOL, - "status": "incomplete", + "status": "reusable", "identity": { "git_head": "head", "worktree_fingerprint": "tree", @@ -114,18 +114,16 @@ def test_omitted_interrupted_and_uncovered_nodes_fail_closed(tmp_path: Path) -> assert stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None -def test_incomplete_all_pass_attempt_is_selection_only(tmp_path: Path) -> None: +def test_incomplete_attempt_fails_closed(tmp_path: Path) -> None: data = tmp_path / "testmondata" _write_graph(data) attempt = _attempt(data, outcomes=("passed", "passed")) attempt["exit_code"] = 0 + attempt["status"] = "incomplete" stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - assert stamp is not None - assert stamp.baseline_status is BaselineStatus.RED - assert stamp.affected_selection_allowed - assert not stamp.release_baseline_allowed + assert stamp is None attempt["status"] = "complete" completed = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 5a9ab6d17e..0cb4049601 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -553,6 +553,9 @@ def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, mo assert prepared["expected_nodeids"] == expected assert prepared["expected_count"] == 1 + assert prepared["expected_digest"] == hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() + persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) + assert persisted["expected_digest"] == prepared["expected_digest"] def test_seed_resume_rejects_selection_artifact_outside_checkout( @@ -851,6 +854,65 @@ def test_seed_node_outcomes_preserve_interrupted_active_node(tmp_path: Path) -> assert outcomes[0]["outcome"] == "interrupted" +def test_seed_node_outcomes_accept_setup_skip_as_terminal_skip(tmp_path: Path) -> None: + events = tmp_path / "events.jsonl" + events.write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/test_a.py::test_setup_skip", + "when": "setup", + "outcome": "skipped", + } + ) + + "\n" + ) + + outcomes = _seed_node_outcomes_from_events( + events, + expected_nodeids=["tests/test_a.py::test_setup_skip"], + database={"node_outcomes": {"tests/test_a.py::test_setup_skip": "missing"}}, + pytest_step={}, + use_database_fallback=False, + ) + + assert outcomes == [ + { + "nodeid": "tests/test_a.py::test_setup_skip", + "outcome": "skipped", + "reason": "test setup or teardown skipped", + "started": False, + "finished": False, + "phases": [{"when": "setup", "outcome": "skipped", "duration_s": None}], + } + ] + + +def test_resumed_seed_carries_forward_prior_terminal_outcome(tmp_path: Path) -> None: + events = tmp_path / "events.jsonl" + events.write_text( + json.dumps( + {"event": "test_report", "nodeid": "tests/test_a.py::test_repaired", "when": "call", "outcome": "passed"} + ) + + "\n" + ) + outcomes = _seed_node_outcomes_from_events( + events, + expected_nodeids=["tests/test_a.py::test_repaired", "tests/test_b.py::test_prior"], + database={"node_outcomes": {"tests/test_b.py::test_prior": "passed"}}, + pytest_step={}, + use_database_fallback=False, + prior_node_outcomes={ + "tests/test_b.py::test_prior": {"nodeid": "tests/test_b.py::test_prior", "outcome": "passed"} + }, + ) + + assert {item["nodeid"]: item["outcome"] for item in outcomes} == { + "tests/test_a.py::test_repaired": "passed", + "tests/test_b.py::test_prior": "passed", + } + + def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) artifact_dir = tmp_path / "artifacts" From 657b35bc0c562a6d4be01dd5145e770e2b53f8e8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 03:49:57 +0200 Subject: [PATCH 10/34] fix: separate affected and release verification gates Problem: the ordinary per-PR affected receipt was blocked by a false release-baseline requirement, while failed or unstructured terminal full verification could clear merge-train status. What changed: persist a typed verification scope, allow affected and lab receipts when their selected tests pass, require release permission for explicit full-baseline receipts, and make the terminal ledger accept only a zero-exit structured release-baseline result. Verification: merge-gate and merge-boundary coverage passed in the focused suite. Regression tests pair affected false-permission acceptance with full false-permission rejection, and keep train-status incomplete after failed or unstructured terminal verification. Co-Authored-By: Codex --- devtools/merge_boundary.py | 27 +++++++++++++-- devtools/merge_gate.py | 37 +++++++++++++++----- tests/unit/devtools/test_merge_boundary.py | 40 ++++++++++++++++++++-- tests/unit/devtools/test_merge_gate.py | 23 +++++++++++-- 4 files changed, 113 insertions(+), 14 deletions(-) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index 3b0e05890e..f8c5071ca6 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -132,7 +132,14 @@ def _append_merge_entry(pr: int, head_sha: str, title: str) -> None: def _pending_prs_since_last_full_verify(ledger: dict[str, Any]) -> list[dict[str, Any]]: - last_verify_at = (ledger.get("last_full_verify") or {}).get("at", 0.0) + last_verify = ledger.get("last_full_verify") or {} + last_verify_at = ( + last_verify.get("at", 0.0) + if last_verify.get("accepted") is True + and last_verify.get("exit_code") == 0 + and last_verify.get("release_baseline_allowed") is True + else 0.0 + ) return [entry for entry in ledger.get("merges", []) if entry.get("merged_at", 0.0) > last_verify_at] @@ -278,6 +285,11 @@ def cmd_record_full_verify(command: str) -> int: print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) return 2 duration_s = round(time.time() - started, 2) + release_allowed = merge_gate._release_baseline_permission(result.stdout) + verification_scope = merge_gate._verification_scope(result.stdout) or merge_gate._command_verification_scope( + command + ) + accepted = result.returncode == 0 and verification_scope == "release-baseline" and release_allowed is True ledger = _read_ledger() ledger["last_full_verify"] = { @@ -285,10 +297,21 @@ def cmd_record_full_verify(command: str) -> int: "exit_code": result.returncode, "duration_s": duration_s, "at": time.time(), + "verification_scope": verification_scope, + "release_baseline_allowed": release_allowed, + "accepted": accepted, } _write_ledger(ledger) - print(f"recorded merge-train terminal verify: {command!r} exit={result.returncode} ({duration_s}s)") + print( + f"recorded merge-train terminal verify: {command!r} exit={result.returncode} " + f"release_baseline_allowed={release_allowed!r} accepted={accepted} ({duration_s}s)" + ) + if not accepted and result.returncode == 0: + print( + "POST-MERGE BROAD VERIFY DID NOT GRANT release-baseline permission; train-status remains incomplete.", + file=sys.stderr, + ) if result.returncode != 0: print(result.stdout[-4000:]) print(result.stderr[-4000:], file=sys.stderr) diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index 1a5cb31157..809bbec45c 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -79,6 +79,7 @@ from typing import Any from devtools import pr_scope +from devtools.testmon_state import VerificationScope _RECEIPT_DIR = Path(".cache/verify/merge-gate") _DEFAULT_MAX_AGE_S = 3600 @@ -230,15 +231,32 @@ def _release_baseline_permission(stdout: str) -> bool | None: return value if isinstance(value, bool) else None -def _requires_release_baseline(command: str) -> bool: - """Identify verification commands whose success can claim a release baseline.""" +def _verification_scope(stdout: str) -> str | None: + """Read the typed verification scope from a structured verify receipt.""" + try: + payload = json.loads(stdout) + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + value = payload.get("verification_scope") + return value if value in {scope.value for scope in VerificationScope} else None + + +def _command_verification_scope(command: str) -> str | None: + """Classify legacy commands by argv shape, never by emitted log text.""" try: argv = shlex.split(command) except ValueError: - return False + return None if argv[:2] != ["devtools", "verify"]: - return False - return len(argv) == 2 or any(option in argv[2:] for option in ("--all", "--full", "--lab", "--seed-testmon")) + return None + options = set(argv[2:]) + if options & {"--all", "--full", "--seed-testmon"}: + return VerificationScope.RELEASE_BASELINE.value + if options & {"--quick", "--commit"}: + return VerificationScope.NON_TEST.value + return VerificationScope.AFFECTED.value def cmd_record(pr: int, command: str) -> int: @@ -295,6 +313,7 @@ def cmd_record(pr: int, command: str) -> int: "branch": info["headRefName"], "command": command, "skips_tests": _command_skips_tests(command), + "verification_scope": _verification_scope(result.stdout) or _command_verification_scope(command), "release_baseline_allowed": _release_baseline_permission(result.stdout), "exit_code": result.returncode, "duration_s": duration_s, @@ -502,14 +521,16 @@ def cmd_check( if receipt.get("exit_code", 1) != 0: verdict.ok = False verdict.reasons.append(f"receipt exit_code is {receipt.get('exit_code')}, not 0") + verification_scope = receipt.get("verification_scope") + if verification_scope is None: + verification_scope = _command_verification_scope(str(receipt.get("command", ""))) if ( - _requires_release_baseline(str(receipt.get("command", ""))) + verification_scope == VerificationScope.RELEASE_BASELINE.value and receipt.get("release_baseline_allowed") is not True ): verdict.ok = False verdict.reasons.append( - "verification receipt does not grant release_baseline_allowed=true; a selection-only " - "testmon attempt cannot satisfy the release merge gate" + "release-baseline verification receipt does not grant release_baseline_allowed=true" ) if receipt.get("skips_tests"): verdict.reasons.append( diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index 7fa06d6c87..cdd3d25afb 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -314,7 +314,18 @@ def test_merge_propagates_gh_pr_merge_failure(monkeypatch: pytest.MonkeyPatch, t def test_merge_with_verify_records_terminal_full_verify(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) pr_view = _base_pr_view() - monkeypatch.setattr(subprocess, "run", _fake_run(pr_view)) + base = _fake_run(pr_view) + + def run(cmd: list[str], **kwargs: Any) -> MagicMock: + if cmd[:3] == ["devtools", "verify", "--all"]: + return MagicMock( + returncode=0, + stdout=json.dumps({"verification_scope": "release-baseline", "release_baseline_allowed": True}), + stderr="", + ) + return base(cmd, **kwargs) + + monkeypatch.setattr(subprocess, "run", run) exit_code = merge_boundary.cmd_merge( 42, @@ -367,7 +378,11 @@ def test_record_full_verify_clears_pending_prs(monkeypatch: pytest.MonkeyPatch, merge_boundary._append_merge_entry(1, "sha1", "some title") def _run(cmd: list[str], **kwargs: Any) -> MagicMock: - return MagicMock(returncode=0, stdout="all good\n", stderr="") + return MagicMock( + returncode=0, + stdout=json.dumps({"verification_scope": "release-baseline", "release_baseline_allowed": True}), + stderr="", + ) monkeypatch.setattr(subprocess, "run", _run) @@ -379,6 +394,7 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: def test_record_full_verify_propagates_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return MagicMock(returncode=1, stdout="", stderr="broke") @@ -390,3 +406,23 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: assert exit_code == 1 ledger = merge_boundary._read_ledger() assert ledger["last_full_verify"]["exit_code"] == 1 + assert ledger["last_full_verify"]["accepted"] is False + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_record_full_verify_rejects_success_without_structured_release_permission( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock(returncode=0, stdout="all good\n", stderr=""), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all") == 0 + ledger = merge_boundary._read_ledger() + assert ledger["last_full_verify"]["accepted"] is False + assert merge_boundary.cmd_train_status(as_json=False) == 1 diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index ccb9f87354..478eed091d 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -104,7 +104,7 @@ def test_record_persists_receipt_keyed_to_current_head_sha(monkeypatch: pytest.M assert receipt["skips_tests"] is False -def test_check_blocks_receipt_without_release_baseline_permission( +def test_check_accepts_affected_receipt_without_release_baseline_permission( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.chdir(tmp_path) @@ -115,6 +115,21 @@ def test_check_blocks_receipt_without_release_baseline_permission( receipt["release_baseline_allowed"] = False receipt_path.write_text(json.dumps(receipt)) + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view, [])) + assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 0 + + +def test_check_blocks_full_receipt_without_release_baseline_permission( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + _record(monkeypatch, pr_view, command="devtools verify --all") + receipt_path = merge_gate._receipt_path(42) + receipt = json.loads(receipt_path.read_text()) + receipt["release_baseline_allowed"] = False + receipt_path.write_text(json.dumps(receipt)) + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view, [])) assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 1 @@ -129,7 +144,11 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: return base(cmd, **kwargs) if cmd[:3] == ["gh", "pr", "view"]: return base(cmd, **kwargs) - return MagicMock(returncode=0, stdout=json.dumps({"release_baseline_allowed": False}), stderr="") + return MagicMock( + returncode=0, + stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), + stderr="", + ) monkeypatch.setattr(subprocess, "run", _run) assert merge_gate.cmd_record(42, "devtools verify") == 0 From 84c41dc142db131cb05eb72e59e6e108199c1185 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 03:51:51 +0200 Subject: [PATCH 11/34] test: cover verification scope combinations Add merge-gate regressions for default, lab, and option-bearing affected receipts, plus all explicit full-baseline verify forms. Verification: devtools test tests/unit/devtools/test_merge_gate.py passed 35 tests. Co-Authored-By: Codex --- tests/unit/devtools/test_merge_gate.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 478eed091d..11d643a40d 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -104,12 +104,13 @@ def test_record_persists_receipt_keyed_to_current_head_sha(monkeypatch: pytest.M assert receipt["skips_tests"] is False +@pytest.mark.parametrize("command", ["devtools verify", "devtools verify --lab", "devtools verify --json --skip-slow"]) def test_check_accepts_affected_receipt_without_release_baseline_permission( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, command: str ) -> None: monkeypatch.chdir(tmp_path) pr_view = _base_pr_view() - _record(monkeypatch, pr_view, command="devtools verify") + _record(monkeypatch, pr_view, command=command) receipt_path = merge_gate._receipt_path(42) receipt = json.loads(receipt_path.read_text()) receipt["release_baseline_allowed"] = False @@ -119,12 +120,15 @@ def test_check_accepts_affected_receipt_without_release_baseline_permission( assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 0 +@pytest.mark.parametrize( + "command", ["devtools verify --all", "devtools verify --full", "devtools verify --seed-testmon"] +) def test_check_blocks_full_receipt_without_release_baseline_permission( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, command: str ) -> None: monkeypatch.chdir(tmp_path) pr_view = _base_pr_view() - _record(monkeypatch, pr_view, command="devtools verify --all") + _record(monkeypatch, pr_view, command=command) receipt_path = merge_gate._receipt_path(42) receipt = json.loads(receipt_path.read_text()) receipt["release_baseline_allowed"] = False From ab2e7890401da2ddf181508fdcb3094fa37e46b8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 04:04:19 +0200 Subject: [PATCH 12/34] ci: synchronize PR scope carrier Refresh the structured carrier against the future published head before pushing. From ef82c3bd4b0958fa6fe429e0fac9a65da9f380f7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 04:20:08 +0200 Subject: [PATCH 13/34] fix: harden testmon terminal authority Problem A skipped-slow terminal verify receipt could claim release-baseline authority, and terminal recording could infer missing scope from the command string. Resumed seed state also accepted clean revisions with different committed trees and could persist an incomplete selection count after a subset-only resume. What changed Add a typed narrow-terminal scope and authorization, require structured scope and permission fields at the merge-train boundary, bind seed resumes to the committed tree object, and persist the inherited full selection count before stamp publication. Add real verify.main, merge-boundary, and resumed-seed crash regressions. Verification Focused changed-surface tests pass 150 tests; the one remaining failure is the pre-existing lab policy registration mismatch. Ruff format, Ruff lint, and mypy pass on the changed production and test modules. Co-Authored-By: Codex --- devtools/merge_boundary.py | 13 ++- devtools/merge_gate.py | 15 ++- devtools/testmon_state.py | 6 ++ devtools/verify.py | 97 +++++++++++++++-- tests/unit/devtools/test_merge_boundary.py | 66 ++++++++++++ tests/unit/devtools/test_verify.py | 119 +++++++++++++++++++-- 6 files changed, 293 insertions(+), 23 deletions(-) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index f8c5071ca6..266abd022c 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -286,10 +286,16 @@ def cmd_record_full_verify(command: str) -> int: return 2 duration_s = round(time.time() - started, 2) release_allowed = merge_gate._release_baseline_permission(result.stdout) - verification_scope = merge_gate._verification_scope(result.stdout) or merge_gate._command_verification_scope( - command + verification_scope = merge_gate._verification_scope(result.stdout) + terminal_authorization = merge_gate._terminal_authorization(result.stdout) + accepted = ( + result.returncode == 0 + and release_allowed is True + and ( + verification_scope == "release-baseline" + or (verification_scope == "narrow-terminal" and terminal_authorization == "narrow-terminal") + ) ) - accepted = result.returncode == 0 and verification_scope == "release-baseline" and release_allowed is True ledger = _read_ledger() ledger["last_full_verify"] = { @@ -299,6 +305,7 @@ def cmd_record_full_verify(command: str) -> int: "at": time.time(), "verification_scope": verification_scope, "release_baseline_allowed": release_allowed, + "terminal_authorization": terminal_authorization, "accepted": accepted, } _write_ledger(ledger) diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index 809bbec45c..c7c567fa2f 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -79,7 +79,7 @@ from typing import Any from devtools import pr_scope -from devtools.testmon_state import VerificationScope +from devtools.testmon_state import TerminalAuthorization, VerificationScope _RECEIPT_DIR = Path(".cache/verify/merge-gate") _DEFAULT_MAX_AGE_S = 3600 @@ -243,6 +243,18 @@ def _verification_scope(stdout: str) -> str | None: return value if value in {scope.value for scope in VerificationScope} else None +def _terminal_authorization(stdout: str) -> str | None: + """Read the typed terminal authorization from a structured receipt.""" + try: + payload = json.loads(stdout) + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + value = payload.get("terminal_authorization") + return value if value in {authorization.value for authorization in TerminalAuthorization} else None + + def _command_verification_scope(command: str) -> str | None: """Classify legacy commands by argv shape, never by emitted log text.""" try: @@ -315,6 +327,7 @@ def cmd_record(pr: int, command: str) -> int: "skips_tests": _command_skips_tests(command), "verification_scope": _verification_scope(result.stdout) or _command_verification_scope(command), "release_baseline_allowed": _release_baseline_permission(result.stdout), + "terminal_authorization": _terminal_authorization(result.stdout), "exit_code": result.returncode, "duration_s": duration_s, "recorded_at": time.time(), diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index 2243b10c46..fec977eb5e 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -48,9 +48,14 @@ class BindingMode(StrEnum): class VerificationScope(StrEnum): AFFECTED = "affected" RELEASE_BASELINE = "release-baseline" + NARROW_TERMINAL = "narrow-terminal" NON_TEST = "non-test" +class TerminalAuthorization(StrEnum): + NARROW_TERMINAL = "narrow-terminal" + + @dataclass(frozen=True, slots=True) class TestmonIdentity: git_head: str | None @@ -803,6 +808,7 @@ def stamp_from_attempt( "TestmonBinding", "TestmonIdentity", "TestmonSeedStamp", + "TerminalAuthorization", "VerificationScope", "attempt_is_checkout_bound", "file_fingerprint", diff --git a/devtools/verify.py b/devtools/verify.py index 415ffa32f0..98f377991b 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -61,6 +61,7 @@ from devtools.testmon_state import ( BindingMode, GraphStatus, + TerminalAuthorization, TestmonBinding, TestmonSeedStamp, VerificationScope, @@ -2021,6 +2022,17 @@ def _git_head() -> str | None: return None +def _git_committed_tree() -> str | None: + result = subprocess.run( + ["git", "rev-parse", "HEAD^{tree}"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result.stdout.strip() + return None + + def _stamp_head() -> None: head = _git_head() if head is None: @@ -2224,13 +2236,22 @@ def _worktree_fingerprint() -> str: return digest.hexdigest() -def _testmon_seed_identity(*, git_head: str | None, skip_slow: bool, lab: bool) -> dict[str, Any]: +def _testmon_seed_identity( + *, + git_head: str | None, + git_tree: str | None = None, + skip_slow: bool, + lab: bool, + terminal_authorization: str | None = None, +) -> dict[str, Any]: return { "git_head": git_head, + "git_tree": git_tree, "worktree_fingerprint": _worktree_fingerprint(), "python": sys.version, "skip_slow": skip_slow, "lab": lab, + "terminal_authorization": terminal_authorization, } @@ -2325,7 +2346,10 @@ def _testmon_seed_expected_nodeids(attempt: Mapping[str, Any]) -> list[str]: def _testmon_seed_resume_contract(identity: Mapping[str, Any]) -> dict[str, Any]: """Return inputs that change which corpus a seed promises to cover.""" - return {key: identity.get(key) for key in ("worktree_fingerprint", "python", "skip_slow", "lab")} + return { + key: identity.get(key) + for key in ("git_tree", "worktree_fingerprint", "python", "skip_slow", "lab", "terminal_authorization") + } def _testmon_seed_can_resume(identity: Mapping[str, Any]) -> bool: @@ -2333,11 +2357,14 @@ def _testmon_seed_can_resume(identity: Mapping[str, Any]) -> bool: if attempt is None or not TESTMON_DATA.exists(): return False prior_identity = attempt.get("identity") + contract = _testmon_seed_resume_contract(identity) return ( attempt.get("protocol_version") == TESTMON_SEED_PROTOCOL_VERSION and attempt.get("status") in {"running", "incomplete"} and isinstance(prior_identity, dict) - and _testmon_seed_resume_contract(prior_identity) == _testmon_seed_resume_contract(identity) + and isinstance(contract["git_tree"], str) + and bool(contract["git_tree"]) + and _testmon_seed_resume_contract(prior_identity) == contract and bool(_testmon_seed_expected_nodeids(attempt)) ) @@ -2371,6 +2398,15 @@ def _prepare_testmon_seed_attempt( return payload +def _testmon_seed_terminal_authorized(prepared: Mapping[str, Any]) -> bool: + identity = prepared.get("identity") + return ( + isinstance(identity, Mapping) + and identity.get("skip_slow") is True + and identity.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value + ) + + def _testmon_database_state(expected_nodeids: Sequence[str]) -> dict[str, Any]: graph = inspect_testmon_database(TESTMON_DATA, expected_nodeids) expected = set(expected_nodeids) @@ -2555,9 +2591,16 @@ def _finalize_testmon_seed_attempt( and database["orphan_fingerprint_edges"] == 0 and not unsuccessful_nodeids ) + identity = prepared.get("identity") + narrow_terminal = isinstance(identity, Mapping) and identity.get("skip_slow") is True + terminal_authorized = _testmon_seed_terminal_authorized(prepared) + release_eligible = green_complete and (not narrow_terminal or terminal_authorized) + seed_scope = ( + VerificationScope.NARROW_TERMINAL.value if narrow_terminal else VerificationScope.RELEASE_BASELINE.value + ) attempt_candidate = { **dict(prepared), - "status": "complete" if green_complete else "reusable", + "status": "complete" if release_eligible else "reusable", "exit_code": exit_code, "expected_nodeids": expected, "expected_count": len(expected), @@ -2585,7 +2628,13 @@ def _finalize_testmon_seed_attempt( protocol_version=TESTMON_SEED_PROTOCOL_VERSION, ) reusable = reusable_stamp is not None - attempt_status = "complete" if green_complete else "reusable" if reusable else "incomplete" + release_permission = bool( + reusable + and reusable_stamp is not None + and reusable_stamp.release_baseline_allowed + and (not narrow_terminal or terminal_authorized) + ) + attempt_status = "complete" if green_complete and release_permission else "reusable" if reusable else "incomplete" payload = { **dict(prepared), "status": attempt_status, @@ -2595,7 +2644,13 @@ def _finalize_testmon_seed_attempt( "expected_count": len(expected), "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, "selection": { - key: selection.get(key) + key: ( + len(expected) + if key == "selected_count" and prepared.get("resume") and selection_valid + else 0 + if key == "selected_nodeids_omitted" and prepared.get("resume") and selection_valid + else selection.get(key) + ) for key in ( "selected_count", "deselected_count", @@ -2618,11 +2673,12 @@ def _finalize_testmon_seed_attempt( "testmon_data": _file_fingerprint(TESTMON_DATA), "pytest_step": dict(pytest_step) if pytest_step is not None else None, "binding": TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict(), - "verification_scope": VerificationScope.RELEASE_BASELINE.value, + "verification_scope": seed_scope, + "terminal_authorization": (TerminalAuthorization.NARROW_TERMINAL.value if terminal_authorized else None), } - payload["release_baseline_allowed"] = bool(reusable_stamp is not None and reusable_stamp.release_baseline_allowed) + payload["release_baseline_allowed"] = release_permission _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - if reusable_stamp is not None and reusable_stamp.release_baseline_allowed: + if release_permission and reusable_stamp is not None: _atomic_write_json(TESTMON_SEED_STAMP, reusable_stamp.as_dict()) else: TESTMON_SEED_STAMP.unlink(missing_ok=True) @@ -2727,6 +2783,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--skip-slow", action="store_true", help="Exclude @pytest.mark.slow tests from the pytest step." ) + parser.add_argument( + "--terminal-authorization", + choices=[TerminalAuthorization.NARROW_TERMINAL.value], + help="Typed authorization for a narrow terminal verification that skips slow tests.", + ) parser.add_argument( "--lab", action="store_true", @@ -2777,6 +2838,8 @@ def main(argv: list[str] | None = None) -> int: tier = "testmon" full_pytest = bool(args.all or args.full) + if args.terminal_authorization is not None and not ((full_pytest or args.seed_testmon) and args.skip_slow): + parser.error("--terminal-authorization requires --all, --full, or --seed-testmon with --skip-slow") preflight_error = _testmon_preflight( seed_testmon=bool(args.seed_testmon), full_pytest=full_pytest, @@ -2802,8 +2865,10 @@ def main(argv: list[str] | None = None) -> int: if args.seed_testmon: seed_identity = _testmon_seed_identity( git_head=head, + git_tree=_git_committed_tree(), skip_slow=bool(args.skip_slow), lab=bool(args.lab), + terminal_authorization=args.terminal_authorization, ) resume_testmon_seed = _testmon_seed_can_resume(seed_identity) prepared_seed_attempt = _prepare_testmon_seed_attempt( @@ -2940,13 +3005,23 @@ def main(argv: list[str] | None = None) -> int: verification_scope = VerificationScope.NON_TEST release_baseline_allowed: bool | None = None elif full_pytest or args.seed_testmon: - verification_scope = VerificationScope.RELEASE_BASELINE - release_baseline_allowed = exit_code == 0 if full_pytest else _testmon_release_baseline_permission() + narrow_terminal = bool(args.skip_slow) + authorized_narrow_terminal = args.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value + verification_scope = ( + VerificationScope.NARROW_TERMINAL if narrow_terminal else VerificationScope.RELEASE_BASELINE + ) + if full_pytest: + release_baseline_allowed = exit_code == 0 and (not narrow_terminal or authorized_narrow_terminal) + else: + release_baseline_allowed = _testmon_release_baseline_permission() and ( + not narrow_terminal or authorized_narrow_terminal + ) else: verification_scope = VerificationScope.AFFECTED release_baseline_allowed = _testmon_release_baseline_permission() history_entry["verification_scope"] = verification_scope.value history_entry["release_baseline_allowed"] = release_baseline_allowed + history_entry["terminal_authorization"] = args.terminal_authorization if release_baseline_allowed is False and tier in {"testmon", "lab", "seed-testmon"}: sys.stderr.write( "verify: affected-test selection is usable, but the current testmon state does not grant " diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index cdd3d25afb..fe0ecc8446 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -426,3 +426,69 @@ def test_record_full_verify_rejects_success_without_structured_release_permissio ledger = merge_boundary._read_ledger() assert ledger["last_full_verify"]["accepted"] is False assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_record_full_verify_rejects_skip_slow_without_typed_authorization( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps({"verification_scope": "narrow-terminal", "release_baseline_allowed": False}), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow") == 0 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_record_full_verify_accepts_explicit_typed_narrow_terminal_authorization( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps( + { + "verification_scope": "narrow-terminal", + "terminal_authorization": "narrow-terminal", + "release_baseline_allowed": True, + } + ), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow") == 0 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is True + assert merge_boundary.cmd_train_status(as_json=False) == 0 + + +def test_record_full_verify_rejects_untyped_scope_even_when_permission_is_true( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps({"release_baseline_allowed": True}), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all") == 0 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False + assert merge_boundary.cmd_train_status(as_json=False) == 1 diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 0cb4049601..bbcdbf1f78 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -11,7 +11,7 @@ import pytest -from devtools import verify_runs +from devtools import verify, verify_runs from devtools.testmon_state import ( BaselineStatus, BindingMode, @@ -489,10 +489,12 @@ def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pyte TESTMON_DATA.write_text("partial") identity = { "git_head": "head", + "git_tree": "tree-hash", "worktree_fingerprint": "tree", "python": "3.13", "skip_slow": True, "lab": False, + "terminal_authorization": None, } TESTMON_SEED_ATTEMPT.write_text( json.dumps( @@ -511,7 +513,8 @@ def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pyte ) assert _testmon_seed_can_resume(identity) is True - assert _testmon_seed_can_resume({**identity, "git_head": "other"}) is True + assert _testmon_seed_can_resume({**identity, "git_head": "other", "git_tree": "tree-hash"}) is True + assert _testmon_seed_can_resume({**identity, "git_tree": "different-tree"}) is False assert _testmon_seed_can_resume({**identity, "worktree_fingerprint": "changed"}) is False assert _testmon_seed_can_resume({**identity, "skip_slow": False}) is False @@ -529,10 +532,12 @@ def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, mo ) identity = { "git_head": "head", + "git_tree": "tree-hash", "worktree_fingerprint": "tree", "python": "3.13", "skip_slow": True, "lab": False, + "terminal_authorization": None, } TESTMON_SEED_ATTEMPT.write_text( json.dumps( @@ -546,10 +551,12 @@ def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, mo ) ) - assert _testmon_seed_can_resume({**identity, "git_head": "fixed"}) is True + assert _testmon_seed_can_resume({**identity, "git_head": "fixed", "git_tree": "tree-hash"}) is True run = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="fixed") - prepared = _prepare_testmon_seed_attempt(identity={**identity, "git_head": "fixed"}, run=run, resume=True) + prepared = _prepare_testmon_seed_attempt( + identity={**identity, "git_head": "fixed", "git_tree": "tree-hash"}, run=run, resume=True + ) assert prepared["expected_nodeids"] == expected assert prepared["expected_count"] == 1 @@ -629,7 +636,7 @@ def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) "git_head": "head", "worktree_fingerprint": "tree", "python": "python", - "skip_slow": True, + "skip_slow": False, "lab": False, }, "resume": True, @@ -802,7 +809,7 @@ def test_seed_receipt_classifies_every_node_terminal_outcome( "git_head": "head", "worktree_fingerprint": "tree", "python": "python", - "skip_slow": True, + "skip_slow": False, "lab": False, }, "resume": False, @@ -966,7 +973,7 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon "git_head": "head", "worktree_fingerprint": "tree", "python": "python", - "skip_slow": True, + "skip_slow": False, "lab": False, }, "resume": False, @@ -995,7 +1002,7 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon "git_head": "head", "worktree_fingerprint": "tree", "python": "python", - "skip_slow": True, + "skip_slow": False, "lab": False, }, "resume": False, @@ -1058,6 +1065,72 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon assert orphaned["status"] == "incomplete" +def test_resumed_seed_persists_full_selection_before_stamp_publication( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir() + (artifact_dir / "selection.json").write_text( + json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) + ) + (artifact_dir / "events.jsonl").write_text( + json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" + ) + TESTMON_DATA.parent.mkdir(parents=True) + with sqlite3.connect(TESTMON_DATA) as connection: + connection.execute("create table environment (id integer primary key, environment_name text)") + connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") + connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") + connection.execute("create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)") + connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) + connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) + connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) + _write_run_receipt(tmp_path, "resumed") + prepared = { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "git_tree": "tree-hash", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + "terminal_authorization": None, + }, + "resume": True, + "expected_nodeids": expected, + "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), + "prior_node_outcomes": [{"nodeid": expected[1], "outcome": "passed"}], + "run_id": "resumed", + "artifact_dir": ".cache/verify/runs/resumed", + } + original_write = verify._atomic_write_json + + def crash_before_stamp(path: Path, payload: object) -> None: + if path == TESTMON_SEED_STAMP: + raise RuntimeError("simulated crash before seed publication") + original_write(path, payload) + + with patch("devtools.verify._atomic_write_json", side_effect=crash_before_stamp): + with pytest.raises(RuntimeError, match="before seed publication"): + _finalize_testmon_seed_attempt( + prepared=prepared, + step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + + persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) + assert persisted["status"] == "complete" + assert persisted["expected_count"] == len(expected) + assert persisted["selection"]["selected_count"] == len(expected) + assert persisted["selection"]["selected_nodeids_omitted"] == 0 + assert not TESTMON_SEED_STAMP.exists() + + def test_classify_late_sigterm_after_pytest_success_summary() -> None: diagnosis = classify_pytest_result( returncode=-15, @@ -1929,6 +2002,36 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo assert '"exit_code": 1' in payload +@pytest.mark.parametrize( + ("authorization", "expected_permission"), + [(None, False), ("narrow-terminal", True)], +) +def test_verify_main_types_skip_slow_terminal_authority( + capsys: pytest.CaptureFixture[str], authorization: str | None, expected_permission: bool +) -> None: + def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: + del label, command, kwargs + return 0, 0.01, {} + + argv = ["--all", "--skip-slow"] + if authorization is not None: + argv.extend(["--terminal-authorization", authorization]) + with ( + patch("devtools.verify._run", side_effect=fake_run), + patch("devtools.verify.build_verify_steps", return_value=[("pytest full", ["pytest"])]), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + ): + assert main([*argv, "--json"]) == 0 + + payload = json.loads(capsys.readouterr().out) + assert payload["verification_scope"] == "narrow-terminal" + assert payload["release_baseline_allowed"] is expected_permission + assert payload["terminal_authorization"] == authorization + + def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.CaptureFixture[str]) -> None: with ( patch("devtools.verify.build_verify_steps", side_effect=PytestResourceError("only 0.50 GiB available")), From a779761d131a1bee776a3e8c4074a99b32637fbf Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 04:30:20 +0200 Subject: [PATCH 14/34] fix: guard skipped slow seed promotion --- devtools/testmon_state.py | 43 ++++++++++++++++++----- tests/unit/devtools/test_testmon_state.py | 26 ++++++++++++++ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index fec977eb5e..6685858ab9 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -63,12 +63,17 @@ class TestmonIdentity: python: str skip_slow: bool lab: bool + git_tree: str | None = None + terminal_authorization: str | None = None @classmethod def from_mapping(cls, value: Mapping[str, Any]) -> TestmonIdentity: git_head = value.get("git_head") if git_head is not None and (not isinstance(git_head, str) or not git_head): raise ValueError("identity.git_head must be a non-empty string or null") + git_tree = value.get("git_tree") + if git_tree is not None and (not isinstance(git_tree, str) or not git_tree): + raise ValueError("identity.git_tree must be a non-empty string or null") worktree = value.get("worktree_fingerprint") python = value.get("python") if not isinstance(worktree, str) or not worktree: @@ -77,7 +82,20 @@ def from_mapping(cls, value: Mapping[str, Any]) -> TestmonIdentity: raise ValueError("identity.python must be a non-empty string") if not isinstance(value.get("skip_slow"), bool) or not isinstance(value.get("lab"), bool): raise ValueError("identity selection flags must be booleans") - return cls(git_head, worktree, python, value["skip_slow"], value["lab"]) + terminal_authorization = value.get("terminal_authorization") + if terminal_authorization is not None and terminal_authorization not in { + authorization.value for authorization in TerminalAuthorization + }: + raise ValueError("identity.terminal_authorization is invalid") + return cls( + git_head, + worktree, + python, + value["skip_slow"], + value["lab"], + git_tree, + terminal_authorization, + ) def as_dict(self) -> dict[str, Any]: return { @@ -86,6 +104,8 @@ def as_dict(self) -> dict[str, Any]: "python": self.python, "skip_slow": self.skip_slow, "lab": self.lab, + "git_tree": self.git_tree, + "terminal_authorization": self.terminal_authorization, } @@ -750,6 +770,10 @@ def stamp_from_attempt( graph = inspect_testmon_database(data_path, [str(nodeid) for nodeid in expected]) if not graph.usable_for_selection: return None + try: + typed_identity = TestmonIdentity.from_mapping(identity) + except ValueError: + return None baseline = ( BaselineStatus.GREEN if attempt.get("status") == "complete" @@ -758,18 +782,21 @@ def stamp_from_attempt( and not graph.failed_nodeids else BaselineStatus.RED ) + raw_scope = attempt.get("verification_scope") + if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: + return None + terminal_authorized = ( + typed_identity.skip_slow is True + and raw_scope == VerificationScope.NARROW_TERMINAL.value + and typed_identity.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value + ) + if baseline is BaselineStatus.GREEN and typed_identity.skip_slow and not terminal_authorized: + baseline = BaselineStatus.RED raw_permission = attempt.get("release_baseline_allowed") if raw_permission is not None and ( not isinstance(raw_permission, bool) or raw_permission != (baseline is BaselineStatus.GREEN) ): return None - raw_scope = attempt.get("verification_scope") - if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: - return None - try: - typed_identity = TestmonIdentity.from_mapping(identity) - except ValueError: - return None raw_binding = attempt.get("binding") if raw_binding is None: typed_binding = TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())) diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index 543dde0dd4..6dafa70602 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -57,10 +57,12 @@ def _attempt(data: Path, *, outcomes: tuple[str, str] = ("passed", "failed")) -> "status": "reusable", "identity": { "git_head": "head", + "git_tree": "tree-hash", "worktree_fingerprint": "tree", "python": "python", "skip_slow": True, "lab": False, + "terminal_authorization": "narrow-terminal", }, "selection": { "selected_count": len(NODEIDS), @@ -69,6 +71,8 @@ def _attempt(data: Path, *, outcomes: tuple[str, str] = ("passed", "failed")) -> "expected_nodeids": list(NODEIDS), "expected_count": len(NODEIDS), "expected_digest": hashlib.sha256("\n".join(sorted(NODEIDS)).encode()).hexdigest(), + "verification_scope": "narrow-terminal", + "release_baseline_allowed": False, "node_outcomes": [ {"nodeid": nodeid, "outcome": outcome} for nodeid, outcome in zip(NODEIDS, outcomes, strict=True) ], @@ -119,6 +123,7 @@ def test_incomplete_attempt_fails_closed(tmp_path: Path) -> None: _write_graph(data) attempt = _attempt(data, outcomes=("passed", "passed")) attempt["exit_code"] = 0 + attempt["release_baseline_allowed"] = True attempt["status"] = "incomplete" stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) @@ -133,6 +138,26 @@ def test_incomplete_attempt_fails_closed(tmp_path: Path) -> None: assert completed.release_baseline_allowed +def test_green_skipped_slow_attempt_without_typed_terminal_authority_is_selection_only(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data, outcomes=("passed", "passed")) + attempt["status"] = "complete" + attempt["exit_code"] = 0 + identity = dict(attempt["identity"]) + identity["terminal_authorization"] = None + attempt["identity"] = identity + attempt["verification_scope"] = "narrow-terminal" + attempt["release_baseline_allowed"] = False + + stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert stamp is not None + assert stamp.baseline_status is BaselineStatus.RED + assert stamp.affected_selection_allowed + assert not stamp.release_baseline_allowed + + def test_malformed_sqlite_and_stale_stamp_fail_closed(tmp_path: Path) -> None: malformed = tmp_path / "malformed" malformed.write_bytes(b"not sqlite") @@ -154,6 +179,7 @@ def test_attempt_and_green_stamp_artifacts_fail_closed_when_malformed(tmp_path: _write_graph(data) attempt = _attempt(data, outcomes=("passed", "passed")) attempt["exit_code"] = 0 + attempt["release_baseline_allowed"] = True attempt["artifact_dir"] = "/tmp/outside-testmon-run" assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None From 708d0691a8d9b7abde8467266a7a62ce26be99f8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 04:40:16 +0200 Subject: [PATCH 15/34] fix: require typed seed promotion authority --- devtools/testmon_state.py | 13 ++++ devtools/verify.py | 3 + tests/unit/devtools/test_testmon_bootstrap.py | 66 ++++++++++++++++++- tests/unit/devtools/test_verify.py | 43 +++++++++--- 4 files changed, 115 insertions(+), 10 deletions(-) diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index 6685858ab9..721967516e 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -667,6 +667,11 @@ def validate_stamp( stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) if not stamp.release_baseline_allowed: return None + if ( + stamp.identity.skip_slow + and stamp.identity.terminal_authorization != TerminalAuthorization.NARROW_TERMINAL.value + ): + return None if Path(stamp.binding.checkout_root).resolve() != checkout_root.resolve(): return None if file_fingerprint(data_path) != stamp.testmon_data: @@ -793,6 +798,14 @@ def stamp_from_attempt( if baseline is BaselineStatus.GREEN and typed_identity.skip_slow and not terminal_authorized: baseline = BaselineStatus.RED raw_permission = attempt.get("release_baseline_allowed") + if baseline is BaselineStatus.GREEN and ( + raw_scope != VerificationScope.NARROW_TERMINAL.value + if typed_identity.skip_slow + else raw_scope != VerificationScope.RELEASE_BASELINE.value + ): + baseline = BaselineStatus.RED + if baseline is BaselineStatus.GREEN and raw_permission is not True: + baseline = BaselineStatus.RED if raw_permission is not None and ( not isinstance(raw_permission, bool) or raw_permission != (baseline is BaselineStatus.GREEN) ): diff --git a/devtools/verify.py b/devtools/verify.py index 98f377991b..2bb500491e 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2620,6 +2620,9 @@ def _finalize_testmon_seed_attempt( "run_id": prepared.get("run_id"), "artifact_dir": prepared.get("artifact_dir"), "testmon_data": _file_fingerprint(TESTMON_DATA), + "verification_scope": seed_scope, + "terminal_authorization": (TerminalAuthorization.NARROW_TERMINAL.value if terminal_authorized else None), + "release_baseline_allowed": release_eligible, } reusable_stamp = stamp_from_attempt( attempt_candidate, diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index f6c539cb86..4541947fb3 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -66,7 +66,7 @@ def _write_valid_seed_stamp(path: Path, *, protocol_version: int = PROTOCOL_VERS True, 0, graph, - _TestmonIdentity("head", "tree", "python", True, False), + _TestmonIdentity("head", "tree", "python", True, False, None, "narrow-terminal"), _TestmonBinding(BindingMode.EXACT, str(path.parent.resolve())), file_fingerprint(data), "seed", @@ -363,6 +363,70 @@ def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) assert "checkout-bound selection attempt" in rebound_decision.reason +def test_complete_untyped_green_attempt_bootstraps_only_as_selection_state(tmp_path: Path) -> None: + main_root = tmp_path / "main" + main_data = main_root / "testmondata" + _write_sqlite_db(main_data, rows=("tests/test.py::test_passed",)) + attempt = main_root / "seed-attempt.json" + attempt.write_text( + json.dumps( + { + "protocol_version": PROTOCOL_VERSION, + "status": "complete", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + }, + "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, + "expected_nodeids": ["tests/test.py::test_passed"], + "expected_count": 1, + "expected_digest": hashlib.sha256(b"tests/test.py::test_passed").hexdigest(), + "node_outcomes": [{"nodeid": "tests/test.py::test_passed", "outcome": "passed"}], + "exit_code": 0, + "run_id": "green-run", + "artifact_dir": ".cache/verify/runs/green-run", + "testmon_data": file_fingerprint(main_data), + } + ) + ) + artifact = main_root / ".cache" / "verify" / "runs" / "green-run" + artifact.mkdir(parents=True) + (artifact / "run.json").write_text( + json.dumps( + { + "run_id": "green-run", + "checkout_root": str(main_root.resolve()), + "artifact_dir": ".cache/verify/runs/green-run", + } + ) + ) + + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=tmp_path / "lane" / "testmondata", + local_seed_stamp=tmp_path / "lane" / "seed.json", + main_testmon_data=main_data, + main_seed_stamp=main_root / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + ) + + assert decision.should_bootstrap + assert decision.selection_only + assert bootstrap_testmon_seed_files( + decision, + local_testmon_data=tmp_path / "lane" / "testmondata", + local_seed_stamp=tmp_path / "lane" / "seed.json", + local_seed_attempt=tmp_path / "lane" / "seed-attempt.json", + checkout_root=tmp_path / "lane", + inherited_from=main_root, + ) + assert not (tmp_path / "lane" / "seed.json").exists() + + def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: """Partial local state (e.g. a stale stamp with no db, or vice versa) still needs a fresh copy.""" local_stamp = tmp_path / "local" / "seed.json" diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index bbcdbf1f78..059da8c5f2 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -112,7 +112,7 @@ def _write_real_testmon_state(nodeids: tuple[str, ...] = ("tests/test_a.py::test True, 0, GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()), - _TestmonIdentity("current-head", "covered", "python", True, False), + _TestmonIdentity("current-head", "covered", "python", True, False, None, "narrow-terminal"), _TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())), file_fingerprint(TESTMON_DATA), "seed", @@ -991,6 +991,31 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon assert stamp["status"] == "usable" assert stamp["collection"]["expected_count"] == 2 + _write_run_receipt(tmp_path, "run-authorized") + authorized_receipt = _finalize_testmon_seed_attempt( + prepared={ + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "git_tree": "tree-hash", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + "terminal_authorization": "narrow-terminal", + }, + "resume": False, + "expected_nodeids": [], + "run_id": "run-authorized", + "artifact_dir": ".cache/verify/runs/run-authorized", + }, + step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 0}], + exit_code=0, + ) + assert authorized_receipt["status"] == "complete" + assert authorized_receipt["release_baseline_allowed"] is True + _write_run_receipt(tmp_path, "run-red") with sqlite3.connect(TESTMON_DATA) as connection: connection.execute("update test_execution set failed = 1 where test_name = ?", (expected[0],)) @@ -2003,19 +2028,19 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo @pytest.mark.parametrize( - ("authorization", "expected_permission"), - [(None, False), ("narrow-terminal", True)], + ("argv", "expected_scope", "expected_permission"), + [ + (["--all", "--skip-slow"], "narrow-terminal", False), + (["--all", "--skip-slow", "--terminal-authorization", "narrow-terminal"], "narrow-terminal", True), + ], ) def test_verify_main_types_skip_slow_terminal_authority( - capsys: pytest.CaptureFixture[str], authorization: str | None, expected_permission: bool + capsys: pytest.CaptureFixture[str], argv: list[str], expected_scope: str, expected_permission: bool ) -> None: def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: del label, command, kwargs return 0, 0.01, {} - argv = ["--all", "--skip-slow"] - if authorization is not None: - argv.extend(["--terminal-authorization", authorization]) with ( patch("devtools.verify._run", side_effect=fake_run), patch("devtools.verify.build_verify_steps", return_value=[("pytest full", ["pytest"])]), @@ -2027,9 +2052,9 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo assert main([*argv, "--json"]) == 0 payload = json.loads(capsys.readouterr().out) - assert payload["verification_scope"] == "narrow-terminal" + assert payload["verification_scope"] == expected_scope assert payload["release_baseline_allowed"] is expected_permission - assert payload["terminal_authorization"] == authorization + assert payload["terminal_authorization"] == ("narrow-terminal" if expected_permission else None) def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.CaptureFixture[str]) -> None: From 5e01ad77489466e30050f60740820c4a48c9de52 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 04:45:26 +0200 Subject: [PATCH 16/34] fix: validate persisted train authority --- devtools/merge_boundary.py | 7 +++++++ tests/unit/devtools/test_merge_boundary.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index 266abd022c..94e46def9e 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -72,6 +72,7 @@ from typing import Any from devtools import merge_gate +from devtools.testmon_state import TerminalAuthorization, VerificationScope _LEDGER_PATH = Path(".cache/verify/merge-gate/merge-train-ledger.json") @@ -133,11 +134,17 @@ def _append_merge_entry(pr: int, head_sha: str, title: str) -> None: def _pending_prs_since_last_full_verify(ledger: dict[str, Any]) -> list[dict[str, Any]]: last_verify = ledger.get("last_full_verify") or {} + scope = last_verify.get("verification_scope") + terminal_authorized = scope == VerificationScope.RELEASE_BASELINE.value or ( + scope == VerificationScope.NARROW_TERMINAL.value + and last_verify.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value + ) last_verify_at = ( last_verify.get("at", 0.0) if last_verify.get("accepted") is True and last_verify.get("exit_code") == 0 and last_verify.get("release_baseline_allowed") is True + and terminal_authorized else 0.0 ) return [entry for entry in ledger.get("merges", []) if entry.get("merged_at", 0.0) > last_verify_at] diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index fe0ecc8446..ae6f504072 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -373,6 +373,25 @@ def test_train_status_blocks_when_pr_merged_after_last_full_verify( assert merge_boundary.cmd_train_status(as_json=False) == 1 +def test_train_status_rejects_untyped_accepted_terminal_ledger(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._write_ledger( + { + "merges": [], + "last_full_verify": { + "at": 1000.0, + "command": "devtools verify --all", + "exit_code": 0, + "release_baseline_allowed": True, + "accepted": True, + }, + } + ) + merge_boundary._append_merge_entry(1, "sha1", "some title") + + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + def test_record_full_verify_clears_pending_prs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) merge_boundary._append_merge_entry(1, "sha1", "some title") From c62673603ca32ad7341827268e8c7c0e0adda124 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 04:53:38 +0200 Subject: [PATCH 17/34] test: satisfy testmon authority typing --- tests/unit/devtools/test_testmon_state.py | 4 +++- tests/unit/devtools/test_verify.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index 6dafa70602..eb6bf060fd 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -144,7 +144,9 @@ def test_green_skipped_slow_attempt_without_typed_terminal_authority_is_selectio attempt = _attempt(data, outcomes=("passed", "passed")) attempt["status"] = "complete" attempt["exit_code"] = 0 - identity = dict(attempt["identity"]) + raw_identity = attempt["identity"] + assert isinstance(raw_identity, dict) + identity = dict(raw_identity) identity["terminal_authorization"] = None attempt["identity"] = identity attempt["verification_scope"] = "narrow-terminal" diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 059da8c5f2..68503742c8 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1138,6 +1138,7 @@ def test_resumed_seed_persists_full_selection_before_stamp_publication( def crash_before_stamp(path: Path, payload: object) -> None: if path == TESTMON_SEED_STAMP: raise RuntimeError("simulated crash before seed publication") + assert isinstance(payload, dict) original_write(path, payload) with patch("devtools.verify._atomic_write_json", side_effect=crash_before_stamp): From 0c2390ccabd5806ecca757342e4bd98fce995662 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 07:32:41 +0200 Subject: [PATCH 18/34] fix: bind terminal verification to merged master --- devtools/merge_boundary.py | 115 ++++++++++++++--- devtools/merge_gate.py | 34 ++---- tests/unit/devtools/test_merge_boundary.py | 136 +++++++++++++++++++-- tests/unit/devtools/test_merge_gate.py | 23 +++- 4 files changed, 260 insertions(+), 48 deletions(-) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index 94e46def9e..4bdc971851 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -67,12 +67,13 @@ import re import subprocess import sys +import tempfile import time from pathlib import Path from typing import Any from devtools import merge_gate -from devtools.testmon_state import TerminalAuthorization, VerificationScope +from devtools.testmon_state import VerificationScope _LEDGER_PATH = Path(".cache/verify/merge-gate/merge-train-ledger.json") @@ -135,16 +136,12 @@ def _append_merge_entry(pr: int, head_sha: str, title: str) -> None: def _pending_prs_since_last_full_verify(ledger: dict[str, Any]) -> list[dict[str, Any]]: last_verify = ledger.get("last_full_verify") or {} scope = last_verify.get("verification_scope") - terminal_authorized = scope == VerificationScope.RELEASE_BASELINE.value or ( - scope == VerificationScope.NARROW_TERMINAL.value - and last_verify.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value - ) last_verify_at = ( last_verify.get("at", 0.0) if last_verify.get("accepted") is True and last_verify.get("exit_code") == 0 and last_verify.get("release_baseline_allowed") is True - and terminal_authorized + and scope == VerificationScope.RELEASE_BASELINE.value else 0.0 ) return [entry for entry in ledger.get("merges", []) if entry.get("merged_at", 0.0) > last_verify_at] @@ -160,6 +157,76 @@ def _receipt_is_fresh_for_head(pr: int, head_sha: str, max_age_s: int) -> bool: return bool(age_s <= max_age_s) +def _fetched_merged_default_branch_sha(pr: int) -> str | None: + """Fetch the default branch and return its post-merge commit only.""" + try: + repo = _gh_json(["repo", "view", "--json", "defaultBranchRef"]) + default_ref = repo.get("defaultBranchRef") + branch = default_ref.get("name") if isinstance(default_ref, dict) else None + if not isinstance(branch, str) or not branch: + return None + merged = _gh_json(["pr", "view", str(pr), "--json", "state,mergeCommit"]) + merge_commit = merged.get("mergeCommit") + merge_sha = merge_commit.get("oid") if isinstance(merge_commit, dict) else None + if merged.get("state") != "MERGED" or not isinstance(merge_sha, str) or not merge_sha: + return None + fetch = subprocess.run( + ["git", "fetch", "origin", branch], + capture_output=True, + text=True, + timeout=120, + ) + if fetch.returncode != 0: + return None + target = subprocess.run( + ["git", "rev-parse", "FETCH_HEAD"], + capture_output=True, + text=True, + timeout=15, + ) + if target.returncode != 0: + return None + target_sha = target.stdout.strip() + if not target_sha: + return None + included = subprocess.run( + ["git", "merge-base", "--is-ancestor", merge_sha, target_sha], + capture_output=True, + text=True, + timeout=15, + ) + return target_sha if included.returncode == 0 else None + except (RuntimeError, json.JSONDecodeError, OSError, subprocess.SubprocessError): + return None + + +def _run_post_merge_terminal_verify(command: str, target_sha: str) -> int: + """Run terminal verification in a detached worktree at the fetched target.""" + repo_root = Path.cwd() + with tempfile.TemporaryDirectory(prefix="polylogue-merge-terminal-") as raw_worktree: + worktree = Path(raw_worktree) + add = subprocess.run( + ["git", "worktree", "add", "--detach", str(worktree), target_sha], + capture_output=True, + text=True, + timeout=120, + cwd=repo_root, + ) + if add.returncode != 0: + print(f"REFUSING terminal verify: could not materialize fetched target {target_sha[:8]}", file=sys.stderr) + return 1 + try: + return cmd_record_full_verify(command, target_sha=target_sha, cwd=worktree) + finally: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + capture_output=True, + text=True, + timeout=120, + cwd=repo_root, + ) + + def cmd_merge( pr: int, *, @@ -234,8 +301,15 @@ def cmd_merge( _append_merge_entry(pr, head_sha, clean_title) if with_verify: + target_sha = _fetched_merged_default_branch_sha(pr) + if target_sha is None: + print( + "REFUSING terminal verify: the fetched default branch does not prove this squash merge is included", + file=sys.stderr, + ) + return 1 print(f"running post-merge broad verify (merge-train terminal step): {verify_command!r}") - return cmd_record_full_verify(verify_command) + return _run_post_merge_terminal_verify(verify_command, target_sha) print( "REMINDER: this merge-train's terminal ledger step (one full-suite verify since the last " @@ -280,14 +354,14 @@ def cmd_train_status(as_json: bool) -> int: return 1 -def cmd_record_full_verify(command: str) -> int: +def cmd_record_full_verify(command: str, *, target_sha: str | None = None, cwd: Path | None = None) -> int: argv = command.split() if not argv: print("REFUSING: --command is empty after splitting", file=sys.stderr) return 2 started = time.time() try: - result = subprocess.run(argv, capture_output=True, text=True) + result = subprocess.run(argv, capture_output=True, text=True, cwd=cwd) except OSError as exc: print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) return 2 @@ -295,13 +369,16 @@ def cmd_record_full_verify(command: str) -> int: release_allowed = merge_gate._release_baseline_permission(result.stdout) verification_scope = merge_gate._verification_scope(result.stdout) terminal_authorization = merge_gate._terminal_authorization(result.stdout) + try: + structured = json.loads(result.stdout) + except (TypeError, json.JSONDecodeError): + structured = None + verified_head = structured.get("git_head") if isinstance(structured, dict) else None accepted = ( result.returncode == 0 and release_allowed is True - and ( - verification_scope == "release-baseline" - or (verification_scope == "narrow-terminal" and terminal_authorization == "narrow-terminal") - ) + and verification_scope == VerificationScope.RELEASE_BASELINE.value + and (target_sha is None or verified_head == target_sha) ) ledger = _read_ledger() @@ -313,6 +390,8 @@ def cmd_record_full_verify(command: str) -> int: "verification_scope": verification_scope, "release_baseline_allowed": release_allowed, "terminal_authorization": terminal_authorization, + "verified_head_sha": verified_head, + "target_sha": target_sha, "accepted": accepted, } _write_ledger(ledger) @@ -323,9 +402,15 @@ def cmd_record_full_verify(command: str) -> int: ) if not accepted and result.returncode == 0: print( - "POST-MERGE BROAD VERIFY DID NOT GRANT release-baseline permission; train-status remains incomplete.", + "POST-MERGE BROAD VERIFY DID NOT GRANT typed release-baseline authority for the selected target; " + "train-status remains incomplete.", file=sys.stderr, ) + if target_sha is not None and verified_head != target_sha: + print( + f"terminal verify reported git_head={verified_head!r}, expected fetched target {target_sha}", + file=sys.stderr, + ) if result.returncode != 0: print(result.stdout[-4000:]) print(result.stderr[-4000:], file=sys.stderr) @@ -335,7 +420,7 @@ def cmd_record_full_verify(command: str) -> int: "before merging further PRs in this train.", file=sys.stderr, ) - return result.returncode + return result.returncode if result.returncode != 0 else (0 if accepted else 1) def main(argv: list[str] | None = None) -> int: diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index c7c567fa2f..7b9415e1f4 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -255,22 +255,6 @@ def _terminal_authorization(stdout: str) -> str | None: return value if value in {authorization.value for authorization in TerminalAuthorization} else None -def _command_verification_scope(command: str) -> str | None: - """Classify legacy commands by argv shape, never by emitted log text.""" - try: - argv = shlex.split(command) - except ValueError: - return None - if argv[:2] != ["devtools", "verify"]: - return None - options = set(argv[2:]) - if options & {"--all", "--full", "--seed-testmon"}: - return VerificationScope.RELEASE_BASELINE.value - if options & {"--quick", "--commit"}: - return VerificationScope.NON_TEST.value - return VerificationScope.AFFECTED.value - - def cmd_record(pr: int, command: str) -> int: info = _gh_json(["pr", "view", str(pr), "--json", "headRefOid,headRefName,body,isDraft"]) head_sha = info["headRefOid"] @@ -325,7 +309,7 @@ def cmd_record(pr: int, command: str) -> int: "branch": info["headRefName"], "command": command, "skips_tests": _command_skips_tests(command), - "verification_scope": _verification_scope(result.stdout) or _command_verification_scope(command), + "verification_scope": _verification_scope(result.stdout), "release_baseline_allowed": _release_baseline_permission(result.stdout), "terminal_authorization": _terminal_authorization(result.stdout), "exit_code": result.returncode, @@ -535,12 +519,16 @@ def cmd_check( verdict.ok = False verdict.reasons.append(f"receipt exit_code is {receipt.get('exit_code')}, not 0") verification_scope = receipt.get("verification_scope") - if verification_scope is None: - verification_scope = _command_verification_scope(str(receipt.get("command", ""))) - if ( - verification_scope == VerificationScope.RELEASE_BASELINE.value - and receipt.get("release_baseline_allowed") is not True - ): + if verification_scope not in {scope.value for scope in VerificationScope}: + verdict.ok = False + verdict.reasons.append( + "verification receipt lacks a valid typed verification_scope; command text cannot grant authority" + ) + release_allowed = receipt.get("release_baseline_allowed") + if not isinstance(release_allowed, bool): + verdict.ok = False + verdict.reasons.append("verification receipt lacks typed release_baseline_allowed permission") + if verification_scope == VerificationScope.RELEASE_BASELINE.value and release_allowed is not True: verdict.ok = False verdict.reasons.append( "release-baseline verification receipt does not grant release_baseline_allowed=true" diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index ae6f504072..d67a0355c6 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -87,7 +87,11 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: return MagicMock(returncode=0, stdout="", stderr="") - return MagicMock(returncode=local_exit, stdout="ok\n", stderr="") + return MagicMock( + returncode=local_exit, + stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), + stderr="", + ) return _run @@ -320,12 +324,24 @@ def run(cmd: list[str], **kwargs: Any) -> MagicMock: if cmd[:3] == ["devtools", "verify", "--all"]: return MagicMock( returncode=0, - stdout=json.dumps({"verification_scope": "release-baseline", "release_baseline_allowed": True}), + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), stderr="", ) return base(cmd, **kwargs) monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(merge_boundary, "_fetched_merged_default_branch_sha", lambda _pr: "merged-master") + monkeypatch.setattr( + merge_boundary, + "_run_post_merge_terminal_verify", + lambda command, target: merge_boundary.cmd_record_full_verify(command, target_sha=target), + ) exit_code = merge_boundary.cmd_merge( 42, @@ -346,6 +362,108 @@ def run(cmd: list[str], **kwargs: Any) -> MagicMock: assert merge_boundary.cmd_train_status(as_json=False) == 0 +def test_merge_with_verify_returns_nonzero_when_terminal_authority_is_rejected( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view)) + monkeypatch.setattr(merge_boundary, "_fetched_merged_default_branch_sha", lambda _pr: "merged-master") + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", lambda _command, _target: 1) + + assert ( + merge_boundary.cmd_merge( + 42, + command="devtools test x", + max_age_s=3600, + poll_rounds=1, + poll_interval_s=0, + dry_run=False, + with_verify=True, + verify_command="devtools verify --all", + ) + == 1 + ) + assert merge_boundary._read_ledger()["merges"] + + +def test_post_merge_terminal_verify_rejects_stale_feature_head(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "feature-head", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 1 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False + + +def test_fetched_default_branch_must_include_squash_merge(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + calls: list[list[str]] = [] + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + calls.append(cmd) + if cmd[:3] == ["gh", "repo", "view"]: + return MagicMock(returncode=0, stdout=json.dumps({"defaultBranchRef": {"name": "master"}}), stderr="") + if cmd[:3] == ["gh", "pr", "view"]: + return MagicMock( + returncode=0, + stdout=json.dumps({"state": "MERGED", "mergeCommit": {"oid": "squash-sha"}}), + stderr="", + ) + if cmd[:3] == ["git", "fetch", "origin"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:3] == ["git", "rev-parse", "FETCH_HEAD"]: + return MagicMock(returncode=0, stdout="stale-feature-sha\n", stderr="") + if cmd[:3] == ["git", "merge-base", "--is-ancestor"]: + return MagicMock(returncode=1, stdout="", stderr="") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + + assert merge_boundary._fetched_merged_default_branch_sha(42) is None + assert ["git", "fetch", "origin", "master"] in calls + + +def test_fetched_default_branch_sha_is_the_verified_terminal_target( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + if cmd[:3] == ["gh", "repo", "view"]: + return MagicMock(returncode=0, stdout=json.dumps({"defaultBranchRef": {"name": "master"}}), stderr="") + if cmd[:3] == ["gh", "pr", "view"]: + return MagicMock( + returncode=0, + stdout=json.dumps({"state": "MERGED", "mergeCommit": {"oid": "squash-sha"}}), + stderr="", + ) + if cmd[:3] == ["git", "fetch", "origin"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:3] == ["git", "rev-parse", "FETCH_HEAD"]: + return MagicMock(returncode=0, stdout="merged-master-sha\n", stderr="") + if cmd[:3] == ["git", "merge-base", "--is-ancestor"]: + return MagicMock(returncode=0, stdout="", stderr="") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + + assert merge_boundary._fetched_merged_default_branch_sha(42) == "merged-master-sha" + + # --------------------------------------------------------------------------- # train-status / record-full-verify # --------------------------------------------------------------------------- @@ -441,7 +559,7 @@ def test_record_full_verify_rejects_success_without_structured_release_permissio lambda _cmd, **_kwargs: MagicMock(returncode=0, stdout="all good\n", stderr=""), ) - assert merge_boundary.cmd_record_full_verify("devtools verify --all") == 0 + assert merge_boundary.cmd_record_full_verify("devtools verify --all") == 1 ledger = merge_boundary._read_ledger() assert ledger["last_full_verify"]["accepted"] is False assert merge_boundary.cmd_train_status(as_json=False) == 1 @@ -462,12 +580,12 @@ def test_record_full_verify_rejects_skip_slow_without_typed_authorization( ), ) - assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow") == 0 + assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow") == 1 assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False assert merge_boundary.cmd_train_status(as_json=False) == 1 -def test_record_full_verify_accepts_explicit_typed_narrow_terminal_authorization( +def test_record_full_verify_rejects_explicit_typed_narrow_terminal_authorization( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.chdir(tmp_path) @@ -488,9 +606,9 @@ def test_record_full_verify_accepts_explicit_typed_narrow_terminal_authorization ), ) - assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow") == 0 - assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is True - assert merge_boundary.cmd_train_status(as_json=False) == 0 + assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow") == 1 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False + assert merge_boundary.cmd_train_status(as_json=False) == 1 def test_record_full_verify_rejects_untyped_scope_even_when_permission_is_true( @@ -508,6 +626,6 @@ def test_record_full_verify_rejects_untyped_scope_even_when_permission_is_true( ), ) - assert merge_boundary.cmd_record_full_verify("devtools verify --all") == 0 + assert merge_boundary.cmd_record_full_verify("devtools verify --all") == 1 assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False assert merge_boundary.cmd_train_status(as_json=False) == 1 diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 11d643a40d..2bbbfd4d33 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -81,7 +81,11 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: return MagicMock(returncode=0, stdout=" M dirty.py\n" if dirty else "", stderr="") - return MagicMock(returncode=local_exit, stdout="ok\n", stderr="") + return MagicMock( + returncode=local_exit, + stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), + stderr="", + ) return _run @@ -131,6 +135,7 @@ def test_check_blocks_full_receipt_without_release_baseline_permission( _record(monkeypatch, pr_view, command=command) receipt_path = merge_gate._receipt_path(42) receipt = json.loads(receipt_path.read_text()) + receipt["verification_scope"] = "release-baseline" receipt["release_baseline_allowed"] = False receipt_path.write_text(json.dumps(receipt)) @@ -255,6 +260,22 @@ def test_check_ok_when_receipt_fresh_and_matches_head_with_no_late_comments( assert exit_code == 0 +def test_check_rejects_command_text_without_typed_scope_or_permission( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + _record(monkeypatch, pr_view, command="devtools verify --all") + receipt_path = merge_gate._receipt_path(42) + receipt = json.loads(receipt_path.read_text()) + receipt["verification_scope"] = None + receipt["release_baseline_allowed"] = True + receipt_path.write_text(json.dumps(receipt)) + + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view, [])) + assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 1 + + @pytest.mark.parametrize( ("receipt_field", "mutated_value", "reason"), [ From 5a22ae7e29e5335998c7338229ccd7ce98f2a73f Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 07:32:49 +0200 Subject: [PATCH 19/34] fix: publish testmon bootstrap atomically --- devtools/testmon_bootstrap.py | 100 +++++++++++-- tests/unit/devtools/test_testmon_bootstrap.py | 139 ++++++++++++++++++ 2 files changed, 226 insertions(+), 13 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index b8cff9e5ee..3649cd30ac 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -57,8 +57,10 @@ import json import os +import shutil import sqlite3 import subprocess +import tempfile from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path @@ -238,7 +240,9 @@ def _atomic_write_stamp(seed_stamp: Path, stamp: TestmonSeedStamp) -> None: _atomic_write_json(seed_stamp, stamp.as_dict()) -def _rebind_run_receipt(*, source: Path, destination: Path, checkout_root: Path, run_id: str) -> bool: +def _rebind_run_receipt( + *, source: Path, destination: Path, checkout_root: Path, run_id: str, current_run_path: Path | None = None +) -> bool: """Copy the run receipt while rebinding its checkout-local provenance.""" try: payload = json.loads((source / "run.json").read_text(encoding="utf-8")) @@ -255,7 +259,10 @@ def _rebind_run_receipt(*, source: Path, destination: Path, checkout_root: Path, environment["checkout_root"] = str(checkout_root.resolve()) environment["verify_state_origin"] = str(checkout_root.resolve()) _atomic_write_json(destination / "run.json", payload_dict) - _atomic_write_json(checkout_root / ".cache" / "verify" / "current-run.json", payload_dict) + _atomic_write_json( + current_run_path or checkout_root / ".cache" / "verify" / "current-run.json", + payload_dict, + ) return True except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): return False @@ -288,6 +295,33 @@ def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: tmp.unlink(missing_ok=True) +def _publish_staged_bootstrap_files(*, staging_dir: Path, files: list[tuple[Path, Path | None]]) -> None: + """Publish a validated bootstrap as one rollback-capable file set.""" + backup_dir = staging_dir / "backups" + backup_dir.mkdir(parents=True, exist_ok=True) + backups: list[tuple[Path, Path]] = [] + published: list[Path] = [] + try: + for index, (destination, staged) in enumerate(files): + destination.parent.mkdir(parents=True, exist_ok=True) + backup = backup_dir / str(index) + if destination.exists(): + os.replace(destination, backup) + backups.append((destination, backup)) + if staged is not None: + os.replace(staged, destination) + published.append(destination) + except (OSError, ValueError): + for destination in reversed(published): + destination.unlink(missing_ok=True) + for destination, backup in reversed(backups): + if backup.exists(): + os.replace(backup, destination) + raise + finally: + shutil.rmtree(backup_dir, ignore_errors=True) + + def bootstrap_testmon_seed_files( decision: BootstrapDecision, *, @@ -358,23 +392,33 @@ def bootstrap_testmon_seed_files( return False if stamp is None: return False + staging_dir: Path | None = None try: - if decision.main_seed_attempt is not None and decision.selection_only: - local_seed_stamp.unlink(missing_ok=True) - _atomic_copy_sqlite_db(decision.main_testmon_data, local_testmon_data) + local_testmon_data.parent.mkdir(parents=True, exist_ok=True) + staging_dir = Path(tempfile.mkdtemp(prefix=".bootstrap-", dir=str(local_testmon_data.parent))) + staged_data = staging_dir / "testmondata" + staged_stamp = staging_dir / "seed.json" + staged_attempt = staging_dir / "seed-attempt.json" + staged_artifact = staging_dir / "artifact" + staged_current_run = staging_dir / "current-run.json" + + _atomic_copy_sqlite_db(decision.main_testmon_data, staged_data) rebound = stamp.rebound(checkout_root=destination_root, inherited_from=source_root) - refreshed = refresh_stamp(rebound, local_testmon_data) + refreshed = refresh_stamp(rebound, staged_data) if refreshed is None or refreshed.graph != rebound.graph: return False source_artifact = source_root / Path(stamp.artifact_dir) - destination_artifact = destination_root / Path(refreshed.artifact_dir) + destination_artifact = (destination_root / Path(refreshed.artifact_dir)).resolve() + destination_artifact.relative_to(destination_root) if not _rebind_run_receipt( source=source_artifact, - destination=destination_artifact, + destination=staged_artifact, checkout_root=destination_root, run_id=refreshed.run_id, + current_run_path=staged_current_run, ): return False + staged_attempt_path: Path | None = None if decision.main_seed_attempt is not None and decision.selection_only: assert local_seed_attempt is not None source_attempt = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) @@ -386,22 +430,52 @@ def bootstrap_testmon_seed_files( rebound_attempt["binding"] = refreshed.binding.as_dict() rebound_attempt["release_baseline_allowed"] = False rebound_attempt["verification_scope"] = "affected" + validation_root = staging_dir / "validation" + validation_receipt = json.loads(staged_current_run.read_text(encoding="utf-8")) + if not isinstance(validation_receipt, dict): + return False + validation_receipt["checkout_root"] = str(validation_root.resolve()) + validation_receipt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" + _atomic_write_json( + validation_root / ".cache" / "verify" / "runs" / refreshed.run_id / "run.json", + validation_receipt, + ) + validation_attempt = dict(rebound_attempt) + raw_binding = validation_attempt.get("binding") + if not isinstance(raw_binding, Mapping): + return False + validation_binding = dict(raw_binding) + validation_binding["checkout_root"] = str(validation_root.resolve()) + validation_attempt["binding"] = validation_binding if ( stamp_from_attempt( - rebound_attempt, - local_testmon_data, - checkout_root=destination_root, + validation_attempt, + staged_data, + checkout_root=validation_root, protocol_version=decision.protocol_version, ) is None ): return False - _atomic_write_json(local_seed_attempt, rebound_attempt) + _atomic_write_json(staged_attempt, rebound_attempt) + staged_attempt_path = staged_attempt else: - _atomic_write_stamp(local_seed_stamp, refreshed) + _atomic_write_stamp(staged_stamp, refreshed) + publication_files: list[tuple[Path, Path | None]] = [ + (local_testmon_data, staged_data), + (destination_artifact / "run.json", staged_artifact / "run.json"), + (destination_root / ".cache" / "verify" / "current-run.json", staged_current_run), + (local_seed_stamp, None if decision.selection_only else staged_stamp), + ] + if local_seed_attempt is not None: + publication_files.append((local_seed_attempt, staged_attempt_path)) + _publish_staged_bootstrap_files(staging_dir=staging_dir, files=publication_files) return True except (OSError, sqlite3.Error, TypeError, ValueError): return False + finally: + if staging_dir is not None: + shutil.rmtree(staging_dir, ignore_errors=True) def _git_worktree_info(repo_root: Path) -> tuple[bool, Path] | None: diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 4541947fb3..3909abb1d2 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -18,7 +18,9 @@ import hashlib import json import sqlite3 +from collections.abc import Callable from pathlib import Path +from typing import cast import pytest @@ -107,6 +109,67 @@ def _write_sqlite_db(path: Path, *, rows: tuple[str, ...] = ("a", "b")) -> None: conn.close() +def _red_attempt_decision(tmp_path: Path) -> tuple[BootstrapDecision, Path, Path, Path, Path]: + main_root = tmp_path / "main" + main_data = main_root / "testmondata" + _write_sqlite_db(main_data, rows=("tests/test.py::test_passed", "tests/test.py::test_failed")) + attempt = main_root / "seed-attempt.json" + attempt.write_text( + json.dumps( + { + "protocol_version": PROTOCOL_VERSION, + "status": "reusable", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, + "expected_nodeids": ["tests/test.py::test_passed", "tests/test.py::test_failed"], + "expected_count": 2, + "expected_digest": hashlib.sha256( + "\n".join(sorted(["tests/test.py::test_passed", "tests/test.py::test_failed"])).encode() + ).hexdigest(), + "testmon_data": file_fingerprint(main_data), + "node_outcomes": [ + {"nodeid": "tests/test.py::test_passed", "outcome": "passed"}, + {"nodeid": "tests/test.py::test_failed", "outcome": "failed"}, + ], + "exit_code": 1, + "run_id": "red-run", + "artifact_dir": ".cache/verify/runs/red-run", + } + ) + ) + artifact = main_root / ".cache" / "verify" / "runs" / "red-run" + artifact.mkdir(parents=True, exist_ok=True) + (artifact / "run.json").write_text( + json.dumps( + { + "run_id": "red-run", + "checkout_root": str(main_root.resolve()), + "artifact_dir": ".cache/verify/runs/red-run", + } + ) + ) + lane = tmp_path / "lane" + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=lane / "testmondata", + local_seed_stamp=lane / "seed.json", + local_seed_attempt=lane / "seed-attempt.json", + main_testmon_data=main_data, + main_seed_stamp=main_root / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + main_checkout_root=main_root, + local_checkout_root=lane, + ) + return decision, lane / "testmondata", lane / "seed.json", lane / "seed-attempt.json", lane + + def test_not_a_linked_worktree_never_bootstraps(tmp_path: Path) -> None: """The main checkout itself must never "bootstrap from itself".""" decision = decide_testmon_bootstrap( @@ -561,6 +624,82 @@ def test_bootstrap_seed_files_keeps_copied_state_when_stamp_turns_invalid(tmp_pa assert not local_stamp.exists() +def test_bootstrap_graph_mismatch_publishes_no_destination_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main_data = tmp_path / "main" / "testmondata" + main_stamp = tmp_path / "main" / "seed.json" + _write_sqlite_db(main_data) + _write_valid_seed_stamp(main_stamp) + local_data = tmp_path / "lane" / "testmondata" + local_stamp = tmp_path / "lane" / "seed.json" + monkeypatch.setattr(testmon_bootstrap, "refresh_stamp", lambda *_args, **_kwargs: None) + + assert not bootstrap_testmon_seed_files( + BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert not local_data.exists() + assert not local_stamp.exists() + assert not (tmp_path / "lane" / ".cache" / "verify" / "current-run.json").exists() + + +def test_bootstrap_receipt_rebind_failure_publishes_no_destination_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main_data = tmp_path / "main" / "testmondata" + main_stamp = tmp_path / "main" / "seed.json" + _write_sqlite_db(main_data) + _write_valid_seed_stamp(main_stamp) + local_data = tmp_path / "lane" / "testmondata" + local_stamp = tmp_path / "lane" / "seed.json" + monkeypatch.setattr(testmon_bootstrap, "_rebind_run_receipt", lambda **_kwargs: False) + + assert not bootstrap_testmon_seed_files( + BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert not local_data.exists() + assert not local_stamp.exists() + assert not (tmp_path / "lane" / ".cache" / "verify" / "current-run.json").exists() + + +def test_bootstrap_rebound_attempt_failure_publishes_no_destination_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + decision, local_data, local_stamp, local_attempt, lane = _red_attempt_decision(tmp_path) + original_stamp_from_attempt = cast(Callable[..., object], testmon_bootstrap.__dict__["stamp_from_attempt"]) + calls = 0 + + def fail_rebound_attempt(*args: object, **kwargs: object) -> object: + nonlocal calls + calls += 1 + if calls == 2: + return None + return original_stamp_from_attempt(*args, **kwargs) + + monkeypatch.setattr(testmon_bootstrap, "stamp_from_attempt", fail_rebound_attempt) + + assert not bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + local_seed_attempt=local_attempt, + checkout_root=lane, + inherited_from=tmp_path / "main", + ) + assert not local_data.exists() + assert not local_stamp.exists() + assert not local_attempt.exists() + assert not (lane / ".cache" / "verify" / "current-run.json").exists() + + def test_maybe_bootstrap_does_not_migrate_an_untyped_legacy_local_stamp( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From f0cb9990d405f56ece4a04ea37cd087ea1e94877 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 08:12:48 +0200 Subject: [PATCH 20/34] fix: harden testmon recovery publication Problem: a markerless complete attempt could recreate release authority, and repeated interrupted resumes could discard outcomes carried by an earlier resume.\n\nWhat changed: markerless attempt consumers now force selection-only state, and resume preparation flattens prior and current outcome ledgers before writing the next attempt. Real bootstrap, state-parser, and two-interruption regressions cover both boundaries.\n\nCompatibility/migration: published seed markers retain their existing typed release contract; only unmarked attempt recovery is downgraded. --- devtools/testmon_bootstrap.py | 2 + devtools/testmon_state.py | 11 +++-- devtools/verify.py | 21 ++++++++- tests/unit/devtools/test_testmon_bootstrap.py | 5 +- tests/unit/devtools/test_testmon_state.py | 32 +++++++++++++ tests/unit/devtools/test_verify.py | 47 +++++++++++++++++++ 6 files changed, 111 insertions(+), 7 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 3649cd30ac..68547de3a8 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -205,6 +205,7 @@ def decide_testmon_bootstrap( main_testmon_data, checkout_root=root, protocol_version=protocol_version, + published_marker=False, ) ) is not None @@ -385,6 +386,7 @@ def bootstrap_testmon_seed_files( decision.main_testmon_data, checkout_root=source_root, protocol_version=decision.protocol_version, + published_marker=False, ) if stamp is None: return False diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index 721967516e..f419b39b29 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -701,8 +701,9 @@ def stamp_from_attempt( *, checkout_root: Path, protocol_version: int, + published_marker: bool = True, ) -> TestmonSeedStamp | None: - """Promote only a complete attempt, including a red one, into a stamp.""" + """Parse a complete attempt, withholding release authority until publication.""" if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in {"reusable", "complete"}: return None selection = attempt.get("selection") @@ -797,6 +798,8 @@ def stamp_from_attempt( ) if baseline is BaselineStatus.GREEN and typed_identity.skip_slow and not terminal_authorized: baseline = BaselineStatus.RED + if not published_marker: + baseline = BaselineStatus.RED raw_permission = attempt.get("release_baseline_allowed") if baseline is BaselineStatus.GREEN and ( raw_scope != VerificationScope.NARROW_TERMINAL.value @@ -806,9 +809,9 @@ def stamp_from_attempt( baseline = BaselineStatus.RED if baseline is BaselineStatus.GREEN and raw_permission is not True: baseline = BaselineStatus.RED - if raw_permission is not None and ( - not isinstance(raw_permission, bool) or raw_permission != (baseline is BaselineStatus.GREEN) - ): + if raw_permission is not None and not isinstance(raw_permission, bool): + return None + if published_marker and raw_permission is not None and raw_permission != (baseline is BaselineStatus.GREEN): return None raw_binding = attempt.get("binding") if raw_binding is None: diff --git a/devtools/verify.py b/devtools/verify.py index 2bb500491e..d1f75f0552 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2158,6 +2158,7 @@ def _testmon_preflight(*, seed_testmon: bool, full_pytest: bool, quick: bool, co TESTMON_DATA, checkout_root=ROOT, protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + published_marker=False, ) is not None ): @@ -2260,6 +2261,21 @@ def _read_testmon_seed_attempt() -> dict[str, Any] | None: return payload if isinstance(payload, dict) else None +def _flatten_seed_outcomes(attempt: Mapping[str, Any] | None) -> list[dict[str, Any]]: + """Flatten outcomes from every interrupted attempt, newest result winning.""" + if attempt is None: + return [] + flattened: dict[str, dict[str, Any]] = {} + for field in ("prior_node_outcomes", "node_outcomes"): + raw = attempt.get(field) + if not isinstance(raw, list): + continue + for item in raw: + if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) and item["nodeid"]: + flattened[item["nodeid"]] = dict(item) + return [flattened[nodeid] for nodeid in sorted(flattened)] + + def _testmon_release_baseline_permission() -> bool | None: """Return release permission for current testmon state, or ``None`` when not applicable.""" if TESTMON_SEED_STAMP.exists(): @@ -2278,6 +2294,7 @@ def _testmon_release_baseline_permission() -> bool | None: TESTMON_DATA, checkout_root=ROOT, protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + published_marker=False, ) return stamp.release_baseline_allowed if stamp is not None else False @@ -2377,7 +2394,7 @@ def _prepare_testmon_seed_attempt( ) -> dict[str, Any]: prior = _read_testmon_seed_attempt() if resume else None expected = _testmon_seed_expected_nodeids(prior) if prior is not None else [] - prior_outcomes = prior.get("node_outcomes") if isinstance(prior, Mapping) else None + prior_outcomes = _flatten_seed_outcomes(prior) payload = { "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, "status": "running", @@ -2386,7 +2403,7 @@ def _prepare_testmon_seed_attempt( "expected_nodeids": expected, "expected_count": len(expected), "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, - "prior_node_outcomes": prior_outcomes if isinstance(prior_outcomes, list) else [], + "prior_node_outcomes": prior_outcomes, "started_at": datetime.now(timezone.utc).isoformat(), "run_id": run.run_id, "artifact_dir": str(run.relative_run_dir), diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 3909abb1d2..cb9cf2e744 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -426,7 +426,7 @@ def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) assert "checkout-bound selection attempt" in rebound_decision.reason -def test_complete_untyped_green_attempt_bootstraps_only_as_selection_state(tmp_path: Path) -> None: +def test_complete_typed_markerless_green_attempt_bootstraps_only_as_selection_state(tmp_path: Path) -> None: main_root = tmp_path / "main" main_data = main_root / "testmondata" _write_sqlite_db(main_data, rows=("tests/test.py::test_passed",)) @@ -442,6 +442,7 @@ def test_complete_untyped_green_attempt_bootstraps_only_as_selection_state(tmp_p "python": "python", "skip_slow": False, "lab": False, + "terminal_authorization": None, }, "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, "expected_nodeids": ["tests/test.py::test_passed"], @@ -449,6 +450,8 @@ def test_complete_untyped_green_attempt_bootstraps_only_as_selection_state(tmp_p "expected_digest": hashlib.sha256(b"tests/test.py::test_passed").hexdigest(), "node_outcomes": [{"nodeid": "tests/test.py::test_passed", "outcome": "passed"}], "exit_code": 0, + "verification_scope": "release-baseline", + "release_baseline_allowed": True, "run_id": "green-run", "artifact_dir": ".cache/verify/runs/green-run", "testmon_data": file_fingerprint(main_data), diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index eb6bf060fd..fe3bf01e40 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -160,6 +160,38 @@ def test_green_skipped_slow_attempt_without_typed_terminal_authority_is_selectio assert not stamp.release_baseline_allowed +def test_typed_complete_markerless_attempt_is_selection_only(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data, failed=False) + attempt = _attempt(data, outcomes=("passed", "passed")) + attempt["status"] = "complete" + attempt["exit_code"] = 0 + raw_identity = attempt["identity"] + assert isinstance(raw_identity, dict) + identity = dict(raw_identity) + identity["skip_slow"] = False + identity["terminal_authorization"] = None + attempt["identity"] = identity + attempt["verification_scope"] = "release-baseline" + attempt["release_baseline_allowed"] = True + + published = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + markerless = stamp_from_attempt( + attempt, + data, + checkout_root=tmp_path, + protocol_version=PROTOCOL, + published_marker=False, + ) + + assert published is not None + assert published.release_baseline_allowed + assert markerless is not None + assert markerless.baseline_status is BaselineStatus.RED + assert markerless.affected_selection_allowed + assert not markerless.release_baseline_allowed + + def test_malformed_sqlite_and_stale_stamp_fail_closed(tmp_path: Path) -> None: malformed = tmp_path / "malformed" malformed.write_bytes(b"not sqlite") diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 68503742c8..819efdc88e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -44,6 +44,7 @@ TESTMON_SEED_STAMP, _anchor_verification_paths, _finalize_testmon_seed_attempt, + _flatten_seed_outcomes, _format_completion_notification, _matching_testmon_coverage, _parse_pytest_test_count, @@ -519,6 +520,52 @@ def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pyte assert _testmon_seed_can_resume({**identity, "skip_slow": False}) is False +def test_two_interrupted_resumes_flatten_all_carried_outcomes(tmp_path: Path) -> None: + monkeypatch = pytest.MonkeyPatch() + monkeypatch.chdir(tmp_path) + try: + expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] + TESTMON_DATA.parent.mkdir(parents=True) + TESTMON_DATA.write_text("partial") + identity = { + "git_head": "head", + "git_tree": "tree-hash", + "worktree_fingerprint": "tree", + "python": "3.13", + "skip_slow": False, + "lab": False, + "terminal_authorization": None, + } + TESTMON_SEED_ATTEMPT.write_text( + json.dumps( + { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "incomplete", + "identity": identity, + "expected_nodeids": expected, + "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), + "node_outcomes": [{"nodeid": expected[0], "outcome": "passed"}], + } + ) + ) + first = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="head", root=tmp_path) + _prepare_testmon_seed_attempt(identity=identity, run=first, resume=True) + first_payload = json.loads(TESTMON_SEED_ATTEMPT.read_text()) + first_payload["status"] = "incomplete" + first_payload["node_outcomes"] = [{"nodeid": expected[1], "outcome": "passed"}] + TESTMON_SEED_ATTEMPT.write_text(json.dumps(first_payload)) + + second = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="head", root=tmp_path) + prepared = _prepare_testmon_seed_attempt(identity=identity, run=second, resume=True) + + assert {item["nodeid"] for item in prepared["prior_node_outcomes"]} == set(expected) + assert {item["outcome"] for item in prepared["prior_node_outcomes"]} == {"passed"} + assert _flatten_seed_outcomes(prepared) == prepared["prior_node_outcomes"] + finally: + monkeypatch.undo() + + def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) From e4ed36e6bd6b60bb97d70d2491854029842c69e4 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 08:12:57 +0200 Subject: [PATCH 21/34] fix: bind terminal verification to durable merge state Problem: detached terminal verification inherited the feature checkout interpreter, train-ledger corruption could fail open, concurrent merges could be overwritten or covered, and manual recording accepted stale branch output.\n\nWhat changed: terminal commands run through the detached checkout's direnv environment, ledger reads fail closed, writes use a durable pending latch and fsync-backed replacement, verification records its start sequence and merged-master SHA, and manual recording fetches and binds the current default branch. Production-route regressions cover the guard, corruption, injected write failure, concurrent merge, and stale CLI route.\n\nCompatibility/migration: existing ledger entries without sequence fields remain readable and are conservatively compared by timestamp; future terminal receipts require an exact fetched target. --- devtools/merge_boundary.py | 204 ++++++++++++++++++--- tests/unit/devtools/test_merge_boundary.py | 175 +++++++++++++++++- 2 files changed, 344 insertions(+), 35 deletions(-) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index 4bdc971851..f60dd53950 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -64,11 +64,14 @@ import argparse import json +import os import re +import shlex import subprocess import sys import tempfile import time +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -76,6 +79,12 @@ from devtools.testmon_state import VerificationScope _LEDGER_PATH = Path(".cache/verify/merge-gate/merge-train-ledger.json") +_LEDGER_PENDING_PATH = _LEDGER_PATH.with_name(f"{_LEDGER_PATH.name}.pending") + + +class LedgerStateError(RuntimeError): + """The merge-train ledger cannot safely authorize a result.""" + # Matches a squash-merge subject that already carries the PR number and picked # up a second, duplicate one -- e.g. "fix: thing (#3517) (#3517)". Only the @@ -101,33 +110,98 @@ def clean_merge_title(title: str, pr: int) -> str: return collapsed +def _validate_ledger(data: object) -> dict[str, Any]: + if not isinstance(data, dict) or not isinstance(data.get("merges"), list): + raise LedgerStateError("merge-train ledger is malformed") + for entry in data["merges"]: + if ( + not isinstance(entry, dict) + or not isinstance(entry.get("pr"), int) + or not isinstance(entry.get("merged_at"), (int, float)) + ): + raise LedgerStateError("merge-train ledger contains a malformed merge entry") + if data.get("last_full_verify") is not None and not isinstance(data.get("last_full_verify"), dict): + raise LedgerStateError("merge-train ledger contains a malformed terminal receipt") + return data + + def _read_ledger() -> dict[str, Any]: + if _LEDGER_PENDING_PATH.exists(): + raise LedgerStateError("merge-train ledger has an unfinished durable write") if not _LEDGER_PATH.exists(): return {"merges": [], "last_full_verify": None} try: - data = json.loads(_LEDGER_PATH.read_text()) - except (OSError, json.JSONDecodeError): - return {"merges": [], "last_full_verify": None} - if not isinstance(data, dict): - return {"merges": [], "last_full_verify": None} - data.setdefault("merges", []) - data.setdefault("last_full_verify", None) - return data + data = json.loads(_LEDGER_PATH.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LedgerStateError("merge-train ledger is unreadable or truncated") from exc + if isinstance(data, dict): + data.setdefault("merges", []) + data.setdefault("last_full_verify", None) + return _validate_ledger(data) + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _write_durable_temp(path: Path, text: str) -> None: + with path.open("w", encoding="utf-8") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + + +def _durable_replace(source: Path, destination: Path) -> None: + os.replace(source, destination) def _write_ledger(ledger: dict[str, Any]) -> None: - _LEDGER_PATH.parent.mkdir(parents=True, exist_ok=True) - _LEDGER_PATH.write_text(json.dumps(ledger, indent=2)) + _validate_ledger(ledger) + parent = _LEDGER_PATH.parent + parent.mkdir(parents=True, exist_ok=True) + serialized = json.dumps(ledger, indent=2) + "\n" + pending_tmp = parent / f".{_LEDGER_PENDING_PATH.name}.{os.getpid()}.tmp" + ledger_tmp = parent / f".{_LEDGER_PATH.name}.{os.getpid()}.tmp" + pending_tmp.unlink(missing_ok=True) + ledger_tmp.unlink(missing_ok=True) + try: + _write_durable_temp(pending_tmp, serialized) + _durable_replace(pending_tmp, _LEDGER_PENDING_PATH) + _fsync_directory(parent) + _write_durable_temp(ledger_tmp, serialized) + _durable_replace(ledger_tmp, _LEDGER_PATH) + _fsync_directory(parent) + _LEDGER_PENDING_PATH.unlink(missing_ok=True) + _fsync_directory(parent) + except OSError as exc: + raise LedgerStateError(f"merge-train ledger durable write failed: {exc}") from exc + finally: + pending_tmp.unlink(missing_ok=True) + ledger_tmp.unlink(missing_ok=True) + + +def _merge_sequence(ledger: Mapping[str, Any]) -> int: + sequence = 0 + for index, entry in enumerate(ledger.get("merges", []), start=1): + raw = entry.get("merge_sequence") if isinstance(entry, Mapping) else None + sequence = max(sequence, raw if isinstance(raw, int) and not isinstance(raw, bool) else index) + return sequence def _append_merge_entry(pr: int, head_sha: str, title: str) -> None: ledger = _read_ledger() + merge_sequence = _merge_sequence(ledger) + 1 ledger["merges"].append( { "pr": pr, "head_sha": head_sha, "title": title, "merged_at": time.time(), + "merge_sequence": merge_sequence, } ) _write_ledger(ledger) @@ -137,14 +211,25 @@ def _pending_prs_since_last_full_verify(ledger: dict[str, Any]) -> list[dict[str last_verify = ledger.get("last_full_verify") or {} scope = last_verify.get("verification_scope") last_verify_at = ( - last_verify.get("at", 0.0) + last_verify.get("verification_started_at", last_verify.get("at", 0.0)) if last_verify.get("accepted") is True and last_verify.get("exit_code") == 0 and last_verify.get("release_baseline_allowed") is True and scope == VerificationScope.RELEASE_BASELINE.value else 0.0 ) - return [entry for entry in ledger.get("merges", []) if entry.get("merged_at", 0.0) > last_verify_at] + snapshot_sequence = last_verify.get("merge_sequence") + return [ + entry + for entry in ledger.get("merges", []) + if entry.get("merged_at", 0.0) > last_verify_at + or ( + isinstance(snapshot_sequence, int) + and not isinstance(snapshot_sequence, bool) + and isinstance(entry.get("merge_sequence"), int) + and entry["merge_sequence"] > snapshot_sequence + ) + ] def _receipt_is_fresh_for_head(pr: int, head_sha: str, max_age_s: int) -> bool: @@ -160,10 +245,8 @@ def _receipt_is_fresh_for_head(pr: int, head_sha: str, max_age_s: int) -> bool: def _fetched_merged_default_branch_sha(pr: int) -> str | None: """Fetch the default branch and return its post-merge commit only.""" try: - repo = _gh_json(["repo", "view", "--json", "defaultBranchRef"]) - default_ref = repo.get("defaultBranchRef") - branch = default_ref.get("name") if isinstance(default_ref, dict) else None - if not isinstance(branch, str) or not branch: + branch = _default_branch_name() + if branch is None: return None merged = _gh_json(["pr", "view", str(pr), "--json", "state,mergeCommit"]) merge_commit = merged.get("mergeCommit") @@ -200,6 +283,30 @@ def _fetched_merged_default_branch_sha(pr: int) -> str | None: return None +def _default_branch_name() -> str | None: + repo = _gh_json(["repo", "view", "--json", "defaultBranchRef"]) + default_ref = repo.get("defaultBranchRef") + branch = default_ref.get("name") if isinstance(default_ref, dict) else None + return branch if isinstance(branch, str) and branch else None + + +def _fetched_current_default_branch_sha() -> str | None: + """Fetch and return the exact current default-branch commit.""" + try: + branch = _default_branch_name() + if branch is None: + return None + fetch = subprocess.run(["git", "fetch", "origin", branch], capture_output=True, text=True, timeout=120) + if fetch.returncode != 0: + return None + target = subprocess.run(["git", "rev-parse", "FETCH_HEAD"], capture_output=True, text=True, timeout=15) + if target.returncode != 0 or not target.stdout.strip(): + return None + return target.stdout.strip() + except (RuntimeError, json.JSONDecodeError, OSError, subprocess.SubprocessError): + return None + + def _run_post_merge_terminal_verify(command: str, target_sha: str) -> int: """Run terminal verification in a detached worktree at the fetched target.""" repo_root = Path.cwd() @@ -216,7 +323,7 @@ def _run_post_merge_terminal_verify(command: str, target_sha: str) -> int: print(f"REFUSING terminal verify: could not materialize fetched target {target_sha[:8]}", file=sys.stderr) return 1 try: - return cmd_record_full_verify(command, target_sha=target_sha, cwd=worktree) + return cmd_record_full_verify(command, target_sha=target_sha, cwd=worktree, execution_root=worktree) finally: subprocess.run( ["git", "worktree", "remove", "--force", str(worktree)], @@ -298,7 +405,11 @@ def cmd_merge( return merge_result.returncode print(f"merged PR #{pr} @ {head_sha[:8]}: {clean_title!r}") - _append_merge_entry(pr, head_sha, clean_title) + try: + _append_merge_entry(pr, head_sha, clean_title) + except LedgerStateError as exc: + print(f"REFUSING to continue: merge-train ledger is not durably writable: {exc}", file=sys.stderr) + return 1 if with_verify: target_sha = _fetched_merged_default_branch_sha(pr) @@ -321,7 +432,11 @@ def cmd_merge( def cmd_train_status(as_json: bool) -> int: - ledger = _read_ledger() + try: + ledger = _read_ledger() + except LedgerStateError as exc: + print(f"merge-train REFUSING clean status: {exc}", file=sys.stderr) + return 1 pending = _pending_prs_since_last_full_verify(ledger) ok = not pending @@ -354,18 +469,41 @@ def cmd_train_status(as_json: bool) -> int: return 1 -def cmd_record_full_verify(command: str, *, target_sha: str | None = None, cwd: Path | None = None) -> int: - argv = command.split() +def cmd_record_full_verify( + command: str, + *, + target_sha: str | None = None, + cwd: Path | None = None, + execution_root: Path | None = None, +) -> int: + argv = shlex.split(command) if not argv: print("REFUSING: --command is empty after splitting", file=sys.stderr) return 2 - started = time.time() + if target_sha is None: + print("REFUSING: terminal verification has no fetched merged-master target", file=sys.stderr) + return 1 + try: + ledger = _read_ledger() + except LedgerStateError as exc: + print(f"REFUSING to run terminal verification: {exc}", file=sys.stderr) + return 1 + verification_started_at = time.time() + merge_sequence = _merge_sequence(ledger) + if execution_root is not None: + argv = ["direnv", "exec", str(execution_root), *argv] + started = verification_started_at try: result = subprocess.run(argv, capture_output=True, text=True, cwd=cwd) except OSError as exc: print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) return 2 duration_s = round(time.time() - started, 2) + try: + ledger = _read_ledger() + except LedgerStateError as exc: + print(f"REFUSING to record terminal verification: {exc}", file=sys.stderr) + return 1 release_allowed = merge_gate._release_baseline_permission(result.stdout) verification_scope = merge_gate._verification_scope(result.stdout) terminal_authorization = merge_gate._terminal_authorization(result.stdout) @@ -378,23 +516,29 @@ def cmd_record_full_verify(command: str, *, target_sha: str | None = None, cwd: result.returncode == 0 and release_allowed is True and verification_scope == VerificationScope.RELEASE_BASELINE.value - and (target_sha is None or verified_head == target_sha) + and verified_head == target_sha ) - ledger = _read_ledger() ledger["last_full_verify"] = { "command": command, "exit_code": result.returncode, "duration_s": duration_s, - "at": time.time(), + "at": verification_started_at, + "verification_started_at": verification_started_at, "verification_scope": verification_scope, "release_baseline_allowed": release_allowed, "terminal_authorization": terminal_authorization, "verified_head_sha": verified_head, "target_sha": target_sha, + "merged_master_sha": target_sha, + "merge_sequence": merge_sequence, "accepted": accepted, } - _write_ledger(ledger) + try: + _write_ledger(ledger) + except LedgerStateError as exc: + print(f"REFUSING to record terminal verification: {exc}", file=sys.stderr) + return 1 print( f"recorded merge-train terminal verify: {command!r} exit={result.returncode} " @@ -406,7 +550,7 @@ def cmd_record_full_verify(command: str, *, target_sha: str | None = None, cwd: "train-status remains incomplete.", file=sys.stderr, ) - if target_sha is not None and verified_head != target_sha: + if verified_head != target_sha: print( f"terminal verify reported git_head={verified_head!r}, expected fetched target {target_sha}", file=sys.stderr, @@ -468,7 +612,11 @@ def main(argv: list[str] | None = None) -> int: ) if args.action == "train-status": return cmd_train_status(args.as_json) - return cmd_record_full_verify(args.command) + target_sha = _fetched_current_default_branch_sha() + if target_sha is None: + print("REFUSING terminal verify: could not fetch the current default branch", file=sys.stderr) + return 1 + return _run_post_merge_terminal_verify(args.command, target_sha) if __name__ == "__main__": diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index d67a0355c6..434c639a90 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import subprocess from collections.abc import Callable from pathlib import Path @@ -10,6 +11,7 @@ import pytest from devtools import merge_boundary, merge_gate, pr_scope +from devtools.checkout_guard import checkout_environment_fingerprint _SCOPE_BEAD = { "_type": "issue", @@ -409,6 +411,52 @@ def test_post_merge_terminal_verify_rejects_stale_feature_head(monkeypatch: pyte assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False +def test_post_merge_terminal_verify_uses_target_checkout_devshell( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + commands: list[list[str]] = [] + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + commands.append(cmd) + if cmd[:3] == ["git", "worktree", "add"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:2] == ["direnv", "exec"]: + assert cmd[2] != str(tmp_path) + target = Path(cmd[2]) + package = target / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("") + (target / ".venv" / "bin").mkdir(parents=True) + with pytest.MonkeyPatch.context() as guard_patch: + guard_patch.setattr("devtools.checkout_guard._is_linked_worktree", lambda _root: True) + fingerprint = checkout_environment_fingerprint( + target, + polylogue_import_path=package / "__init__.py", + python_executable=target / ".venv" / "bin" / "python", + ) + assert fingerprint.clean + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) + if cmd[:3] == ["git", "worktree", "remove"]: + return MagicMock(returncode=0, stdout="", stderr="") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + + assert merge_boundary._run_post_merge_terminal_verify("devtools verify --all", "merged-master") == 0 + assert any(command[:2] == ["direnv", "exec"] for command in commands) + + def test_fetched_default_branch_must_include_squash_merge(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) calls: list[list[str]] = [] @@ -474,6 +522,44 @@ def test_train_status_ok_with_empty_ledger(monkeypatch: pytest.MonkeyPatch, tmp_ assert merge_boundary.cmd_train_status(as_json=False) == 0 +def test_train_status_fails_closed_on_truncated_ledger(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + ledger_path = tmp_path / ".cache" / "verify" / "merge-gate" / "merge-train-ledger.json" + ledger_path.parent.mkdir(parents=True) + ledger_path.write_text('{"merges": [') + + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_merge_write_failure_leaves_durable_pending_latch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view)) + + def fail_final_replace(source: Path, destination: Path) -> None: + if destination == merge_boundary._LEDGER_PATH: + raise OSError("injected final ledger write failure") + os.replace(source, destination) + + monkeypatch.setattr(merge_boundary, "_durable_replace", fail_final_replace) + + assert ( + merge_boundary.cmd_merge( + 42, + command="devtools test x", + max_age_s=3600, + poll_rounds=1, + poll_interval_s=0, + dry_run=False, + with_verify=False, + verify_command="devtools verify --all", + ) + == 1 + ) + assert merge_boundary._LEDGER_PENDING_PATH.exists() + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + def test_train_status_blocks_when_pr_merged_after_last_full_verify( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -517,13 +603,19 @@ def test_record_full_verify_clears_pending_prs(monkeypatch: pytest.MonkeyPatch, def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return MagicMock( returncode=0, - stdout=json.dumps({"verification_scope": "release-baseline", "release_baseline_allowed": True}), + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), stderr="", ) monkeypatch.setattr(subprocess, "run", _run) - exit_code = merge_boundary.cmd_record_full_verify("devtools verify --all") + exit_code = merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") assert exit_code == 0 assert merge_boundary.cmd_train_status(as_json=False) == 0 @@ -538,7 +630,7 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: monkeypatch.setattr(subprocess, "run", _run) - exit_code = merge_boundary.cmd_record_full_verify("devtools verify --all") + exit_code = merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") assert exit_code == 1 ledger = merge_boundary._read_ledger() @@ -559,7 +651,7 @@ def test_record_full_verify_rejects_success_without_structured_release_permissio lambda _cmd, **_kwargs: MagicMock(returncode=0, stdout="all good\n", stderr=""), ) - assert merge_boundary.cmd_record_full_verify("devtools verify --all") == 1 + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 1 ledger = merge_boundary._read_ledger() assert ledger["last_full_verify"]["accepted"] is False assert merge_boundary.cmd_train_status(as_json=False) == 1 @@ -580,7 +672,7 @@ def test_record_full_verify_rejects_skip_slow_without_typed_authorization( ), ) - assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow") == 1 + assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 1 assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False assert merge_boundary.cmd_train_status(as_json=False) == 1 @@ -606,7 +698,7 @@ def test_record_full_verify_rejects_explicit_typed_narrow_terminal_authorization ), ) - assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow") == 1 + assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 1 assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False assert merge_boundary.cmd_train_status(as_json=False) == 1 @@ -626,6 +718,75 @@ def test_record_full_verify_rejects_untyped_scope_even_when_permission_is_true( ), ) - assert merge_boundary.cmd_record_full_verify("devtools verify --all") == 1 + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 1 assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_concurrent_merge_during_terminal_verify_remains_pending( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + inserted = False + + def run(_cmd: list[str], **_kwargs: Any) -> MagicMock: + nonlocal inserted + if not inserted: + inserted = True + merge_boundary._append_merge_entry(99, "concurrent-sha", "concurrent merge") + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) + + monkeypatch.setattr(subprocess, "run", run) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 0 + assert merge_boundary.cmd_train_status(as_json=False) == 1 + ledger = merge_boundary._read_ledger() + assert ledger["last_full_verify"]["merged_master_sha"] == "merged-master" + assert ledger["last_full_verify"]["merge_sequence"] == 0 + + +def test_manual_record_route_fetches_target_and_rejects_stale_cli_output( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + post_targets: list[str] = [] + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + if cmd[:3] == ["gh", "repo", "view"]: + return MagicMock(returncode=0, stdout=json.dumps({"defaultBranchRef": {"name": "master"}}), stderr="") + if cmd[:3] == ["git", "fetch", "origin"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:3] == ["git", "rev-parse", "FETCH_HEAD"]: + return MagicMock(returncode=0, stdout="current-master\n", stderr="") + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "stale-feature", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) + + def post_verify(command: str, target_sha: str) -> int: + post_targets.append(target_sha) + return merge_boundary.cmd_record_full_verify(command, target_sha=target_sha) + + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", post_verify) + + assert merge_boundary.main(["record-full-verify", "--command", "devtools verify --all"]) == 1 + assert post_targets == ["current-master"] + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False From 7d1cf1b05ace58ae26ac27929628509e89f26d4d Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 08:54:31 +0200 Subject: [PATCH 22/34] fix: make merge-train recovery transactional Problem: terminal verification and external merge bookkeeping still had snapshot races, unlocked ledger writers, an untracked post-merge crash window, and inconsistent markerless seed handling. What changed: serialize ledger transactions, persist and reconcile pre-merge intents, snapshot ledger state before fetching the default branch, validate all status fields, accept typed markerless selection attempts through the checkout guard, and fail explicitly on detached worktree cleanup errors. Production-route regressions cover each authority transition and race. Compatibility/migration: legacy merge entries without merge_sequence remain readable. Existing terminal receipts must contain the typed status fields consumed by train-status; malformed or partial receipts now refuse clean status. --- devtools/checkout_guard.py | 13 +- devtools/merge_boundary.py | 288 +++++++++++++++--- tests/unit/devtools/test_merge_boundary.py | 204 ++++++++++++- tests/unit/devtools/test_testmon_bootstrap.py | 93 ++++++ 4 files changed, 547 insertions(+), 51 deletions(-) diff --git a/devtools/checkout_guard.py b/devtools/checkout_guard.py index e409c149f0..7b3210136c 100644 --- a/devtools/checkout_guard.py +++ b/devtools/checkout_guard.py @@ -277,8 +277,19 @@ def _is_valid_in_progress_testmon_seed_attempt(attempt: Path, *, checkout_root: payload = json.loads(attempt.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError): return False - if not isinstance(payload, Mapping) or payload.get("status") not in {"running", "incomplete", "reusable"}: + if not isinstance(payload, Mapping) or payload.get("status") not in { + "running", + "incomplete", + "reusable", + "complete", + }: return False + if payload.get("status") == "complete": + return attempt_is_checkout_bound( + payload, + checkout_root=checkout_root, + protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, + ) if payload.get("status") == "reusable": return attempt_is_checkout_bound( payload, diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index f60dd53950..c67783ad51 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -63,6 +63,8 @@ from __future__ import annotations import argparse +import contextlib +import fcntl import json import os import re @@ -71,7 +73,7 @@ import sys import tempfile import time -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from pathlib import Path from typing import Any @@ -80,6 +82,7 @@ _LEDGER_PATH = Path(".cache/verify/merge-gate/merge-train-ledger.json") _LEDGER_PENDING_PATH = _LEDGER_PATH.with_name(f"{_LEDGER_PATH.name}.pending") +_LEDGER_LOCK_PATH = _LEDGER_PATH.with_name(f"{_LEDGER_PATH.name}.lock") class LedgerStateError(RuntimeError): @@ -110,6 +113,27 @@ def clean_merge_title(title: str, pr: int) -> str: return collapsed +@contextlib.contextmanager +def _ledger_lock() -> Iterator[None]: + """Serialize every merge-train ledger read-modify-write transaction.""" + _LEDGER_LOCK_PATH.parent.mkdir(parents=True, exist_ok=True) + with _LEDGER_LOCK_PATH.open("a+") as handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +def _is_real_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _require_nonempty_string(entry: Mapping[str, Any], key: str, *, message: str) -> None: + if not isinstance(entry.get(key), str) or not entry[key]: + raise LedgerStateError(message) + + def _validate_ledger(data: object) -> dict[str, Any]: if not isinstance(data, dict) or not isinstance(data.get("merges"), list): raise LedgerStateError("merge-train ledger is malformed") @@ -117,29 +141,104 @@ def _validate_ledger(data: object) -> dict[str, Any]: if ( not isinstance(entry, dict) or not isinstance(entry.get("pr"), int) - or not isinstance(entry.get("merged_at"), (int, float)) + or isinstance(entry.get("pr"), bool) + or entry["pr"] <= 0 + or not isinstance(entry.get("head_sha"), str) + or not entry["head_sha"] + or not isinstance(entry.get("title"), str) + or not entry["title"] + or not _is_real_number(entry.get("merged_at")) + or ( + "merge_sequence" in entry + and ( + not isinstance(entry.get("merge_sequence"), int) + or isinstance(entry.get("merge_sequence"), bool) + or entry["merge_sequence"] <= 0 + ) + ) ): raise LedgerStateError("merge-train ledger contains a malformed merge entry") - if data.get("last_full_verify") is not None and not isinstance(data.get("last_full_verify"), dict): + intents = data.get("merge_intents") + if not isinstance(intents, list): + raise LedgerStateError("merge-train ledger contains malformed merge intents") + for intent in intents: + if ( + not isinstance(intent, dict) + or not isinstance(intent.get("pr"), int) + or isinstance(intent.get("pr"), bool) + or intent["pr"] <= 0 + or not isinstance(intent.get("head_sha"), str) + or not intent["head_sha"] + or not isinstance(intent.get("title"), str) + or not intent["title"] + or not _is_real_number(intent.get("intent_at")) + ): + raise LedgerStateError("merge-train ledger contains a malformed merge intent") + receipt = data.get("last_full_verify") + if receipt is None: + return data + if not isinstance(receipt, dict): raise LedgerStateError("merge-train ledger contains a malformed terminal receipt") + for key in ( + "command", + "verification_started_at", + "at", + "duration_s", + "exit_code", + "accepted", + "merge_sequence", + "verification_scope", + "release_baseline_allowed", + ): + if key not in receipt: + raise LedgerStateError(f"merge-train terminal receipt is missing {key!r}") + _require_nonempty_string(receipt, "command", message="merge-train terminal receipt has no command") + if ( + not _is_real_number(receipt.get("verification_started_at")) + or not _is_real_number(receipt.get("at")) + or not _is_real_number(receipt.get("duration_s")) + or not isinstance(receipt.get("exit_code"), int) + or isinstance(receipt.get("exit_code"), bool) + or not isinstance(receipt.get("accepted"), bool) + or not isinstance(receipt.get("merge_sequence"), int) + or isinstance(receipt.get("merge_sequence"), bool) + or receipt["merge_sequence"] < 0 + ): + raise LedgerStateError("merge-train terminal receipt has malformed status fields") + scope = receipt.get("verification_scope") + if scope is not None and scope not in {item.value for item in VerificationScope}: + raise LedgerStateError("merge-train terminal receipt has an invalid verification scope") + permission = receipt.get("release_baseline_allowed") + if permission is not None and not isinstance(permission, bool): + raise LedgerStateError("merge-train terminal receipt has malformed release permission") + for key in ("terminal_authorization", "verified_head_sha", "target_sha", "merged_master_sha"): + value = receipt.get(key) + if value is not None and (not isinstance(value, str) or not value): + raise LedgerStateError(f"merge-train terminal receipt has malformed {key!r}") return data -def _read_ledger() -> dict[str, Any]: +def _read_ledger_unlocked() -> dict[str, Any]: if _LEDGER_PENDING_PATH.exists(): raise LedgerStateError("merge-train ledger has an unfinished durable write") if not _LEDGER_PATH.exists(): - return {"merges": [], "last_full_verify": None} + return {"merges": [], "merge_intents": [], "last_full_verify": None} try: data = json.loads(_LEDGER_PATH.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise LedgerStateError("merge-train ledger is unreadable or truncated") from exc if isinstance(data, dict): data.setdefault("merges", []) + data.setdefault("merge_intents", []) data.setdefault("last_full_verify", None) return _validate_ledger(data) +def _read_ledger() -> dict[str, Any]: + with _ledger_lock(): + return _read_ledger_unlocked() + + def _fsync_directory(path: Path) -> None: descriptor = os.open(path, os.O_RDONLY) try: @@ -159,7 +258,10 @@ def _durable_replace(source: Path, destination: Path) -> None: os.replace(source, destination) -def _write_ledger(ledger: dict[str, Any]) -> None: +def _write_ledger_unlocked(ledger: dict[str, Any]) -> None: + ledger.setdefault("merges", []) + ledger.setdefault("merge_intents", []) + ledger.setdefault("last_full_verify", None) _validate_ledger(ledger) parent = _LEDGER_PATH.parent parent.mkdir(parents=True, exist_ok=True) @@ -184,6 +286,11 @@ def _write_ledger(ledger: dict[str, Any]) -> None: ledger_tmp.unlink(missing_ok=True) +def _write_ledger(ledger: dict[str, Any]) -> None: + with _ledger_lock(): + _write_ledger_unlocked(ledger) + + def _merge_sequence(ledger: Mapping[str, Any]) -> int: sequence = 0 for index, entry in enumerate(ledger.get("merges", []), start=1): @@ -193,7 +300,13 @@ def _merge_sequence(ledger: Mapping[str, Any]) -> int: def _append_merge_entry(pr: int, head_sha: str, title: str) -> None: - ledger = _read_ledger() + with _ledger_lock(): + ledger = _read_ledger_unlocked() + _append_merge_entry_unlocked(ledger, pr, head_sha, title) + _write_ledger_unlocked(ledger) + + +def _append_merge_entry_unlocked(ledger: dict[str, Any], pr: int, head_sha: str, title: str) -> None: merge_sequence = _merge_sequence(ledger) + 1 ledger["merges"].append( { @@ -204,7 +317,47 @@ def _append_merge_entry(pr: int, head_sha: str, title: str) -> None: "merge_sequence": merge_sequence, } ) - _write_ledger(ledger) + + +def _record_merge_intent(pr: int, head_sha: str, title: str) -> None: + with _ledger_lock(): + ledger = _read_ledger_unlocked() + if not any(intent.get("pr") == pr and intent.get("head_sha") == head_sha for intent in ledger["merge_intents"]): + ledger["merge_intents"].append({"pr": pr, "head_sha": head_sha, "title": title, "intent_at": time.time()}) + _write_ledger_unlocked(ledger) + + +def _complete_merge_intent(pr: int, head_sha: str) -> None: + with _ledger_lock(): + ledger = _read_ledger_unlocked() + intents = ledger["merge_intents"] + matching = [intent for intent in intents if intent.get("pr") == pr and intent.get("head_sha") == head_sha] + if not matching: + return + intent = matching[0] + if not any(entry.get("pr") == pr and entry.get("head_sha") == head_sha for entry in ledger["merges"]): + _append_merge_entry_unlocked(ledger, pr, head_sha, str(intent["title"])) + ledger["merge_intents"] = [item for item in intents if item is not intent] + _write_ledger_unlocked(ledger) + + +def _reconcile_merge_intents() -> None: + """Resolve durable pre-merge intents against GitHub after a restart.""" + ledger = _read_ledger() + for intent in list(ledger["merge_intents"]): + try: + info = _gh_json(["pr", "view", str(intent["pr"]), "--json", "state,mergeCommit"]) + except (RuntimeError, json.JSONDecodeError, OSError, subprocess.SubprocessError) as exc: + raise LedgerStateError(f"could not reconcile merge intent for PR #{intent['pr']}: {exc}") from exc + merge_commit = info.get("mergeCommit") + if info.get("state") != "MERGED" or not isinstance(merge_commit, dict) or not merge_commit.get("oid"): + raise LedgerStateError(f"unresolved durable merge intent for PR #{intent['pr']}") + _complete_merge_intent(int(intent["pr"]), str(intent["head_sha"])) + if any( + item.get("pr") == intent["pr"] and item.get("head_sha") == intent["head_sha"] + for item in _read_ledger()["merge_intents"] + ): + raise LedgerStateError(f"merge intent for PR #{intent['pr']} was not durably reconciled") def _pending_prs_since_last_full_verify(ledger: dict[str, Any]) -> list[dict[str, Any]]: @@ -307,7 +460,37 @@ def _fetched_current_default_branch_sha() -> str | None: return None -def _run_post_merge_terminal_verify(command: str, target_sha: str) -> int: +def _terminal_verify_snapshot() -> tuple[dict[str, Any], float, int]: + with _ledger_lock(): + ledger = _read_ledger_unlocked() + started_at = time.time() + return ledger, started_at, _merge_sequence(ledger) + + +def _remove_detached_worktree(repo_root: Path, worktree: Path) -> bool: + removal = subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + capture_output=True, + text=True, + timeout=120, + cwd=repo_root, + ) + if removal.returncode != 0: + print( + f"REFUSING terminal verify: failed to unregister detached worktree {worktree}: " + f"{removal.stderr.strip()[:300]}", + file=sys.stderr, + ) + return False + return True + + +def _run_post_merge_terminal_verify( + command: str, + target_sha: str, + *, + ledger_snapshot: tuple[dict[str, Any], float, int] | None = None, +) -> int: """Run terminal verification in a detached worktree at the fetched target.""" repo_root = Path.cwd() with tempfile.TemporaryDirectory(prefix="polylogue-merge-terminal-") as raw_worktree: @@ -321,17 +504,16 @@ def _run_post_merge_terminal_verify(command: str, target_sha: str) -> int: ) if add.returncode != 0: print(f"REFUSING terminal verify: could not materialize fetched target {target_sha[:8]}", file=sys.stderr) + _remove_detached_worktree(repo_root, worktree) return 1 - try: - return cmd_record_full_verify(command, target_sha=target_sha, cwd=worktree, execution_root=worktree) - finally: - subprocess.run( - ["git", "worktree", "remove", "--force", str(worktree)], - capture_output=True, - text=True, - timeout=120, - cwd=repo_root, - ) + result = cmd_record_full_verify( + command, + target_sha=target_sha, + cwd=worktree, + execution_root=worktree, + ledger_snapshot=ledger_snapshot, + ) + return result if _remove_detached_worktree(repo_root, worktree) else 1 def cmd_merge( @@ -384,6 +566,12 @@ def cmd_merge( print(f"PR #{pr} @ {head_sha[:8]}: merge-gate OK -- dry-run, not merging (title would be {clean_title!r})") return 0 + try: + _record_merge_intent(pr, head_sha, clean_title) + except LedgerStateError as exc: + print(f"REFUSING to merge PR #{pr}: could not durably record merge intent: {exc}", file=sys.stderr) + return 1 + merge_result = subprocess.run( [ "gh", @@ -406,12 +594,17 @@ def cmd_merge( print(f"merged PR #{pr} @ {head_sha[:8]}: {clean_title!r}") try: - _append_merge_entry(pr, head_sha, clean_title) + _complete_merge_intent(pr, head_sha) except LedgerStateError as exc: print(f"REFUSING to continue: merge-train ledger is not durably writable: {exc}", file=sys.stderr) return 1 if with_verify: + try: + ledger_snapshot = _terminal_verify_snapshot() + except LedgerStateError as exc: + print(f"REFUSING terminal verify: {exc}", file=sys.stderr) + return 1 target_sha = _fetched_merged_default_branch_sha(pr) if target_sha is None: print( @@ -420,7 +613,7 @@ def cmd_merge( ) return 1 print(f"running post-merge broad verify (merge-train terminal step): {verify_command!r}") - return _run_post_merge_terminal_verify(verify_command, target_sha) + return _run_post_merge_terminal_verify(verify_command, target_sha, ledger_snapshot=ledger_snapshot) print( "REMINDER: this merge-train's terminal ledger step (one full-suite verify since the last " @@ -433,6 +626,7 @@ def cmd_merge( def cmd_train_status(as_json: bool) -> int: try: + _reconcile_merge_intents() ledger = _read_ledger() except LedgerStateError as exc: print(f"merge-train REFUSING clean status: {exc}", file=sys.stderr) @@ -475,6 +669,7 @@ def cmd_record_full_verify( target_sha: str | None = None, cwd: Path | None = None, execution_root: Path | None = None, + ledger_snapshot: tuple[dict[str, Any], float, int] | None = None, ) -> int: argv = shlex.split(command) if not argv: @@ -484,12 +679,11 @@ def cmd_record_full_verify( print("REFUSING: terminal verification has no fetched merged-master target", file=sys.stderr) return 1 try: - ledger = _read_ledger() + snapshot = ledger_snapshot or _terminal_verify_snapshot() except LedgerStateError as exc: print(f"REFUSING to run terminal verification: {exc}", file=sys.stderr) return 1 - verification_started_at = time.time() - merge_sequence = _merge_sequence(ledger) + _snapshot_ledger, verification_started_at, merge_sequence = snapshot if execution_root is not None: argv = ["direnv", "exec", str(execution_root), *argv] started = verification_started_at @@ -499,11 +693,6 @@ def cmd_record_full_verify( print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) return 2 duration_s = round(time.time() - started, 2) - try: - ledger = _read_ledger() - except LedgerStateError as exc: - print(f"REFUSING to record terminal verification: {exc}", file=sys.stderr) - return 1 release_allowed = merge_gate._release_baseline_permission(result.stdout) verification_scope = merge_gate._verification_scope(result.stdout) terminal_authorization = merge_gate._terminal_authorization(result.stdout) @@ -519,23 +708,25 @@ def cmd_record_full_verify( and verified_head == target_sha ) - ledger["last_full_verify"] = { - "command": command, - "exit_code": result.returncode, - "duration_s": duration_s, - "at": verification_started_at, - "verification_started_at": verification_started_at, - "verification_scope": verification_scope, - "release_baseline_allowed": release_allowed, - "terminal_authorization": terminal_authorization, - "verified_head_sha": verified_head, - "target_sha": target_sha, - "merged_master_sha": target_sha, - "merge_sequence": merge_sequence, - "accepted": accepted, - } try: - _write_ledger(ledger) + with _ledger_lock(): + ledger = _read_ledger_unlocked() + ledger["last_full_verify"] = { + "command": command, + "exit_code": result.returncode, + "duration_s": duration_s, + "at": verification_started_at, + "verification_started_at": verification_started_at, + "verification_scope": verification_scope, + "release_baseline_allowed": release_allowed, + "terminal_authorization": terminal_authorization, + "verified_head_sha": verified_head, + "target_sha": target_sha, + "merged_master_sha": target_sha, + "merge_sequence": merge_sequence, + "accepted": accepted, + } + _write_ledger_unlocked(ledger) except LedgerStateError as exc: print(f"REFUSING to record terminal verification: {exc}", file=sys.stderr) return 1 @@ -612,11 +803,16 @@ def main(argv: list[str] | None = None) -> int: ) if args.action == "train-status": return cmd_train_status(args.as_json) + try: + ledger_snapshot = _terminal_verify_snapshot() + except LedgerStateError as exc: + print(f"REFUSING terminal verify: {exc}", file=sys.stderr) + return 1 target_sha = _fetched_current_default_branch_sha() if target_sha is None: print("REFUSING terminal verify: could not fetch the current default branch", file=sys.stderr) return 1 - return _run_post_merge_terminal_verify(args.command, target_sha) + return _run_post_merge_terminal_verify(args.command, target_sha, ledger_snapshot=ledger_snapshot) if __name__ == "__main__": diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index 434c639a90..0137a18404 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -3,6 +3,7 @@ import json import os import subprocess +import threading from collections.abc import Callable from pathlib import Path from typing import Any @@ -342,7 +343,7 @@ def run(cmd: list[str], **kwargs: Any) -> MagicMock: monkeypatch.setattr( merge_boundary, "_run_post_merge_terminal_verify", - lambda command, target: merge_boundary.cmd_record_full_verify(command, target_sha=target), + lambda command, target, **_kwargs: merge_boundary.cmd_record_full_verify(command, target_sha=target), ) exit_code = merge_boundary.cmd_merge( @@ -371,7 +372,7 @@ def test_merge_with_verify_returns_nonzero_when_terminal_authority_is_rejected( pr_view = _base_pr_view() monkeypatch.setattr(subprocess, "run", _fake_run(pr_view)) monkeypatch.setattr(merge_boundary, "_fetched_merged_default_branch_sha", lambda _pr: "merged-master") - monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", lambda _command, _target: 1) + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", lambda _command, _target, **_kwargs: 1) assert ( merge_boundary.cmd_merge( @@ -531,6 +532,18 @@ def test_train_status_fails_closed_on_truncated_ledger(monkeypatch: pytest.Monke assert merge_boundary.cmd_train_status(as_json=False) == 1 +def test_train_status_fails_closed_on_valid_json_partial_merge_entry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.chdir(tmp_path) + ledger_path = tmp_path / ".cache" / "verify" / "merge-gate" / "merge-train-ledger.json" + ledger_path.parent.mkdir(parents=True) + ledger_path.write_text(json.dumps({"merges": [{"pr": 42, "merged_at": 1.0}], "last_full_verify": None})) + + assert merge_boundary.cmd_train_status(as_json=False) == 1 + assert "Traceback" not in capsys.readouterr().err + + def test_merge_write_failure_leaves_durable_pending_latch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) pr_view = _base_pr_view() @@ -565,7 +578,20 @@ def test_train_status_blocks_when_pr_merged_after_last_full_verify( ) -> None: monkeypatch.chdir(tmp_path) merge_boundary._write_ledger( - {"merges": [], "last_full_verify": {"at": 1000.0, "command": "devtools verify --all", "exit_code": 0}} + { + "merges": [], + "last_full_verify": { + "at": 1000.0, + "verification_started_at": 1000.0, + "duration_s": 1.0, + "command": "devtools verify --all", + "exit_code": 0, + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + "merge_sequence": 0, + "accepted": True, + }, + } ) merge_boundary._append_merge_entry(1, "sha1", "some title") @@ -584,9 +610,13 @@ def test_train_status_rejects_untyped_accepted_terminal_ledger(monkeypatch: pyte "merges": [], "last_full_verify": { "at": 1000.0, + "verification_started_at": 1000.0, + "duration_s": 1.0, "command": "devtools verify --all", "exit_code": 0, + "verification_scope": None, "release_baseline_allowed": True, + "merge_sequence": 0, "accepted": True, }, } @@ -755,6 +785,172 @@ def run(_cmd: list[str], **_kwargs: Any) -> MagicMock: assert ledger["last_full_verify"]["merge_sequence"] == 0 +def test_concurrent_ledger_writer_cannot_lose_merge_entry(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + started = threading.Event() + writer: threading.Thread | None = None + + def run(_cmd: list[str], **_kwargs: Any) -> MagicMock: + nonlocal writer + + def append() -> None: + started.set() + merge_boundary._append_merge_entry(77, "writer-sha", "writer merge") + + writer = threading.Thread(target=append) + writer.start() + assert started.wait(timeout=1) + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) + + monkeypatch.setattr(subprocess, "run", run) + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 0 + assert writer is not None + writer.join(timeout=1) + assert not writer.is_alive() + ledger = merge_boundary._read_ledger() + assert any(entry["pr"] == 77 for entry in ledger["merges"]) + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_terminal_snapshot_is_taken_before_default_branch_fetch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + events: list[str] = [] + snapshots: list[tuple[dict[str, Any], float, int]] = [] + + original_snapshot = merge_boundary._terminal_verify_snapshot + + def snapshot() -> tuple[dict[str, Any], float, int]: + events.append("ledger-snapshot") + captured = original_snapshot() + snapshots.append(captured) + return captured + + def fetch() -> str: + events.append("target-fetch") + merge_boundary._append_merge_entry(88, "after-fetch-sha", "merge after target fetch") + return "merged-master" + + def post_verify( + _command: str, _target: str, *, ledger_snapshot: tuple[dict[str, Any], float, int] | None = None + ) -> int: + assert ledger_snapshot is not None + assert ledger_snapshot[2] == 0 + return 1 + + monkeypatch.setattr(merge_boundary, "_terminal_verify_snapshot", snapshot) + monkeypatch.setattr(merge_boundary, "_fetched_current_default_branch_sha", fetch) + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", post_verify) + + assert merge_boundary.main(["record-full-verify", "--command", "devtools verify --all"]) == 1 + assert events == ["ledger-snapshot", "target-fetch"] + assert snapshots[0][2] == 0 + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_external_merge_before_completion_is_reconciled_from_durable_intent( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + base_run = _fake_run(pr_view) + merged = False + + def run(cmd: list[str], **kwargs: Any) -> MagicMock: + if cmd[:3] == ["gh", "pr", "view"] and merged: + return MagicMock( + returncode=0, + stdout=json.dumps({"state": "MERGED", "mergeCommit": {"oid": "merge-commit"}}), + stderr="", + ) + return base_run(cmd, **kwargs) + + complete_merge_intent = merge_boundary._complete_merge_intent + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(merge_boundary, "_complete_merge_intent", lambda _pr, _head_sha: None) + assert ( + merge_boundary.cmd_merge( + 42, + command="devtools test x", + max_age_s=3600, + poll_rounds=1, + poll_interval_s=0, + dry_run=False, + with_verify=False, + verify_command="devtools verify --all", + ) + == 0 + ) + merged = True + assert merge_boundary._read_ledger()["merge_intents"] + monkeypatch.setattr(merge_boundary, "_complete_merge_intent", complete_merge_intent) + assert merge_boundary.cmd_train_status(as_json=False) == 1 + ledger = merge_boundary._read_ledger() + assert not ledger["merge_intents"] + assert ledger["merges"][0]["pr"] == 42 + + +def test_external_merge_completion_write_failure_keeps_recovery_latch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._record_merge_intent(42, "feature-sha", "fix: thing (#42)") + + def fail_completion_replace(source: Path, destination: Path) -> None: + if destination == merge_boundary._LEDGER_PATH: + raise OSError("injected completion publication failure") + os.replace(source, destination) + + monkeypatch.setattr(merge_boundary, "_durable_replace", fail_completion_replace) + with pytest.raises(merge_boundary.LedgerStateError): + merge_boundary._complete_merge_intent(42, "feature-sha") + assert merge_boundary._LEDGER_PENDING_PATH.exists() + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_detached_worktree_add_failure_attempts_cleanup(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + commands: list[list[str]] = [] + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + commands.append(cmd) + if cmd[:3] == ["git", "worktree", "add"]: + return MagicMock(returncode=1, stdout="", stderr="add failed") + if cmd[:3] == ["git", "worktree", "remove"]: + return MagicMock(returncode=0, stdout="", stderr="") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + assert merge_boundary._run_post_merge_terminal_verify("devtools verify --all", "merged-master") == 1 + assert any(command[:3] == ["git", "worktree", "remove"] for command in commands) + + +def test_detached_worktree_cleanup_failure_is_explicit(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + if cmd[:3] == ["git", "worktree", "add"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:3] == ["git", "worktree", "remove"]: + return MagicMock(returncode=2, stdout="", stderr="remove failed") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(merge_boundary, "cmd_record_full_verify", lambda *_args, **_kwargs: 0) + assert merge_boundary._run_post_merge_terminal_verify("devtools verify --all", "merged-master") == 1 + + def test_manual_record_route_fetches_target_and_rejects_stale_cli_output( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -780,7 +976,7 @@ def run(cmd: list[str], **_kwargs: Any) -> MagicMock: stderr="", ) - def post_verify(command: str, target_sha: str) -> int: + def post_verify(command: str, target_sha: str, **_kwargs: Any) -> int: post_targets.append(target_sha) return merge_boundary.cmd_record_full_verify(command, target_sha=target_sha) diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index cb9cf2e744..3d987e642e 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -24,7 +24,9 @@ import pytest +import devtools.checkout_guard as checkout_guard import devtools.testmon_bootstrap as testmon_bootstrap +import devtools.verify as verify from devtools.testmon_bootstrap import ( BootstrapDecision, bootstrap_testmon_seed_files, @@ -493,6 +495,97 @@ def test_complete_typed_markerless_green_attempt_bootstraps_only_as_selection_st assert not (tmp_path / "lane" / "seed.json").exists() +def test_markerless_complete_bootstrap_passes_guard_and_verify_preflight( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main_root = tmp_path / "main" + main_data = main_root / "testmondata" + nodeid = "tests/test.py::test_passed" + _write_sqlite_db(main_data, rows=(nodeid,)) + attempt = main_root / "seed-attempt.json" + attempt.write_text( + json.dumps( + { + "protocol_version": PROTOCOL_VERSION, + "status": "complete", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + "terminal_authorization": None, + }, + "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, + "expected_nodeids": [nodeid], + "expected_count": 1, + "expected_digest": hashlib.sha256(nodeid.encode()).hexdigest(), + "node_outcomes": [{"nodeid": nodeid, "outcome": "passed"}], + "exit_code": 0, + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + "run_id": "green-run", + "artifact_dir": ".cache/verify/runs/green-run", + "testmon_data": file_fingerprint(main_data), + } + ) + ) + artifact = main_root / ".cache" / "verify" / "runs" / "green-run" + artifact.mkdir(parents=True) + (artifact / "run.json").write_text( + json.dumps( + { + "run_id": "green-run", + "checkout_root": str(main_root.resolve()), + "artifact_dir": ".cache/verify/runs/green-run", + } + ) + ) + + lane = tmp_path / "lane" + lane.mkdir() + (lane / ".git").write_text("gitdir: /main/.git/worktrees/lane\n") + (lane / ".venv" / "bin").mkdir(parents=True) + package = lane / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("") + local_data = lane / ".cache" / "testmon" / "testmondata" + local_stamp = lane / ".cache" / "testmon" / "seed.json" + local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + main_testmon_data=main_data, + main_seed_stamp=main_root / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + ) + assert decision.selection_only + assert bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + local_seed_attempt=local_attempt, + checkout_root=lane, + inherited_from=main_root, + ) + + monkeypatch.setattr(checkout_guard, "_is_linked_worktree", lambda _root: True) + fingerprint = checkout_guard.checkout_environment_fingerprint( + lane, + polylogue_import_path=package / "__init__.py", + python_executable=lane / ".venv" / "bin" / "python", + ) + assert fingerprint.clean + monkeypatch.setattr(verify, "ROOT", lane) + monkeypatch.setattr(verify, "TESTMON_DATA", local_data) + monkeypatch.setattr(verify, "TESTMON_SEED_STAMP", local_stamp) + monkeypatch.setattr(verify, "TESTMON_SEED_ATTEMPT", local_attempt) + assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is None + assert json.loads(local_attempt.read_text())["release_baseline_allowed"] is False + + def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: """Partial local state (e.g. a stale stamp with no db, or vice versa) still needs a fresh copy.""" local_stamp = tmp_path / "local" / "seed.json" From 7f5840b6ea98e7414becb41e9b5ee514f056b34b Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 02:37:16 +0200 Subject: [PATCH 23/34] fix(testmon): cap seed worker concurrency Problem: seed-testmon discarded its declared four-worker bound and could start the full adaptive pool. The recorded resume reached 85 processes, 9.7 GiB PSS, 4.5 GiB swap PSS, and the 2,700-second containment timeout. What changed: retain adaptive sizing for ordinary pytest lanes, but cap only seed-testmon at four workers. The focused harness now proves a 12-worker adaptive result is bounded to four on the production command path. --- devtools/verify.py | 17 +++++++++-------- tests/unit/devtools/test_verify.py | 13 ++++++++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index d1f75f0552..ba67bbb1ef 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -234,7 +234,6 @@ def _format_completion_notification( DEFAULT_PYTEST_STALL_TIMEOUT_S = 10 * 60.0 DEFAULT_PYTEST_TERM_GRACE_S = 5.0 DEFAULT_PYTEST_RESOURCE_INTERVAL_S = 2.0 -DEFAULT_TESTMON_WORKERS = "4" def _load_history() -> list[dict[str, Any]]: @@ -1912,7 +1911,7 @@ def build_verify_steps( else: pytest_cmd.append("--testmon-noselect") label = "pytest seed-testmon" - pytest_cmd.extend(_pytest_worker_args(default="4")) + pytest_cmd.extend(_pytest_worker_args(maximum=4)) steps.append((label, pytest_cmd)) elif full_pytest: # #1775: the full diagnostic runs as two lanes. The bulk lane keeps @@ -1927,7 +1926,7 @@ def build_verify_steps( *pytest_cmd, "-m", f"({base_marker}) and not load_sensitive and not tui", - *_pytest_worker_args(default="4"), + *_pytest_worker_args(), ] steps.append(("pytest full (parallel)", bulk_cmd)) @@ -1944,8 +1943,7 @@ def _isolated_report_arg(arg: str) -> str: isolated_cmd.extend(["-m", f"({base_marker}) and (load_sensitive or tui)", "-p", "no:randomly", "-n", "0"]) steps.append(("pytest load-sensitive (isolated)", isolated_cmd)) else: - default_workers = DEFAULT_TESTMON_WORKERS - pytest_cmd.extend(["-m", base_marker, "--testmon", *_pytest_worker_args(default=default_workers)]) + pytest_cmd.extend(["-m", base_marker, "--testmon", *_pytest_worker_args()]) pytest_cmd.append("--testmon-forceselect") label = "pytest testmon (broad)" if broad_testmon else "pytest testmon" steps.append((label, pytest_cmd)) @@ -2055,9 +2053,12 @@ def _file_fingerprint(path: Path) -> str: return h.hexdigest() -def _pytest_worker_args(*, default: str) -> list[str]: - del default - return ["-n", str(adaptive_pytest_worker_count(os.environ))] +def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: + """Return the managed worker count, optionally capped for a bounded lane.""" + workers = adaptive_pytest_worker_count(os.environ) + if maximum is not None: + workers = min(workers, maximum) + return ["-n", str(workers)] _BROAD_TESTMON_CHANGED_PATHS = { diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 819efdc88e..59fbcc5e75 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -236,7 +236,18 @@ def test_seed_testmon_runs_full_collection_without_selection(monkeypatch: pytest assert "--testmon" in command assert "--testmon-noselect" in command assert "-n" in command - assert command[command.index("-n") + 1] == "8" + assert command[command.index("-n") + 1] == "4" + + +def test_seed_testmon_caps_adaptive_workers(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("POLYLOGUE_PYTEST_WORKERS", raising=False) + monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 12) + + steps = build_verify_steps(quick=False, lab=False, skip_slow=False, seed_testmon=True) + + label, command = steps[-1] + assert label == "pytest seed-testmon" + assert command[command.index("-n") + 1] == "4" def test_resumed_seed_uses_affected_selection_for_remaining_tests() -> None: From 4590cda74a20b3bb47e500e19cd125c86ec47df4 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 05:16:50 +0200 Subject: [PATCH 24/34] fix(verify): record capped pytest concurrency Problem: seed-testmon caps the executed pytest command at four workers, while workload receipts recorded the uncapped adaptive policy. This made resource evidence disagree with the process that actually ran. What changed: derive workload receipt concurrency from the final pytest command. A mutation test proves an uncapped twelve-worker policy records four for a capped seed command. --- devtools/verify.py | 14 +++++++++++++- tests/unit/devtools/test_verify.py | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/devtools/verify.py b/devtools/verify.py index ba67bbb1ef..fd5eebc6fc 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -1694,7 +1694,7 @@ def _run( last_resource_sample=last_resource_row, tmpfs_budget_mb=pytest_tmpfs_budget_mb, basetemp_cleanup=basetemp_cleanup, - concurrency=runtime_policy.workers if runtime_policy is not None else 1, + concurrency=_pytest_command_concurrency(cmd), ) metadata["workload_receipt"] = workload_receipt if artifacts is not None: @@ -2061,6 +2061,18 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: return ["-n", str(workers)] +def _pytest_command_concurrency(cmd: Sequence[str]) -> int: + """Return the worker count actually requested by the final pytest command.""" + for index in range(len(cmd) - 2, -1, -1): + if cmd[index] != "-n": + continue + try: + return max(1, int(cmd[index + 1])) + except ValueError: + return 1 + return 1 + + _BROAD_TESTMON_CHANGED_PATHS = { "pyproject.toml", "tests/conftest.py", diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 59fbcc5e75..524d2a33fc 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1673,6 +1673,27 @@ def test_run_records_managed_basetemp_cleanup_metadata(tmp_path: Path) -> None: assert metadata["basetemp_cleanup"] == str(cleaned) +def test_run_receipt_uses_capped_pytest_command_concurrency() -> None: + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + + class UncappedPolicy: + workers = 12 + + def to_dict(self) -> dict[str, int]: + return {"workers": self.workers} + + with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, UncappedPolicy())), + patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), + patch("devtools.verify._read_pytest_report", return_value=None), + ): + rc, _elapsed, metadata = _run("pytest seed-testmon", ["pytest", "--testmon", "-n", "4"]) + + assert rc == 0 + assert metadata["pytest_runtime_policy"] == {"workers": 12} + assert metadata["workload_receipt"]["spec"]["concurrency"] == 4 + + def test_run_forces_subprocesses_to_current_checkout(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("POLYLOGUE_ROOT", "/stale/main") monkeypatch.setenv("POLYLOGUE_REPO_ROOT", "/stale/main") From 7b3b29bc166792550874b40eb616ebd3615b245a Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 05:58:17 +0200 Subject: [PATCH 25/34] fix(testmon): close validated SQLite reads Problem: read-only graph inspection left SQLite connections open, while stamp creation re-read a database after it had already proved its fingerprint. What changed: close inspection connections deterministically and retain the validated fingerprint in the resulting stamp. A focused mutation test proves stamp construction makes one fingerprint read. --- devtools/testmon_state.py | 5 +++-- tests/unit/devtools/test_testmon_state.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index f419b39b29..f94bd97770 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -14,6 +14,7 @@ from __future__ import annotations +import contextlib import hashlib import json import sqlite3 @@ -530,7 +531,7 @@ def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> Gra GraphStatus.INCOMPLETE, 0, 0, expected, 0, 0, "missing or malformed expected nodeids", () ) try: - with sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True) as connection: + with contextlib.closing(sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)) as connection: if connection.execute("PRAGMA integrity_check").fetchone() != ("ok",): return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "sqlite integrity check failed", ()) required = {"test_execution", "test_execution_file_fp", "file_fp"} @@ -836,7 +837,7 @@ def stamp_from_attempt( graph, typed_identity, typed_binding, - file_fingerprint(data_path), + recorded_data, run_id, artifact_dir, ) diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index fe3bf01e40..94b8eb0a7e 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -4,6 +4,7 @@ import json import sqlite3 from pathlib import Path +from unittest.mock import patch import pytest @@ -118,6 +119,18 @@ def test_omitted_interrupted_and_uncovered_nodes_fail_closed(tmp_path: Path) -> assert stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None +def test_stamp_from_attempt_does_not_reopen_the_validated_database(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data) + + with patch("devtools.testmon_state.file_fingerprint", return_value=attempt["testmon_data"]) as fingerprint: + stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert stamp is not None + fingerprint.assert_called_once_with(data) + + def test_incomplete_attempt_fails_closed(tmp_path: Path) -> None: data = tmp_path / "testmondata" _write_graph(data) From 10265a13bd3e9fdd177814789fb21b7d39671641 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 06:03:59 +0200 Subject: [PATCH 26/34] fix(testmon): validate local bootstrap attempts Bootstrap now checks local markerless attempts against their SQLite graph and fingerprint, and it carries the caller's attempt relpath through both roots. Publication uses one selection-only condition so it cannot discard a staged marker under a divergent future decision. --- devtools/testmon_bootstrap.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 68547de3a8..1a99070505 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -68,7 +68,6 @@ from devtools.testmon_state import ( TestmonSeedStamp, - attempt_is_checkout_bound, refresh_stamp, stamp_from_attempt, validate_stamp, @@ -158,10 +157,16 @@ def decide_testmon_bootstrap( local_attempt = json.loads(local_seed_attempt.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError): local_attempt = None - if isinstance(local_attempt, Mapping) and attempt_is_checkout_bound( - local_attempt, - checkout_root=local_root, - protocol_version=protocol_version, + if ( + isinstance(local_attempt, Mapping) + and stamp_from_attempt( + local_attempt, + local_testmon_data, + checkout_root=local_root, + protocol_version=protocol_version, + published_marker=False, + ) + is not None ): return BootstrapDecision(False, "local .cache/testmon already has a checkout-bound selection attempt") if not main_testmon_data.is_file(): @@ -421,7 +426,8 @@ def bootstrap_testmon_seed_files( ): return False staged_attempt_path: Path | None = None - if decision.main_seed_attempt is not None and decision.selection_only: + publishes_selection_attempt = decision.main_seed_attempt is not None and decision.selection_only + if publishes_selection_attempt: assert local_seed_attempt is not None source_attempt = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) if not isinstance(source_attempt, dict): @@ -467,7 +473,7 @@ def bootstrap_testmon_seed_files( (local_testmon_data, staged_data), (destination_artifact / "run.json", staged_artifact / "run.json"), (destination_root / ".cache" / "verify" / "current-run.json", staged_current_run), - (local_seed_stamp, None if decision.selection_only else staged_stamp), + (local_seed_stamp, None if publishes_selection_attempt else staged_stamp), ] if local_seed_attempt is not None: publication_files.append((local_seed_attempt, staged_attempt_path)) @@ -515,6 +521,7 @@ def maybe_bootstrap_testmon_seed( *, testmon_data_relpath: str = TESTMON_DATA_RELPATH, seed_stamp_relpath: str = TESTMON_SEED_STAMP_RELPATH, + seed_attempt_relpath: str = TESTMON_SEED_ATTEMPT_RELPATH, protocol_version: int, ) -> str | None: """Bootstrap `repo_root`'s testmon seed from its main checkout if warranted. @@ -533,10 +540,10 @@ def maybe_bootstrap_testmon_seed( return None local_testmon_data = repo_root / testmon_data_relpath local_seed_stamp = repo_root / seed_stamp_relpath - local_seed_attempt = repo_root / TESTMON_SEED_ATTEMPT_RELPATH + local_seed_attempt = repo_root / seed_attempt_relpath main_testmon_data = main_checkout / testmon_data_relpath main_seed_stamp = main_checkout / seed_stamp_relpath - main_seed_attempt = main_checkout / TESTMON_SEED_ATTEMPT_RELPATH + main_seed_attempt = main_checkout / seed_attempt_relpath decision = decide_testmon_bootstrap( is_linked_worktree=is_linked_worktree, local_testmon_data=local_testmon_data, @@ -561,8 +568,8 @@ def maybe_bootstrap_testmon_seed( ) if not stamped: return ( - f"verify: bootstrapped pytest-testmon seed into {local_testmon_data.parent}, " - "but could not record its checkout provenance" + f"verify: refused pytest-testmon bootstrap into {local_testmon_data.parent}; " + "no local state was published because provenance validation failed" ) if decision.main_seed_attempt is not None and decision.selection_only: return ( From 14ea1cedc5aa56860898a4eb8e168cdfa7220f29 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 06:05:12 +0200 Subject: [PATCH 27/34] fix(testmon): prove selection attempt source Assert the non-null decision field in the selection-only publication branch so the static type contract matches the branch predicate. --- devtools/testmon_bootstrap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 1a99070505..52cdadcbfc 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -428,6 +428,7 @@ def bootstrap_testmon_seed_files( staged_attempt_path: Path | None = None publishes_selection_attempt = decision.main_seed_attempt is not None and decision.selection_only if publishes_selection_attempt: + assert decision.main_seed_attempt is not None assert local_seed_attempt is not None source_attempt = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) if not isinstance(source_attempt, dict): From 1d0b1c5305bdc093c59f845eac7c8a1c6c85c1e8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 06:51:21 +0200 Subject: [PATCH 28/34] fix(testmon): bind reusable state to runtime environment --- devtools/checkout_guard.py | 2 +- devtools/testmon_state.py | 109 +++++++++++++++++- devtools/verify.py | 43 +++++-- tests/unit/devtools/test_testmon_bootstrap.py | 4 +- tests/unit/devtools/test_testmon_state.py | 26 +++++ tests/unit/devtools/test_verify.py | 24 +++- 6 files changed, 194 insertions(+), 14 deletions(-) diff --git a/devtools/checkout_guard.py b/devtools/checkout_guard.py index 7b3210136c..cc6cc76e28 100644 --- a/devtools/checkout_guard.py +++ b/devtools/checkout_guard.py @@ -132,7 +132,7 @@ def as_dict(self) -> dict[str, object]: _TESTMON_STATE_DIR = Path(".cache/testmon") _TESTMON_STATE_MARKER = _TESTMON_STATE_DIR / "seed.json" _TESTMON_SEED_ATTEMPT = _TESTMON_STATE_DIR / "seed-attempt.json" -_TESTMON_SEED_PROTOCOL_VERSION = 4 +_TESTMON_SEED_PROTOCOL_VERSION = 5 _VERIFY_STATE_DIR = Path(".cache/verify") _VERIFY_STATE_MARKER = _VERIFY_STATE_DIR / "current-run.json" diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index f94bd97770..d664b6f69a 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -16,7 +16,9 @@ import contextlib import hashlib +import importlib.metadata import json +import os import sqlite3 from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace @@ -66,6 +68,8 @@ class TestmonIdentity: lab: bool git_tree: str | None = None terminal_authorization: str | None = None + dependency_environment: str = "" + pytest_harness: str = "" @classmethod def from_mapping(cls, value: Mapping[str, Any]) -> TestmonIdentity: @@ -81,6 +85,16 @@ def from_mapping(cls, value: Mapping[str, Any]) -> TestmonIdentity: raise ValueError("identity.worktree_fingerprint must be a non-empty string") if not isinstance(python, str) or not python: raise ValueError("identity.python must be a non-empty string") + dependency_environment = value.get("dependency_environment") + pytest_harness = value.get("pytest_harness") + if dependency_environment is None: + dependency_environment = "" + if pytest_harness is None: + pytest_harness = "" + if not isinstance(dependency_environment, str): + raise ValueError("identity.dependency_environment must be a string") + if not isinstance(pytest_harness, str): + raise ValueError("identity.pytest_harness must be a string") if not isinstance(value.get("skip_slow"), bool) or not isinstance(value.get("lab"), bool): raise ValueError("identity selection flags must be booleans") terminal_authorization = value.get("terminal_authorization") @@ -96,6 +110,8 @@ def from_mapping(cls, value: Mapping[str, Any]) -> TestmonIdentity: value["lab"], git_tree, terminal_authorization, + dependency_environment, + pytest_harness, ) def as_dict(self) -> dict[str, Any]: @@ -107,9 +123,91 @@ def as_dict(self) -> dict[str, Any]: "lab": self.lab, "git_tree": self.git_tree, "terminal_authorization": self.terminal_authorization, + "dependency_environment": self.dependency_environment, + "pytest_harness": self.pytest_harness, } +def _fingerprint_files(checkout_root: Path, relative_paths: Sequence[str]) -> str: + """Hash named checkout inputs, preserving absent inputs as typed state.""" + digest = hashlib.sha256() + for relative_path in relative_paths: + digest.update(relative_path.encode()) + digest.update(b"\0") + try: + contents = (checkout_root / relative_path).read_bytes() + except OSError: + digest.update(b"missing") + else: + digest.update(contents) + digest.update(b"\0") + return digest.hexdigest() + + +def _installed_distributions() -> tuple[tuple[str, str], ...] | None: + """Return the active environment's normalized installed distributions.""" + try: + distributions = [] + for distribution in importlib.metadata.distributions(): + name = distribution.metadata.get("Name") + version = distribution.version + if not isinstance(name, str) or not name or not isinstance(version, str) or not version: + return None + distributions.append((name.casefold(), version)) + except (OSError, TypeError, ValueError, importlib.metadata.PackageNotFoundError): + return None + return tuple(sorted(distributions)) + + +def testmon_runtime_identity(checkout_root: Path) -> tuple[str, str] | None: + """Identify the lock, installed dependencies, and pytest execution harness. + + A testmon graph is reusable only under this exact dependency environment. + The application lock catches declared changes; installed distributions and + pytest-specific configuration catch a stale or differently provisioned + virtual environment even when ``sys.version`` is unchanged. + """ + distributions = _installed_distributions() + if distributions is None: + return None + normalized_root = checkout_root.resolve() + dependency_payload = { + "lock_inputs": _fingerprint_files(normalized_root, ("uv.lock", "pyproject.toml")), + "distributions": distributions, + } + harness_payload = { + "configuration": _fingerprint_files( + normalized_root, + ("pyproject.toml", "pytest.ini", "tox.ini", "setup.cfg", "tests/conftest.py"), + ), + "environment": { + key: os.environ.get(key) for key in ("PYTEST_ADDOPTS", "PYTEST_DISABLE_PLUGIN_AUTOLOAD", "PYTEST_PLUGINS") + }, + "pytest_distributions": tuple( + item for item in distributions if item[0] in {"pytest", "pytest-testmon", "pytest-xdist", "pluggy"} + ), + } + return ( + hashlib.sha256(json.dumps(dependency_payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + hashlib.sha256(json.dumps(harness_payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + ) + + +def _identity_matches_runtime(identity: TestmonIdentity, *, checkout_root: Path, protocol_version: int) -> bool: + """Keep pre-binding protocol receipts parseable but never reusable today.""" + if protocol_version < 5: + return True + runtime_identity = testmon_runtime_identity(checkout_root) + return ( + runtime_identity is not None + and ( + identity.dependency_environment, + identity.pytest_harness, + ) + == runtime_identity + ) + + @dataclass(frozen=True, slots=True) class TestmonBinding: mode: BindingMode @@ -464,9 +562,11 @@ def attempt_is_checkout_bound( if attempt.get("expected_digest") != expected_digest: return False try: - TestmonIdentity.from_mapping(identity) + typed_identity = TestmonIdentity.from_mapping(identity) except ValueError: return False + if not _identity_matches_runtime(typed_identity, checkout_root=checkout_root, protocol_version=protocol_version): + return False omitted = selection.get("selected_nodeids_omitted") selected_count = selection.get("selected_count") if ( @@ -673,6 +773,10 @@ def validate_stamp( and stamp.identity.terminal_authorization != TerminalAuthorization.NARROW_TERMINAL.value ): return None + if not _identity_matches_runtime( + stamp.identity, checkout_root=checkout_root, protocol_version=protocol_version + ): + return None if Path(stamp.binding.checkout_root).resolve() != checkout_root.resolve(): return None if file_fingerprint(data_path) != stamp.testmon_data: @@ -781,6 +885,8 @@ def stamp_from_attempt( typed_identity = TestmonIdentity.from_mapping(identity) except ValueError: return None + if not _identity_matches_runtime(typed_identity, checkout_root=checkout_root, protocol_version=protocol_version): + return None baseline = ( BaselineStatus.GREEN if attempt.get("status") == "complete" @@ -860,5 +966,6 @@ def stamp_from_attempt( "refresh_stamp", "seed_marker_is_checkout_bound", "stamp_from_attempt", + "testmon_runtime_identity", "validate_stamp", ] diff --git a/devtools/verify.py b/devtools/verify.py index fd5eebc6fc..3a8a83f5a3 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -68,6 +68,7 @@ inspect_testmon_database, refresh_stamp, stamp_from_attempt, + testmon_runtime_identity, validate_stamp, ) from devtools.verify_runs import ( @@ -212,7 +213,7 @@ def _format_completion_notification( TESTMON_SEED_STAMP = Path(".cache/testmon/seed.json") TESTMON_SEED_ATTEMPT = Path(".cache/testmon/seed-attempt.json") TESTMON_AFFECTED_STAMP = Path(".cache/testmon/affected.json") -TESTMON_SEED_PROTOCOL_VERSION = 4 +TESTMON_SEED_PROTOCOL_VERSION = 5 PYTEST_REPORT_DIR = Path(".cache/verify") PYTEST_REPORT_PATH = PYTEST_REPORT_DIR / "last-pytest.json" PYTEST_JUNIT_REPORT_DIR = Path(".cache/test-reports") @@ -2258,6 +2259,10 @@ def _testmon_seed_identity( lab: bool, terminal_authorization: str | None = None, ) -> dict[str, Any]: + runtime_identity = testmon_runtime_identity(ROOT) + if runtime_identity is None: + raise RuntimeError("could not identify the active dependency environment and pytest harness") + dependency_environment, pytest_harness = runtime_identity return { "git_head": git_head, "git_tree": git_tree, @@ -2266,6 +2271,8 @@ def _testmon_seed_identity( "skip_slow": skip_slow, "lab": lab, "terminal_authorization": terminal_authorization, + "dependency_environment": dependency_environment, + "pytest_harness": pytest_harness, } @@ -2378,7 +2385,16 @@ def _testmon_seed_resume_contract(identity: Mapping[str, Any]) -> dict[str, Any] """Return inputs that change which corpus a seed promises to cover.""" return { key: identity.get(key) - for key in ("git_tree", "worktree_fingerprint", "python", "skip_slow", "lab", "terminal_authorization") + for key in ( + "git_tree", + "worktree_fingerprint", + "python", + "skip_slow", + "lab", + "terminal_authorization", + "dependency_environment", + "pytest_harness", + ) } @@ -2896,13 +2912,22 @@ def main(argv: list[str] | None = None) -> int: resume_testmon_seed = False prepared_seed_attempt: dict[str, Any] | None = None if args.seed_testmon: - seed_identity = _testmon_seed_identity( - git_head=head, - git_tree=_git_committed_tree(), - skip_slow=bool(args.skip_slow), - lab=bool(args.lab), - terminal_authorization=args.terminal_authorization, - ) + try: + seed_identity = _testmon_seed_identity( + git_head=head, + git_tree=_git_committed_tree(), + skip_slow=bool(args.skip_slow), + lab=bool(args.lab), + terminal_authorization=args.terminal_authorization, + ) + except RuntimeError as exc: + sys.stderr.write(f"verify: {exc}\n") + verify_run.finish( + exit_code=125, + duration_s=time.monotonic() - t0, + diagnosis="testmon_environment_identity_unavailable", + ) + return 125 resume_testmon_seed = _testmon_seed_can_resume(seed_identity) prepared_seed_attempt = _prepare_testmon_seed_attempt( identity=seed_identity, diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 3d987e642e..2fde1706a0 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -577,12 +577,12 @@ def test_markerless_complete_bootstrap_passes_guard_and_verify_preflight( polylogue_import_path=package / "__init__.py", python_executable=lane / ".venv" / "bin" / "python", ) - assert fingerprint.clean + assert not fingerprint.clean monkeypatch.setattr(verify, "ROOT", lane) monkeypatch.setattr(verify, "TESTMON_DATA", local_data) monkeypatch.setattr(verify, "TESTMON_SEED_STAMP", local_stamp) monkeypatch.setattr(verify, "TESTMON_SEED_ATTEMPT", local_attempt) - assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is None + assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is not None assert json.loads(local_attempt.read_text())["release_baseline_allowed"] is False diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index 94b8eb0a7e..a4c882fdbe 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -8,6 +8,7 @@ import pytest +import devtools.testmon_state as testmon_state from devtools.testmon_state import ( BaselineStatus, GraphStatus, @@ -151,6 +152,31 @@ def test_incomplete_attempt_fails_closed(tmp_path: Path) -> None: assert completed.release_baseline_allowed +def test_reusable_attempt_rejects_a_changed_dependency_or_pytest_harness( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Reusable graphs belong to the environment that captured them.""" + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data) + identity = attempt["identity"] + assert isinstance(identity, dict) + identity["dependency_environment"] = "dependency-environment" + identity["pytest_harness"] = "pytest-harness" + attempt["protocol_version"] = 5 + monkeypatch.setattr( + testmon_state, + "testmon_runtime_identity", + lambda _root: ("dependency-environment", "pytest-harness"), + raising=False, + ) + + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=5) is not None + + identity["dependency_environment"] = "different-environment" + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=5) is None + + def test_green_skipped_slow_attempt_without_typed_terminal_authority_is_selection_only(tmp_path: Path) -> None: data = tmp_path / "testmondata" _write_graph(data) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 524d2a33fc..c713c8dfb6 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -93,6 +93,13 @@ def _pytest_marker_expr(command: list[str]) -> str: return command[marker_indexes[-1] + 1] +def _testmon_runtime_identity_fields(checkout_root: Path = ROOT) -> dict[str, str]: + runtime_identity = verify.testmon_runtime_identity(checkout_root) + assert runtime_identity is not None + dependency_environment, pytest_harness = runtime_identity + return {"dependency_environment": dependency_environment, "pytest_harness": pytest_harness} + + def _write_real_testmon_state(nodeids: tuple[str, ...] = ("tests/test_a.py::test_one",)) -> Path: TESTMON_DATA.parent.mkdir(parents=True, exist_ok=True) with sqlite3.connect(TESTMON_DATA) as conn: @@ -113,7 +120,16 @@ def _write_real_testmon_state(nodeids: tuple[str, ...] = ("tests/test_a.py::test True, 0, GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()), - _TestmonIdentity("current-head", "covered", "python", True, False, None, "narrow-terminal"), + _TestmonIdentity( + "current-head", + "covered", + "python", + True, + False, + None, + "narrow-terminal", + **_testmon_runtime_identity_fields(), + ), _TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())), file_fingerprint(TESTMON_DATA), "seed", @@ -696,6 +712,7 @@ def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) "python": "python", "skip_slow": False, "lab": False, + **_testmon_runtime_identity_fields(Path.cwd()), }, "resume": True, "expected_nodeids": expected, @@ -869,6 +886,7 @@ def test_seed_receipt_classifies_every_node_terminal_outcome( "python": "python", "skip_slow": False, "lab": False, + **_testmon_runtime_identity_fields(Path.cwd()), }, "resume": False, "expected_nodeids": [], @@ -1033,6 +1051,7 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon "python": "python", "skip_slow": False, "lab": False, + **_testmon_runtime_identity_fields(Path.cwd()), }, "resume": False, "expected_nodeids": [], @@ -1062,6 +1081,7 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon "skip_slow": True, "lab": False, "terminal_authorization": "narrow-terminal", + **_testmon_runtime_identity_fields(Path.cwd()), }, "resume": False, "expected_nodeids": [], @@ -1087,6 +1107,7 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon "python": "python", "skip_slow": False, "lab": False, + **_testmon_runtime_identity_fields(Path.cwd()), }, "resume": False, "expected_nodeids": [], @@ -1182,6 +1203,7 @@ def test_resumed_seed_persists_full_selection_before_stamp_publication( "skip_slow": False, "lab": False, "terminal_authorization": None, + **_testmon_runtime_identity_fields(Path.cwd()), }, "resume": True, "expected_nodeids": expected, From 3ac00e879fbfa15f9fb4cda31c4e9fa417fd6e4a Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 06:51:40 +0200 Subject: [PATCH 29/34] fix(merge): recover terminal verification state --- devtools/merge_boundary.py | 61 +++++++++++++++++++--- tests/unit/devtools/test_merge_boundary.py | 51 ++++++++++++++++-- 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index c67783ad51..f97f8a595d 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -78,7 +78,7 @@ from typing import Any from devtools import merge_gate -from devtools.testmon_state import VerificationScope +from devtools.testmon_state import TerminalAuthorization, VerificationScope _LEDGER_PATH = Path(".cache/verify/merge-gate/merge-train-ledger.json") _LEDGER_PENDING_PATH = _LEDGER_PATH.with_name(f"{_LEDGER_PATH.name}.pending") @@ -219,8 +219,7 @@ def _validate_ledger(data: object) -> dict[str, Any]: def _read_ledger_unlocked() -> dict[str, Any]: - if _LEDGER_PENDING_PATH.exists(): - raise LedgerStateError("merge-train ledger has an unfinished durable write") + _recover_pending_ledger_unlocked() if not _LEDGER_PATH.exists(): return {"merges": [], "merge_intents": [], "last_full_verify": None} try: @@ -234,6 +233,34 @@ def _read_ledger_unlocked() -> dict[str, Any]: return _validate_ledger(data) +def _read_ledger_file(path: Path, *, description: str) -> dict[str, Any]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LedgerStateError(f"{description} is unreadable or truncated") from exc + if isinstance(data, dict): + data.setdefault("merges", []) + data.setdefault("merge_intents", []) + data.setdefault("last_full_verify", None) + return _validate_ledger(data) + + +def _recover_pending_ledger_unlocked() -> None: + """Finish or clear the write-ahead ledger journal after interruption.""" + if not _LEDGER_PENDING_PATH.exists(): + return + pending = _read_ledger_file(_LEDGER_PENDING_PATH, description="merge-train pending ledger write") + parent = _LEDGER_PATH.parent + try: + if _LEDGER_PATH.exists() and pending == _read_ledger_file(_LEDGER_PATH, description="merge-train ledger"): + _LEDGER_PENDING_PATH.unlink() + else: + _durable_replace(_LEDGER_PENDING_PATH, _LEDGER_PATH) + _fsync_directory(parent) + except OSError as exc: + raise LedgerStateError(f"merge-train ledger pending-write recovery failed: {exc}") from exc + + def _read_ledger() -> dict[str, Any]: with _ledger_lock(): return _read_ledger_unlocked() @@ -368,7 +395,13 @@ def _pending_prs_since_last_full_verify(ledger: dict[str, Any]) -> list[dict[str if last_verify.get("accepted") is True and last_verify.get("exit_code") == 0 and last_verify.get("release_baseline_allowed") is True - and scope == VerificationScope.RELEASE_BASELINE.value + and ( + scope == VerificationScope.RELEASE_BASELINE.value + or ( + scope == VerificationScope.NARROW_TERMINAL.value + and last_verify.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value + ) + ) else 0.0 ) snapshot_sequence = last_verify.get("merge_sequence") @@ -467,6 +500,12 @@ def _terminal_verify_snapshot() -> tuple[dict[str, Any], float, int]: return ledger, started_at, _merge_sequence(ledger) +def _reconciled_terminal_verify_snapshot() -> tuple[dict[str, Any], float, int]: + """Recover merged intents before fixing the terminal verification boundary.""" + _reconcile_merge_intents() + return _terminal_verify_snapshot() + + def _remove_detached_worktree(repo_root: Path, worktree: Path) -> bool: removal = subprocess.run( ["git", "worktree", "remove", "--force", str(worktree)], @@ -601,7 +640,7 @@ def cmd_merge( if with_verify: try: - ledger_snapshot = _terminal_verify_snapshot() + ledger_snapshot = _reconciled_terminal_verify_snapshot() except LedgerStateError as exc: print(f"REFUSING terminal verify: {exc}", file=sys.stderr) return 1 @@ -679,7 +718,7 @@ def cmd_record_full_verify( print("REFUSING: terminal verification has no fetched merged-master target", file=sys.stderr) return 1 try: - snapshot = ledger_snapshot or _terminal_verify_snapshot() + snapshot = ledger_snapshot or _reconciled_terminal_verify_snapshot() except LedgerStateError as exc: print(f"REFUSING to run terminal verification: {exc}", file=sys.stderr) return 1 @@ -704,7 +743,13 @@ def cmd_record_full_verify( accepted = ( result.returncode == 0 and release_allowed is True - and verification_scope == VerificationScope.RELEASE_BASELINE.value + and ( + verification_scope == VerificationScope.RELEASE_BASELINE.value + or ( + verification_scope == VerificationScope.NARROW_TERMINAL.value + and terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value + ) + ) and verified_head == target_sha ) @@ -804,7 +849,7 @@ def main(argv: list[str] | None = None) -> int: if args.action == "train-status": return cmd_train_status(args.as_json) try: - ledger_snapshot = _terminal_verify_snapshot() + ledger_snapshot = _reconciled_terminal_verify_snapshot() except LedgerStateError as exc: print(f"REFUSING terminal verify: {exc}", file=sys.stderr) return 1 diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index 0137a18404..7b98a5fdf6 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -544,7 +544,7 @@ def test_train_status_fails_closed_on_valid_json_partial_merge_entry( assert "Traceback" not in capsys.readouterr().err -def test_merge_write_failure_leaves_durable_pending_latch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +def test_merge_write_failure_recovers_valid_pending_ledger(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) pr_view = _base_pr_view() monkeypatch.setattr(subprocess, "run", _fake_run(pr_view)) @@ -570,7 +570,20 @@ def fail_final_replace(source: Path, destination: Path) -> None: == 1 ) assert merge_boundary._LEDGER_PENDING_PATH.exists() + monkeypatch.setattr(merge_boundary, "_durable_replace", os.replace) assert merge_boundary.cmd_train_status(as_json=False) == 1 + assert not merge_boundary._LEDGER_PENDING_PATH.exists() + assert merge_boundary._read_ledger()["merge_intents"] + + +def test_read_ledger_clears_byte_identical_pending_write(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._write_ledger({"merges": [], "merge_intents": [], "last_full_verify": None}) + serialized = merge_boundary._LEDGER_PATH.read_text() + merge_boundary._LEDGER_PENDING_PATH.write_text(serialized) + + assert merge_boundary._read_ledger() == {"merges": [], "merge_intents": [], "last_full_verify": None} + assert not merge_boundary._LEDGER_PENDING_PATH.exists() def test_train_status_blocks_when_pr_merged_after_last_full_verify( @@ -707,7 +720,7 @@ def test_record_full_verify_rejects_skip_slow_without_typed_authorization( assert merge_boundary.cmd_train_status(as_json=False) == 1 -def test_record_full_verify_rejects_explicit_typed_narrow_terminal_authorization( +def test_record_full_verify_accepts_explicit_typed_narrow_terminal_authorization( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.chdir(tmp_path) @@ -719,6 +732,7 @@ def test_record_full_verify_rejects_explicit_typed_narrow_terminal_authorization returncode=0, stdout=json.dumps( { + "git_head": "merged-master", "verification_scope": "narrow-terminal", "terminal_authorization": "narrow-terminal", "release_baseline_allowed": True, @@ -728,9 +742,9 @@ def test_record_full_verify_rejects_explicit_typed_narrow_terminal_authorization ), ) - assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 1 - assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False - assert merge_boundary.cmd_train_status(as_json=False) == 1 + assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 0 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is True + assert merge_boundary.cmd_train_status(as_json=False) == 0 def test_record_full_verify_rejects_untyped_scope_even_when_permission_is_true( @@ -901,6 +915,33 @@ def run(cmd: list[str], **kwargs: Any) -> MagicMock: assert ledger["merges"][0]["pr"] == 42 +def test_record_full_verify_reconciles_durable_intents_before_snapshot( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._record_merge_intent(42, "pr-head", "merged before recovery") + monkeypatch.setattr( + merge_boundary, + "_gh_json", + lambda _args: {"state": "MERGED", "mergeCommit": {"oid": "merge-commit"}}, + ) + monkeypatch.setattr(merge_boundary, "_fetched_current_default_branch_sha", lambda: "merged-master") + snapshots: list[tuple[dict[str, Any], float, int]] = [] + + def post_verify( + _command: str, _target: str, *, ledger_snapshot: tuple[dict[str, Any], float, int] | None = None + ) -> int: + assert ledger_snapshot is not None + snapshots.append(ledger_snapshot) + return 0 + + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", post_verify) + + assert merge_boundary.main(["record-full-verify", "--command", "devtools verify --all"]) == 0 + assert snapshots[0][2] == 1 + assert not merge_boundary._read_ledger()["merge_intents"] + + def test_external_merge_completion_write_failure_keeps_recovery_latch( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 4d6f93ef426eefdeb664b86d4fff574d54c93f18 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 06:55:40 +0200 Subject: [PATCH 30/34] fix(testmon): type runtime fingerprint metadata --- devtools/testmon_state.py | 7 +++++-- tests/unit/devtools/test_verify.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index d664b6f69a..dd6118051c 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -149,9 +149,12 @@ def _installed_distributions() -> tuple[tuple[str, str], ...] | None: try: distributions = [] for distribution in importlib.metadata.distributions(): - name = distribution.metadata.get("Name") + try: + name = distribution.metadata["Name"] + except KeyError: + return None version = distribution.version - if not isinstance(name, str) or not name or not isinstance(version, str) or not version: + if not name or not version: return None distributions.append((name.casefold(), version)) except (OSError, TypeError, ValueError, importlib.metadata.PackageNotFoundError): diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index c713c8dfb6..410c4ccddc 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -29,6 +29,9 @@ from devtools.testmon_state import ( TestmonSeedStamp as _TestmonSeedStamp, ) +from devtools.testmon_state import ( + testmon_runtime_identity as _testmon_runtime_identity, +) from devtools.verify import ( PYTEST_CONTAINMENT_PATH, PYTEST_EVENTS_PATH, @@ -94,7 +97,7 @@ def _pytest_marker_expr(command: list[str]) -> str: def _testmon_runtime_identity_fields(checkout_root: Path = ROOT) -> dict[str, str]: - runtime_identity = verify.testmon_runtime_identity(checkout_root) + runtime_identity = _testmon_runtime_identity(checkout_root) assert runtime_identity is not None dependency_environment, pytest_harness = runtime_identity return {"dependency_environment": dependency_environment, "pytest_harness": pytest_harness} From 5bcbb41c7eaa3c24e9c9da58dff14e1b4533d588 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 09:09:02 +0200 Subject: [PATCH 31/34] fix(testmon): validate staged bootstrap runtime inputs --- devtools/testmon_bootstrap.py | 19 +++++++++++++++++++ .../devtools/test_testmon_seed_recovery.py | 10 ++++++++-- tests/unit/devtools/test_checkout_guard.py | 13 +++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 52cdadcbfc..7397bbe294 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -328,6 +328,24 @@ def _publish_staged_bootstrap_files(*, staging_dir: Path, files: list[tuple[Path shutil.rmtree(backup_dir, ignore_errors=True) +def _copy_runtime_identity_inputs(*, source_root: Path, destination_root: Path) -> None: + """Mirror the inputs used to validate a staged testmon receipt.""" + for relative_path in ( + "uv.lock", + "pyproject.toml", + "pytest.ini", + "tox.ini", + "setup.cfg", + "tests/conftest.py", + ): + source = source_root / relative_path + if not source.is_file(): + continue + destination = destination_root / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + def bootstrap_testmon_seed_files( decision: BootstrapDecision, *, @@ -445,6 +463,7 @@ def bootstrap_testmon_seed_files( return False validation_receipt["checkout_root"] = str(validation_root.resolve()) validation_receipt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" + _copy_runtime_identity_inputs(source_root=destination_root, destination_root=validation_root) _atomic_write_json( validation_root / ".cache" / "verify" / "runs" / refreshed.run_id / "run.json", validation_receipt, diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py index 4c0b0638bd..412ae2f6b6 100644 --- a/tests/integration/devtools/test_testmon_seed_recovery.py +++ b/tests/integration/devtools/test_testmon_seed_recovery.py @@ -10,7 +10,7 @@ import pytest -from devtools import testmon_bootstrap, verify +from devtools import testmon_bootstrap, testmon_state, verify from devtools.testmon_state import file_fingerprint, inspect_testmon_database @@ -21,6 +21,7 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( ) -> None: source = tmp_path / "source" source.mkdir() + (source / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") (source / "test_sample.py").write_text( "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 2\n", encoding="utf-8", @@ -40,8 +41,11 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( assert run.returncode != 0 expected = ("test_sample.py::test_passed", "test_sample.py::test_failed") assert inspect_testmon_database(data, expected).usable_for_selection + runtime_identity = testmon_state.testmon_runtime_identity(source) + assert runtime_identity is not None + dependency_environment, pytest_harness = runtime_identity attempt = { - "protocol_version": 4, + "protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, "status": "reusable", "identity": { "git_head": "head", @@ -49,6 +53,8 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( "python": sys.version, "skip_slow": False, "lab": False, + "dependency_environment": dependency_environment, + "pytest_harness": pytest_harness, }, "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, "expected_nodeids": list(expected), diff --git a/tests/unit/devtools/test_checkout_guard.py b/tests/unit/devtools/test_checkout_guard.py index 93d6c21f84..a5ed1b0916 100644 --- a/tests/unit/devtools/test_checkout_guard.py +++ b/tests/unit/devtools/test_checkout_guard.py @@ -17,6 +17,7 @@ import devtools.click_dispatch as click_dispatch import devtools.run_tests as run_tests +import devtools.testmon_state as testmon_state import devtools.verify as verify import polylogue from devtools.checkout_guard import ( @@ -186,8 +187,20 @@ def _write_in_progress_seed_attempt(root: Path, *, status: str = "running", **ov } if status == "reusable": nodeid = "tests/test.py::test_one" + runtime_identity = testmon_state.testmon_runtime_identity(root) + assert runtime_identity is not None + dependency_environment, pytest_harness = runtime_identity payload.update( { + "identity": { + "git_head": "head", + "worktree_fingerprint": "fingerprint", + "python": "3.14", + "skip_slow": True, + "lab": False, + "dependency_environment": dependency_environment, + "pytest_harness": pytest_harness, + }, "expected_nodeids": [nodeid], "expected_count": 1, "expected_digest": hashlib.sha256(nodeid.encode()).hexdigest(), From 2d95500457d6e12c53758f8da1203b25468cdc03 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 11:09:41 +0200 Subject: [PATCH 32/34] fix: bind testmon state to test behavior environment --- devtools/testmon_state.py | 9 ++++++++- tests/unit/devtools/test_testmon_state.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index dd6118051c..b54c6dad62 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -184,7 +184,14 @@ def testmon_runtime_identity(checkout_root: Path) -> tuple[str, str] | None: ("pyproject.toml", "pytest.ini", "tox.ini", "setup.cfg", "tests/conftest.py"), ), "environment": { - key: os.environ.get(key) for key in ("PYTEST_ADDOPTS", "PYTEST_DISABLE_PLUGIN_AUTOLOAD", "PYTEST_PLUGINS") + key: os.environ.get(key) + for key in ( + "PYTEST_ADDOPTS", + "PYTEST_DISABLE_PLUGIN_AUTOLOAD", + "PYTEST_PLUGINS", + "HYPOTHESIS_PROFILE", + "POLYLOGUE_CI", + ) }, "pytest_distributions": tuple( item for item in distributions if item[0] in {"pytest", "pytest-testmon", "pytest-xdist", "pluggy"} diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index a4c882fdbe..36180fe946 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -177,6 +177,21 @@ def test_reusable_attempt_rejects_a_changed_dependency_or_pytest_harness( assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=5) is None +def test_runtime_identity_includes_test_behavior_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(testmon_state, "_installed_distributions", lambda: (("pytest", "9"),)) + monkeypatch.setenv("HYPOTHESIS_PROFILE", "ci") + monkeypatch.setenv("POLYLOGUE_CI", "1") + first = testmon_state.testmon_runtime_identity(tmp_path) + + monkeypatch.setenv("HYPOTHESIS_PROFILE", "default") + second = testmon_state.testmon_runtime_identity(tmp_path) + + assert first is not None + assert second is not None + assert first[0] == second[0] + assert first[1] != second[1] + + def test_green_skipped_slow_attempt_without_typed_terminal_authority_is_selection_only(tmp_path: Path) -> None: data = tmp_path / "testmondata" _write_graph(data) From e27caeecf98dc19c6dd9b3b36362b6f89f94568c Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 12:15:00 +0200 Subject: [PATCH 33/34] fix: type focused verification receipts --- devtools/run_tests.py | 8 +++++++- devtools/verify.py | 13 +++++++++++++ devtools/verify_runs.py | 15 ++++++++++++++- tests/unit/devtools/test_verify.py | 15 +++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 61fcec8fc2..08686f43d9 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -162,7 +162,13 @@ def main(argv: list[str] | None = None) -> int: ) started = time.monotonic() rc, _elapsed, metadata = _run("pytest focused", cmd, cwd=str(ROOT), run=run) - run.finish(exit_code=rc, duration_s=time.monotonic() - started, diagnosis=metadata.get("diagnosis")) + run.finish( + exit_code=rc, + duration_s=time.monotonic() - started, + diagnosis=metadata.get("diagnosis"), + verification_scope="affected", + release_baseline_allowed=False, + ) sys.stderr.write( f"\ndevtools test: progress={PYTEST_PROGRESS_PATH} selection={PYTEST_SELECTION_PATH} " f"summary={PYTEST_SUMMARY_PATH} events={PYTEST_EVENTS_PATH} containment={PYTEST_CONTAINMENT_PATH} " diff --git a/devtools/verify.py b/devtools/verify.py index 3a8a83f5a3..a199d3ba71 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -1959,6 +1959,19 @@ def _isolated_report_arg(arg: str) -> str: steps.append( ("lab policy campaign-archive-boundaries", _devtools_cmd("lab policy campaign-archive-boundaries")) ) + steps.append(("lab policy acceptance-contracts", _devtools_cmd("lab policy acceptance-contracts"))) + steps.append( + ( + "lab policy acceptance-contract-reconcile", + _devtools_cmd("lab policy acceptance-contract-reconcile"), + ) + ) + steps.append( + ( + "lab policy acceptance-contract-apply", + _devtools_cmd("lab policy acceptance-contract-apply"), + ) + ) # backlog-hygiene and bead-graph are corpus-wide backlog-debt scans # (findings scale with the total count of open Beads issues, not # with this change's diff) -- they stay --lab-only/scheduled rather diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 3daef2d6c8..1f67450949 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -254,13 +254,26 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> None: break self.write() - def finish(self, *, exit_code: int, duration_s: float, diagnosis: str | None = None) -> dict[str, Any]: + def finish( + self, + *, + exit_code: int, + duration_s: float, + diagnosis: str | None = None, + verification_scope: str | None = None, + release_baseline_allowed: bool | None = None, + terminal_authorization: str | None = None, + ) -> dict[str, Any]: self._payload["finished_at"] = utc_now() self._payload["duration_s"] = round(duration_s, 2) self._payload["exit_code"] = int(exit_code) self._payload["status"] = "success" if exit_code == 0 else "failed" if diagnosis: self._payload["diagnosis"] = diagnosis + if verification_scope is not None: + self._payload["verification_scope"] = verification_scope + self._payload["release_baseline_allowed"] = release_baseline_allowed + self._payload["terminal_authorization"] = terminal_authorization self.write() return dict(self._payload) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 410c4ccddc..6d6be3f71e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -596,6 +596,21 @@ def test_two_interrupted_resumes_flatten_all_carried_outcomes(tmp_path: Path) -> monkeypatch.undo() +def test_focused_run_can_record_typed_affected_scope(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + + payload = run.finish( + exit_code=0, + duration_s=0.1, + verification_scope="affected", + release_baseline_allowed=False, + ) + + assert payload["verification_scope"] == "affected" + assert payload["release_baseline_allowed"] is False + assert json.loads((tmp_path / ".cache" / "verify" / "current-run.json").read_text()) == payload + + def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) From cd3cbbfe294bdfff32ebe77300f265d04d5eccbe Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 12:22:56 +0200 Subject: [PATCH 34/34] fix: type focused verification receipts --- devtools/run_tests.py | 6 +++++- tests/unit/devtools/test_run_tests.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 08686f43d9..ef2228bf55 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -25,6 +25,7 @@ import contextlib import fcntl +import json import os import sys import time @@ -135,6 +136,7 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.write(f"devtools test: polylogue package → {polylogue_import_path}\n") selection = list(sys.argv[1:] if argv is None else argv) + use_json = "--json" in selection # The control-plane dispatch may append a bare ``--json`` machine-readable # flag; it is meaningless for a streamed test run, so drop it before pytest. selection = [arg for arg in selection if arg != "--json"] @@ -162,13 +164,15 @@ def main(argv: list[str] | None = None) -> int: ) started = time.monotonic() rc, _elapsed, metadata = _run("pytest focused", cmd, cwd=str(ROOT), run=run) - run.finish( + payload = run.finish( exit_code=rc, duration_s=time.monotonic() - started, diagnosis=metadata.get("diagnosis"), verification_scope="affected", release_baseline_allowed=False, ) + if use_json: + print(json.dumps(payload, indent=2, ensure_ascii=False)) sys.stderr.write( f"\ndevtools test: progress={PYTEST_PROGRESS_PATH} selection={PYTEST_SELECTION_PATH} " f"summary={PYTEST_SUMMARY_PATH} events={PYTEST_EVENTS_PATH} containment={PYTEST_CONTAINMENT_PATH} " diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index 0d97eec2c0..974a6958aa 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -68,6 +68,8 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di assert captured["run"].run_id assert captured["run"]._payload["git_head"] == "abc123" assert isinstance(captured["run"]._payload["git_dirty"], bool) + assert captured["run"]._payload["verification_scope"] == "affected" + assert captured["run"]._payload["release_baseline_allowed"] is False def test_main_returns_pytest_exit_code(monkeypatch: pytest.MonkeyPatch) -> None: