diff --git a/devtools/checkout_guard.py b/devtools/checkout_guard.py index 0b56e9a315..cc6cc76e28 100644 --- a/devtools/checkout_guard.py +++ b/devtools/checkout_guard.py @@ -68,6 +68,8 @@ import tomllib +from devtools.testmon_state import attempt_is_checkout_bound, seed_marker_is_checkout_bound + class CheckoutImportMismatchError(RuntimeError): """``import polylogue`` resolved to a package outside the invoking checkout.""" @@ -130,6 +132,7 @@ def as_dict(self) -> dict[str, object]: _TESTMON_STATE_DIR = Path(".cache/testmon") _TESTMON_STATE_MARKER = _TESTMON_STATE_DIR / "seed.json" _TESTMON_SEED_ATTEMPT = _TESTMON_STATE_DIR / "seed-attempt.json" +_TESTMON_SEED_PROTOCOL_VERSION = 5 _VERIFY_STATE_DIR = Path(".cache/verify") _VERIFY_STATE_MARKER = _VERIFY_STATE_DIR / "current-run.json" @@ -249,6 +252,10 @@ def _marker_origin(marker: Path) -> Path | None: if not isinstance(payload, Mapping): return None raw = payload.get("checkout_root") + if raw is None: + binding = payload.get("binding") + if isinstance(binding, Mapping): + raw = binding.get("checkout_root") if raw is None: fingerprint = payload.get("environment_fingerprint") if isinstance(fingerprint, Mapping): @@ -258,7 +265,7 @@ def _marker_origin(marker: Path) -> Path | None: return Path(raw).resolve() -def _is_valid_in_progress_testmon_seed_attempt(attempt: Path) -> bool: +def _is_valid_in_progress_testmon_seed_attempt(attempt: Path, *, checkout_root: Path) -> bool: """Recognize the live seed ledger before its completion marker exists. ``verify --seed-testmon`` writes this receipt before pytest starts and @@ -270,8 +277,25 @@ def _is_valid_in_progress_testmon_seed_attempt(attempt: Path) -> bool: payload = json.loads(attempt.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError): return False - if not isinstance(payload, Mapping) or payload.get("status") not in {"running", "incomplete"}: + if not isinstance(payload, Mapping) or payload.get("status") not in { + "running", + "incomplete", + "reusable", + "complete", + }: return False + if payload.get("status") == "complete": + return attempt_is_checkout_bound( + payload, + checkout_root=checkout_root, + protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, + ) + if payload.get("status") == "reusable": + return attempt_is_checkout_bound( + payload, + checkout_root=checkout_root, + protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, + ) protocol_version = payload.get("protocol_version") if not isinstance(protocol_version, int) or isinstance(protocol_version, bool) or protocol_version <= 0: return False @@ -331,12 +355,32 @@ def _cache_artifact( marker_path = repo_root / marker origin = _marker_origin(marker_path) if origin == repo_root: + if state_dir == _TESTMON_STATE_DIR and not seed_marker_is_checkout_bound( + marker_path, + checkout_root=repo_root, + protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, + ): + return ( + origin, + EnvironmentArtifact( + kind="invalid_testmon_seed", + path=marker_path, + detail="testmon seed marker is stale, malformed, or its SQLite graph is incomplete", + remediation=( + f"remove {state_path} and run `devtools verify --seed-testmon` " + "to rebuild the typed testmon state" + ), + ), + ) return origin, None if ( origin is None and not marker_path.exists() and state_dir == _TESTMON_STATE_DIR - and _is_valid_in_progress_testmon_seed_attempt(repo_root / _TESTMON_SEED_ATTEMPT) + and _is_valid_in_progress_testmon_seed_attempt( + repo_root / _TESTMON_SEED_ATTEMPT, + checkout_root=repo_root, + ) ): return None, None if origin is None: diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index 3b0e05890e..f97f8a595d 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -63,17 +63,31 @@ from __future__ import annotations import argparse +import contextlib +import fcntl import json +import os import re +import shlex import subprocess import sys +import tempfile import time +from collections.abc import Iterator, Mapping from pathlib import Path from typing import Any from devtools import merge_gate +from devtools.testmon_state import TerminalAuthorization, VerificationScope _LEDGER_PATH = Path(".cache/verify/merge-gate/merge-train-ledger.json") +_LEDGER_PENDING_PATH = _LEDGER_PATH.with_name(f"{_LEDGER_PATH.name}.pending") +_LEDGER_LOCK_PATH = _LEDGER_PATH.with_name(f"{_LEDGER_PATH.name}.lock") + + +class LedgerStateError(RuntimeError): + """The merge-train ledger cannot safely authorize a result.""" + # Matches a squash-merge subject that already carries the PR number and picked # up a second, duplicate one -- e.g. "fix: thing (#3517) (#3517)". Only the @@ -99,41 +113,309 @@ def clean_merge_title(title: str, pr: int) -> str: return collapsed -def _read_ledger() -> dict[str, Any]: +@contextlib.contextmanager +def _ledger_lock() -> Iterator[None]: + """Serialize every merge-train ledger read-modify-write transaction.""" + _LEDGER_LOCK_PATH.parent.mkdir(parents=True, exist_ok=True) + with _LEDGER_LOCK_PATH.open("a+") as handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +def _is_real_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _require_nonempty_string(entry: Mapping[str, Any], key: str, *, message: str) -> None: + if not isinstance(entry.get(key), str) or not entry[key]: + raise LedgerStateError(message) + + +def _validate_ledger(data: object) -> dict[str, Any]: + if not isinstance(data, dict) or not isinstance(data.get("merges"), list): + raise LedgerStateError("merge-train ledger is malformed") + for entry in data["merges"]: + if ( + not isinstance(entry, dict) + or not isinstance(entry.get("pr"), int) + or isinstance(entry.get("pr"), bool) + or entry["pr"] <= 0 + or not isinstance(entry.get("head_sha"), str) + or not entry["head_sha"] + or not isinstance(entry.get("title"), str) + or not entry["title"] + or not _is_real_number(entry.get("merged_at")) + or ( + "merge_sequence" in entry + and ( + not isinstance(entry.get("merge_sequence"), int) + or isinstance(entry.get("merge_sequence"), bool) + or entry["merge_sequence"] <= 0 + ) + ) + ): + raise LedgerStateError("merge-train ledger contains a malformed merge entry") + intents = data.get("merge_intents") + if not isinstance(intents, list): + raise LedgerStateError("merge-train ledger contains malformed merge intents") + for intent in intents: + if ( + not isinstance(intent, dict) + or not isinstance(intent.get("pr"), int) + or isinstance(intent.get("pr"), bool) + or intent["pr"] <= 0 + or not isinstance(intent.get("head_sha"), str) + or not intent["head_sha"] + or not isinstance(intent.get("title"), str) + or not intent["title"] + or not _is_real_number(intent.get("intent_at")) + ): + raise LedgerStateError("merge-train ledger contains a malformed merge intent") + receipt = data.get("last_full_verify") + if receipt is None: + return data + if not isinstance(receipt, dict): + raise LedgerStateError("merge-train ledger contains a malformed terminal receipt") + for key in ( + "command", + "verification_started_at", + "at", + "duration_s", + "exit_code", + "accepted", + "merge_sequence", + "verification_scope", + "release_baseline_allowed", + ): + if key not in receipt: + raise LedgerStateError(f"merge-train terminal receipt is missing {key!r}") + _require_nonempty_string(receipt, "command", message="merge-train terminal receipt has no command") + if ( + not _is_real_number(receipt.get("verification_started_at")) + or not _is_real_number(receipt.get("at")) + or not _is_real_number(receipt.get("duration_s")) + or not isinstance(receipt.get("exit_code"), int) + or isinstance(receipt.get("exit_code"), bool) + or not isinstance(receipt.get("accepted"), bool) + or not isinstance(receipt.get("merge_sequence"), int) + or isinstance(receipt.get("merge_sequence"), bool) + or receipt["merge_sequence"] < 0 + ): + raise LedgerStateError("merge-train terminal receipt has malformed status fields") + scope = receipt.get("verification_scope") + if scope is not None and scope not in {item.value for item in VerificationScope}: + raise LedgerStateError("merge-train terminal receipt has an invalid verification scope") + permission = receipt.get("release_baseline_allowed") + if permission is not None and not isinstance(permission, bool): + raise LedgerStateError("merge-train terminal receipt has malformed release permission") + for key in ("terminal_authorization", "verified_head_sha", "target_sha", "merged_master_sha"): + value = receipt.get(key) + if value is not None and (not isinstance(value, str) or not value): + raise LedgerStateError(f"merge-train terminal receipt has malformed {key!r}") + return data + + +def _read_ledger_unlocked() -> dict[str, Any]: + _recover_pending_ledger_unlocked() if not _LEDGER_PATH.exists(): - return {"merges": [], "last_full_verify": None} + return {"merges": [], "merge_intents": [], "last_full_verify": None} try: - data = json.loads(_LEDGER_PATH.read_text()) - except (OSError, json.JSONDecodeError): - return {"merges": [], "last_full_verify": None} - if not isinstance(data, dict): - return {"merges": [], "last_full_verify": None} - data.setdefault("merges", []) - data.setdefault("last_full_verify", None) - return data + data = json.loads(_LEDGER_PATH.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LedgerStateError("merge-train ledger is unreadable or truncated") from exc + if isinstance(data, dict): + data.setdefault("merges", []) + data.setdefault("merge_intents", []) + data.setdefault("last_full_verify", None) + return _validate_ledger(data) + + +def _read_ledger_file(path: Path, *, description: str) -> dict[str, Any]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LedgerStateError(f"{description} is unreadable or truncated") from exc + if isinstance(data, dict): + data.setdefault("merges", []) + data.setdefault("merge_intents", []) + data.setdefault("last_full_verify", None) + return _validate_ledger(data) + + +def _recover_pending_ledger_unlocked() -> None: + """Finish or clear the write-ahead ledger journal after interruption.""" + if not _LEDGER_PENDING_PATH.exists(): + return + pending = _read_ledger_file(_LEDGER_PENDING_PATH, description="merge-train pending ledger write") + parent = _LEDGER_PATH.parent + try: + if _LEDGER_PATH.exists() and pending == _read_ledger_file(_LEDGER_PATH, description="merge-train ledger"): + _LEDGER_PENDING_PATH.unlink() + else: + _durable_replace(_LEDGER_PENDING_PATH, _LEDGER_PATH) + _fsync_directory(parent) + except OSError as exc: + raise LedgerStateError(f"merge-train ledger pending-write recovery failed: {exc}") from exc + + +def _read_ledger() -> dict[str, Any]: + with _ledger_lock(): + return _read_ledger_unlocked() + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _write_durable_temp(path: Path, text: str) -> None: + with path.open("w", encoding="utf-8") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + + +def _durable_replace(source: Path, destination: Path) -> None: + os.replace(source, destination) + + +def _write_ledger_unlocked(ledger: dict[str, Any]) -> None: + ledger.setdefault("merges", []) + ledger.setdefault("merge_intents", []) + ledger.setdefault("last_full_verify", None) + _validate_ledger(ledger) + parent = _LEDGER_PATH.parent + parent.mkdir(parents=True, exist_ok=True) + serialized = json.dumps(ledger, indent=2) + "\n" + pending_tmp = parent / f".{_LEDGER_PENDING_PATH.name}.{os.getpid()}.tmp" + ledger_tmp = parent / f".{_LEDGER_PATH.name}.{os.getpid()}.tmp" + pending_tmp.unlink(missing_ok=True) + ledger_tmp.unlink(missing_ok=True) + try: + _write_durable_temp(pending_tmp, serialized) + _durable_replace(pending_tmp, _LEDGER_PENDING_PATH) + _fsync_directory(parent) + _write_durable_temp(ledger_tmp, serialized) + _durable_replace(ledger_tmp, _LEDGER_PATH) + _fsync_directory(parent) + _LEDGER_PENDING_PATH.unlink(missing_ok=True) + _fsync_directory(parent) + except OSError as exc: + raise LedgerStateError(f"merge-train ledger durable write failed: {exc}") from exc + finally: + pending_tmp.unlink(missing_ok=True) + ledger_tmp.unlink(missing_ok=True) def _write_ledger(ledger: dict[str, Any]) -> None: - _LEDGER_PATH.parent.mkdir(parents=True, exist_ok=True) - _LEDGER_PATH.write_text(json.dumps(ledger, indent=2)) + with _ledger_lock(): + _write_ledger_unlocked(ledger) + + +def _merge_sequence(ledger: Mapping[str, Any]) -> int: + sequence = 0 + for index, entry in enumerate(ledger.get("merges", []), start=1): + raw = entry.get("merge_sequence") if isinstance(entry, Mapping) else None + sequence = max(sequence, raw if isinstance(raw, int) and not isinstance(raw, bool) else index) + return sequence def _append_merge_entry(pr: int, head_sha: str, title: str) -> None: - ledger = _read_ledger() + with _ledger_lock(): + ledger = _read_ledger_unlocked() + _append_merge_entry_unlocked(ledger, pr, head_sha, title) + _write_ledger_unlocked(ledger) + + +def _append_merge_entry_unlocked(ledger: dict[str, Any], pr: int, head_sha: str, title: str) -> None: + merge_sequence = _merge_sequence(ledger) + 1 ledger["merges"].append( { "pr": pr, "head_sha": head_sha, "title": title, "merged_at": time.time(), + "merge_sequence": merge_sequence, } ) - _write_ledger(ledger) + + +def _record_merge_intent(pr: int, head_sha: str, title: str) -> None: + with _ledger_lock(): + ledger = _read_ledger_unlocked() + if not any(intent.get("pr") == pr and intent.get("head_sha") == head_sha for intent in ledger["merge_intents"]): + ledger["merge_intents"].append({"pr": pr, "head_sha": head_sha, "title": title, "intent_at": time.time()}) + _write_ledger_unlocked(ledger) + + +def _complete_merge_intent(pr: int, head_sha: str) -> None: + with _ledger_lock(): + ledger = _read_ledger_unlocked() + intents = ledger["merge_intents"] + matching = [intent for intent in intents if intent.get("pr") == pr and intent.get("head_sha") == head_sha] + if not matching: + return + intent = matching[0] + if not any(entry.get("pr") == pr and entry.get("head_sha") == head_sha for entry in ledger["merges"]): + _append_merge_entry_unlocked(ledger, pr, head_sha, str(intent["title"])) + ledger["merge_intents"] = [item for item in intents if item is not intent] + _write_ledger_unlocked(ledger) + + +def _reconcile_merge_intents() -> None: + """Resolve durable pre-merge intents against GitHub after a restart.""" + ledger = _read_ledger() + for intent in list(ledger["merge_intents"]): + try: + info = _gh_json(["pr", "view", str(intent["pr"]), "--json", "state,mergeCommit"]) + except (RuntimeError, json.JSONDecodeError, OSError, subprocess.SubprocessError) as exc: + raise LedgerStateError(f"could not reconcile merge intent for PR #{intent['pr']}: {exc}") from exc + merge_commit = info.get("mergeCommit") + if info.get("state") != "MERGED" or not isinstance(merge_commit, dict) or not merge_commit.get("oid"): + raise LedgerStateError(f"unresolved durable merge intent for PR #{intent['pr']}") + _complete_merge_intent(int(intent["pr"]), str(intent["head_sha"])) + if any( + item.get("pr") == intent["pr"] and item.get("head_sha") == intent["head_sha"] + for item in _read_ledger()["merge_intents"] + ): + raise LedgerStateError(f"merge intent for PR #{intent['pr']} was not durably reconciled") def _pending_prs_since_last_full_verify(ledger: dict[str, Any]) -> list[dict[str, Any]]: - last_verify_at = (ledger.get("last_full_verify") or {}).get("at", 0.0) - return [entry for entry in ledger.get("merges", []) if entry.get("merged_at", 0.0) > last_verify_at] + last_verify = ledger.get("last_full_verify") or {} + scope = last_verify.get("verification_scope") + last_verify_at = ( + last_verify.get("verification_started_at", last_verify.get("at", 0.0)) + if last_verify.get("accepted") is True + and last_verify.get("exit_code") == 0 + and last_verify.get("release_baseline_allowed") is True + and ( + scope == VerificationScope.RELEASE_BASELINE.value + or ( + scope == VerificationScope.NARROW_TERMINAL.value + and last_verify.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value + ) + ) + else 0.0 + ) + snapshot_sequence = last_verify.get("merge_sequence") + return [ + entry + for entry in ledger.get("merges", []) + if entry.get("merged_at", 0.0) > last_verify_at + or ( + isinstance(snapshot_sequence, int) + and not isinstance(snapshot_sequence, bool) + and isinstance(entry.get("merge_sequence"), int) + and entry["merge_sequence"] > snapshot_sequence + ) + ] def _receipt_is_fresh_for_head(pr: int, head_sha: str, max_age_s: int) -> bool: @@ -146,6 +428,133 @@ def _receipt_is_fresh_for_head(pr: int, head_sha: str, max_age_s: int) -> bool: return bool(age_s <= max_age_s) +def _fetched_merged_default_branch_sha(pr: int) -> str | None: + """Fetch the default branch and return its post-merge commit only.""" + try: + branch = _default_branch_name() + if branch is None: + return None + merged = _gh_json(["pr", "view", str(pr), "--json", "state,mergeCommit"]) + merge_commit = merged.get("mergeCommit") + merge_sha = merge_commit.get("oid") if isinstance(merge_commit, dict) else None + if merged.get("state") != "MERGED" or not isinstance(merge_sha, str) or not merge_sha: + return None + fetch = subprocess.run( + ["git", "fetch", "origin", branch], + capture_output=True, + text=True, + timeout=120, + ) + if fetch.returncode != 0: + return None + target = subprocess.run( + ["git", "rev-parse", "FETCH_HEAD"], + capture_output=True, + text=True, + timeout=15, + ) + if target.returncode != 0: + return None + target_sha = target.stdout.strip() + if not target_sha: + return None + included = subprocess.run( + ["git", "merge-base", "--is-ancestor", merge_sha, target_sha], + capture_output=True, + text=True, + timeout=15, + ) + return target_sha if included.returncode == 0 else None + except (RuntimeError, json.JSONDecodeError, OSError, subprocess.SubprocessError): + return None + + +def _default_branch_name() -> str | None: + repo = _gh_json(["repo", "view", "--json", "defaultBranchRef"]) + default_ref = repo.get("defaultBranchRef") + branch = default_ref.get("name") if isinstance(default_ref, dict) else None + return branch if isinstance(branch, str) and branch else None + + +def _fetched_current_default_branch_sha() -> str | None: + """Fetch and return the exact current default-branch commit.""" + try: + branch = _default_branch_name() + if branch is None: + return None + fetch = subprocess.run(["git", "fetch", "origin", branch], capture_output=True, text=True, timeout=120) + if fetch.returncode != 0: + return None + target = subprocess.run(["git", "rev-parse", "FETCH_HEAD"], capture_output=True, text=True, timeout=15) + if target.returncode != 0 or not target.stdout.strip(): + return None + return target.stdout.strip() + except (RuntimeError, json.JSONDecodeError, OSError, subprocess.SubprocessError): + return None + + +def _terminal_verify_snapshot() -> tuple[dict[str, Any], float, int]: + with _ledger_lock(): + ledger = _read_ledger_unlocked() + started_at = time.time() + return ledger, started_at, _merge_sequence(ledger) + + +def _reconciled_terminal_verify_snapshot() -> tuple[dict[str, Any], float, int]: + """Recover merged intents before fixing the terminal verification boundary.""" + _reconcile_merge_intents() + return _terminal_verify_snapshot() + + +def _remove_detached_worktree(repo_root: Path, worktree: Path) -> bool: + removal = subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + capture_output=True, + text=True, + timeout=120, + cwd=repo_root, + ) + if removal.returncode != 0: + print( + f"REFUSING terminal verify: failed to unregister detached worktree {worktree}: " + f"{removal.stderr.strip()[:300]}", + file=sys.stderr, + ) + return False + return True + + +def _run_post_merge_terminal_verify( + command: str, + target_sha: str, + *, + ledger_snapshot: tuple[dict[str, Any], float, int] | None = None, +) -> int: + """Run terminal verification in a detached worktree at the fetched target.""" + repo_root = Path.cwd() + with tempfile.TemporaryDirectory(prefix="polylogue-merge-terminal-") as raw_worktree: + worktree = Path(raw_worktree) + add = subprocess.run( + ["git", "worktree", "add", "--detach", str(worktree), target_sha], + capture_output=True, + text=True, + timeout=120, + cwd=repo_root, + ) + if add.returncode != 0: + print(f"REFUSING terminal verify: could not materialize fetched target {target_sha[:8]}", file=sys.stderr) + _remove_detached_worktree(repo_root, worktree) + return 1 + result = cmd_record_full_verify( + command, + target_sha=target_sha, + cwd=worktree, + execution_root=worktree, + ledger_snapshot=ledger_snapshot, + ) + return result if _remove_detached_worktree(repo_root, worktree) else 1 + + def cmd_merge( pr: int, *, @@ -196,6 +605,12 @@ def cmd_merge( print(f"PR #{pr} @ {head_sha[:8]}: merge-gate OK -- dry-run, not merging (title would be {clean_title!r})") return 0 + try: + _record_merge_intent(pr, head_sha, clean_title) + except LedgerStateError as exc: + print(f"REFUSING to merge PR #{pr}: could not durably record merge intent: {exc}", file=sys.stderr) + return 1 + merge_result = subprocess.run( [ "gh", @@ -217,11 +632,27 @@ def cmd_merge( return merge_result.returncode print(f"merged PR #{pr} @ {head_sha[:8]}: {clean_title!r}") - _append_merge_entry(pr, head_sha, clean_title) + try: + _complete_merge_intent(pr, head_sha) + except LedgerStateError as exc: + print(f"REFUSING to continue: merge-train ledger is not durably writable: {exc}", file=sys.stderr) + return 1 if with_verify: + try: + ledger_snapshot = _reconciled_terminal_verify_snapshot() + except LedgerStateError as exc: + print(f"REFUSING terminal verify: {exc}", file=sys.stderr) + return 1 + target_sha = _fetched_merged_default_branch_sha(pr) + if target_sha is None: + print( + "REFUSING terminal verify: the fetched default branch does not prove this squash merge is included", + file=sys.stderr, + ) + return 1 print(f"running post-merge broad verify (merge-train terminal step): {verify_command!r}") - return cmd_record_full_verify(verify_command) + return _run_post_merge_terminal_verify(verify_command, target_sha, ledger_snapshot=ledger_snapshot) print( "REMINDER: this merge-train's terminal ledger step (one full-suite verify since the last " @@ -233,7 +664,12 @@ def cmd_merge( def cmd_train_status(as_json: bool) -> int: - ledger = _read_ledger() + try: + _reconcile_merge_intents() + ledger = _read_ledger() + except LedgerStateError as exc: + print(f"merge-train REFUSING clean status: {exc}", file=sys.stderr) + return 1 pending = _pending_prs_since_last_full_verify(ledger) ok = not pending @@ -266,29 +702,95 @@ def cmd_train_status(as_json: bool) -> int: return 1 -def cmd_record_full_verify(command: str) -> int: - argv = command.split() +def cmd_record_full_verify( + command: str, + *, + target_sha: str | None = None, + cwd: Path | None = None, + execution_root: Path | None = None, + ledger_snapshot: tuple[dict[str, Any], float, int] | None = None, +) -> int: + argv = shlex.split(command) if not argv: print("REFUSING: --command is empty after splitting", file=sys.stderr) return 2 - started = time.time() + if target_sha is None: + print("REFUSING: terminal verification has no fetched merged-master target", file=sys.stderr) + return 1 + try: + snapshot = ledger_snapshot or _reconciled_terminal_verify_snapshot() + except LedgerStateError as exc: + print(f"REFUSING to run terminal verification: {exc}", file=sys.stderr) + return 1 + _snapshot_ledger, verification_started_at, merge_sequence = snapshot + if execution_root is not None: + argv = ["direnv", "exec", str(execution_root), *argv] + started = verification_started_at try: - result = subprocess.run(argv, capture_output=True, text=True) + result = subprocess.run(argv, capture_output=True, text=True, cwd=cwd) except OSError as exc: print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) return 2 duration_s = round(time.time() - started, 2) + release_allowed = merge_gate._release_baseline_permission(result.stdout) + verification_scope = merge_gate._verification_scope(result.stdout) + terminal_authorization = merge_gate._terminal_authorization(result.stdout) + try: + structured = json.loads(result.stdout) + except (TypeError, json.JSONDecodeError): + structured = None + verified_head = structured.get("git_head") if isinstance(structured, dict) else None + accepted = ( + result.returncode == 0 + and release_allowed is True + and ( + verification_scope == VerificationScope.RELEASE_BASELINE.value + or ( + verification_scope == VerificationScope.NARROW_TERMINAL.value + and terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value + ) + ) + and verified_head == target_sha + ) - ledger = _read_ledger() - ledger["last_full_verify"] = { - "command": command, - "exit_code": result.returncode, - "duration_s": duration_s, - "at": time.time(), - } - _write_ledger(ledger) - - print(f"recorded merge-train terminal verify: {command!r} exit={result.returncode} ({duration_s}s)") + try: + with _ledger_lock(): + ledger = _read_ledger_unlocked() + ledger["last_full_verify"] = { + "command": command, + "exit_code": result.returncode, + "duration_s": duration_s, + "at": verification_started_at, + "verification_started_at": verification_started_at, + "verification_scope": verification_scope, + "release_baseline_allowed": release_allowed, + "terminal_authorization": terminal_authorization, + "verified_head_sha": verified_head, + "target_sha": target_sha, + "merged_master_sha": target_sha, + "merge_sequence": merge_sequence, + "accepted": accepted, + } + _write_ledger_unlocked(ledger) + except LedgerStateError as exc: + print(f"REFUSING to record terminal verification: {exc}", file=sys.stderr) + return 1 + + print( + f"recorded merge-train terminal verify: {command!r} exit={result.returncode} " + f"release_baseline_allowed={release_allowed!r} accepted={accepted} ({duration_s}s)" + ) + if not accepted and result.returncode == 0: + print( + "POST-MERGE BROAD VERIFY DID NOT GRANT typed release-baseline authority for the selected target; " + "train-status remains incomplete.", + file=sys.stderr, + ) + if verified_head != target_sha: + print( + f"terminal verify reported git_head={verified_head!r}, expected fetched target {target_sha}", + file=sys.stderr, + ) if result.returncode != 0: print(result.stdout[-4000:]) print(result.stderr[-4000:], file=sys.stderr) @@ -298,7 +800,7 @@ def cmd_record_full_verify(command: str) -> int: "before merging further PRs in this train.", file=sys.stderr, ) - return result.returncode + return result.returncode if result.returncode != 0 else (0 if accepted else 1) def main(argv: list[str] | None = None) -> int: @@ -346,7 +848,16 @@ def main(argv: list[str] | None = None) -> int: ) if args.action == "train-status": return cmd_train_status(args.as_json) - return cmd_record_full_verify(args.command) + try: + ledger_snapshot = _reconciled_terminal_verify_snapshot() + except LedgerStateError as exc: + print(f"REFUSING terminal verify: {exc}", file=sys.stderr) + return 1 + target_sha = _fetched_current_default_branch_sha() + if target_sha is None: + print("REFUSING terminal verify: could not fetch the current default branch", file=sys.stderr) + return 1 + return _run_post_merge_terminal_verify(args.command, target_sha, ledger_snapshot=ledger_snapshot) if __name__ == "__main__": diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index b728d54c3e..7b9415e1f4 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -79,6 +79,7 @@ from typing import Any from devtools import pr_scope +from devtools.testmon_state import TerminalAuthorization, VerificationScope _RECEIPT_DIR = Path(".cache/verify/merge-gate") _DEFAULT_MAX_AGE_S = 3600 @@ -218,6 +219,42 @@ def _command_skips_tests(command: str) -> bool: return not any(marker in lowered for marker in _LOOKS_LIKE_TESTS_MARKERS) +def _release_baseline_permission(stdout: str) -> bool | None: + """Read the structured verify decision when the command emitted one.""" + try: + payload = json.loads(stdout) + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + value = payload.get("release_baseline_allowed") + return value if isinstance(value, bool) else None + + +def _verification_scope(stdout: str) -> str | None: + """Read the typed verification scope from a structured verify receipt.""" + try: + payload = json.loads(stdout) + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + value = payload.get("verification_scope") + return value if value in {scope.value for scope in VerificationScope} else None + + +def _terminal_authorization(stdout: str) -> str | None: + """Read the typed terminal authorization from a structured receipt.""" + try: + payload = json.loads(stdout) + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + value = payload.get("terminal_authorization") + return value if value in {authorization.value for authorization in TerminalAuthorization} else None + + def cmd_record(pr: int, command: str) -> int: info = _gh_json(["pr", "view", str(pr), "--json", "headRefOid,headRefName,body,isDraft"]) head_sha = info["headRefOid"] @@ -272,6 +309,9 @@ def cmd_record(pr: int, command: str) -> int: "branch": info["headRefName"], "command": command, "skips_tests": _command_skips_tests(command), + "verification_scope": _verification_scope(result.stdout), + "release_baseline_allowed": _release_baseline_permission(result.stdout), + "terminal_authorization": _terminal_authorization(result.stdout), "exit_code": result.returncode, "duration_s": duration_s, "recorded_at": time.time(), @@ -478,6 +518,21 @@ def cmd_check( if receipt.get("exit_code", 1) != 0: verdict.ok = False verdict.reasons.append(f"receipt exit_code is {receipt.get('exit_code')}, not 0") + verification_scope = receipt.get("verification_scope") + if verification_scope not in {scope.value for scope in VerificationScope}: + verdict.ok = False + verdict.reasons.append( + "verification receipt lacks a valid typed verification_scope; command text cannot grant authority" + ) + release_allowed = receipt.get("release_baseline_allowed") + if not isinstance(release_allowed, bool): + verdict.ok = False + verdict.reasons.append("verification receipt lacks typed release_baseline_allowed permission") + if verification_scope == VerificationScope.RELEASE_BASELINE.value and release_allowed is not True: + verdict.ok = False + verdict.reasons.append( + "release-baseline verification receipt does not grant release_baseline_allowed=true" + ) if receipt.get("skips_tests"): verdict.reasons.append( f"advisory: receipt command {receipt.get('command')!r} does not look like it ran tests " diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 61fcec8fc2..ef2228bf55 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -25,6 +25,7 @@ import contextlib import fcntl +import json import os import sys import time @@ -135,6 +136,7 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.write(f"devtools test: polylogue package → {polylogue_import_path}\n") selection = list(sys.argv[1:] if argv is None else argv) + use_json = "--json" in selection # The control-plane dispatch may append a bare ``--json`` machine-readable # flag; it is meaningless for a streamed test run, so drop it before pytest. selection = [arg for arg in selection if arg != "--json"] @@ -162,7 +164,15 @@ def main(argv: list[str] | None = None) -> int: ) started = time.monotonic() rc, _elapsed, metadata = _run("pytest focused", cmd, cwd=str(ROOT), run=run) - run.finish(exit_code=rc, duration_s=time.monotonic() - started, diagnosis=metadata.get("diagnosis")) + payload = run.finish( + exit_code=rc, + duration_s=time.monotonic() - started, + diagnosis=metadata.get("diagnosis"), + verification_scope="affected", + release_baseline_allowed=False, + ) + if use_json: + print(json.dumps(payload, indent=2, ensure_ascii=False)) sys.stderr.write( f"\ndevtools test: progress={PYTEST_PROGRESS_PATH} selection={PYTEST_SELECTION_PATH} " f"summary={PYTEST_SUMMARY_PATH} events={PYTEST_EVENTS_PATH} containment={PYTEST_CONTAINMENT_PATH} " diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index de41011860..7397bbe294 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -17,14 +17,22 @@ differs between the worktree and the main checkout at copy time self-invalidates (its ``fsha`` won't match), and testmon correctly treats the tests that depend on it as affected on the very next run. No merge or rewrite -is needed -- a straight copy is a valid seed. +is needed for the relative file fingerprints. + +The reusable stamp is typed. It records collection completeness, graph +completeness, baseline color, and whether the graph is exact or rebound to a +new checkout. A red graph is allowed for affected selection only. Bootstrap +revalidates the SQLite graph after the online backup and recomputes its file +fingerprint because SQLite backup can produce a byte-different equivalent +database. This module owns exactly one decision and one action: -- :func:`decide_testmon_bootstrap` -- pure decision, no I/O beyond reading the - two candidate seed files (local + main). Testable with plain tmp dirs. +- :func:`decide_testmon_bootstrap` -- pure decision, no subprocess beyond the + caller. It validates the main stamp or a complete red seed attempt. - :func:`bootstrap_testmon_seed_files` -- the copy action once bootstrapping - has been decided. + has been decided. A red attempt is copied as a rebound attempt receipt and + never synthesized into ``seed.json``. - :func:`maybe_bootstrap_testmon_seed` -- the orchestrator `devtools verify` calls: detects whether ``repo_root`` is a linked worktree (via ``git rev-parse --absolute-git-dir --git-common-dir``, the same mechanism @@ -38,8 +46,8 @@ copies it through :meth:`sqlite3.Connection.backup`, sqlite's own online-backup API -- built for copying a live database without an exclusive lock, immune to concurrent writers by design. ``seed.json`` is a small file written atomically -by ``verify.py`` (write-temp-then-rename), so a plain read-then-atomic-write -copy is enough for it; there is no partial-write window to observe. +by ``verify.py`` (write-temp-then-rename), so bootstrap writes its newly bound +stamp atomically after the copied graph has been revalidated. This module NEVER writes to the main checkout's copy of either file -- only reads from main, only writes to ``repo_root``. @@ -49,13 +57,25 @@ import json import os +import shutil import sqlite3 import subprocess +import tempfile +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path +from typing import Any + +from devtools.testmon_state import ( + TestmonSeedStamp, + refresh_stamp, + stamp_from_attempt, + validate_stamp, +) TESTMON_DATA_RELPATH = ".cache/testmon/testmondata" TESTMON_SEED_STAMP_RELPATH = ".cache/testmon/seed.json" +TESTMON_SEED_ATTEMPT_RELPATH = ".cache/testmon/seed-attempt.json" @dataclass(frozen=True) @@ -66,19 +86,37 @@ class BootstrapDecision: reason: str main_testmon_data: Path | None = None main_seed_stamp: Path | None = None + main_seed_attempt: Path | None = None + main_checkout_root: Path | None = None + protocol_version: int = 4 + selection_only: bool = False -def _is_valid_complete_seed_stamp(seed_stamp: Path, *, protocol_version: int) -> bool: - """Mirror the validity check `devtools.verify._testmon_preflight` applies.""" - if not seed_stamp.is_file(): - return False - try: - stamp = json.loads(seed_stamp.read_text()) - except (OSError, json.JSONDecodeError): - return False - if not isinstance(stamp, dict): - return False - return stamp.get("protocol_version") == protocol_version and stamp.get("status") == "complete" +def _checkout_root_for_data(data_path: Path) -> Path: + """Resolve the checkout root for canonical and test-local cache layouts.""" + resolved = data_path.resolve() + if resolved.parent.name == "testmon" and resolved.parent.parent.name == ".cache": + return resolved.parents[2] + return resolved.parent + + +def _is_valid_complete_seed_stamp( + seed_stamp: Path, + testmon_data: Path, + *, + protocol_version: int, + checkout_root: Path, +) -> bool: + """Validate both the typed stamp and the real SQLite graph it describes.""" + return ( + validate_stamp( + seed_stamp, + testmon_data, + checkout_root=checkout_root, + protocol_version=protocol_version, + ) + is not None + ) def decide_testmon_bootstrap( @@ -89,6 +127,10 @@ def decide_testmon_bootstrap( main_testmon_data: Path, main_seed_stamp: Path, protocol_version: int, + main_seed_attempt: Path | None = None, + main_checkout_root: Path | None = None, + local_checkout_root: Path | None = None, + local_seed_attempt: Path | None = None, ) -> BootstrapDecision: """Decide whether to copy the main checkout's testmon seed into a worktree. @@ -98,57 +140,138 @@ def decide_testmon_bootstrap( """ if not is_linked_worktree: return BootstrapDecision(False, "repo_root is not a linked worktree; nothing to bootstrap") - if local_testmon_data.is_file() and local_seed_stamp.is_file(): - return BootstrapDecision(False, "local .cache/testmon already has a testmondata + seed stamp") - if not _is_valid_complete_seed_stamp(main_seed_stamp, protocol_version=protocol_version): - return BootstrapDecision( - False, - "main checkout has no valid complete testmon seed stamp to bootstrap from", + local_root = (local_checkout_root or _checkout_root_for_data(local_testmon_data)).resolve() + if ( + local_testmon_data.is_file() + and local_seed_stamp.is_file() + and _is_valid_complete_seed_stamp( + local_seed_stamp, + local_testmon_data, + protocol_version=protocol_version, + checkout_root=local_root, ) + ): + return BootstrapDecision(False, "local .cache/testmon already has a validated testmondata + seed stamp") + if local_testmon_data.is_file() and local_seed_attempt is not None and local_seed_attempt.is_file(): + try: + local_attempt = json.loads(local_seed_attempt.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + local_attempt = None + if ( + isinstance(local_attempt, Mapping) + and stamp_from_attempt( + local_attempt, + local_testmon_data, + checkout_root=local_root, + protocol_version=protocol_version, + published_marker=False, + ) + is not None + ): + return BootstrapDecision(False, "local .cache/testmon already has a checkout-bound selection attempt") if not main_testmon_data.is_file(): return BootstrapDecision( False, - "main checkout seed stamp is valid but its testmondata file is missing", + "main checkout has no valid testmon graph because its testmondata file is missing", ) + root = main_checkout_root or _checkout_root_for_data(main_testmon_data) + root = root.resolve() + try: + main_testmon_data.resolve().relative_to(root) + main_seed_stamp.resolve().relative_to(root) + if main_seed_attempt is not None: + main_seed_attempt.resolve().relative_to(root) + except ValueError: + return BootstrapDecision(False, "main testmon paths are not bound to the declared checkout root") + if _is_valid_complete_seed_stamp( + main_seed_stamp, + main_testmon_data, + protocol_version=protocol_version, + checkout_root=root, + ): + return BootstrapDecision( + True, + f"main checkout has a validated testmon graph ({main_seed_stamp}); bootstrapping worktree cache", + main_testmon_data=main_testmon_data, + main_seed_stamp=main_seed_stamp, + main_checkout_root=root, + protocol_version=protocol_version, + ) + if main_seed_attempt is not None and main_seed_attempt.is_file(): + try: + attempt = json.loads(main_seed_attempt.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + attempt = None + if ( + isinstance(attempt, dict) + and ( + attempt_stamp := stamp_from_attempt( + attempt, + main_testmon_data, + checkout_root=root, + protocol_version=protocol_version, + published_marker=False, + ) + ) + is not None + ): + return BootstrapDecision( + True, + "main checkout has a validated complete graph from a red seed attempt; bootstrapping worktree cache", + main_testmon_data=main_testmon_data, + main_seed_attempt=main_seed_attempt, + main_checkout_root=root, + protocol_version=protocol_version, + selection_only=not attempt_stamp.release_baseline_allowed, + ) + if main_seed_stamp.is_file(): + return BootstrapDecision(False, "main checkout seed stamp is stale, malformed, or graph-incomplete") return BootstrapDecision( - True, - f"main checkout has a valid complete testmon seed ({main_seed_stamp}); bootstrapping worktree cache", - main_testmon_data=main_testmon_data, - main_seed_stamp=main_seed_stamp, + False, + "main checkout has no validated reusable testmon state", ) -def _atomic_copy_bytes(src: Path, dst: Path) -> None: - dst.parent.mkdir(parents=True, exist_ok=True) - tmp = dst.with_name(f"{dst.name}.{os.getpid()}.tmp") - tmp.write_bytes(src.read_bytes()) - tmp.replace(dst) +def _atomic_write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + try: + tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + tmp.replace(path) + finally: + tmp.unlink(missing_ok=True) -def _stamp_seed_checkout_origin( - seed_stamp: Path, - *, - checkout_root: Path, - inherited_from: Path | None = None, +def _atomic_write_stamp(seed_stamp: Path, stamp: TestmonSeedStamp) -> None: + _atomic_write_json(seed_stamp, stamp.as_dict()) + + +def _rebind_run_receipt( + *, source: Path, destination: Path, checkout_root: Path, run_id: str, current_run_path: Path | None = None ) -> bool: - """Mark a copied seed with its destination checkout and source provenance.""" + """Copy the run receipt while rebinding its checkout-local provenance.""" try: - payload = json.loads(seed_stamp.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - return False - if not isinstance(payload, dict): - return False - payload["checkout_root"] = str(checkout_root.resolve()) - if inherited_from is not None: - payload["inherited_from"] = str(inherited_from.resolve()) - tmp = seed_stamp.with_name(f"{seed_stamp.name}.{os.getpid()}.tmp") - try: - tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - tmp.replace(seed_stamp) - except OSError: - tmp.unlink(missing_ok=True) + payload = json.loads((source / "run.json").read_text(encoding="utf-8")) + if not isinstance(payload, Mapping) or payload.get("run_id") != run_id: + return False + source_root = payload.get("checkout_root") + if not isinstance(source_root, str) or Path(source_root).resolve() != source.parents[3].resolve(): + return False + payload_dict: dict[str, Any] = dict(payload) + payload_dict["checkout_root"] = str(checkout_root.resolve()) + payload_dict["artifact_dir"] = str(Path(".cache") / "verify" / "runs" / run_id) + environment = payload_dict.get("environment_fingerprint") + if isinstance(environment, dict): + environment["checkout_root"] = str(checkout_root.resolve()) + environment["verify_state_origin"] = str(checkout_root.resolve()) + _atomic_write_json(destination / "run.json", payload_dict) + _atomic_write_json( + current_run_path or checkout_root / ".cache" / "verify" / "current-run.json", + payload_dict, + ) + return True + except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): return False - return True def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: @@ -163,16 +286,64 @@ def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: dst.parent.mkdir(parents=True, exist_ok=True) tmp = dst.with_name(f"{dst.name}.{os.getpid()}.tmp") tmp.unlink(missing_ok=True) - src_conn = sqlite3.connect(f"file:{src}?mode=ro", uri=True) try: - dst_conn = sqlite3.connect(tmp) + src_conn = sqlite3.connect(f"{src.resolve().as_uri()}?mode=ro", uri=True) try: - src_conn.backup(dst_conn) + dst_conn = sqlite3.connect(tmp) + try: + src_conn.backup(dst_conn) + finally: + dst_conn.close() finally: - dst_conn.close() + src_conn.close() + tmp.replace(dst) finally: - src_conn.close() - tmp.replace(dst) + tmp.unlink(missing_ok=True) + + +def _publish_staged_bootstrap_files(*, staging_dir: Path, files: list[tuple[Path, Path | None]]) -> None: + """Publish a validated bootstrap as one rollback-capable file set.""" + backup_dir = staging_dir / "backups" + backup_dir.mkdir(parents=True, exist_ok=True) + backups: list[tuple[Path, Path]] = [] + published: list[Path] = [] + try: + for index, (destination, staged) in enumerate(files): + destination.parent.mkdir(parents=True, exist_ok=True) + backup = backup_dir / str(index) + if destination.exists(): + os.replace(destination, backup) + backups.append((destination, backup)) + if staged is not None: + os.replace(staged, destination) + published.append(destination) + except (OSError, ValueError): + for destination in reversed(published): + destination.unlink(missing_ok=True) + for destination, backup in reversed(backups): + if backup.exists(): + os.replace(backup, destination) + raise + finally: + shutil.rmtree(backup_dir, ignore_errors=True) + + +def _copy_runtime_identity_inputs(*, source_root: Path, destination_root: Path) -> None: + """Mirror the inputs used to validate a staged testmon receipt.""" + for relative_path in ( + "uv.lock", + "pyproject.toml", + "pytest.ini", + "tox.ini", + "setup.cfg", + "tests/conftest.py", + ): + source = source_root / relative_path + if not source.is_file(): + continue + destination = destination_root / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) def bootstrap_testmon_seed_files( @@ -180,6 +351,7 @@ def bootstrap_testmon_seed_files( *, local_testmon_data: Path, local_seed_stamp: Path, + local_seed_attempt: Path | None = None, checkout_root: Path | None = None, inherited_from: Path | None = None, ) -> bool: @@ -187,12 +359,151 @@ def bootstrap_testmon_seed_files( if not decision.should_bootstrap: return True assert decision.main_testmon_data is not None - assert decision.main_seed_stamp is not None - _atomic_copy_sqlite_db(decision.main_testmon_data, local_testmon_data) - _atomic_copy_bytes(decision.main_seed_stamp, local_seed_stamp) - if checkout_root is not None and inherited_from is not None: - return _stamp_seed_checkout_origin(local_seed_stamp, checkout_root=checkout_root, inherited_from=inherited_from) - return True + if decision.main_seed_stamp is None and decision.main_seed_attempt is None: + return False + if decision.main_seed_attempt is not None and local_seed_attempt is None: + return False + if checkout_root is None or inherited_from is None: + return False + stamp: TestmonSeedStamp | None = None + try: + source_root = (decision.main_checkout_root or inherited_from).resolve() + destination_root = checkout_root.resolve() + if source_root == destination_root: + return False + if inherited_from.resolve() != source_root: + return False + if decision.main_testmon_data.resolve() == local_testmon_data.resolve(): + return False + decision.main_testmon_data.resolve().relative_to(source_root) + local_testmon_data.resolve().relative_to(destination_root) + local_seed_stamp.resolve().relative_to(destination_root) + if local_seed_stamp.resolve() == local_testmon_data.resolve(): + return False + if local_seed_attempt is not None: + local_seed_attempt.resolve().relative_to(destination_root) + if local_seed_attempt.resolve() in {local_testmon_data.resolve(), local_seed_stamp.resolve()}: + return False + if decision.main_seed_stamp is not None: + decision.main_seed_stamp.resolve().relative_to(source_root) + if decision.main_seed_stamp.resolve() == decision.main_testmon_data.resolve(): + return False + if decision.main_seed_attempt is not None: + decision.main_seed_attempt.resolve().relative_to(source_root) + if decision.main_seed_attempt.resolve() == decision.main_testmon_data.resolve(): + return False + if decision.main_seed_stamp is not None: + stamp = validate_stamp( + decision.main_seed_stamp, + decision.main_testmon_data, + checkout_root=source_root, + protocol_version=decision.protocol_version, + ) + else: + assert decision.main_seed_attempt is not None + source = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) + if not isinstance(source, dict): + return False + stamp = stamp_from_attempt( + source, + decision.main_testmon_data, + checkout_root=source_root, + protocol_version=decision.protocol_version, + published_marker=False, + ) + if stamp is None: + return False + except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError, TypeError, sqlite3.Error): + return False + if stamp is None: + return False + staging_dir: Path | None = None + try: + local_testmon_data.parent.mkdir(parents=True, exist_ok=True) + staging_dir = Path(tempfile.mkdtemp(prefix=".bootstrap-", dir=str(local_testmon_data.parent))) + staged_data = staging_dir / "testmondata" + staged_stamp = staging_dir / "seed.json" + staged_attempt = staging_dir / "seed-attempt.json" + staged_artifact = staging_dir / "artifact" + staged_current_run = staging_dir / "current-run.json" + + _atomic_copy_sqlite_db(decision.main_testmon_data, staged_data) + rebound = stamp.rebound(checkout_root=destination_root, inherited_from=source_root) + refreshed = refresh_stamp(rebound, staged_data) + if refreshed is None or refreshed.graph != rebound.graph: + return False + source_artifact = source_root / Path(stamp.artifact_dir) + destination_artifact = (destination_root / Path(refreshed.artifact_dir)).resolve() + destination_artifact.relative_to(destination_root) + if not _rebind_run_receipt( + source=source_artifact, + destination=staged_artifact, + checkout_root=destination_root, + run_id=refreshed.run_id, + current_run_path=staged_current_run, + ): + return False + staged_attempt_path: Path | None = None + publishes_selection_attempt = decision.main_seed_attempt is not None and decision.selection_only + if publishes_selection_attempt: + assert decision.main_seed_attempt is not None + assert local_seed_attempt is not None + source_attempt = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) + if not isinstance(source_attempt, dict): + return False + rebound_attempt = dict(source_attempt) + rebound_attempt["testmon_data"] = refreshed.testmon_data + rebound_attempt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" + rebound_attempt["binding"] = refreshed.binding.as_dict() + rebound_attempt["release_baseline_allowed"] = False + rebound_attempt["verification_scope"] = "affected" + validation_root = staging_dir / "validation" + validation_receipt = json.loads(staged_current_run.read_text(encoding="utf-8")) + if not isinstance(validation_receipt, dict): + return False + validation_receipt["checkout_root"] = str(validation_root.resolve()) + validation_receipt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" + _copy_runtime_identity_inputs(source_root=destination_root, destination_root=validation_root) + _atomic_write_json( + validation_root / ".cache" / "verify" / "runs" / refreshed.run_id / "run.json", + validation_receipt, + ) + validation_attempt = dict(rebound_attempt) + raw_binding = validation_attempt.get("binding") + if not isinstance(raw_binding, Mapping): + return False + validation_binding = dict(raw_binding) + validation_binding["checkout_root"] = str(validation_root.resolve()) + validation_attempt["binding"] = validation_binding + if ( + stamp_from_attempt( + validation_attempt, + staged_data, + checkout_root=validation_root, + protocol_version=decision.protocol_version, + ) + is None + ): + return False + _atomic_write_json(staged_attempt, rebound_attempt) + staged_attempt_path = staged_attempt + else: + _atomic_write_stamp(staged_stamp, refreshed) + publication_files: list[tuple[Path, Path | None]] = [ + (local_testmon_data, staged_data), + (destination_artifact / "run.json", staged_artifact / "run.json"), + (destination_root / ".cache" / "verify" / "current-run.json", staged_current_run), + (local_seed_stamp, None if publishes_selection_attempt else staged_stamp), + ] + if local_seed_attempt is not None: + publication_files.append((local_seed_attempt, staged_attempt_path)) + _publish_staged_bootstrap_files(staging_dir=staging_dir, files=publication_files) + return True + except (OSError, sqlite3.Error, TypeError, ValueError): + return False + finally: + if staging_dir is not None: + shutil.rmtree(staging_dir, ignore_errors=True) def _git_worktree_info(repo_root: Path) -> tuple[bool, Path] | None: @@ -230,6 +541,7 @@ def maybe_bootstrap_testmon_seed( *, testmon_data_relpath: str = TESTMON_DATA_RELPATH, seed_stamp_relpath: str = TESTMON_SEED_STAMP_RELPATH, + seed_attempt_relpath: str = TESTMON_SEED_ATTEMPT_RELPATH, protocol_version: int, ) -> str | None: """Bootstrap `repo_root`'s testmon seed from its main checkout if warranted. @@ -248,8 +560,10 @@ def maybe_bootstrap_testmon_seed( return None local_testmon_data = repo_root / testmon_data_relpath local_seed_stamp = repo_root / seed_stamp_relpath + local_seed_attempt = repo_root / seed_attempt_relpath main_testmon_data = main_checkout / testmon_data_relpath main_seed_stamp = main_checkout / seed_stamp_relpath + main_seed_attempt = main_checkout / seed_attempt_relpath decision = decide_testmon_bootstrap( is_linked_worktree=is_linked_worktree, local_testmon_data=local_testmon_data, @@ -257,33 +571,30 @@ def maybe_bootstrap_testmon_seed( main_testmon_data=main_testmon_data, main_seed_stamp=main_seed_stamp, protocol_version=protocol_version, + main_seed_attempt=main_seed_attempt, + main_checkout_root=main_checkout, + local_checkout_root=repo_root, + local_seed_attempt=local_seed_attempt, ) if not decision.should_bootstrap: - if local_testmon_data.is_file() and _is_valid_complete_seed_stamp( - local_seed_stamp, protocol_version=protocol_version - ): - try: - local_payload = json.loads(local_seed_stamp.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - return None - if ( - isinstance(local_payload, dict) - and not local_payload.get("checkout_root") - and _stamp_seed_checkout_origin(local_seed_stamp, checkout_root=repo_root) - ): - return f"verify: migrated legacy pytest-testmon seed marker in {local_seed_stamp.parent}" return None stamped = bootstrap_testmon_seed_files( decision, local_testmon_data=local_testmon_data, local_seed_stamp=local_seed_stamp, + local_seed_attempt=local_seed_attempt, checkout_root=repo_root, inherited_from=main_checkout, ) if not stamped: return ( - f"verify: bootstrapped pytest-testmon seed into {local_testmon_data.parent}, " - "but could not record its checkout provenance" + f"verify: refused pytest-testmon bootstrap into {local_testmon_data.parent}; " + "no local state was published because provenance validation failed" + ) + if decision.main_seed_attempt is not None and decision.selection_only: + return ( + f"verify: bootstrapped pytest-testmon graph from main checkout {main_checkout} " + f"into {local_testmon_data.parent} as a selection-only attempt receipt (no seed.json)" ) return ( f"verify: bootstrapped pytest-testmon seed from main checkout {main_checkout} " @@ -294,6 +605,7 @@ def maybe_bootstrap_testmon_seed( __all__ = [ "TESTMON_DATA_RELPATH", "TESTMON_SEED_STAMP_RELPATH", + "TESTMON_SEED_ATTEMPT_RELPATH", "BootstrapDecision", "decide_testmon_bootstrap", "bootstrap_testmon_seed_files", diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py new file mode 100644 index 0000000000..b54c6dad62 --- /dev/null +++ b/devtools/testmon_state.py @@ -0,0 +1,981 @@ +"""Typed safety contract for reusable pytest-testmon state. + +The testmon database answers two different questions which must not share a +boolean marker: + +* did collection and dependency capture cover every promised node? +* did that run establish a green release baseline? + +A failed test can still have a complete dependency graph. Such a graph is +usable for affected-test selection, but it is never evidence that the suite +is releasable. This module is the single parser and SQLite validator used by +verification, worktree bootstrap, and the checkout guard. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import importlib.metadata +import json +import os +import sqlite3 +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from enum import StrEnum +from pathlib import Path +from typing import Any + + +class CollectionStatus(StrEnum): + COMPLETE = "complete" + INCOMPLETE = "incomplete" + + +class GraphStatus(StrEnum): + COMPLETE = "complete" + INCOMPLETE = "incomplete" + INVALID = "invalid" + + +class BaselineStatus(StrEnum): + GREEN = "green" + RED = "red" + + +class BindingMode(StrEnum): + EXACT = "exact" + RELATIVE_FILE_FINGERPRINTS = "relative-file-fingerprints" + + +class VerificationScope(StrEnum): + AFFECTED = "affected" + RELEASE_BASELINE = "release-baseline" + NARROW_TERMINAL = "narrow-terminal" + NON_TEST = "non-test" + + +class TerminalAuthorization(StrEnum): + NARROW_TERMINAL = "narrow-terminal" + + +@dataclass(frozen=True, slots=True) +class TestmonIdentity: + git_head: str | None + worktree_fingerprint: str + python: str + skip_slow: bool + lab: bool + git_tree: str | None = None + terminal_authorization: str | None = None + dependency_environment: str = "" + pytest_harness: str = "" + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> TestmonIdentity: + git_head = value.get("git_head") + if git_head is not None and (not isinstance(git_head, str) or not git_head): + raise ValueError("identity.git_head must be a non-empty string or null") + git_tree = value.get("git_tree") + if git_tree is not None and (not isinstance(git_tree, str) or not git_tree): + raise ValueError("identity.git_tree must be a non-empty string or null") + worktree = value.get("worktree_fingerprint") + python = value.get("python") + if not isinstance(worktree, str) or not worktree: + raise ValueError("identity.worktree_fingerprint must be a non-empty string") + if not isinstance(python, str) or not python: + raise ValueError("identity.python must be a non-empty string") + dependency_environment = value.get("dependency_environment") + pytest_harness = value.get("pytest_harness") + if dependency_environment is None: + dependency_environment = "" + if pytest_harness is None: + pytest_harness = "" + if not isinstance(dependency_environment, str): + raise ValueError("identity.dependency_environment must be a string") + if not isinstance(pytest_harness, str): + raise ValueError("identity.pytest_harness must be a string") + if not isinstance(value.get("skip_slow"), bool) or not isinstance(value.get("lab"), bool): + raise ValueError("identity selection flags must be booleans") + terminal_authorization = value.get("terminal_authorization") + if terminal_authorization is not None and terminal_authorization not in { + authorization.value for authorization in TerminalAuthorization + }: + raise ValueError("identity.terminal_authorization is invalid") + return cls( + git_head, + worktree, + python, + value["skip_slow"], + value["lab"], + git_tree, + terminal_authorization, + dependency_environment, + pytest_harness, + ) + + def as_dict(self) -> dict[str, Any]: + return { + "git_head": self.git_head, + "worktree_fingerprint": self.worktree_fingerprint, + "python": self.python, + "skip_slow": self.skip_slow, + "lab": self.lab, + "git_tree": self.git_tree, + "terminal_authorization": self.terminal_authorization, + "dependency_environment": self.dependency_environment, + "pytest_harness": self.pytest_harness, + } + + +def _fingerprint_files(checkout_root: Path, relative_paths: Sequence[str]) -> str: + """Hash named checkout inputs, preserving absent inputs as typed state.""" + digest = hashlib.sha256() + for relative_path in relative_paths: + digest.update(relative_path.encode()) + digest.update(b"\0") + try: + contents = (checkout_root / relative_path).read_bytes() + except OSError: + digest.update(b"missing") + else: + digest.update(contents) + digest.update(b"\0") + return digest.hexdigest() + + +def _installed_distributions() -> tuple[tuple[str, str], ...] | None: + """Return the active environment's normalized installed distributions.""" + try: + distributions = [] + for distribution in importlib.metadata.distributions(): + try: + name = distribution.metadata["Name"] + except KeyError: + return None + version = distribution.version + if not name or not version: + return None + distributions.append((name.casefold(), version)) + except (OSError, TypeError, ValueError, importlib.metadata.PackageNotFoundError): + return None + return tuple(sorted(distributions)) + + +def testmon_runtime_identity(checkout_root: Path) -> tuple[str, str] | None: + """Identify the lock, installed dependencies, and pytest execution harness. + + A testmon graph is reusable only under this exact dependency environment. + The application lock catches declared changes; installed distributions and + pytest-specific configuration catch a stale or differently provisioned + virtual environment even when ``sys.version`` is unchanged. + """ + distributions = _installed_distributions() + if distributions is None: + return None + normalized_root = checkout_root.resolve() + dependency_payload = { + "lock_inputs": _fingerprint_files(normalized_root, ("uv.lock", "pyproject.toml")), + "distributions": distributions, + } + harness_payload = { + "configuration": _fingerprint_files( + normalized_root, + ("pyproject.toml", "pytest.ini", "tox.ini", "setup.cfg", "tests/conftest.py"), + ), + "environment": { + key: os.environ.get(key) + for key in ( + "PYTEST_ADDOPTS", + "PYTEST_DISABLE_PLUGIN_AUTOLOAD", + "PYTEST_PLUGINS", + "HYPOTHESIS_PROFILE", + "POLYLOGUE_CI", + ) + }, + "pytest_distributions": tuple( + item for item in distributions if item[0] in {"pytest", "pytest-testmon", "pytest-xdist", "pluggy"} + ), + } + return ( + hashlib.sha256(json.dumps(dependency_payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + hashlib.sha256(json.dumps(harness_payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + ) + + +def _identity_matches_runtime(identity: TestmonIdentity, *, checkout_root: Path, protocol_version: int) -> bool: + """Keep pre-binding protocol receipts parseable but never reusable today.""" + if protocol_version < 5: + return True + runtime_identity = testmon_runtime_identity(checkout_root) + return ( + runtime_identity is not None + and ( + identity.dependency_environment, + identity.pytest_harness, + ) + == runtime_identity + ) + + +@dataclass(frozen=True, slots=True) +class TestmonBinding: + mode: BindingMode + checkout_root: str + source_checkout_root: str | None = None + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> TestmonBinding: + raw_mode = value.get("mode") + if not isinstance(raw_mode, str): + raise ValueError("binding.mode is invalid") + try: + mode = BindingMode(raw_mode) + except ValueError as exc: + raise ValueError("binding.mode is invalid") from exc + checkout_root = value.get("checkout_root") + source = value.get("source_checkout_root") + if not isinstance(checkout_root, str) or not checkout_root: + raise ValueError("binding.checkout_root must be a non-empty string") + if not Path(checkout_root).is_absolute(): + raise ValueError("binding.checkout_root must be absolute") + if source is not None and (not isinstance(source, str) or not source): + raise ValueError("binding.source_checkout_root must be a non-empty string or null") + if source is not None and not Path(source).is_absolute(): + raise ValueError("binding.source_checkout_root must be absolute") + if mode is BindingMode.EXACT and source is not None: + raise ValueError("exact bindings cannot have a source checkout") + if mode is BindingMode.RELATIVE_FILE_FINGERPRINTS: + if source is None: + raise ValueError("rebound bindings require a source checkout") + if Path(source).resolve() == Path(checkout_root).resolve(): + raise ValueError("rebound binding source and destination must differ") + return cls(mode, checkout_root, source) + + def as_dict(self) -> dict[str, Any]: + return { + "mode": self.mode.value, + "checkout_root": self.checkout_root, + "source_checkout_root": self.source_checkout_root, + } + + +@dataclass(frozen=True, slots=True) +class GraphInspection: + status: GraphStatus + recorded_count: int + dependency_edge_count: int + missing_nodeids: tuple[str, ...] + orphan_execution_edges: int + orphan_fingerprint_edges: int + error: str | None + failed_nodeids: tuple[str, ...] + + @property + def usable_for_selection(self) -> bool: + return self.status is GraphStatus.COMPLETE + + def as_dict(self) -> dict[str, Any]: + return { + "status": self.status.value, + "recorded_count": self.recorded_count, + "dependency_edge_count": self.dependency_edge_count, + "missing_nodeids": list(self.missing_nodeids), + "orphan_execution_edges": self.orphan_execution_edges, + "orphan_fingerprint_edges": self.orphan_fingerprint_edges, + "error": self.error, + "failed_nodeids": list(self.failed_nodeids), + } + + +@dataclass(frozen=True, slots=True) +class TestmonSeedStamp: + protocol_version: int + collection_status: CollectionStatus + expected_nodeids: tuple[str, ...] + selected_nodeids_omitted: int + baseline_status: BaselineStatus + release_baseline_allowed: bool + baseline_exit_code: int + graph: GraphInspection + identity: TestmonIdentity + binding: TestmonBinding + testmon_data: str + run_id: str + artifact_dir: str + + @property + def affected_selection_allowed(self) -> bool: + return ( + self.collection_status is CollectionStatus.COMPLETE + and self.selected_nodeids_omitted == 0 + and self.graph.usable_for_selection + ) + + @property + def expected_digest(self) -> str: + return hashlib.sha256("\n".join(sorted(self.expected_nodeids)).encode()).hexdigest() + + def as_dict(self) -> dict[str, Any]: + return { + "protocol_version": self.protocol_version, + "status": "usable", + "collection": { + "status": self.collection_status.value, + "expected_count": len(self.expected_nodeids), + "expected_digest": self.expected_digest, + "selected_nodeids": list(self.expected_nodeids), + "selected_nodeids_omitted": self.selected_nodeids_omitted, + }, + "baseline": { + "status": self.baseline_status.value, + "exit_code": self.baseline_exit_code, + "release_baseline_allowed": self.release_baseline_allowed, + }, + "graph": self.graph.as_dict(), + "identity": self.identity.as_dict(), + "binding": self.binding.as_dict(), + "testmon_data": self.testmon_data, + "run_id": self.run_id, + "artifact_dir": self.artifact_dir, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, Any], *, protocol_version: int) -> TestmonSeedStamp: + if value.get("protocol_version") != protocol_version or value.get("status") != "usable": + raise ValueError("seed stamp is not a current usable testmon stamp") + collection = value.get("collection") + baseline = value.get("baseline") + graph = value.get("graph") + identity = value.get("identity") + binding = value.get("binding") + if not all(isinstance(item, Mapping) for item in (collection, baseline, graph, identity, binding)): + raise ValueError("seed stamp has incomplete typed state") + assert isinstance(collection, Mapping) + assert isinstance(baseline, Mapping) + assert isinstance(graph, Mapping) + assert isinstance(identity, Mapping) + assert isinstance(binding, Mapping) + if collection.get("status") != CollectionStatus.COMPLETE.value: + raise ValueError("seed stamp collection is not complete") + nodeids = collection.get("selected_nodeids") + if ( + not isinstance(nodeids, list) + or not nodeids + or any(not isinstance(item, str) or not item for item in nodeids) + ): + raise ValueError("seed stamp selected nodeids are missing or malformed") + if len(set(nodeids)) != len(nodeids): + raise ValueError("seed stamp selected nodeids are not unique") + omitted = collection.get("selected_nodeids_omitted") + if not isinstance(omitted, int) or isinstance(omitted, bool) or omitted != 0: + raise ValueError("seed stamp has controlled collection omissions") + if collection.get("expected_count") != len(nodeids): + raise ValueError("seed stamp expected count does not match selected nodeids") + expected_digest = hashlib.sha256("\n".join(sorted(nodeids)).encode()).hexdigest() + if collection.get("expected_digest") != expected_digest: + raise ValueError("seed stamp expected nodeid digest is stale") + raw_baseline_status = baseline.get("status") + if not isinstance(raw_baseline_status, str): + raise ValueError("seed stamp baseline status is invalid") + try: + baseline_status = BaselineStatus(raw_baseline_status) + except ValueError as exc: + raise ValueError("seed stamp baseline status is invalid") from exc + exit_code = baseline.get("exit_code") + release_allowed = baseline.get("release_baseline_allowed") + if not isinstance(exit_code, int) or isinstance(exit_code, bool) or not isinstance(release_allowed, bool): + raise ValueError("seed stamp baseline fields are malformed") + if release_allowed != (baseline_status is BaselineStatus.GREEN): + raise ValueError("release permission does not match baseline status") + if baseline_status is BaselineStatus.GREEN and exit_code != 0: + raise ValueError("green seed stamp must have a zero exit code") + graph_status = graph.get("status") + if not isinstance(graph_status, str): + raise ValueError("seed stamp graph status is invalid") + try: + status = GraphStatus(graph_status) + except ValueError as exc: + raise ValueError("seed stamp graph status is invalid") from exc + if status is not GraphStatus.COMPLETE: + raise ValueError("seed stamp graph is not complete") + graph_expected = [ + "recorded_count", + "dependency_edge_count", + "orphan_execution_edges", + "orphan_fingerprint_edges", + ] + graph_counts = {key: graph.get(key) for key in graph_expected} + if any(not isinstance(item, int) or isinstance(item, bool) or item < 0 for item in graph_counts.values()): + raise ValueError("seed stamp graph counts are malformed") + dependency_edge_count = graph.get("dependency_edge_count") + if not isinstance(dependency_edge_count, int) or isinstance(dependency_edge_count, bool): + raise ValueError("seed stamp dependency edge count is malformed") + if ( + graph.get("recorded_count") != len(nodeids) + or dependency_edge_count < len(nodeids) + or graph.get("orphan_execution_edges") != 0 + or graph.get("orphan_fingerprint_edges") != 0 + ): + raise ValueError("seed stamp graph coverage is incomplete") + missing_nodeids = graph.get("missing_nodeids") + if ( + not isinstance(missing_nodeids, list) + or any(not isinstance(item, str) or not item for item in missing_nodeids) + or not set(missing_nodeids).issubset(nodeids) + ): + raise ValueError("seed stamp missing-node ledger is malformed") + if graph.get("error") is not None or missing_nodeids: + raise ValueError("seed stamp graph has missing or erroneous nodes") + graph_nodeids = graph.get("failed_nodeids", []) + if ( + not isinstance(graph_nodeids, list) + or any(not isinstance(item, str) or not item for item in graph_nodeids) + or not set(graph_nodeids).issubset(nodeids) + or len(set(graph_nodeids)) != len(graph_nodeids) + ): + raise ValueError("seed stamp graph failure ledger is malformed") + if baseline_status is BaselineStatus.GREEN and graph_nodeids: + raise ValueError("green seed stamp cannot contain failed graph nodes") + testmon_data = value.get("testmon_data") + run_id = value.get("run_id") + artifact_dir = value.get("artifact_dir") + if not all(isinstance(item, str) and item for item in (testmon_data, run_id, artifact_dir)): + raise ValueError("seed stamp provenance is incomplete") + assert isinstance(testmon_data, str) + assert isinstance(run_id, str) + assert isinstance(artifact_dir, str) + typed_binding = TestmonBinding.from_mapping(binding) + if not _is_bound_run_artifact( + artifact_dir, + checkout_root=Path(typed_binding.checkout_root), + run_id=run_id, + ): + raise ValueError("seed stamp artifact directory is not checkout-bound") + return cls( + protocol_version, + CollectionStatus.COMPLETE, + tuple(nodeids), + 0, + baseline_status, + release_allowed, + exit_code, + GraphInspection( + status, + graph["recorded_count"], + dependency_edge_count, + tuple(graph.get("missing_nodeids", [])), + graph["orphan_execution_edges"], + graph["orphan_fingerprint_edges"], + graph.get("error"), + tuple(graph_nodeids), + ), + TestmonIdentity.from_mapping(identity), + typed_binding, + testmon_data, + run_id, + artifact_dir, + ) + + def rebound(self, *, checkout_root: Path, inherited_from: Path) -> TestmonSeedStamp: + return replace( + self, + binding=TestmonBinding( + BindingMode.RELATIVE_FILE_FINGERPRINTS, + str(checkout_root.resolve()), + str(inherited_from.resolve()), + ), + ) + + +def file_fingerprint(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _is_bound_run_artifact(raw: object, *, checkout_root: Path, run_id: str) -> bool: + if not isinstance(raw, str) or not raw or not run_id: + return False + path = Path(raw) + if path.is_absolute() or path.parts[:3] != (".cache", "verify", "runs"): + return False + if path.parts[3:] != (run_id,): + return False + try: + artifact_dir = (checkout_root / path).resolve() + artifact_dir.relative_to((checkout_root / ".cache" / "verify" / "runs" / run_id).resolve()) + receipt = json.loads((artifact_dir / "run.json").read_text(encoding="utf-8")) + if not isinstance(receipt, Mapping): + return False + return ( + receipt.get("run_id") == run_id + and isinstance(receipt.get("checkout_root"), str) + and Path(receipt["checkout_root"]).resolve() == checkout_root.resolve() + and receipt.get("artifact_dir") == str(Path(".cache") / "verify" / "runs" / run_id) + ) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): + return False + + +def seed_marker_is_checkout_bound( + marker_path: Path, + *, + checkout_root: Path, + protocol_version: int, +) -> bool: + """Validate only the typed ownership envelope of a seed marker. + + This intentionally does not open or fingerprint SQLite. The checkout guard + uses this cheap predicate for every entrypoint; verify preflight performs + the exhaustive graph validation before authorizing selection. + """ + try: + payload = json.loads(marker_path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + return False + stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) + return Path(stamp.binding.checkout_root).resolve() == checkout_root.resolve() + except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): + return False + + +def attempt_is_checkout_bound( + attempt: Mapping[str, Any], + *, + checkout_root: Path, + protocol_version: int, + reusable_only: bool = True, +) -> bool: + """Check a seed-attempt receipt without inspecting its SQLite graph.""" + allowed_statuses = {"reusable", "complete"} if reusable_only else {"running", "incomplete", "reusable", "complete"} + if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in allowed_statuses: + return False + identity = attempt.get("identity") + expected = attempt.get("expected_nodeids") + selection = attempt.get("selection") + if not isinstance(identity, Mapping) or not isinstance(expected, list) or not isinstance(selection, Mapping): + return False + if not expected or any(not isinstance(nodeid, str) or not nodeid for nodeid in expected): + return False + if len(set(expected)) != len(expected): + return False + if ( + not isinstance(attempt.get("expected_count"), int) + or isinstance(attempt.get("expected_count"), bool) + or attempt.get("expected_count") != len(expected) + ): + return False + expected_digest = hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() + if attempt.get("expected_digest") != expected_digest: + return False + try: + typed_identity = TestmonIdentity.from_mapping(identity) + except ValueError: + return False + if not _identity_matches_runtime(typed_identity, checkout_root=checkout_root, protocol_version=protocol_version): + return False + omitted = selection.get("selected_nodeids_omitted") + selected_count = selection.get("selected_count") + if ( + not isinstance(omitted, int) + or isinstance(omitted, bool) + or omitted != 0 + or not isinstance(selected_count, int) + or isinstance(selected_count, bool) + or selected_count != len(expected) + ): + return False + recorded_data = attempt.get("testmon_data") + run_id = attempt.get("run_id") + artifact_dir = attempt.get("artifact_dir") + if ( + not isinstance(recorded_data, str) + or not recorded_data + or not isinstance(run_id, str) + or not run_id + or not isinstance(artifact_dir, str) + or not artifact_dir + or not _is_bound_run_artifact(artifact_dir, checkout_root=checkout_root, run_id=run_id) + ): + return False + raw_binding = attempt.get("binding") + if raw_binding is None: + binding = TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())) + elif isinstance(raw_binding, Mapping): + try: + binding = TestmonBinding.from_mapping(raw_binding) + except ValueError: + return False + else: + return False + if Path(binding.checkout_root).resolve() != checkout_root.resolve(): + return False + raw_permission = attempt.get("release_baseline_allowed") + if raw_permission is not None and not isinstance(raw_permission, bool): + return False + raw_scope = attempt.get("verification_scope") + if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: + return False + if reusable_only and raw_permission is not False: + return False + if reusable_only: + outcomes = attempt.get("node_outcomes") + if not isinstance(outcomes, list) or len(outcomes) != len(expected): + return False + nodeids = [item.get("nodeid") for item in outcomes if isinstance(item, Mapping)] + if len(nodeids) != len(outcomes) or set(nodeids) != set(expected) or len(set(nodeids)) != len(nodeids): + return False + if any(item.get("outcome") not in {"passed", "failed", "error", "skipped"} for item in outcomes): + return False + return True + + +def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> GraphInspection: + """Validate the real testmon schema and every expected dependency edge.""" + expected = tuple(expected_nodeids) + if not path.is_file() or not expected or len(set(expected)) != len(expected): + return GraphInspection( + GraphStatus.INCOMPLETE, 0, 0, expected, 0, 0, "missing or malformed expected nodeids", () + ) + try: + with contextlib.closing(sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)) as connection: + if connection.execute("PRAGMA integrity_check").fetchone() != ("ok",): + return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "sqlite integrity check failed", ()) + required = {"test_execution", "test_execution_file_fp", "file_fp"} + tables = {str(row[0]) for row in connection.execute("select name from sqlite_master where type='table'")} + if not required <= tables: + return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon schema is incomplete", ()) + required_columns = { + "test_execution": {"id", "test_name", "failed"}, + "test_execution_file_fp": {"test_execution_id", "fingerprint_id"}, + "file_fp": {"id", "filename", "fsha"}, + } + for table, columns in required_columns.items(): + actual = {str(row[1]) for row in connection.execute(f"pragma table_info({table})")} + if not columns <= actual: + return GraphInspection( + GraphStatus.INVALID, + 0, + 0, + expected, + 0, + 0, + f"testmon schema is missing columns from {table}", + (), + ) + executions = connection.execute( + "select id, test_name, failed from test_execution where test_name is not null" + ).fetchall() + latest: dict[str, tuple[int, bool]] = {} + execution_ids: set[int] = set() + for execution_id, test_name, failed in executions: + if ( + not isinstance(execution_id, int) + or isinstance(execution_id, bool) + or execution_id <= 0 + or not isinstance(test_name, str) + or not test_name + or not isinstance(failed, int) + or isinstance(failed, bool) + or failed not in (0, 1) + ): + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon execution row is malformed", () + ) + if execution_id in execution_ids: + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon execution ids are not unique", () + ) + execution_ids.add(execution_id) + name = test_name + prior = latest.get(name) + if prior is None or execution_id > prior[0]: + latest[name] = (execution_id, failed == 1) + missing = tuple(sorted(set(expected) - latest.keys())) + expected_ids = {latest[nodeid][0] for nodeid in expected if nodeid in latest} + edge_rows = connection.execute( + "select test_execution_id, fingerprint_id from test_execution_file_fp" + ).fetchall() + fingerprints = connection.execute("select id, filename, fsha from file_fp").fetchall() + fingerprint_ids: set[int] = set() + for fingerprint_id, filename, fsha in fingerprints: + if ( + not isinstance(fingerprint_id, int) + or isinstance(fingerprint_id, bool) + or fingerprint_id <= 0 + or not isinstance(filename, str) + or not filename + or Path(filename).is_absolute() + or ".." in Path(filename).parts + or not isinstance(fsha, str) + or not fsha + ): + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon fingerprint row is malformed", () + ) + if fingerprint_id in fingerprint_ids: + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon fingerprint ids are not unique", () + ) + fingerprint_ids.add(fingerprint_id) + for execution_id, fingerprint_id in edge_rows: + if ( + not isinstance(execution_id, int) + or isinstance(execution_id, bool) + or execution_id <= 0 + or not isinstance(fingerprint_id, int) + or isinstance(fingerprint_id, bool) + or fingerprint_id <= 0 + ): + return GraphInspection( + GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon dependency edge is malformed", () + ) + orphan_execution_edges = sum(1 for row in edge_rows if row[0] not in execution_ids) + orphan_fingerprint_edges = sum(1 for row in edge_rows if row[1] not in fingerprint_ids) + edge_counts: dict[int, int] = {} + for execution_id, _fingerprint_id in edge_rows: + edge_counts[execution_id] = edge_counts.get(execution_id, 0) + 1 + uncovered = tuple( + sorted(nodeid for nodeid in expected if nodeid in latest and edge_counts.get(latest[nodeid][0], 0) == 0) + ) + missing = tuple(sorted(set(missing) | set(uncovered))) + failed = tuple(sorted(nodeid for nodeid in expected if nodeid in latest and latest[nodeid][1])) + edge_count = sum(edge_counts.get(execution_id, 0) for execution_id in expected_ids) + status = ( + GraphStatus.COMPLETE + if not missing and not orphan_execution_edges and not orphan_fingerprint_edges + else GraphStatus.INCOMPLETE + ) + return GraphInspection( + status, + len(expected) - len(missing), + edge_count, + missing, + orphan_execution_edges, + orphan_fingerprint_edges, + None, + failed, + ) + except (OSError, sqlite3.Error, UnicodeError, TypeError, ValueError, OverflowError) as exc: + return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, str(exc), ()) + + +def validate_stamp( + stamp_path: Path, + data_path: Path, + *, + checkout_root: Path, + protocol_version: int, +) -> TestmonSeedStamp | None: + """Parse and re-check a stamp against its current SQLite graph.""" + try: + payload = json.loads(stamp_path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + return None + stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) + if not stamp.release_baseline_allowed: + return None + if ( + stamp.identity.skip_slow + and stamp.identity.terminal_authorization != TerminalAuthorization.NARROW_TERMINAL.value + ): + return None + if not _identity_matches_runtime( + stamp.identity, checkout_root=checkout_root, protocol_version=protocol_version + ): + return None + if Path(stamp.binding.checkout_root).resolve() != checkout_root.resolve(): + return None + if file_fingerprint(data_path) != stamp.testmon_data: + return None + graph = inspect_testmon_database(data_path, stamp.expected_nodeids) + if graph != stamp.graph: + return None + return stamp + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return None + + +def refresh_stamp(stamp: TestmonSeedStamp, data_path: Path) -> TestmonSeedStamp | None: + """Refresh mutable SQLite provenance after a successful affected run.""" + graph = inspect_testmon_database(data_path, stamp.expected_nodeids) + if not graph.usable_for_selection: + return None + try: + return replace(stamp, graph=graph, testmon_data=file_fingerprint(data_path)) + except OSError: + return None + + +def stamp_from_attempt( + attempt: Mapping[str, Any], + data_path: Path, + *, + checkout_root: Path, + protocol_version: int, + published_marker: bool = True, +) -> TestmonSeedStamp | None: + """Parse a complete attempt, withholding release authority until publication.""" + if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in {"reusable", "complete"}: + return None + selection = attempt.get("selection") + expected = attempt.get("expected_nodeids") + identity = attempt.get("identity") + if not isinstance(selection, Mapping) or not isinstance(expected, list) or not isinstance(identity, Mapping): + return None + assert isinstance(selection, Mapping) + assert isinstance(identity, Mapping) + omitted = selection.get("selected_nodeids_omitted") + selected_count = selection.get("selected_count") + if ( + not isinstance(omitted, int) + or isinstance(omitted, bool) + or omitted != 0 + or not isinstance(selected_count, int) + or isinstance(selected_count, bool) + or selected_count != len(expected) + or not expected + or any(not isinstance(nodeid, str) or not nodeid for nodeid in expected) + or len(set(expected)) != len(expected) + ): + return None + expected_count = attempt.get("expected_count") + if not isinstance(expected_count, int) or isinstance(expected_count, bool) or expected_count != len(expected): + return None + expected_digest = attempt.get("expected_digest") + if ( + not isinstance(expected_digest, str) + or expected_digest != hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() + ): + return None + recorded_data = attempt.get("testmon_data") + if not isinstance(recorded_data, str) or not recorded_data or not data_path.is_file(): + return None + try: + if file_fingerprint(data_path) != recorded_data: + return None + except OSError: + return None + run_id = attempt.get("run_id") + artifact_dir = attempt.get("artifact_dir") + if not isinstance(run_id, str) or not run_id or not isinstance(artifact_dir, str) or not artifact_dir: + return None + if not _is_bound_run_artifact(artifact_dir, checkout_root=checkout_root, run_id=run_id): + return None + outcomes = attempt.get("node_outcomes") + if not isinstance(outcomes, list) or len(outcomes) != len(expected): + return None + if any(not isinstance(item, Mapping) for item in outcomes): + return None + outcome_items = [item for item in outcomes if isinstance(item, Mapping)] + if any( + not isinstance(item.get("nodeid"), str) or not item.get("nodeid") or item.get("nodeid") not in expected + for item in outcome_items + ): + return None + outcome_by_node = {item["nodeid"]: item.get("outcome") for item in outcome_items} + if set(outcome_by_node) != set(expected): + return None + if len(outcome_by_node) != len(outcomes) or any( + not isinstance(nodeid, str) or not nodeid for nodeid in outcome_by_node + ): + return None + if any(outcome not in {"passed", "failed", "error", "skipped"} for outcome in outcome_by_node.values()): + return None + exit_code = attempt.get("exit_code") + if not isinstance(exit_code, int) or isinstance(exit_code, bool): + return None + graph = inspect_testmon_database(data_path, [str(nodeid) for nodeid in expected]) + if not graph.usable_for_selection: + return None + try: + typed_identity = TestmonIdentity.from_mapping(identity) + except ValueError: + return None + if not _identity_matches_runtime(typed_identity, checkout_root=checkout_root, protocol_version=protocol_version): + return None + baseline = ( + BaselineStatus.GREEN + if attempt.get("status") == "complete" + and exit_code == 0 + and all(outcome in {"passed", "skipped"} for outcome in outcome_by_node.values()) + and not graph.failed_nodeids + else BaselineStatus.RED + ) + raw_scope = attempt.get("verification_scope") + if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: + return None + terminal_authorized = ( + typed_identity.skip_slow is True + and raw_scope == VerificationScope.NARROW_TERMINAL.value + and typed_identity.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value + ) + if baseline is BaselineStatus.GREEN and typed_identity.skip_slow and not terminal_authorized: + baseline = BaselineStatus.RED + if not published_marker: + baseline = BaselineStatus.RED + raw_permission = attempt.get("release_baseline_allowed") + if baseline is BaselineStatus.GREEN and ( + raw_scope != VerificationScope.NARROW_TERMINAL.value + if typed_identity.skip_slow + else raw_scope != VerificationScope.RELEASE_BASELINE.value + ): + baseline = BaselineStatus.RED + if baseline is BaselineStatus.GREEN and raw_permission is not True: + baseline = BaselineStatus.RED + if raw_permission is not None and not isinstance(raw_permission, bool): + return None + if published_marker and raw_permission is not None and raw_permission != (baseline is BaselineStatus.GREEN): + return None + raw_binding = attempt.get("binding") + if raw_binding is None: + typed_binding = TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())) + elif isinstance(raw_binding, Mapping): + try: + typed_binding = TestmonBinding.from_mapping(raw_binding) + except ValueError: + return None + if Path(typed_binding.checkout_root).resolve() != checkout_root.resolve(): + return None + else: + return None + return TestmonSeedStamp( + protocol_version, + CollectionStatus.COMPLETE, + tuple(str(nodeid) for nodeid in expected), + 0, + baseline, + baseline is BaselineStatus.GREEN, + exit_code, + graph, + typed_identity, + typed_binding, + recorded_data, + run_id, + artifact_dir, + ) + + +__all__ = [ + "BaselineStatus", + "BindingMode", + "CollectionStatus", + "GraphInspection", + "GraphStatus", + "TestmonBinding", + "TestmonIdentity", + "TestmonSeedStamp", + "TerminalAuthorization", + "VerificationScope", + "attempt_is_checkout_bound", + "file_fingerprint", + "inspect_testmon_database", + "refresh_stamp", + "seed_marker_is_checkout_bound", + "stamp_from_attempt", + "testmon_runtime_identity", + "validate_stamp", +] diff --git a/devtools/verify.py b/devtools/verify.py index c07bde6b74..a199d3ba71 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -31,7 +31,6 @@ import shlex import shutil import signal -import sqlite3 import stat import subprocess import sys @@ -59,6 +58,19 @@ write_termination_request, ) from devtools.testmon_bootstrap import maybe_bootstrap_testmon_seed +from devtools.testmon_state import ( + BindingMode, + GraphStatus, + TerminalAuthorization, + TestmonBinding, + TestmonSeedStamp, + VerificationScope, + inspect_testmon_database, + refresh_stamp, + stamp_from_attempt, + testmon_runtime_identity, + validate_stamp, +) from devtools.verify_runs import ( CURRENT_CONTAINMENT_PATH, CURRENT_EVENTS_DIR, @@ -95,6 +107,17 @@ ROOT = Path(__file__).resolve().parents[1] + +def _anchor_verification_paths() -> None: + """Use the checkout root for relative verification state when invoked inside it.""" + current = Path.cwd().resolve() + try: + current.relative_to(ROOT.resolve()) + except ValueError: + return + os.chdir(ROOT) + + # ── mypy daemon probe ────────────────────────────────────────────── @@ -190,7 +213,7 @@ def _format_completion_notification( TESTMON_SEED_STAMP = Path(".cache/testmon/seed.json") TESTMON_SEED_ATTEMPT = Path(".cache/testmon/seed-attempt.json") TESTMON_AFFECTED_STAMP = Path(".cache/testmon/affected.json") -TESTMON_SEED_PROTOCOL_VERSION = 3 +TESTMON_SEED_PROTOCOL_VERSION = 5 PYTEST_REPORT_DIR = Path(".cache/verify") PYTEST_REPORT_PATH = PYTEST_REPORT_DIR / "last-pytest.json" PYTEST_JUNIT_REPORT_DIR = Path(".cache/test-reports") @@ -212,7 +235,6 @@ def _format_completion_notification( DEFAULT_PYTEST_STALL_TIMEOUT_S = 10 * 60.0 DEFAULT_PYTEST_TERM_GRACE_S = 5.0 DEFAULT_PYTEST_RESOURCE_INTERVAL_S = 2.0 -DEFAULT_TESTMON_WORKERS = "4" def _load_history() -> list[dict[str, Any]]: @@ -1673,7 +1695,7 @@ def _run( last_resource_sample=last_resource_row, tmpfs_budget_mb=pytest_tmpfs_budget_mb, basetemp_cleanup=basetemp_cleanup, - concurrency=runtime_policy.workers if runtime_policy is not None else 1, + concurrency=_pytest_command_concurrency(cmd), ) metadata["workload_receipt"] = workload_receipt if artifacts is not None: @@ -1890,7 +1912,7 @@ def build_verify_steps( else: pytest_cmd.append("--testmon-noselect") label = "pytest seed-testmon" - pytest_cmd.extend(_pytest_worker_args(default="4")) + pytest_cmd.extend(_pytest_worker_args(maximum=4)) steps.append((label, pytest_cmd)) elif full_pytest: # #1775: the full diagnostic runs as two lanes. The bulk lane keeps @@ -1905,7 +1927,7 @@ def build_verify_steps( *pytest_cmd, "-m", f"({base_marker}) and not load_sensitive and not tui", - *_pytest_worker_args(default="4"), + *_pytest_worker_args(), ] steps.append(("pytest full (parallel)", bulk_cmd)) @@ -1922,8 +1944,7 @@ def _isolated_report_arg(arg: str) -> str: isolated_cmd.extend(["-m", f"({base_marker}) and (load_sensitive or tui)", "-p", "no:randomly", "-n", "0"]) steps.append(("pytest load-sensitive (isolated)", isolated_cmd)) else: - default_workers = DEFAULT_TESTMON_WORKERS - pytest_cmd.extend(["-m", base_marker, "--testmon", *_pytest_worker_args(default=default_workers)]) + pytest_cmd.extend(["-m", base_marker, "--testmon", *_pytest_worker_args()]) pytest_cmd.append("--testmon-forceselect") label = "pytest testmon (broad)" if broad_testmon else "pytest testmon" steps.append((label, pytest_cmd)) @@ -1938,6 +1959,19 @@ def _isolated_report_arg(arg: str) -> str: steps.append( ("lab policy campaign-archive-boundaries", _devtools_cmd("lab policy campaign-archive-boundaries")) ) + steps.append(("lab policy acceptance-contracts", _devtools_cmd("lab policy acceptance-contracts"))) + steps.append( + ( + "lab policy acceptance-contract-reconcile", + _devtools_cmd("lab policy acceptance-contract-reconcile"), + ) + ) + steps.append( + ( + "lab policy acceptance-contract-apply", + _devtools_cmd("lab policy acceptance-contract-apply"), + ) + ) # backlog-hygiene and bead-graph are corpus-wide backlog-debt scans # (findings scale with the total count of open Beads issues, not # with this change's diff) -- they stay --lab-only/scheduled rather @@ -2000,6 +2034,17 @@ def _git_head() -> str | None: return None +def _git_committed_tree() -> str | None: + result = subprocess.run( + ["git", "rev-parse", "HEAD^{tree}"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result.stdout.strip() + return None + + def _stamp_head() -> None: head = _git_head() if head is None: @@ -2022,9 +2067,24 @@ def _file_fingerprint(path: Path) -> str: return h.hexdigest() -def _pytest_worker_args(*, default: str) -> list[str]: - del default - return ["-n", str(adaptive_pytest_worker_count(os.environ))] +def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: + """Return the managed worker count, optionally capped for a bounded lane.""" + workers = adaptive_pytest_worker_count(os.environ) + if maximum is not None: + workers = min(workers, maximum) + return ["-n", str(workers)] + + +def _pytest_command_concurrency(cmd: Sequence[str]) -> int: + """Return the worker count actually requested by the final pytest command.""" + for index in range(len(cmd) - 2, -1, -1): + if cmd[index] != "-n": + continue + try: + return max(1, int(cmd[index + 1])) + except ValueError: + return 1 + return 1 _BROAD_TESTMON_CHANGED_PATHS = { @@ -2072,18 +2132,21 @@ def _testmon_coverage_identity(executable_paths: Sequence[str]) -> dict[str, Any def _matching_testmon_coverage(executable_paths: Sequence[str]) -> str | None: """Return the receipt kind proving that zero new selection is legitimate.""" identity = _testmon_coverage_identity(executable_paths) - seed = _read_json_artifact(TESTMON_SEED_STAMP) - if isinstance(seed, dict): - seed_identity = seed.get("identity") - if ( - seed.get("protocol_version") == TESTMON_SEED_PROTOCOL_VERSION - and seed.get("status") == "complete" - and isinstance(seed_identity, dict) - and seed_identity.get("worktree_fingerprint") == identity["worktree_fingerprint"] - ): - return "complete_seed" affected = _read_json_artifact(TESTMON_AFFECTED_STAMP) - if isinstance(affected, dict) and affected.get("identity") == identity: + selected_count = affected.get("selected_count") if isinstance(affected, dict) else None + if ( + isinstance(affected, dict) + and affected.get("protocol_version") == 1 + and affected.get("status") == "complete" + and isinstance(affected.get("timestamp"), str) + and bool(affected.get("timestamp")) + and isinstance(affected.get("run_id"), str) + and bool(affected.get("run_id")) + and isinstance(selected_count, int) + and not isinstance(selected_count, bool) + and selected_count > 0 + and affected.get("identity") == identity + ): return "successful_affected_run" return None @@ -2111,36 +2174,37 @@ def _testmon_preflight(*, seed_testmon: bool, full_pytest: bool, quick: bool, co "to create .cache/testmon/testmondata and .cache/testmon/seed.json " "before using the default affected-test path.\n" ) - if not TESTMON_DATA.exists() or not TESTMON_SEED_STAMP.exists(): + if not TESTMON_DATA.exists(): return seed_message - try: - stamp = json.loads(TESTMON_SEED_STAMP.read_text()) - except (OSError, json.JSONDecodeError): - return ( - "verify: pytest-testmon seed stamp is unreadable; run `devtools verify --seed-testmon` " - "to refresh .cache/testmon/testmondata and .cache/testmon/seed.json.\n" - ) - if not isinstance(stamp, dict): - return ( - "verify: pytest-testmon seed stamp has an invalid shape; run `devtools verify --seed-testmon` " - "to refresh .cache/testmon/testmondata and .cache/testmon/seed.json.\n" - ) - if stamp.get("protocol_version") != TESTMON_SEED_PROTOCOL_VERSION or stamp.get("status") != "complete": + if not TESTMON_SEED_STAMP.exists(): + attempt = _read_testmon_seed_attempt() + if ( + attempt is not None + and stamp_from_attempt( + attempt, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + published_marker=False, + ) + is not None + ): + sys.stderr.write( + "verify: using a validated complete pytest-testmon graph from a red seed attempt; " + "the release baseline remains red.\n" + ) + return None + return seed_message + stamp = validate_stamp( + TESTMON_SEED_STAMP, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + if stamp is None: return ( - "verify: pytest-testmon has no validated complete seed receipt; run " - "`devtools verify --seed-testmon` to resume or rebuild the dependency baseline.\n" - ) - current_head = _git_head() - stamped_head = stamp.get("git_head") - if current_head is not None and stamped_head != current_head: - sys.stderr.write( - "verify: pytest-testmon seed was recorded for a different git head; " - "continuing with the existing dependency database and recording affected-test evidence.\n" - ) - if stamp.get("testmon_data") != _file_fingerprint(TESTMON_DATA): - sys.stderr.write( - "verify: pytest-testmon database changed after the seed stamp; " - "continuing because testmon updates its dependency database during normal affected runs.\n" + "verify: pytest-testmon seed state is unreadable, stale, malformed, or not graph-complete; run " + "`devtools verify --seed-testmon` to rebuild the dependency baseline.\n" ) return None @@ -2200,13 +2264,28 @@ def _worktree_fingerprint() -> str: return digest.hexdigest() -def _testmon_seed_identity(*, git_head: str | None, skip_slow: bool, lab: bool) -> dict[str, Any]: +def _testmon_seed_identity( + *, + git_head: str | None, + git_tree: str | None = None, + skip_slow: bool, + lab: bool, + terminal_authorization: str | None = None, +) -> dict[str, Any]: + runtime_identity = testmon_runtime_identity(ROOT) + if runtime_identity is None: + raise RuntimeError("could not identify the active dependency environment and pytest harness") + dependency_environment, pytest_harness = runtime_identity return { "git_head": git_head, + "git_tree": git_tree, "worktree_fingerprint": _worktree_fingerprint(), "python": sys.version, "skip_slow": skip_slow, "lab": lab, + "terminal_authorization": terminal_authorization, + "dependency_environment": dependency_environment, + "pytest_harness": pytest_harness, } @@ -2215,29 +2294,121 @@ def _read_testmon_seed_attempt() -> dict[str, Any] | None: return payload if isinstance(payload, dict) else None +def _flatten_seed_outcomes(attempt: Mapping[str, Any] | None) -> list[dict[str, Any]]: + """Flatten outcomes from every interrupted attempt, newest result winning.""" + if attempt is None: + return [] + flattened: dict[str, dict[str, Any]] = {} + for field in ("prior_node_outcomes", "node_outcomes"): + raw = attempt.get(field) + if not isinstance(raw, list): + continue + for item in raw: + if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) and item["nodeid"]: + flattened[item["nodeid"]] = dict(item) + return [flattened[nodeid] for nodeid in sorted(flattened)] + + +def _testmon_release_baseline_permission() -> bool | None: + """Return release permission for current testmon state, or ``None`` when not applicable.""" + if TESTMON_SEED_STAMP.exists(): + stamp = validate_stamp( + TESTMON_SEED_STAMP, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + return stamp.release_baseline_allowed if stamp is not None else False + attempt = _read_testmon_seed_attempt() + if attempt is None: + return False + stamp = stamp_from_attempt( + attempt, + TESTMON_DATA, + checkout_root=ROOT, + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + published_marker=False, + ) + return stamp.release_baseline_allowed if stamp is not None else False + + +def _safe_testmon_artifact_dir(raw: object, *, require_run_root: bool = False) -> Path | None: + if not isinstance(raw, str) or not raw: + return None + path = Path(raw) + checkout_root = Path.cwd().resolve() + if require_run_root and path.is_absolute(): + return None + resolved = (path if path.is_absolute() else checkout_root / path).resolve() + try: + resolved.relative_to(checkout_root) + if require_run_root: + resolved.relative_to((checkout_root / ".cache" / "verify" / "runs").resolve()) + except ValueError: + return None + return resolved + + def _testmon_seed_expected_nodeids(attempt: Mapping[str, Any]) -> list[str]: """Recover the seed ledger, including after an abrupt outer-run exit.""" expected = attempt.get("expected_nodeids") - if isinstance(expected, list) and expected and all(isinstance(nodeid, str) for nodeid in expected): + if isinstance(expected, list) and expected: + if ( + any(not isinstance(nodeid, str) or not nodeid for nodeid in expected) + or len(set(expected)) != len(expected) + or not isinstance(attempt.get("expected_count"), int) + or isinstance(attempt.get("expected_count"), bool) + or attempt.get("expected_count") != len(expected) + or not isinstance(attempt.get("expected_digest"), str) + or attempt.get("expected_digest") != hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() + ): + return [] return list(expected) - artifact_dir_raw = attempt.get("artifact_dir") - if not isinstance(artifact_dir_raw, str): + artifact_dir = _safe_testmon_artifact_dir(attempt.get("artifact_dir"), require_run_root=True) + if artifact_dir is None: return [] - artifact_dir = Path(artifact_dir_raw) for selection_path in sorted(artifact_dir.glob("steps/*/selection.json")): selection = _read_json_artifact(selection_path) - if not isinstance(selection, dict) or int(selection.get("selected_nodeids_omitted") or 0) != 0: + if not isinstance(selection, dict): + continue + omitted = selection.get("selected_nodeids_omitted") + selected_count = selection.get("selected_count") + if ( + not isinstance(omitted, int) + or isinstance(omitted, bool) + or omitted != 0 + or not isinstance(selected_count, int) + or isinstance(selected_count, bool) + ): continue selected = selection.get("selected_nodeids") - if isinstance(selected, list) and selected and all(isinstance(nodeid, str) for nodeid in selected): + if ( + isinstance(selected, list) + and selected + and all(isinstance(nodeid, str) and nodeid for nodeid in selected) + and len(set(selected)) == len(selected) + and selected_count == len(selected) + ): return list(selected) return [] def _testmon_seed_resume_contract(identity: Mapping[str, Any]) -> dict[str, Any]: """Return inputs that change which corpus a seed promises to cover.""" - return {key: identity.get(key) for key in ("worktree_fingerprint", "python", "skip_slow", "lab")} + return { + key: identity.get(key) + for key in ( + "git_tree", + "worktree_fingerprint", + "python", + "skip_slow", + "lab", + "terminal_authorization", + "dependency_environment", + "pytest_harness", + ) + } def _testmon_seed_can_resume(identity: Mapping[str, Any]) -> bool: @@ -2245,11 +2416,14 @@ def _testmon_seed_can_resume(identity: Mapping[str, Any]) -> bool: if attempt is None or not TESTMON_DATA.exists(): return False prior_identity = attempt.get("identity") + contract = _testmon_seed_resume_contract(identity) return ( attempt.get("protocol_version") == TESTMON_SEED_PROTOCOL_VERSION and attempt.get("status") in {"running", "incomplete"} and isinstance(prior_identity, dict) - and _testmon_seed_resume_contract(prior_identity) == _testmon_seed_resume_contract(identity) + and isinstance(contract["git_tree"], str) + and bool(contract["git_tree"]) + and _testmon_seed_resume_contract(prior_identity) == contract and bool(_testmon_seed_expected_nodeids(attempt)) ) @@ -2262,6 +2436,7 @@ def _prepare_testmon_seed_attempt( ) -> dict[str, Any]: prior = _read_testmon_seed_attempt() if resume else None expected = _testmon_seed_expected_nodeids(prior) if prior is not None else [] + prior_outcomes = _flatten_seed_outcomes(prior) payload = { "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, "status": "running", @@ -2269,61 +2444,46 @@ def _prepare_testmon_seed_attempt( "resume": resume, "expected_nodeids": expected, "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, + "prior_node_outcomes": prior_outcomes, "started_at": datetime.now(timezone.utc).isoformat(), "run_id": run.run_id, "artifact_dir": str(run.relative_run_dir), "testmon_data_before": _file_fingerprint(TESTMON_DATA), + "binding": TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict(), } TESTMON_SEED_STAMP.unlink(missing_ok=True) _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) return payload +def _testmon_seed_terminal_authorized(prepared: Mapping[str, Any]) -> bool: + identity = prepared.get("identity") + return ( + isinstance(identity, Mapping) + and identity.get("skip_slow") is True + and identity.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value + ) + + def _testmon_database_state(expected_nodeids: Sequence[str]) -> dict[str, Any]: - if not TESTMON_DATA.exists(): - return { - "recorded_count": 0, - "failed_count": 0, - "missing_nodeids": list(expected_nodeids), - "failed_nodeids": [], - "node_outcomes": dict.fromkeys(expected_nodeids, "missing"), - "error": "missing", - } - try: - with sqlite3.connect(TESTMON_DATA) as conn: - rows = conn.execute( - """ - SELECT current.test_name, current.failed - FROM test_execution AS current - JOIN ( - SELECT test_name, MAX(id) AS latest_id - FROM test_execution - GROUP BY test_name - ) AS latest ON latest.latest_id = current.id - """ - ).fetchall() - except sqlite3.Error as exc: - return { - "recorded_count": 0, - "failed_count": 0, - "missing_nodeids": list(expected_nodeids), - "failed_nodeids": [], - "node_outcomes": dict.fromkeys(expected_nodeids, "missing"), - "error": str(exc), - } - recorded = {str(name): bool(failed) for name, failed in rows} + graph = inspect_testmon_database(TESTMON_DATA, expected_nodeids) expected = set(expected_nodeids) - failed = sorted(nodeid for nodeid in expected if recorded.get(nodeid) is True) + failed = list(graph.failed_nodeids) return { - "recorded_count": len(recorded), - "failed_count": sum(recorded.values()), - "missing_nodeids": sorted(expected - recorded.keys()), + "recorded_count": graph.recorded_count, + "failed_count": len(failed), + "dependency_edge_count": graph.dependency_edge_count, + "missing_nodeids": list(graph.missing_nodeids), "failed_nodeids": failed, "node_outcomes": { - nodeid: ("failed" if recorded.get(nodeid) is True else "passed" if nodeid in recorded else "missing") + nodeid: ("failed" if nodeid in failed else "passed" if nodeid not in graph.missing_nodeids else "missing") for nodeid in sorted(expected) }, - "error": None, + "error": graph.error, + "graph_status": graph.status.value, + "orphan_execution_edges": graph.orphan_execution_edges, + "orphan_fingerprint_edges": graph.orphan_fingerprint_edges, } @@ -2333,6 +2493,8 @@ def _seed_node_outcomes_from_events( expected_nodeids: Sequence[str], database: Mapping[str, Any], pytest_step: Mapping[str, Any] | None, + use_database_fallback: bool = True, + prior_node_outcomes: Mapping[str, Mapping[str, Any]] | None = None, ) -> list[dict[str, Any]]: """Classify every promised seed node into one explicit terminal state.""" reports: dict[str, list[dict[str, Any]]] = {} @@ -2378,6 +2540,8 @@ def _seed_node_outcomes_from_events( outcome, reason = "passed", "test call passed" elif any(report.get("outcome") == "skipped" for report in call_reports): outcome, reason = "skipped", "test call skipped" + elif any(report.get("outcome") == "skipped" for report in node_reports): + outcome, reason = "skipped", "test setup or teardown skipped" elif nodeid in started and nodeid not in finished and "timeout" in diagnosis: outcome, reason = "timeout", "supervisor timed out while node was active" elif nodeid in started and nodeid not in finished and "worker" in diagnosis: @@ -2388,10 +2552,17 @@ def _seed_node_outcomes_from_events( and any(marker in diagnosis for marker in ("interrupt", "signal", "terminated")) ): outcome, reason = "interrupted", "run ended while node was active" - elif recorded.get(nodeid) == "passed": + elif use_database_fallback and recorded.get(nodeid) == "passed": outcome, reason = "passed", "testmon database recorded success" - elif recorded.get(nodeid) == "failed": + elif use_database_fallback and recorded.get(nodeid) == "failed": outcome, reason = "failed", "testmon database recorded failure" + elif prior_node_outcomes is not None and nodeid in prior_node_outcomes: + prior = prior_node_outcomes[nodeid] + prior_outcome = prior.get("outcome") + if prior_outcome in {"passed", "failed", "error", "skipped"}: + outcome, reason = str(prior_outcome), "terminal outcome carried from the prior seed attempt" + else: + outcome, reason = "missing", "prior seed attempt has no terminal outcome" else: outcome, reason = "missing", "no terminal report or testmon execution row" results.append( @@ -2426,46 +2597,122 @@ def _finalize_testmon_seed_attempt( selection: dict[str, Any] = {} events_path: Path | None = None if pytest_step is not None: - artifact_dir_raw = pytest_step.get("artifact_dir") - if isinstance(artifact_dir_raw, str): - artifact_dir = Path(artifact_dir_raw) + artifact_dir = _safe_testmon_artifact_dir(pytest_step.get("artifact_dir")) + if artifact_dir is not None: selection_payload = _read_json_artifact(artifact_dir / "selection.json") if isinstance(selection_payload, dict): selection = selection_payload events_path = artifact_dir / "events.jsonl" - expected_raw = prepared.get("expected_nodeids") if prepared.get("resume") else selection.get("selected_nodeids") - expected = [str(nodeid) for nodeid in expected_raw] if isinstance(expected_raw, list) else [] - omitted = int(selection.get("selected_nodeids_omitted") or 0) + raw_omitted = selection.get("selected_nodeids_omitted") + raw_selected_count = selection.get("selected_count") + selected_nodeids = selection.get("selected_nodeids") + selection_valid = ( + isinstance(raw_omitted, int) + and not isinstance(raw_omitted, bool) + and raw_omitted >= 0 + and isinstance(raw_selected_count, int) + and not isinstance(raw_selected_count, bool) + and isinstance(selected_nodeids, list) + and all(isinstance(nodeid, str) and nodeid for nodeid in selected_nodeids) + and len(set(selected_nodeids)) == len(selected_nodeids) + and raw_selected_count == len(selected_nodeids) + ) + expected_raw = prepared.get("expected_nodeids") if prepared.get("resume") else selected_nodeids + expected = list(expected_raw) if isinstance(expected_raw, list) else [] + omitted = raw_omitted if selection_valid else 1 database = _testmon_database_state(expected) node_outcomes = _seed_node_outcomes_from_events( events_path or Path(".missing-testmon-events"), expected_nodeids=expected, database=database, pytest_step=pytest_step, + use_database_fallback=False, + prior_node_outcomes={ + str(item["nodeid"]): item + for item in prepared.get("prior_node_outcomes", []) + if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) + }, ) unsuccessful_nodeids = [ str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} ] - complete = ( + green_complete = ( exit_code == 0 and bool(expected) - and (bool(prepared.get("resume")) or omitted == 0) + and selection_valid + and omitted == 0 and database["error"] is None + and database["graph_status"] == "complete" and not database["missing_nodeids"] and not database["failed_nodeids"] + and database["orphan_execution_edges"] == 0 + and database["orphan_fingerprint_edges"] == 0 and not unsuccessful_nodeids ) + identity = prepared.get("identity") + narrow_terminal = isinstance(identity, Mapping) and identity.get("skip_slow") is True + terminal_authorized = _testmon_seed_terminal_authorized(prepared) + release_eligible = green_complete and (not narrow_terminal or terminal_authorized) + seed_scope = ( + VerificationScope.NARROW_TERMINAL.value if narrow_terminal else VerificationScope.RELEASE_BASELINE.value + ) + attempt_candidate = { + **dict(prepared), + "status": "complete" if release_eligible else "reusable", + "exit_code": exit_code, + "expected_nodeids": expected, + "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, + "selection": { + **selection, + # A resumed run inherits the complete collection ledger from its + # original selection. The current pytest step may select only a + # subset while it repairs missing graph edges. + "selected_count": len(expected) + if prepared.get("resume") and selection_valid + else selection.get("selected_count"), + "selected_nodeids_omitted": 0 if prepared.get("resume") and selection_valid else omitted, + }, + "node_outcomes": node_outcomes, + "identity": prepared.get("identity"), + "run_id": prepared.get("run_id"), + "artifact_dir": prepared.get("artifact_dir"), + "testmon_data": _file_fingerprint(TESTMON_DATA), + "verification_scope": seed_scope, + "terminal_authorization": (TerminalAuthorization.NARROW_TERMINAL.value if terminal_authorized else None), + "release_baseline_allowed": release_eligible, + } + reusable_stamp = stamp_from_attempt( + attempt_candidate, + TESTMON_DATA, + checkout_root=Path.cwd(), + protocol_version=TESTMON_SEED_PROTOCOL_VERSION, + ) + reusable = reusable_stamp is not None + release_permission = bool( + reusable + and reusable_stamp is not None + and reusable_stamp.release_baseline_allowed + and (not narrow_terminal or terminal_authorized) + ) + attempt_status = "complete" if green_complete and release_permission else "reusable" if reusable else "incomplete" payload = { **dict(prepared), - "status": "complete" if complete else "incomplete", + "status": attempt_status, "finished_at": datetime.now(timezone.utc).isoformat(), "exit_code": exit_code, "expected_nodeids": expected, "expected_count": len(expected), "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, "selection": { - key: selection.get(key) + key: ( + len(expected) + if key == "selected_count" and prepared.get("resume") and selection_valid + else 0 + if key == "selected_nodeids_omitted" and prepared.get("resume") and selection_valid + else selection.get(key) + ) for key in ( "selected_count", "deselected_count", @@ -2487,30 +2734,96 @@ def _finalize_testmon_seed_attempt( "unsuccessful_nodeids": unsuccessful_nodeids, "testmon_data": _file_fingerprint(TESTMON_DATA), "pytest_step": dict(pytest_step) if pytest_step is not None else None, + "binding": TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict(), + "verification_scope": seed_scope, + "terminal_authorization": (TerminalAuthorization.NARROW_TERMINAL.value if terminal_authorized else None), } + payload["release_baseline_allowed"] = release_permission _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - if complete: - stamp = { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "timestamp": payload["finished_at"], - "checkout_root": str(ROOT.resolve()), - "git_head": dict(prepared["identity"]).get("git_head"), - "identity": prepared["identity"], - "expected_count": payload["expected_count"], - "expected_digest": payload["expected_digest"], - "testmon_data": payload["testmon_data"], - "database": { - "recorded_count": database["recorded_count"], - "failed_count": database["failed_count"], - }, - "run_id": payload["run_id"], - "artifact_dir": payload["artifact_dir"], - } - _atomic_write_json(TESTMON_SEED_STAMP, stamp) + if release_permission and reusable_stamp is not None: + _atomic_write_json(TESTMON_SEED_STAMP, reusable_stamp.as_dict()) + else: + TESTMON_SEED_STAMP.unlink(missing_ok=True) return payload +def _refresh_testmon_selection_attempt( + *, + step: Mapping[str, Any], + run: VerifyRun, + exit_code: int, +) -> None: + """Refresh a reusable red graph after every completed affected run.""" + attempt = _read_testmon_seed_attempt() + if attempt is None or attempt.get("release_baseline_allowed") is True: + return + expected = _testmon_seed_expected_nodeids(attempt) + if not expected: + return + database = _testmon_database_state(expected) + artifact_dir = _safe_testmon_artifact_dir(step.get("artifact_dir")) + events_path = artifact_dir / "events.jsonl" if artifact_dir is not None else Path(".missing-testmon-events") + prior = { + str(item["nodeid"]): item + for item in attempt.get("node_outcomes", []) + if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) + } + node_outcomes = _seed_node_outcomes_from_events( + events_path, + expected_nodeids=expected, + database=database, + pytest_step=step, + use_database_fallback=False, + prior_node_outcomes=prior, + ) + graph_complete = ( + database.get("graph_status") == GraphStatus.COMPLETE.value + and not database.get("missing_nodeids") + and database.get("error") is None + and database.get("orphan_execution_edges") == 0 + and database.get("orphan_fingerprint_edges") == 0 + ) + terminal = all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in node_outcomes) + prior_selection = attempt.get("selection") + payload = { + **attempt, + "status": "reusable" if graph_complete and terminal else "incomplete", + "finished_at": datetime.now(timezone.utc).isoformat(), + "exit_code": exit_code, + "expected_nodeids": expected, + "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), + "selection": { + **(dict(prior_selection) if isinstance(prior_selection, Mapping) else {}), + "selected_count": len(expected), + "selected_nodeids_omitted": 0, + }, + "database": database, + "node_outcomes": node_outcomes, + "node_outcome_counts": dict( + sorted( + { + outcome: sum(1 for item in node_outcomes if item.get("outcome") == outcome) + for outcome in {str(item.get("outcome")) for item in node_outcomes} + }.items() + ) + ), + "unsuccessful_nodeids": [ + str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} + ], + "testmon_data": _file_fingerprint(TESTMON_DATA), + "run_id": run.run_id, + "artifact_dir": str(run.relative_run_dir), + "pytest_step": dict(step), + "release_baseline_allowed": False, + "verification_scope": VerificationScope.AFFECTED.value, + } + raw_binding = attempt.get("binding") + if not isinstance(raw_binding, Mapping): + payload["binding"] = TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict() + _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) + + # ── main ──────────────────────────────────────────────────────────── @@ -2532,6 +2845,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--skip-slow", action="store_true", help="Exclude @pytest.mark.slow tests from the pytest step." ) + parser.add_argument( + "--terminal-authorization", + choices=[TerminalAuthorization.NARROW_TERMINAL.value], + help="Typed authorization for a narrow terminal verification that skips slow tests.", + ) parser.add_argument( "--lab", action="store_true", @@ -2544,6 +2862,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--json", action="store_true", default=None, help="Write structured JSON to stdout.") args = parser.parse_args(sys.argv[1:] if argv is None else argv) + _anchor_verification_paths() bootstrap_message = maybe_bootstrap_testmon_seed( ROOT, protocol_version=TESTMON_SEED_PROTOCOL_VERSION, @@ -2581,6 +2900,8 @@ def main(argv: list[str] | None = None) -> int: tier = "testmon" full_pytest = bool(args.all or args.full) + if args.terminal_authorization is not None and not ((full_pytest or args.seed_testmon) and args.skip_slow): + parser.error("--terminal-authorization requires --all, --full, or --seed-testmon with --skip-slow") preflight_error = _testmon_preflight( seed_testmon=bool(args.seed_testmon), full_pytest=full_pytest, @@ -2604,11 +2925,22 @@ def main(argv: list[str] | None = None) -> int: resume_testmon_seed = False prepared_seed_attempt: dict[str, Any] | None = None if args.seed_testmon: - seed_identity = _testmon_seed_identity( - git_head=head, - skip_slow=bool(args.skip_slow), - lab=bool(args.lab), - ) + try: + seed_identity = _testmon_seed_identity( + git_head=head, + git_tree=_git_committed_tree(), + skip_slow=bool(args.skip_slow), + lab=bool(args.lab), + terminal_authorization=args.terminal_authorization, + ) + except RuntimeError as exc: + sys.stderr.write(f"verify: {exc}\n") + verify_run.finish( + exit_code=125, + duration_s=time.monotonic() - t0, + diagnosis="testmon_environment_identity_unavailable", + ) + return 125 resume_testmon_seed = _testmon_seed_can_resume(seed_identity) prepared_seed_attempt = _prepare_testmon_seed_attempt( identity=seed_identity, @@ -2649,6 +2981,19 @@ def main(argv: list[str] | None = None) -> int: _warn_low_memory() # check again right before the heavy step rc, elapsed, metadata = _run(label, cmd, run=verify_run) if rc == 0 and label in {"pytest testmon", "pytest testmon (broad)"}: + raw_stamp = _read_json_artifact(TESTMON_SEED_STAMP) + try: + current_stamp = ( + TestmonSeedStamp.from_mapping(raw_stamp, protocol_version=TESTMON_SEED_PROTOCOL_VERSION) + if isinstance(raw_stamp, Mapping) + else None + ) + except ValueError: + current_stamp = None + if current_stamp is not None: + refreshed_stamp = refresh_stamp(current_stamp, TESTMON_DATA) + if refreshed_stamp is not None: + _atomic_write_json(TESTMON_SEED_STAMP, refreshed_stamp.as_dict()) executable_paths = _changed_executable_paths() selected_count = metadata.get("selected_count") if selected_count == 0 and executable_paths: @@ -2673,6 +3018,8 @@ def main(argv: list[str] | None = None) -> int: step_result: dict[str, Any] = {"name": label, "duration_s": round(elapsed, 2), "exit": rc} step_result.update(metadata) step_results.append(step_result) + if label in {"pytest testmon", "pytest testmon (broad)"} and not args.seed_testmon and not full_pytest: + _refresh_testmon_selection_attempt(step=step_result, run=verify_run, exit_code=rc) if rc != 0: exit_code = rc if _stop_after_failed_step(label): @@ -2721,9 +3068,37 @@ def main(argv: list[str] | None = None) -> int: "resume": seed_receipt["resume"], "expected_count": seed_receipt["expected_count"], "attempt_path": str(TESTMON_SEED_ATTEMPT), - "stamp_path": str(TESTMON_SEED_STAMP) if seed_receipt["status"] == "complete" else None, + "stamp_path": str(TESTMON_SEED_STAMP) if seed_receipt["release_baseline_allowed"] else None, + "release_baseline_allowed": seed_receipt["release_baseline_allowed"], } + if args.quick or args.commit: + verification_scope = VerificationScope.NON_TEST + release_baseline_allowed: bool | None = None + elif full_pytest or args.seed_testmon: + narrow_terminal = bool(args.skip_slow) + authorized_narrow_terminal = args.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value + verification_scope = ( + VerificationScope.NARROW_TERMINAL if narrow_terminal else VerificationScope.RELEASE_BASELINE + ) + if full_pytest: + release_baseline_allowed = exit_code == 0 and (not narrow_terminal or authorized_narrow_terminal) + else: + release_baseline_allowed = _testmon_release_baseline_permission() and ( + not narrow_terminal or authorized_narrow_terminal + ) + else: + verification_scope = VerificationScope.AFFECTED + release_baseline_allowed = _testmon_release_baseline_permission() + history_entry["verification_scope"] = verification_scope.value + history_entry["release_baseline_allowed"] = release_baseline_allowed + history_entry["terminal_authorization"] = args.terminal_authorization + if release_baseline_allowed is False and tier in {"testmon", "lab", "seed-testmon"}: + sys.stderr.write( + "verify: affected-test selection is usable, but the current testmon state does not grant " + "release-baseline permission.\n" + ) + if use_json: _print_json(history_entry) else: diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 3daef2d6c8..1f67450949 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -254,13 +254,26 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> None: break self.write() - def finish(self, *, exit_code: int, duration_s: float, diagnosis: str | None = None) -> dict[str, Any]: + def finish( + self, + *, + exit_code: int, + duration_s: float, + diagnosis: str | None = None, + verification_scope: str | None = None, + release_baseline_allowed: bool | None = None, + terminal_authorization: str | None = None, + ) -> dict[str, Any]: self._payload["finished_at"] = utc_now() self._payload["duration_s"] = round(duration_s, 2) self._payload["exit_code"] = int(exit_code) self._payload["status"] = "success" if exit_code == 0 else "failed" if diagnosis: self._payload["diagnosis"] = diagnosis + if verification_scope is not None: + self._payload["verification_scope"] = verification_scope + self._payload["release_baseline_allowed"] = release_baseline_allowed + self._payload["terminal_authorization"] = terminal_authorization self.write() return dict(self._payload) diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py new file mode 100644 index 0000000000..412ae2f6b6 --- /dev/null +++ b/tests/integration/devtools/test_testmon_seed_recovery.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest + +from devtools import testmon_bootstrap, testmon_state, verify +from devtools.testmon_state import file_fingerprint, inspect_testmon_database + + +def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") + (source / "test_sample.py").write_text( + "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 2\n", + encoding="utf-8", + ) + data = source / ".cache" / "testmon" / "testmondata" + data.parent.mkdir(parents=True) + env = os.environ.copy() + env["TESTMON_DATAFILE"] = str(data) + run = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "--testmon", "--testmon-noselect"], + cwd=source, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert run.returncode != 0 + expected = ("test_sample.py::test_passed", "test_sample.py::test_failed") + assert inspect_testmon_database(data, expected).usable_for_selection + runtime_identity = testmon_state.testmon_runtime_identity(source) + assert runtime_identity is not None + dependency_environment, pytest_harness = runtime_identity + attempt = { + "protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, + "status": "reusable", + "identity": { + "git_head": "head", + "worktree_fingerprint": "source-tree", + "python": sys.version, + "skip_slow": False, + "lab": False, + "dependency_environment": dependency_environment, + "pytest_harness": pytest_harness, + }, + "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, + "expected_nodeids": list(expected), + "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), + "node_outcomes": [ + {"nodeid": expected[0], "outcome": "passed"}, + {"nodeid": expected[1], "outcome": "failed"}, + ], + "exit_code": 1, + "run_id": "real-testmon", + "artifact_dir": ".cache/verify/runs/real-testmon", + "testmon_data": file_fingerprint(data), + } + artifact_dir = source / ".cache" / "verify" / "runs" / "real-testmon" + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": "real-testmon", + "checkout_root": str(source.resolve()), + "artifact_dir": ".cache/verify/runs/real-testmon", + } + ), + encoding="utf-8", + ) + source_attempt = source / ".cache" / "testmon" / "seed-attempt.json" + source_attempt.parent.mkdir(parents=True, exist_ok=True) + source_attempt.write_text(json.dumps(attempt), encoding="utf-8") + + lane = tmp_path / "lane" + lane.mkdir() + (lane / "test_sample.py").write_text( + "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 1\n", + encoding="utf-8", + ) + monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, source)) + copy_calls: list[tuple[Path, Path]] = [] + original_copy = testmon_bootstrap._atomic_copy_sqlite_db + + def counted_copy(src: Path, dst: Path) -> None: + copy_calls.append((src, dst)) + original_copy(src, dst) + + monkeypatch.setattr(testmon_bootstrap, "_atomic_copy_sqlite_db", counted_copy) + (lane / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") + (lane / "polylogue" / "cli").mkdir(parents=True) + (lane / "polylogue" / "__init__.py").write_text("", encoding="utf-8") + (lane / "polylogue" / "cli" / "click_app.py").write_text("", encoding="utf-8") + + local_data = lane / ".cache" / "testmon" / "testmondata" + local_stamp = lane / ".cache" / "testmon" / "seed.json" + local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" + + monkeypatch.chdir(lane) + monkeypatch.setattr(verify, "ROOT", lane) + monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: lane / "polylogue" / "__init__.py") + monkeypatch.setattr("devtools.checkout_guard._is_linked_worktree", lambda _root: True) + monkeypatch.setattr("devtools.checkout_guard._python_environment_root", lambda _executable: lane) + monkeypatch.setattr(verify, "build_verify_steps", lambda **_kwargs: [("pytest testmon", ["pytest"])]) + run_count = 0 + + def fake_run(*_args: object, **_kwargs: object) -> tuple[int, float, dict[str, object]]: + nonlocal run_count + run_count += 1 + if run_count == 1: + with sqlite3.connect(local_data) as connection: + connection.execute("update test_execution set failed = 0 where test_name = ?", (expected[1],)) + return 1, 0.01, {"selected_count": 1} + return 0, 0.01, {"selected_count": 1} + + monkeypatch.setattr(verify, "_run", fake_run) + monkeypatch.setattr(verify, "_changed_executable_paths", lambda: ()) + monkeypatch.setattr(verify, "_stamp_head", lambda: None) + + assert verify.main([]) == 1 + result = json.loads(capsys.readouterr().out) + assert local_data.is_file() + assert local_attempt.is_file() + assert not local_stamp.exists() + assert result["steps"][0]["selected_count"] == 1 + assert result["release_baseline_allowed"] is False + assert verify._testmon_release_baseline_permission() is False + assert not local_stamp.exists() + assert len(copy_calls) == 1 + assert (lane / ".cache" / "verify" / "current-run.json").is_file() + refreshed_attempt = json.loads(local_attempt.read_text()) + assert refreshed_attempt["testmon_data"] == file_fingerprint(local_data) + current_run = json.loads((lane / ".cache" / "verify" / "current-run.json").read_text()) + assert refreshed_attempt["run_id"] == current_run["run_id"] + + assert verify.main([]) == 0 + second = json.loads(capsys.readouterr().out) + assert second["steps"][0]["selected_count"] == 1 + assert len(copy_calls) == 1 + + assert verify.main([]) == 0 + third = json.loads(capsys.readouterr().out) + assert third["steps"][0]["selected_count"] == 1 + assert len(copy_calls) == 1 + + with sqlite3.connect(local_data) as connection: + connection.execute("delete from test_execution_file_fp") + assert verify.main([]) == 2 diff --git a/tests/unit/devtools/test_checkout_guard.py b/tests/unit/devtools/test_checkout_guard.py index 42f7e622b5..a5ed1b0916 100644 --- a/tests/unit/devtools/test_checkout_guard.py +++ b/tests/unit/devtools/test_checkout_guard.py @@ -9,6 +9,7 @@ from __future__ import annotations +import hashlib import json from pathlib import Path @@ -16,6 +17,7 @@ import devtools.click_dispatch as click_dispatch import devtools.run_tests as run_tests +import devtools.testmon_state as testmon_state import devtools.verify as verify import polylogue from devtools.checkout_guard import ( @@ -183,10 +185,63 @@ def _write_in_progress_seed_attempt(root: Path, *, status: str = "running", **ov "artifact_dir": ".cache/verify/runs/seed-testmon-20260805T120000Z", "testmon_data_before": "missing", } + if status == "reusable": + nodeid = "tests/test.py::test_one" + runtime_identity = testmon_state.testmon_runtime_identity(root) + assert runtime_identity is not None + dependency_environment, pytest_harness = runtime_identity + payload.update( + { + "identity": { + "git_head": "head", + "worktree_fingerprint": "fingerprint", + "python": "3.14", + "skip_slow": True, + "lab": False, + "dependency_environment": dependency_environment, + "pytest_harness": pytest_harness, + }, + "expected_nodeids": [nodeid], + "expected_count": 1, + "expected_digest": hashlib.sha256(nodeid.encode()).hexdigest(), + "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, + "node_outcomes": [{"nodeid": nodeid, "outcome": "failed"}], + "exit_code": 1, + "testmon_data": "fingerprint", + "release_baseline_allowed": False, + "verification_scope": "affected", + "binding": { + "mode": "exact", + "checkout_root": str(root.resolve()), + "source_checkout_root": None, + }, + } + ) payload.update(overrides) attempt = root / ".cache" / "testmon" / "seed-attempt.json" attempt.parent.mkdir(parents=True, exist_ok=True) attempt.write_text(json.dumps(payload)) + if status == "reusable": + run_dir = root / ".cache" / "verify" / "runs" / str(payload["run_id"]) + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "run.json").write_text( + json.dumps( + { + "run_id": payload["run_id"], + "checkout_root": str(root.resolve()), + "artifact_dir": f".cache/verify/runs/{payload['run_id']}", + } + ) + ) + (root / ".cache" / "verify" / "current-run.json").write_text( + json.dumps( + { + "run_id": payload["run_id"], + "checkout_root": str(root.resolve()), + "artifact_dir": f".cache/verify/runs/{payload['run_id']}", + } + ) + ) return attempt @@ -223,6 +278,26 @@ def test_checkout_environment_fingerprint_accepts_current_in_progress_seed_attem assert attempt.is_file() +def test_checkout_environment_fingerprint_accepts_finalized_selection_attempt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = _fake_linked_checkout(tmp_path) + attempt = _write_in_progress_seed_attempt(root, status="reusable") + (root / ".cache" / "testmon" / "testmondata").write_text("complete graph") + package_path = root / "polylogue" / "__init__.py" + monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) + + fingerprint = assert_polylogue_matches_checkout( + root, + context="affected selection", + python_executable=root / ".venv" / "bin" / "python", + ) + + assert fingerprint.clean + assert fingerprint.testmon_state_origin is None + assert attempt.is_file() + + @pytest.mark.parametrize( ("status", "overrides"), [ diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index 7fa06d6c87..7b98a5fdf6 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -1,7 +1,9 @@ from __future__ import annotations import json +import os import subprocess +import threading from collections.abc import Callable from pathlib import Path from typing import Any @@ -10,6 +12,7 @@ import pytest from devtools import merge_boundary, merge_gate, pr_scope +from devtools.checkout_guard import checkout_environment_fingerprint _SCOPE_BEAD = { "_type": "issue", @@ -87,7 +90,11 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: return MagicMock(returncode=0, stdout="", stderr="") - return MagicMock(returncode=local_exit, stdout="ok\n", stderr="") + return MagicMock( + returncode=local_exit, + stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), + stderr="", + ) return _run @@ -314,7 +321,30 @@ def test_merge_propagates_gh_pr_merge_failure(monkeypatch: pytest.MonkeyPatch, t def test_merge_with_verify_records_terminal_full_verify(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) pr_view = _base_pr_view() - monkeypatch.setattr(subprocess, "run", _fake_run(pr_view)) + base = _fake_run(pr_view) + + def run(cmd: list[str], **kwargs: Any) -> MagicMock: + if cmd[:3] == ["devtools", "verify", "--all"]: + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) + return base(cmd, **kwargs) + + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(merge_boundary, "_fetched_merged_default_branch_sha", lambda _pr: "merged-master") + monkeypatch.setattr( + merge_boundary, + "_run_post_merge_terminal_verify", + lambda command, target, **_kwargs: merge_boundary.cmd_record_full_verify(command, target_sha=target), + ) exit_code = merge_boundary.cmd_merge( 42, @@ -335,6 +365,154 @@ def test_merge_with_verify_records_terminal_full_verify(monkeypatch: pytest.Monk assert merge_boundary.cmd_train_status(as_json=False) == 0 +def test_merge_with_verify_returns_nonzero_when_terminal_authority_is_rejected( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view)) + monkeypatch.setattr(merge_boundary, "_fetched_merged_default_branch_sha", lambda _pr: "merged-master") + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", lambda _command, _target, **_kwargs: 1) + + assert ( + merge_boundary.cmd_merge( + 42, + command="devtools test x", + max_age_s=3600, + poll_rounds=1, + poll_interval_s=0, + dry_run=False, + with_verify=True, + verify_command="devtools verify --all", + ) + == 1 + ) + assert merge_boundary._read_ledger()["merges"] + + +def test_post_merge_terminal_verify_rejects_stale_feature_head(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "feature-head", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 1 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False + + +def test_post_merge_terminal_verify_uses_target_checkout_devshell( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + commands: list[list[str]] = [] + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + commands.append(cmd) + if cmd[:3] == ["git", "worktree", "add"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:2] == ["direnv", "exec"]: + assert cmd[2] != str(tmp_path) + target = Path(cmd[2]) + package = target / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("") + (target / ".venv" / "bin").mkdir(parents=True) + with pytest.MonkeyPatch.context() as guard_patch: + guard_patch.setattr("devtools.checkout_guard._is_linked_worktree", lambda _root: True) + fingerprint = checkout_environment_fingerprint( + target, + polylogue_import_path=package / "__init__.py", + python_executable=target / ".venv" / "bin" / "python", + ) + assert fingerprint.clean + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) + if cmd[:3] == ["git", "worktree", "remove"]: + return MagicMock(returncode=0, stdout="", stderr="") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + + assert merge_boundary._run_post_merge_terminal_verify("devtools verify --all", "merged-master") == 0 + assert any(command[:2] == ["direnv", "exec"] for command in commands) + + +def test_fetched_default_branch_must_include_squash_merge(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + calls: list[list[str]] = [] + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + calls.append(cmd) + if cmd[:3] == ["gh", "repo", "view"]: + return MagicMock(returncode=0, stdout=json.dumps({"defaultBranchRef": {"name": "master"}}), stderr="") + if cmd[:3] == ["gh", "pr", "view"]: + return MagicMock( + returncode=0, + stdout=json.dumps({"state": "MERGED", "mergeCommit": {"oid": "squash-sha"}}), + stderr="", + ) + if cmd[:3] == ["git", "fetch", "origin"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:3] == ["git", "rev-parse", "FETCH_HEAD"]: + return MagicMock(returncode=0, stdout="stale-feature-sha\n", stderr="") + if cmd[:3] == ["git", "merge-base", "--is-ancestor"]: + return MagicMock(returncode=1, stdout="", stderr="") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + + assert merge_boundary._fetched_merged_default_branch_sha(42) is None + assert ["git", "fetch", "origin", "master"] in calls + + +def test_fetched_default_branch_sha_is_the_verified_terminal_target( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + if cmd[:3] == ["gh", "repo", "view"]: + return MagicMock(returncode=0, stdout=json.dumps({"defaultBranchRef": {"name": "master"}}), stderr="") + if cmd[:3] == ["gh", "pr", "view"]: + return MagicMock( + returncode=0, + stdout=json.dumps({"state": "MERGED", "mergeCommit": {"oid": "squash-sha"}}), + stderr="", + ) + if cmd[:3] == ["git", "fetch", "origin"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:3] == ["git", "rev-parse", "FETCH_HEAD"]: + return MagicMock(returncode=0, stdout="merged-master-sha\n", stderr="") + if cmd[:3] == ["git", "merge-base", "--is-ancestor"]: + return MagicMock(returncode=0, stdout="", stderr="") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + + assert merge_boundary._fetched_merged_default_branch_sha(42) == "merged-master-sha" + + # --------------------------------------------------------------------------- # train-status / record-full-verify # --------------------------------------------------------------------------- @@ -345,12 +523,88 @@ def test_train_status_ok_with_empty_ledger(monkeypatch: pytest.MonkeyPatch, tmp_ assert merge_boundary.cmd_train_status(as_json=False) == 0 +def test_train_status_fails_closed_on_truncated_ledger(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + ledger_path = tmp_path / ".cache" / "verify" / "merge-gate" / "merge-train-ledger.json" + ledger_path.parent.mkdir(parents=True) + ledger_path.write_text('{"merges": [') + + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_train_status_fails_closed_on_valid_json_partial_merge_entry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.chdir(tmp_path) + ledger_path = tmp_path / ".cache" / "verify" / "merge-gate" / "merge-train-ledger.json" + ledger_path.parent.mkdir(parents=True) + ledger_path.write_text(json.dumps({"merges": [{"pr": 42, "merged_at": 1.0}], "last_full_verify": None})) + + assert merge_boundary.cmd_train_status(as_json=False) == 1 + assert "Traceback" not in capsys.readouterr().err + + +def test_merge_write_failure_recovers_valid_pending_ledger(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view)) + + def fail_final_replace(source: Path, destination: Path) -> None: + if destination == merge_boundary._LEDGER_PATH: + raise OSError("injected final ledger write failure") + os.replace(source, destination) + + monkeypatch.setattr(merge_boundary, "_durable_replace", fail_final_replace) + + assert ( + merge_boundary.cmd_merge( + 42, + command="devtools test x", + max_age_s=3600, + poll_rounds=1, + poll_interval_s=0, + dry_run=False, + with_verify=False, + verify_command="devtools verify --all", + ) + == 1 + ) + assert merge_boundary._LEDGER_PENDING_PATH.exists() + monkeypatch.setattr(merge_boundary, "_durable_replace", os.replace) + assert merge_boundary.cmd_train_status(as_json=False) == 1 + assert not merge_boundary._LEDGER_PENDING_PATH.exists() + assert merge_boundary._read_ledger()["merge_intents"] + + +def test_read_ledger_clears_byte_identical_pending_write(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._write_ledger({"merges": [], "merge_intents": [], "last_full_verify": None}) + serialized = merge_boundary._LEDGER_PATH.read_text() + merge_boundary._LEDGER_PENDING_PATH.write_text(serialized) + + assert merge_boundary._read_ledger() == {"merges": [], "merge_intents": [], "last_full_verify": None} + assert not merge_boundary._LEDGER_PENDING_PATH.exists() + + def test_train_status_blocks_when_pr_merged_after_last_full_verify( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.chdir(tmp_path) merge_boundary._write_ledger( - {"merges": [], "last_full_verify": {"at": 1000.0, "command": "devtools verify --all", "exit_code": 0}} + { + "merges": [], + "last_full_verify": { + "at": 1000.0, + "verification_started_at": 1000.0, + "duration_s": 1.0, + "command": "devtools verify --all", + "exit_code": 0, + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + "merge_sequence": 0, + "accepted": True, + }, + } ) merge_boundary._append_merge_entry(1, "sha1", "some title") @@ -362,16 +616,49 @@ def test_train_status_blocks_when_pr_merged_after_last_full_verify( assert merge_boundary.cmd_train_status(as_json=False) == 1 +def test_train_status_rejects_untyped_accepted_terminal_ledger(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._write_ledger( + { + "merges": [], + "last_full_verify": { + "at": 1000.0, + "verification_started_at": 1000.0, + "duration_s": 1.0, + "command": "devtools verify --all", + "exit_code": 0, + "verification_scope": None, + "release_baseline_allowed": True, + "merge_sequence": 0, + "accepted": True, + }, + } + ) + merge_boundary._append_merge_entry(1, "sha1", "some title") + + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + def test_record_full_verify_clears_pending_prs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) merge_boundary._append_merge_entry(1, "sha1", "some title") def _run(cmd: list[str], **kwargs: Any) -> MagicMock: - return MagicMock(returncode=0, stdout="all good\n", stderr="") + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) monkeypatch.setattr(subprocess, "run", _run) - exit_code = merge_boundary.cmd_record_full_verify("devtools verify --all") + exit_code = merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") assert exit_code == 0 assert merge_boundary.cmd_train_status(as_json=False) == 0 @@ -379,14 +666,364 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: def test_record_full_verify_propagates_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return MagicMock(returncode=1, stdout="", stderr="broke") monkeypatch.setattr(subprocess, "run", _run) - exit_code = merge_boundary.cmd_record_full_verify("devtools verify --all") + exit_code = merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") assert exit_code == 1 ledger = merge_boundary._read_ledger() assert ledger["last_full_verify"]["exit_code"] == 1 + assert ledger["last_full_verify"]["accepted"] is False + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_record_full_verify_rejects_success_without_structured_release_permission( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock(returncode=0, stdout="all good\n", stderr=""), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 1 + ledger = merge_boundary._read_ledger() + assert ledger["last_full_verify"]["accepted"] is False + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_record_full_verify_rejects_skip_slow_without_typed_authorization( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps({"verification_scope": "narrow-terminal", "release_baseline_allowed": False}), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 1 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_record_full_verify_accepts_explicit_typed_narrow_terminal_authorization( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "narrow-terminal", + "terminal_authorization": "narrow-terminal", + "release_baseline_allowed": True, + } + ), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 0 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is True + assert merge_boundary.cmd_train_status(as_json=False) == 0 + + +def test_record_full_verify_rejects_untyped_scope_even_when_permission_is_true( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps({"release_baseline_allowed": True}), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 1 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_concurrent_merge_during_terminal_verify_remains_pending( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + inserted = False + + def run(_cmd: list[str], **_kwargs: Any) -> MagicMock: + nonlocal inserted + if not inserted: + inserted = True + merge_boundary._append_merge_entry(99, "concurrent-sha", "concurrent merge") + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) + + monkeypatch.setattr(subprocess, "run", run) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 0 + assert merge_boundary.cmd_train_status(as_json=False) == 1 + ledger = merge_boundary._read_ledger() + assert ledger["last_full_verify"]["merged_master_sha"] == "merged-master" + assert ledger["last_full_verify"]["merge_sequence"] == 0 + + +def test_concurrent_ledger_writer_cannot_lose_merge_entry(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + started = threading.Event() + writer: threading.Thread | None = None + + def run(_cmd: list[str], **_kwargs: Any) -> MagicMock: + nonlocal writer + + def append() -> None: + started.set() + merge_boundary._append_merge_entry(77, "writer-sha", "writer merge") + + writer = threading.Thread(target=append) + writer.start() + assert started.wait(timeout=1) + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) + + monkeypatch.setattr(subprocess, "run", run) + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 0 + assert writer is not None + writer.join(timeout=1) + assert not writer.is_alive() + ledger = merge_boundary._read_ledger() + assert any(entry["pr"] == 77 for entry in ledger["merges"]) + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_terminal_snapshot_is_taken_before_default_branch_fetch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + events: list[str] = [] + snapshots: list[tuple[dict[str, Any], float, int]] = [] + + original_snapshot = merge_boundary._terminal_verify_snapshot + + def snapshot() -> tuple[dict[str, Any], float, int]: + events.append("ledger-snapshot") + captured = original_snapshot() + snapshots.append(captured) + return captured + + def fetch() -> str: + events.append("target-fetch") + merge_boundary._append_merge_entry(88, "after-fetch-sha", "merge after target fetch") + return "merged-master" + + def post_verify( + _command: str, _target: str, *, ledger_snapshot: tuple[dict[str, Any], float, int] | None = None + ) -> int: + assert ledger_snapshot is not None + assert ledger_snapshot[2] == 0 + return 1 + + monkeypatch.setattr(merge_boundary, "_terminal_verify_snapshot", snapshot) + monkeypatch.setattr(merge_boundary, "_fetched_current_default_branch_sha", fetch) + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", post_verify) + + assert merge_boundary.main(["record-full-verify", "--command", "devtools verify --all"]) == 1 + assert events == ["ledger-snapshot", "target-fetch"] + assert snapshots[0][2] == 0 + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_external_merge_before_completion_is_reconciled_from_durable_intent( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + base_run = _fake_run(pr_view) + merged = False + + def run(cmd: list[str], **kwargs: Any) -> MagicMock: + if cmd[:3] == ["gh", "pr", "view"] and merged: + return MagicMock( + returncode=0, + stdout=json.dumps({"state": "MERGED", "mergeCommit": {"oid": "merge-commit"}}), + stderr="", + ) + return base_run(cmd, **kwargs) + + complete_merge_intent = merge_boundary._complete_merge_intent + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(merge_boundary, "_complete_merge_intent", lambda _pr, _head_sha: None) + assert ( + merge_boundary.cmd_merge( + 42, + command="devtools test x", + max_age_s=3600, + poll_rounds=1, + poll_interval_s=0, + dry_run=False, + with_verify=False, + verify_command="devtools verify --all", + ) + == 0 + ) + merged = True + assert merge_boundary._read_ledger()["merge_intents"] + monkeypatch.setattr(merge_boundary, "_complete_merge_intent", complete_merge_intent) + assert merge_boundary.cmd_train_status(as_json=False) == 1 + ledger = merge_boundary._read_ledger() + assert not ledger["merge_intents"] + assert ledger["merges"][0]["pr"] == 42 + + +def test_record_full_verify_reconciles_durable_intents_before_snapshot( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._record_merge_intent(42, "pr-head", "merged before recovery") + monkeypatch.setattr( + merge_boundary, + "_gh_json", + lambda _args: {"state": "MERGED", "mergeCommit": {"oid": "merge-commit"}}, + ) + monkeypatch.setattr(merge_boundary, "_fetched_current_default_branch_sha", lambda: "merged-master") + snapshots: list[tuple[dict[str, Any], float, int]] = [] + + def post_verify( + _command: str, _target: str, *, ledger_snapshot: tuple[dict[str, Any], float, int] | None = None + ) -> int: + assert ledger_snapshot is not None + snapshots.append(ledger_snapshot) + return 0 + + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", post_verify) + + assert merge_boundary.main(["record-full-verify", "--command", "devtools verify --all"]) == 0 + assert snapshots[0][2] == 1 + assert not merge_boundary._read_ledger()["merge_intents"] + + +def test_external_merge_completion_write_failure_keeps_recovery_latch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._record_merge_intent(42, "feature-sha", "fix: thing (#42)") + + def fail_completion_replace(source: Path, destination: Path) -> None: + if destination == merge_boundary._LEDGER_PATH: + raise OSError("injected completion publication failure") + os.replace(source, destination) + + monkeypatch.setattr(merge_boundary, "_durable_replace", fail_completion_replace) + with pytest.raises(merge_boundary.LedgerStateError): + merge_boundary._complete_merge_intent(42, "feature-sha") + assert merge_boundary._LEDGER_PENDING_PATH.exists() + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + +def test_detached_worktree_add_failure_attempts_cleanup(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + commands: list[list[str]] = [] + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + commands.append(cmd) + if cmd[:3] == ["git", "worktree", "add"]: + return MagicMock(returncode=1, stdout="", stderr="add failed") + if cmd[:3] == ["git", "worktree", "remove"]: + return MagicMock(returncode=0, stdout="", stderr="") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + assert merge_boundary._run_post_merge_terminal_verify("devtools verify --all", "merged-master") == 1 + assert any(command[:3] == ["git", "worktree", "remove"] for command in commands) + + +def test_detached_worktree_cleanup_failure_is_explicit(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + if cmd[:3] == ["git", "worktree", "add"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:3] == ["git", "worktree", "remove"]: + return MagicMock(returncode=2, stdout="", stderr="remove failed") + raise AssertionError(cmd) + + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(merge_boundary, "cmd_record_full_verify", lambda *_args, **_kwargs: 0) + assert merge_boundary._run_post_merge_terminal_verify("devtools verify --all", "merged-master") == 1 + + +def test_manual_record_route_fetches_target_and_rejects_stale_cli_output( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + post_targets: list[str] = [] + + def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + if cmd[:3] == ["gh", "repo", "view"]: + return MagicMock(returncode=0, stdout=json.dumps({"defaultBranchRef": {"name": "master"}}), stderr="") + if cmd[:3] == ["git", "fetch", "origin"]: + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[:3] == ["git", "rev-parse", "FETCH_HEAD"]: + return MagicMock(returncode=0, stdout="current-master\n", stderr="") + return MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "stale-feature", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ) + + def post_verify(command: str, target_sha: str, **_kwargs: Any) -> int: + post_targets.append(target_sha) + return merge_boundary.cmd_record_full_verify(command, target_sha=target_sha) + + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", post_verify) + + assert merge_boundary.main(["record-full-verify", "--command", "devtools verify --all"]) == 1 + assert post_targets == ["current-master"] + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 6d26285f94..2bbbfd4d33 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -4,6 +4,7 @@ import subprocess from collections.abc import Callable from pathlib import Path +from typing import cast from unittest.mock import MagicMock import pytest @@ -80,7 +81,11 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: return MagicMock(returncode=0, stdout=" M dirty.py\n" if dirty else "", stderr="") - return MagicMock(returncode=local_exit, stdout="ok\n", stderr="") + return MagicMock( + returncode=local_exit, + stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), + stderr="", + ) return _run @@ -103,6 +108,63 @@ def test_record_persists_receipt_keyed_to_current_head_sha(monkeypatch: pytest.M assert receipt["skips_tests"] is False +@pytest.mark.parametrize("command", ["devtools verify", "devtools verify --lab", "devtools verify --json --skip-slow"]) +def test_check_accepts_affected_receipt_without_release_baseline_permission( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, command: str +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + _record(monkeypatch, pr_view, command=command) + receipt_path = merge_gate._receipt_path(42) + receipt = json.loads(receipt_path.read_text()) + receipt["release_baseline_allowed"] = False + receipt_path.write_text(json.dumps(receipt)) + + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view, [])) + assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 0 + + +@pytest.mark.parametrize( + "command", ["devtools verify --all", "devtools verify --full", "devtools verify --seed-testmon"] +) +def test_check_blocks_full_receipt_without_release_baseline_permission( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, command: str +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + _record(monkeypatch, pr_view, command=command) + receipt_path = merge_gate._receipt_path(42) + receipt = json.loads(receipt_path.read_text()) + receipt["verification_scope"] = "release-baseline" + receipt["release_baseline_allowed"] = False + receipt_path.write_text(json.dumps(receipt)) + + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view, [])) + assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 1 + + +def test_record_consumes_structured_verify_release_permission(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + return MagicMock( + returncode=0, + stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), + stderr="", + ) + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools verify") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["release_baseline_allowed"] is False + + def test_record_captures_nonzero_local_command_exit(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) monkeypatch.setattr( @@ -198,6 +260,22 @@ def test_check_ok_when_receipt_fresh_and_matches_head_with_no_late_comments( assert exit_code == 0 +def test_check_rejects_command_text_without_typed_scope_or_permission( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view = _base_pr_view() + _record(monkeypatch, pr_view, command="devtools verify --all") + receipt_path = merge_gate._receipt_path(42) + receipt = json.loads(receipt_path.read_text()) + receipt["verification_scope"] = None + receipt["release_baseline_allowed"] = True + receipt_path.write_text(json.dumps(receipt)) + + monkeypatch.setattr(subprocess, "run", _fake_run(pr_view, [])) + assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 1 + + @pytest.mark.parametrize( ("receipt_field", "mutated_value", "reason"), [ diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index 0d97eec2c0..974a6958aa 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -68,6 +68,8 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di assert captured["run"].run_id assert captured["run"]._payload["git_head"] == "abc123" assert isinstance(captured["run"]._payload["git_dirty"], bool) + assert captured["run"]._payload["verification_scope"] == "affected" + assert captured["run"]._payload["release_baseline_allowed"] is False def test_main_returns_pytest_exit_code(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index f5b1042ef3..2fde1706a0 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -15,38 +15,163 @@ from __future__ import annotations +import hashlib import json import sqlite3 +from collections.abc import Callable from pathlib import Path +from typing import cast import pytest +import devtools.checkout_guard as checkout_guard import devtools.testmon_bootstrap as testmon_bootstrap +import devtools.verify as verify from devtools.testmon_bootstrap import ( BootstrapDecision, bootstrap_testmon_seed_files, decide_testmon_bootstrap, ) +from devtools.testmon_state import ( + BaselineStatus, + BindingMode, + CollectionStatus, + GraphInspection, + GraphStatus, + file_fingerprint, +) +from devtools.testmon_state import ( + TestmonBinding as _TestmonBinding, +) +from devtools.testmon_state import ( + TestmonIdentity as _TestmonIdentity, +) +from devtools.testmon_state import ( + TestmonSeedStamp as _TestmonSeedStamp, +) -PROTOCOL_VERSION = 3 +PROTOCOL_VERSION = 4 def _write_valid_seed_stamp(path: Path, *, protocol_version: int = PROTOCOL_VERSION) -> None: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps({"protocol_version": protocol_version, "status": "complete"})) + data = path.parent / "testmondata" + if not data.exists(): + _write_sqlite_db(data) + with sqlite3.connect(data) as conn: + nodeids = tuple(row[0] for row in conn.execute("select test_name from test_execution")) + graph = GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()) + stamp = _TestmonSeedStamp( + protocol_version, + CollectionStatus.COMPLETE, + nodeids, + 0, + BaselineStatus.GREEN, + True, + 0, + graph, + _TestmonIdentity("head", "tree", "python", True, False, None, "narrow-terminal"), + _TestmonBinding(BindingMode.EXACT, str(path.parent.resolve())), + file_fingerprint(data), + "seed", + ".cache/verify/runs/seed", + ) + artifact_dir = path.parent / ".cache" / "verify" / "runs" / "seed" + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": "seed", + "checkout_root": str(path.parent.resolve()), + "artifact_dir": ".cache/verify/runs/seed", + } + ) + ) + path.write_text(json.dumps(stamp.as_dict())) def _write_sqlite_db(path: Path, *, rows: tuple[str, ...] = ("a", "b")) -> None: path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path) try: - conn.execute("CREATE TABLE file_fp (path TEXT, fsha TEXT)") - conn.executemany("INSERT INTO file_fp VALUES (?, ?)", [(row, f"sha-{row}") for row in rows]) + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") + conn.execute( + "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, environment_id INTEGER, test_name TEXT, failed INTEGER)" + ) + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") + conn.executemany("INSERT INTO file_fp(filename, fsha) VALUES (?, ?)", [(row, f"sha-{row}") for row in rows]) + conn.executemany("INSERT INTO test_execution(test_name, failed) VALUES (?, 0)", [(row,) for row in rows]) + conn.executemany( + "INSERT INTO test_execution_file_fp VALUES (?, ?)", + [(index, index) for index, _row in enumerate(rows, start=1)], + ) conn.commit() finally: conn.close() +def _red_attempt_decision(tmp_path: Path) -> tuple[BootstrapDecision, Path, Path, Path, Path]: + main_root = tmp_path / "main" + main_data = main_root / "testmondata" + _write_sqlite_db(main_data, rows=("tests/test.py::test_passed", "tests/test.py::test_failed")) + attempt = main_root / "seed-attempt.json" + attempt.write_text( + json.dumps( + { + "protocol_version": PROTOCOL_VERSION, + "status": "reusable", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, + "expected_nodeids": ["tests/test.py::test_passed", "tests/test.py::test_failed"], + "expected_count": 2, + "expected_digest": hashlib.sha256( + "\n".join(sorted(["tests/test.py::test_passed", "tests/test.py::test_failed"])).encode() + ).hexdigest(), + "testmon_data": file_fingerprint(main_data), + "node_outcomes": [ + {"nodeid": "tests/test.py::test_passed", "outcome": "passed"}, + {"nodeid": "tests/test.py::test_failed", "outcome": "failed"}, + ], + "exit_code": 1, + "run_id": "red-run", + "artifact_dir": ".cache/verify/runs/red-run", + } + ) + ) + artifact = main_root / ".cache" / "verify" / "runs" / "red-run" + artifact.mkdir(parents=True, exist_ok=True) + (artifact / "run.json").write_text( + json.dumps( + { + "run_id": "red-run", + "checkout_root": str(main_root.resolve()), + "artifact_dir": ".cache/verify/runs/red-run", + } + ) + ) + lane = tmp_path / "lane" + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=lane / "testmondata", + local_seed_stamp=lane / "seed.json", + local_seed_attempt=lane / "seed-attempt.json", + main_testmon_data=main_data, + main_seed_stamp=main_root / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + main_checkout_root=main_root, + local_checkout_root=lane, + ) + return decision, lane / "testmondata", lane / "seed.json", lane / "seed-attempt.json", lane + + def test_not_a_linked_worktree_never_bootstraps(tmp_path: Path) -> None: """The main checkout itself must never "bootstrap from itself".""" decision = decide_testmon_bootstrap( @@ -84,6 +209,29 @@ def test_local_seed_already_present_skips_bootstrap(tmp_path: Path) -> None: assert "already has" in decision.reason +def test_invalid_local_seed_does_not_block_valid_main_bootstrap(tmp_path: Path) -> None: + local_data = tmp_path / "local" / "testmondata" + local_stamp = tmp_path / "local" / "seed.json" + _write_sqlite_db(local_data) + _write_valid_seed_stamp(local_stamp) + local_data.write_bytes(local_data.read_bytes() + b"stale") + main_data = tmp_path / "main" / "testmondata" + main_stamp = tmp_path / "main" / "seed.json" + _write_sqlite_db(main_data) + _write_valid_seed_stamp(main_stamp) + + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + main_testmon_data=main_data, + main_seed_stamp=main_stamp, + protocol_version=PROTOCOL_VERSION, + ) + + assert decision.should_bootstrap + + def test_main_seed_absent_skips_bootstrap(tmp_path: Path) -> None: decision = decide_testmon_bootstrap( is_linked_worktree=True, @@ -94,13 +242,12 @@ def test_main_seed_absent_skips_bootstrap(tmp_path: Path) -> None: protocol_version=PROTOCOL_VERSION, ) assert not decision.should_bootstrap - assert "no valid complete testmon seed stamp" in decision.reason + assert "testmondata file is missing" in decision.reason def test_main_seed_stamp_wrong_protocol_version_skips_bootstrap(tmp_path: Path) -> None: main_stamp = tmp_path / "main" / "seed.json" _write_valid_seed_stamp(main_stamp, protocol_version=PROTOCOL_VERSION + 1) - _write_sqlite_db(tmp_path / "main" / "testmondata") decision = decide_testmon_bootstrap( is_linked_worktree=True, @@ -111,7 +258,7 @@ def test_main_seed_stamp_wrong_protocol_version_skips_bootstrap(tmp_path: Path) protocol_version=PROTOCOL_VERSION, ) assert not decision.should_bootstrap - assert "no valid complete testmon seed stamp" in decision.reason + assert "stale" in decision.reason or "no validated" in decision.reason def test_main_seed_stamp_incomplete_status_skips_bootstrap(tmp_path: Path) -> None: @@ -151,7 +298,8 @@ def test_main_seed_stamp_unreadable_json_skips_bootstrap(tmp_path: Path) -> None def test_valid_seed_stamp_but_missing_testmondata_skips_bootstrap(tmp_path: Path) -> None: """A seed stamp claims completeness but the db file itself vanished -- don't copy nothing.""" main_stamp = tmp_path / "main" / "seed.json" - _write_valid_seed_stamp(main_stamp) + main_stamp.parent.mkdir(parents=True, exist_ok=True) + main_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) decision = decide_testmon_bootstrap( is_linked_worktree=True, @@ -184,10 +332,265 @@ def test_valid_main_seed_and_empty_local_bootstraps(tmp_path: Path) -> None: assert decision.main_seed_stamp == main_stamp +def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) -> None: + main_data = tmp_path / "main" / "testmondata" + _write_sqlite_db(main_data, rows=("tests/test.py::test_passed", "tests/test.py::test_failed")) + attempt = tmp_path / "main" / "seed-attempt.json" + attempt.write_text( + json.dumps( + { + "protocol_version": PROTOCOL_VERSION, + "status": "reusable", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, + "expected_nodeids": ["tests/test.py::test_passed", "tests/test.py::test_failed"], + "expected_count": 2, + "expected_digest": hashlib.sha256( + "\n".join(sorted(["tests/test.py::test_passed", "tests/test.py::test_failed"])).encode() + ).hexdigest(), + "testmon_data": file_fingerprint(main_data), + "node_outcomes": [ + {"nodeid": "tests/test.py::test_passed", "outcome": "passed"}, + {"nodeid": "tests/test.py::test_failed", "outcome": "failed"}, + ], + "exit_code": 1, + "run_id": "red-run", + "artifact_dir": ".cache/verify/runs/red-run", + } + ) + ) + red_artifact = tmp_path / "main" / ".cache" / "verify" / "runs" / "red-run" + red_artifact.mkdir(parents=True, exist_ok=True) + (red_artifact / "run.json").write_text( + json.dumps( + { + "run_id": "red-run", + "checkout_root": str((tmp_path / "main").resolve()), + "artifact_dir": ".cache/verify/runs/red-run", + } + ) + ) + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=tmp_path / "lane" / "testmondata", + local_seed_stamp=tmp_path / "lane" / "seed.json", + main_testmon_data=main_data, + main_seed_stamp=tmp_path / "main" / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + ) + + assert decision.should_bootstrap + assert decision.main_seed_attempt == attempt + local_data = tmp_path / "lane" / "testmondata" + local_stamp = tmp_path / "lane" / "seed.json" + local_attempt = tmp_path / "lane" / "seed-attempt.json" + assert bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + local_seed_attempt=local_attempt, + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert not local_stamp.exists() + rebound_attempt = json.loads(local_attempt.read_text()) + assert rebound_attempt["artifact_dir"] == ".cache/verify/runs/red-run" + assert rebound_attempt["testmon_data"] == file_fingerprint(local_data) + rebound_receipt = json.loads( + (tmp_path / "lane" / ".cache" / "verify" / "runs" / "red-run" / "run.json").read_text() + ) + assert rebound_receipt["run_id"] == "red-run" + assert rebound_receipt["checkout_root"] == str((tmp_path / "lane").resolve()) + current_run = json.loads((tmp_path / "lane" / ".cache" / "verify" / "current-run.json").read_text()) + assert current_run["run_id"] == "red-run" + assert current_run["checkout_root"] == str((tmp_path / "lane").resolve()) + + rebound_decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + local_seed_attempt=local_attempt, + main_testmon_data=main_data, + main_seed_stamp=tmp_path / "main" / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + main_checkout_root=tmp_path / "main", + local_checkout_root=tmp_path / "lane", + ) + assert not rebound_decision.should_bootstrap + assert "checkout-bound selection attempt" in rebound_decision.reason + + +def test_complete_typed_markerless_green_attempt_bootstraps_only_as_selection_state(tmp_path: Path) -> None: + main_root = tmp_path / "main" + main_data = main_root / "testmondata" + _write_sqlite_db(main_data, rows=("tests/test.py::test_passed",)) + attempt = main_root / "seed-attempt.json" + attempt.write_text( + json.dumps( + { + "protocol_version": PROTOCOL_VERSION, + "status": "complete", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + "terminal_authorization": None, + }, + "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, + "expected_nodeids": ["tests/test.py::test_passed"], + "expected_count": 1, + "expected_digest": hashlib.sha256(b"tests/test.py::test_passed").hexdigest(), + "node_outcomes": [{"nodeid": "tests/test.py::test_passed", "outcome": "passed"}], + "exit_code": 0, + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + "run_id": "green-run", + "artifact_dir": ".cache/verify/runs/green-run", + "testmon_data": file_fingerprint(main_data), + } + ) + ) + artifact = main_root / ".cache" / "verify" / "runs" / "green-run" + artifact.mkdir(parents=True) + (artifact / "run.json").write_text( + json.dumps( + { + "run_id": "green-run", + "checkout_root": str(main_root.resolve()), + "artifact_dir": ".cache/verify/runs/green-run", + } + ) + ) + + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=tmp_path / "lane" / "testmondata", + local_seed_stamp=tmp_path / "lane" / "seed.json", + main_testmon_data=main_data, + main_seed_stamp=main_root / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + ) + + assert decision.should_bootstrap + assert decision.selection_only + assert bootstrap_testmon_seed_files( + decision, + local_testmon_data=tmp_path / "lane" / "testmondata", + local_seed_stamp=tmp_path / "lane" / "seed.json", + local_seed_attempt=tmp_path / "lane" / "seed-attempt.json", + checkout_root=tmp_path / "lane", + inherited_from=main_root, + ) + assert not (tmp_path / "lane" / "seed.json").exists() + + +def test_markerless_complete_bootstrap_passes_guard_and_verify_preflight( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main_root = tmp_path / "main" + main_data = main_root / "testmondata" + nodeid = "tests/test.py::test_passed" + _write_sqlite_db(main_data, rows=(nodeid,)) + attempt = main_root / "seed-attempt.json" + attempt.write_text( + json.dumps( + { + "protocol_version": PROTOCOL_VERSION, + "status": "complete", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + "terminal_authorization": None, + }, + "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, + "expected_nodeids": [nodeid], + "expected_count": 1, + "expected_digest": hashlib.sha256(nodeid.encode()).hexdigest(), + "node_outcomes": [{"nodeid": nodeid, "outcome": "passed"}], + "exit_code": 0, + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + "run_id": "green-run", + "artifact_dir": ".cache/verify/runs/green-run", + "testmon_data": file_fingerprint(main_data), + } + ) + ) + artifact = main_root / ".cache" / "verify" / "runs" / "green-run" + artifact.mkdir(parents=True) + (artifact / "run.json").write_text( + json.dumps( + { + "run_id": "green-run", + "checkout_root": str(main_root.resolve()), + "artifact_dir": ".cache/verify/runs/green-run", + } + ) + ) + + lane = tmp_path / "lane" + lane.mkdir() + (lane / ".git").write_text("gitdir: /main/.git/worktrees/lane\n") + (lane / ".venv" / "bin").mkdir(parents=True) + package = lane / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("") + local_data = lane / ".cache" / "testmon" / "testmondata" + local_stamp = lane / ".cache" / "testmon" / "seed.json" + local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" + decision = decide_testmon_bootstrap( + is_linked_worktree=True, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + main_testmon_data=main_data, + main_seed_stamp=main_root / "seed.json", + main_seed_attempt=attempt, + protocol_version=PROTOCOL_VERSION, + ) + assert decision.selection_only + assert bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + local_seed_attempt=local_attempt, + checkout_root=lane, + inherited_from=main_root, + ) + + monkeypatch.setattr(checkout_guard, "_is_linked_worktree", lambda _root: True) + fingerprint = checkout_guard.checkout_environment_fingerprint( + lane, + polylogue_import_path=package / "__init__.py", + python_executable=lane / ".venv" / "bin" / "python", + ) + assert not fingerprint.clean + monkeypatch.setattr(verify, "ROOT", lane) + monkeypatch.setattr(verify, "TESTMON_DATA", local_data) + monkeypatch.setattr(verify, "TESTMON_SEED_STAMP", local_stamp) + monkeypatch.setattr(verify, "TESTMON_SEED_ATTEMPT", local_attempt) + assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is not None + assert json.loads(local_attempt.read_text())["release_baseline_allowed"] is False + + def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: """Partial local state (e.g. a stale stamp with no db, or vice versa) still needs a fresh copy.""" local_stamp = tmp_path / "local" / "seed.json" - _write_valid_seed_stamp(local_stamp) + local_stamp.parent.mkdir(parents=True, exist_ok=True) + local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) main_data = tmp_path / "main" / "testmondata" main_stamp = tmp_path / "main" / "seed.json" _write_sqlite_db(main_data) @@ -205,8 +608,8 @@ def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" + main_data = tmp_path / "main?fragment#1" / "testmondata" + main_stamp = tmp_path / "main?fragment#1" / "seed.json" _write_sqlite_db(main_data, rows=("x", "y", "z")) _write_valid_seed_stamp(main_stamp) local_data = tmp_path / "local" / "testmondata" @@ -218,17 +621,28 @@ def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: main_testmon_data=main_data, main_seed_stamp=main_stamp, ) - bootstrap_testmon_seed_files(decision, local_testmon_data=local_data, local_seed_stamp=local_stamp) + assert bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + checkout_root=tmp_path / "local", + inherited_from=tmp_path / "main?fragment#1", + ) - assert local_stamp.read_text() == main_stamp.read_text() + local_payload = json.loads(local_stamp.read_text()) + source_payload = json.loads(main_stamp.read_text()) + comparable_keys = set(source_payload) - {"binding", "testmon_data"} + assert {key: local_payload[key] for key in comparable_keys} == {key: source_payload[key] for key in comparable_keys} + assert local_payload["binding"]["checkout_root"] == str(tmp_path / "local") + assert local_payload["binding"]["source_checkout_root"] == str(tmp_path / "main?fragment#1") conn = sqlite3.connect(local_data) try: - rows = conn.execute("SELECT path, fsha FROM file_fp ORDER BY path").fetchall() + rows = conn.execute("SELECT filename, fsha FROM file_fp ORDER BY filename").fetchall() finally: conn.close() assert rows == [("x", "sha-x"), ("y", "sha-y"), ("z", "sha-z")] # No temp files left behind. - assert sorted(p.name for p in local_data.parent.iterdir()) == ["seed.json", "testmondata"] + assert sorted(p.name for p in local_data.parent.iterdir()) == [".cache", "seed.json", "testmondata"] def test_bootstrap_seed_files_marks_destination_and_source_checkout(tmp_path: Path) -> None: @@ -248,10 +662,40 @@ def test_bootstrap_seed_files_marks_destination_and_source_checkout(tmp_path: Pa ) payload = json.loads(local_stamp.read_text()) - assert payload["checkout_root"] == str((tmp_path / "lane").resolve()) - assert payload["inherited_from"] == str((tmp_path / "main").resolve()) + assert payload["binding"]["checkout_root"] == str((tmp_path / "lane").resolve()) + assert payload["binding"]["source_checkout_root"] == str((tmp_path / "main").resolve()) source = json.loads(main_stamp.read_text()) - assert {key: payload[key] for key in source} == source + assert {key: payload[key] for key in source if key not in {"binding", "testmon_data"}} == { + key: source[key] for key in source if key not in {"binding", "testmon_data"} + } + + +def test_bootstrap_seed_files_rejects_paths_outside_or_colliding_with_destination(tmp_path: Path) -> None: + main_data = tmp_path / "main" / "testmondata" + main_stamp = tmp_path / "main" / "seed.json" + _write_sqlite_db(main_data) + _write_valid_seed_stamp(main_stamp) + decision = BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp) + local_data = tmp_path / "lane" / "testmondata" + + assert not bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=tmp_path / "outside" / "seed.json", + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert not (tmp_path / "outside" / "seed.json").exists() + assert not local_data.exists() + + assert not bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_data, + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert not local_data.exists() def test_bootstrap_seed_files_keeps_copied_state_when_stamp_turns_invalid(tmp_path: Path) -> None: @@ -272,17 +716,96 @@ def test_bootstrap_seed_files_keeps_copied_state_when_stamp_turns_invalid(tmp_pa ) assert stamped is False - assert local_data.is_file() - assert local_stamp.read_text() == "{concurrent rewrite" + assert not local_data.exists() + assert not local_stamp.exists() + + +def test_bootstrap_graph_mismatch_publishes_no_destination_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main_data = tmp_path / "main" / "testmondata" + main_stamp = tmp_path / "main" / "seed.json" + _write_sqlite_db(main_data) + _write_valid_seed_stamp(main_stamp) + local_data = tmp_path / "lane" / "testmondata" + local_stamp = tmp_path / "lane" / "seed.json" + monkeypatch.setattr(testmon_bootstrap, "refresh_stamp", lambda *_args, **_kwargs: None) + + assert not bootstrap_testmon_seed_files( + BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert not local_data.exists() + assert not local_stamp.exists() + assert not (tmp_path / "lane" / ".cache" / "verify" / "current-run.json").exists() + + +def test_bootstrap_receipt_rebind_failure_publishes_no_destination_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main_data = tmp_path / "main" / "testmondata" + main_stamp = tmp_path / "main" / "seed.json" + _write_sqlite_db(main_data) + _write_valid_seed_stamp(main_stamp) + local_data = tmp_path / "lane" / "testmondata" + local_stamp = tmp_path / "lane" / "seed.json" + monkeypatch.setattr(testmon_bootstrap, "_rebind_run_receipt", lambda **_kwargs: False) + assert not bootstrap_testmon_seed_files( + BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + checkout_root=tmp_path / "lane", + inherited_from=tmp_path / "main", + ) + assert not local_data.exists() + assert not local_stamp.exists() + assert not (tmp_path / "lane" / ".cache" / "verify" / "current-run.json").exists() + + +def test_bootstrap_rebound_attempt_failure_publishes_no_destination_state( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + decision, local_data, local_stamp, local_attempt, lane = _red_attempt_decision(tmp_path) + original_stamp_from_attempt = cast(Callable[..., object], testmon_bootstrap.__dict__["stamp_from_attempt"]) + calls = 0 + + def fail_rebound_attempt(*args: object, **kwargs: object) -> object: + nonlocal calls + calls += 1 + if calls == 2: + return None + return original_stamp_from_attempt(*args, **kwargs) -def test_maybe_bootstrap_migrates_a_valid_legacy_local_stamp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(testmon_bootstrap, "stamp_from_attempt", fail_rebound_attempt) + + assert not bootstrap_testmon_seed_files( + decision, + local_testmon_data=local_data, + local_seed_stamp=local_stamp, + local_seed_attempt=local_attempt, + checkout_root=lane, + inherited_from=tmp_path / "main", + ) + assert not local_data.exists() + assert not local_stamp.exists() + assert not local_attempt.exists() + assert not (lane / ".cache" / "verify" / "current-run.json").exists() + + +def test_maybe_bootstrap_does_not_migrate_an_untyped_legacy_local_stamp( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: lane = tmp_path / "lane" main = tmp_path / "main" local_data = lane / "cache" / "testmondata" local_stamp = lane / "cache" / "seed.json" _write_sqlite_db(local_data) - _write_valid_seed_stamp(local_stamp) + local_stamp.parent.mkdir(parents=True, exist_ok=True) + local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "complete"})) monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, main)) message = testmon_bootstrap.maybe_bootstrap_testmon_seed( @@ -292,11 +815,8 @@ def test_maybe_bootstrap_migrates_a_valid_legacy_local_stamp(tmp_path: Path, mon protocol_version=PROTOCOL_VERSION, ) - payload = json.loads(local_stamp.read_text()) - assert message is not None and "migrated legacy" in message - assert payload["checkout_root"] == str(lane.resolve()) - assert payload["protocol_version"] == PROTOCOL_VERSION - assert payload["status"] == "complete" + assert message is None + assert json.loads(local_stamp.read_text())["status"] == "complete" def test_bootstrap_seed_files_noop_when_decision_says_no(tmp_path: Path) -> None: diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py new file mode 100644 index 0000000000..36180fe946 --- /dev/null +++ b/tests/unit/devtools/test_testmon_state.py @@ -0,0 +1,362 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +import pytest + +import devtools.testmon_state as testmon_state +from devtools.testmon_state import ( + BaselineStatus, + GraphStatus, + file_fingerprint, + inspect_testmon_database, + stamp_from_attempt, + validate_stamp, +) +from devtools.testmon_state import ( + TestmonSeedStamp as _TestmonSeedStamp, +) + +PROTOCOL = 4 +NODEIDS = ("tests/test_seed.py::test_passed", "tests/test_seed.py::test_failed") + + +def _write_graph(path: Path, *, failed: bool = False, with_edges: bool = True) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(path) as connection: + connection.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + connection.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") + connection.execute("CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT, failed INTEGER)") + connection.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") + for index, nodeid in enumerate(NODEIDS, start=1): + connection.execute("INSERT INTO file_fp VALUES (?, ?, ?)", (index, f"file-{index}.py", f"sha-{index}")) + connection.execute( + "INSERT INTO test_execution VALUES (?, ?, ?)", + (index, nodeid, int(failed and index == 2)), + ) + if with_edges: + connection.execute("INSERT INTO test_execution_file_fp VALUES (?, ?)", (index, index)) + + +def _attempt(data: Path, *, outcomes: tuple[str, str] = ("passed", "failed")) -> dict[str, object]: + artifact_dir = data.parent / ".cache" / "verify" / "runs" / "run-red" + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": "run-red", + "checkout_root": str(data.parent.resolve()), + "artifact_dir": ".cache/verify/runs/run-red", + } + ) + ) + return { + "protocol_version": PROTOCOL, + "status": "reusable", + "identity": { + "git_head": "head", + "git_tree": "tree-hash", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + "terminal_authorization": "narrow-terminal", + }, + "selection": { + "selected_count": len(NODEIDS), + "selected_nodeids_omitted": 0, + }, + "expected_nodeids": list(NODEIDS), + "expected_count": len(NODEIDS), + "expected_digest": hashlib.sha256("\n".join(sorted(NODEIDS)).encode()).hexdigest(), + "verification_scope": "narrow-terminal", + "release_baseline_allowed": False, + "node_outcomes": [ + {"nodeid": nodeid, "outcome": outcome} for nodeid, outcome in zip(NODEIDS, outcomes, strict=True) + ], + "exit_code": 1, + "run_id": "run-red", + "artifact_dir": ".cache/verify/runs/run-red", + "testmon_data": file_fingerprint(data), + } + + +def test_failed_complete_graph_is_selection_only_and_rebindable(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data, failed=True) + stamp = stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert stamp is not None + assert stamp.baseline_status is BaselineStatus.RED + assert stamp.affected_selection_allowed + assert not stamp.release_baseline_allowed + + passed_outcomes = stamp_from_attempt( + _attempt(data, outcomes=("passed", "passed")), data, checkout_root=tmp_path, protocol_version=PROTOCOL + ) + assert passed_outcomes is not None + assert passed_outcomes.baseline_status is BaselineStatus.RED + assert not passed_outcomes.release_baseline_allowed + + stamp_path = tmp_path / "seed.json" + stamp_path.write_text(json.dumps(stamp.as_dict())) + assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + +def test_omitted_interrupted_and_uncovered_nodes_fail_closed(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + omitted = _attempt(data) + omitted["selection"] = {"selected_count": 1, "selected_nodeids_omitted": 1} + assert stamp_from_attempt(omitted, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + interrupted = _attempt(data, outcomes=("passed", "interrupted")) + assert stamp_from_attempt(interrupted, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + data.unlink() + _write_graph(data, with_edges=False) + assert stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + +def test_stamp_from_attempt_does_not_reopen_the_validated_database(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data) + + with patch("devtools.testmon_state.file_fingerprint", return_value=attempt["testmon_data"]) as fingerprint: + stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert stamp is not None + fingerprint.assert_called_once_with(data) + + +def test_incomplete_attempt_fails_closed(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data, outcomes=("passed", "passed")) + attempt["exit_code"] = 0 + attempt["release_baseline_allowed"] = True + attempt["status"] = "incomplete" + + stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert stamp is None + + attempt["status"] = "complete" + completed = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert completed is not None + assert completed.baseline_status is BaselineStatus.GREEN + assert completed.release_baseline_allowed + + +def test_reusable_attempt_rejects_a_changed_dependency_or_pytest_harness( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Reusable graphs belong to the environment that captured them.""" + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data) + identity = attempt["identity"] + assert isinstance(identity, dict) + identity["dependency_environment"] = "dependency-environment" + identity["pytest_harness"] = "pytest-harness" + attempt["protocol_version"] = 5 + monkeypatch.setattr( + testmon_state, + "testmon_runtime_identity", + lambda _root: ("dependency-environment", "pytest-harness"), + raising=False, + ) + + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=5) is not None + + identity["dependency_environment"] = "different-environment" + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=5) is None + + +def test_runtime_identity_includes_test_behavior_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(testmon_state, "_installed_distributions", lambda: (("pytest", "9"),)) + monkeypatch.setenv("HYPOTHESIS_PROFILE", "ci") + monkeypatch.setenv("POLYLOGUE_CI", "1") + first = testmon_state.testmon_runtime_identity(tmp_path) + + monkeypatch.setenv("HYPOTHESIS_PROFILE", "default") + second = testmon_state.testmon_runtime_identity(tmp_path) + + assert first is not None + assert second is not None + assert first[0] == second[0] + assert first[1] != second[1] + + +def test_green_skipped_slow_attempt_without_typed_terminal_authority_is_selection_only(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data, outcomes=("passed", "passed")) + attempt["status"] = "complete" + attempt["exit_code"] = 0 + raw_identity = attempt["identity"] + assert isinstance(raw_identity, dict) + identity = dict(raw_identity) + identity["terminal_authorization"] = None + attempt["identity"] = identity + attempt["verification_scope"] = "narrow-terminal" + attempt["release_baseline_allowed"] = False + + stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert stamp is not None + assert stamp.baseline_status is BaselineStatus.RED + assert stamp.affected_selection_allowed + assert not stamp.release_baseline_allowed + + +def test_typed_complete_markerless_attempt_is_selection_only(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data, failed=False) + attempt = _attempt(data, outcomes=("passed", "passed")) + attempt["status"] = "complete" + attempt["exit_code"] = 0 + raw_identity = attempt["identity"] + assert isinstance(raw_identity, dict) + identity = dict(raw_identity) + identity["skip_slow"] = False + identity["terminal_authorization"] = None + attempt["identity"] = identity + attempt["verification_scope"] = "release-baseline" + attempt["release_baseline_allowed"] = True + + published = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + markerless = stamp_from_attempt( + attempt, + data, + checkout_root=tmp_path, + protocol_version=PROTOCOL, + published_marker=False, + ) + + assert published is not None + assert published.release_baseline_allowed + assert markerless is not None + assert markerless.baseline_status is BaselineStatus.RED + assert markerless.affected_selection_allowed + assert not markerless.release_baseline_allowed + + +def test_malformed_sqlite_and_stale_stamp_fail_closed(tmp_path: Path) -> None: + malformed = tmp_path / "malformed" + malformed.write_bytes(b"not sqlite") + inspection = inspect_testmon_database(malformed, NODEIDS) + assert inspection.status is GraphStatus.INVALID + + data = tmp_path / "testmondata" + _write_graph(data) + stamp = stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) + assert stamp is not None + stamp_path = tmp_path / "seed.json" + stamp_path.write_text(json.dumps(stamp.as_dict())) + data.write_bytes(data.read_bytes() + b"stale") + assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + +def test_attempt_and_green_stamp_artifacts_fail_closed_when_malformed(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data, outcomes=("passed", "passed")) + attempt["exit_code"] = 0 + attempt["release_baseline_allowed"] = True + attempt["artifact_dir"] = "/tmp/outside-testmon-run" + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + attempt["artifact_dir"] = ".cache/verify/runs/run-red" + attempt["status"] = "complete" + stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + assert stamp is not None + receipt = tmp_path / ".cache" / "verify" / "runs" / "run-red" / "run.json" + receipt.unlink() + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + receipt.write_text( + json.dumps( + { + "run_id": "wrong-run", + "checkout_root": str(tmp_path.resolve()), + "artifact_dir": ".cache/verify/runs/run-red", + } + ) + ) + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + receipt.write_text( + json.dumps( + { + "run_id": "run-red", + "checkout_root": str(tmp_path.resolve()), + "artifact_dir": ".cache/verify/runs/run-red", + } + ) + ) + stamp_path = tmp_path / ".cache" / "testmon" / "seed.json" + stamp_path.parent.mkdir(parents=True) + payload = stamp.as_dict() + payload["baseline"]["exit_code"] = 1 + stamp_path.write_text(json.dumps(payload)) + assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + payload = stamp.as_dict() + payload["graph"]["failed_nodeids"] = [NODEIDS[0]] + stamp_path.write_text(json.dumps(payload)) + assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + +def test_malformed_sqlite_values_fail_closed(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + with sqlite3.connect(data) as connection: + connection.execute("update test_execution set failed = 'bad' where id = 1") + + inspection = inspect_testmon_database(data, NODEIDS) + + assert inspection.status is GraphStatus.INVALID + + +def test_sqlite_paths_with_uri_characters_are_inspected_safely(tmp_path: Path) -> None: + data = tmp_path / "checkout?fragment#1" / "testmondata" + _write_graph(data) + + inspection = inspect_testmon_database(data, NODEIDS) + + assert inspection.status is GraphStatus.COMPLETE + + +@pytest.mark.parametrize("filename", ["../outside.py", "/tmp/outside.py"]) +def test_unsafe_testmon_fingerprint_paths_fail_closed(tmp_path: Path, filename: str) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + with sqlite3.connect(data) as connection: + connection.execute("update file_fp set filename = ? where id = 1", (filename,)) + + assert inspect_testmon_database(data, NODEIDS).status is GraphStatus.INVALID + + +def test_attempt_status_must_be_promotable(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data) + attempt["status"] = "running" + + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + attempt = _attempt(data) + attempt["run_id"] = None + assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None + + +def test_stamp_parser_rejects_untyped_or_non_graph_state() -> None: + try: + _TestmonSeedStamp.from_mapping({"protocol_version": PROTOCOL, "status": "complete"}, protocol_version=PROTOCOL) + except ValueError: + pass + else: + raise AssertionError("legacy green-looking stamp must not be accepted") diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 44388ae490..6d6be3f71e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -11,7 +11,27 @@ import pytest -from devtools import verify_runs +from devtools import verify, verify_runs +from devtools.testmon_state import ( + BaselineStatus, + BindingMode, + CollectionStatus, + GraphInspection, + GraphStatus, + file_fingerprint, +) +from devtools.testmon_state import ( + TestmonBinding as _TestmonBinding, +) +from devtools.testmon_state import ( + TestmonIdentity as _TestmonIdentity, +) +from devtools.testmon_state import ( + TestmonSeedStamp as _TestmonSeedStamp, +) +from devtools.testmon_state import ( + testmon_runtime_identity as _testmon_runtime_identity, +) from devtools.verify import ( PYTEST_CONTAINMENT_PATH, PYTEST_EVENTS_PATH, @@ -25,7 +45,9 @@ TESTMON_SEED_ATTEMPT, TESTMON_SEED_PROTOCOL_VERSION, TESTMON_SEED_STAMP, + _anchor_verification_paths, _finalize_testmon_seed_attempt, + _flatten_seed_outcomes, _format_completion_notification, _matching_testmon_coverage, _parse_pytest_test_count, @@ -74,6 +96,78 @@ def _pytest_marker_expr(command: list[str]) -> str: return command[marker_indexes[-1] + 1] +def _testmon_runtime_identity_fields(checkout_root: Path = ROOT) -> dict[str, str]: + runtime_identity = _testmon_runtime_identity(checkout_root) + assert runtime_identity is not None + dependency_environment, pytest_harness = runtime_identity + return {"dependency_environment": dependency_environment, "pytest_harness": pytest_harness} + + +def _write_real_testmon_state(nodeids: tuple[str, ...] = ("tests/test_a.py::test_one",)) -> Path: + TESTMON_DATA.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(TESTMON_DATA) as conn: + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") + conn.execute("CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT, failed INTEGER)") + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") + for index, nodeid in enumerate(nodeids, start=1): + conn.execute("INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", (index, nodeid, f"sha-{index}")) + conn.execute("INSERT INTO test_execution(id, test_name, failed) VALUES (?, ?, 0)", (index, nodeid)) + conn.execute("INSERT INTO test_execution_file_fp VALUES (?, ?)", (index, index)) + stamp = _TestmonSeedStamp( + TESTMON_SEED_PROTOCOL_VERSION, + CollectionStatus.COMPLETE, + nodeids, + 0, + BaselineStatus.GREEN, + True, + 0, + GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()), + _TestmonIdentity( + "current-head", + "covered", + "python", + True, + False, + None, + "narrow-terminal", + **_testmon_runtime_identity_fields(), + ), + _TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())), + file_fingerprint(TESTMON_DATA), + "seed", + ".cache/verify/runs/seed", + ) + TESTMON_SEED_STAMP.parent.mkdir(parents=True, exist_ok=True) + artifact_dir = ROOT / ".cache" / "verify" / "runs" / "seed" + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": "seed", + "checkout_root": str(ROOT.resolve()), + "artifact_dir": ".cache/verify/runs/seed", + } + ) + ) + TESTMON_SEED_STAMP.write_text(json.dumps(stamp.as_dict())) + return TESTMON_DATA + + +def _write_run_receipt(root: Path, run_id: str) -> None: + artifact_dir = root / ".cache" / "verify" / "runs" / run_id + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "run.json").write_text( + json.dumps( + { + "run_id": run_id, + "checkout_root": str(root.resolve()), + "artifact_dir": f".cache/verify/runs/{run_id}", + } + ) + ) + + def test_quick_verify_omits_pytest() -> None: steps = build_verify_steps(quick=True, lab=False, skip_slow=False) @@ -161,7 +255,18 @@ def test_seed_testmon_runs_full_collection_without_selection(monkeypatch: pytest assert "--testmon" in command assert "--testmon-noselect" in command assert "-n" in command - assert command[command.index("-n") + 1] == "8" + assert command[command.index("-n") + 1] == "4" + + +def test_seed_testmon_caps_adaptive_workers(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("POLYLOGUE_PYTEST_WORKERS", raising=False) + monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 12) + + steps = build_verify_steps(quick=False, lab=False, skip_slow=False, seed_testmon=True) + + label, command = steps[-1] + assert label == "pytest seed-testmon" + assert command[command.index("-n") + 1] == "4" def test_resumed_seed_uses_affected_selection_for_remaining_tests() -> None: @@ -344,52 +449,26 @@ def test_testmon_preflight_requires_seed_stamp(tmp_path: Path, monkeypatch: pyte def test_testmon_preflight_accepts_seeded_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("seeded") - seed_stamp = tmp_path / ".cache" / "testmon" / "seed.json" - seed_stamp.parent.mkdir(parents=True, exist_ok=True) - seed_stamp.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "git_head": "current-head", - "testmon_data": hashlib.sha256(b"seeded").hexdigest(), - } - ) - ) - monkeypatch.setattr("devtools.verify._git_head", lambda: "current-head") + _write_real_testmon_state() assert _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is None -def test_testmon_preflight_warns_on_stale_git_head( +def test_testmon_preflight_rejects_stale_database_fingerprint( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("seeded") - seed_stamp = tmp_path / ".cache" / "testmon" / "seed.json" - seed_stamp.parent.mkdir(parents=True, exist_ok=True) - seed_stamp.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "git_head": "old-head", - "testmon_data": hashlib.sha256(b"seeded").hexdigest(), - } - ) - ) - monkeypatch.setattr("devtools.verify._git_head", lambda: "current-head") + _write_real_testmon_state() + TESTMON_DATA.write_bytes(TESTMON_DATA.read_bytes() + b"stale") message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - assert message is None - assert "different git head" in capsys.readouterr().err + assert message is not None + assert "stale" in message + assert capsys.readouterr().err == "" -def test_testmon_preflight_warns_on_database_fingerprint_drift( +def test_testmon_preflight_rejects_malformed_sqlite_state( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: monkeypatch.chdir(tmp_path) @@ -401,18 +480,16 @@ def test_testmon_preflight_warns_on_database_fingerprint_drift( json.dumps( { "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "git_head": "current-head", - "testmon_data": hashlib.sha256(b"seeded").hexdigest(), + "status": "usable", } ) ) - monkeypatch.setattr("devtools.verify._git_head", lambda: "current-head") message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - assert message is None - assert "database changed" in capsys.readouterr().err + assert message is not None + assert "stale" in message or "malformed" in message + assert capsys.readouterr().err == "" def test_testmon_preflight_rejects_incomplete_seed_receipt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -434,7 +511,7 @@ def test_testmon_preflight_rejects_incomplete_seed_receipt(tmp_path: Path, monke message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) assert message is not None - assert "no validated complete seed receipt" in message + assert "stale" in message or "malformed" in message def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -443,10 +520,12 @@ def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pyte TESTMON_DATA.write_text("partial") identity = { "git_head": "head", + "git_tree": "tree-hash", "worktree_fingerprint": "tree", "python": "3.13", "skip_slow": True, "lab": False, + "terminal_authorization": None, } TESTMON_SEED_ATTEMPT.write_text( json.dumps( @@ -455,16 +534,83 @@ def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pyte "status": "incomplete", "identity": identity, "expected_nodeids": ["tests/unit/test_example.py::test_one"], + "expected_count": 1, + "expected_digest": hashlib.sha256(b"tests/unit/test_example.py::test_one").hexdigest(), + "run_id": "interrupted", + "started_at": "2026-08-05T12:00:00+00:00", + "testmon_data_before": "partial", } ) ) assert _testmon_seed_can_resume(identity) is True - assert _testmon_seed_can_resume({**identity, "git_head": "other"}) is True + assert _testmon_seed_can_resume({**identity, "git_head": "other", "git_tree": "tree-hash"}) is True + assert _testmon_seed_can_resume({**identity, "git_tree": "different-tree"}) is False assert _testmon_seed_can_resume({**identity, "worktree_fingerprint": "changed"}) is False assert _testmon_seed_can_resume({**identity, "skip_slow": False}) is False +def test_two_interrupted_resumes_flatten_all_carried_outcomes(tmp_path: Path) -> None: + monkeypatch = pytest.MonkeyPatch() + monkeypatch.chdir(tmp_path) + try: + expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] + TESTMON_DATA.parent.mkdir(parents=True) + TESTMON_DATA.write_text("partial") + identity = { + "git_head": "head", + "git_tree": "tree-hash", + "worktree_fingerprint": "tree", + "python": "3.13", + "skip_slow": False, + "lab": False, + "terminal_authorization": None, + } + TESTMON_SEED_ATTEMPT.write_text( + json.dumps( + { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "incomplete", + "identity": identity, + "expected_nodeids": expected, + "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), + "node_outcomes": [{"nodeid": expected[0], "outcome": "passed"}], + } + ) + ) + first = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="head", root=tmp_path) + _prepare_testmon_seed_attempt(identity=identity, run=first, resume=True) + first_payload = json.loads(TESTMON_SEED_ATTEMPT.read_text()) + first_payload["status"] = "incomplete" + first_payload["node_outcomes"] = [{"nodeid": expected[1], "outcome": "passed"}] + TESTMON_SEED_ATTEMPT.write_text(json.dumps(first_payload)) + + second = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="head", root=tmp_path) + prepared = _prepare_testmon_seed_attempt(identity=identity, run=second, resume=True) + + assert {item["nodeid"] for item in prepared["prior_node_outcomes"]} == set(expected) + assert {item["outcome"] for item in prepared["prior_node_outcomes"]} == {"passed"} + assert _flatten_seed_outcomes(prepared) == prepared["prior_node_outcomes"] + finally: + monkeypatch.undo() + + +def test_focused_run_can_record_typed_affected_scope(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + + payload = run.finish( + exit_code=0, + duration_s=0.1, + verification_scope="affected", + release_baseline_allowed=False, + ) + + assert payload["verification_scope"] == "affected" + assert payload["release_baseline_allowed"] is False + assert json.loads((tmp_path / ".cache" / "verify" / "current-run.json").read_text()) == payload + + def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) @@ -473,13 +619,17 @@ def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, mo step_dir = artifact_dir / "steps" / "17-pytest-seed-testmon" step_dir.mkdir(parents=True) expected = ["tests/unit/test_example.py::test_one"] - (step_dir / "selection.json").write_text(json.dumps({"selected_nodeids": expected, "selected_nodeids_omitted": 0})) + (step_dir / "selection.json").write_text( + json.dumps({"selected_nodeids": expected, "selected_nodeids_omitted": 0, "selected_count": 1}) + ) identity = { "git_head": "head", + "git_tree": "tree-hash", "worktree_fingerprint": "tree", "python": "3.13", "skip_slow": True, "lab": False, + "terminal_authorization": None, } TESTMON_SEED_ATTEMPT.write_text( json.dumps( @@ -488,18 +638,135 @@ def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, mo "status": "running", "identity": identity, "expected_nodeids": [], - "artifact_dir": str(artifact_dir), + "artifact_dir": str(artifact_dir.relative_to(tmp_path)), } ) ) - assert _testmon_seed_can_resume({**identity, "git_head": "fixed"}) is True + assert _testmon_seed_can_resume({**identity, "git_head": "fixed", "git_tree": "tree-hash"}) is True run = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="fixed") - prepared = _prepare_testmon_seed_attempt(identity={**identity, "git_head": "fixed"}, run=run, resume=True) + prepared = _prepare_testmon_seed_attempt( + identity={**identity, "git_head": "fixed", "git_tree": "tree-hash"}, run=run, resume=True + ) assert prepared["expected_nodeids"] == expected assert prepared["expected_count"] == 1 + assert prepared["expected_digest"] == hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() + persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) + assert persisted["expected_digest"] == prepared["expected_digest"] + + +def test_seed_resume_rejects_selection_artifact_outside_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + TESTMON_DATA.parent.mkdir(parents=True) + TESTMON_DATA.write_text("partial") + outside = tmp_path.parent / "outside-testmon-artifacts" + step_dir = outside / "steps" / "17-pytest-seed-testmon" + step_dir.mkdir(parents=True) + (step_dir / "selection.json").write_text( + json.dumps( + { + "selected_nodeids": ["tests/unit/test_example.py::test_one"], + "selected_nodeids_omitted": 0, + "selected_count": 1, + } + ) + ) + identity = { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "3.13", + "skip_slow": True, + "lab": False, + } + TESTMON_SEED_ATTEMPT.write_text( + json.dumps( + { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": identity, + "expected_nodeids": [], + "artifact_dir": str(outside), + } + ) + ) + + assert _testmon_seed_can_resume(identity) is False + + +def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) -> None: + monkeypatch = pytest.MonkeyPatch() + monkeypatch.chdir(tmp_path) + try: + expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir() + (artifact_dir / "selection.json").write_text( + json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) + ) + (artifact_dir / "events.jsonl").write_text( + json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" + ) + TESTMON_DATA.parent.mkdir(parents=True) + with sqlite3.connect(TESTMON_DATA) as connection: + connection.execute("create table environment (id integer primary key, environment_name text)") + connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") + connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") + connection.execute( + "create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)" + ) + connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) + connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) + connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) + prepared = { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + **_testmon_runtime_identity_fields(Path.cwd()), + }, + "resume": True, + "expected_nodeids": expected, + "run_id": "resume", + "artifact_dir": ".cache/verify/runs/resume", + } + _write_run_receipt(tmp_path, "resume") + + receipt = _finalize_testmon_seed_attempt( + prepared=prepared, + step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + + assert receipt["status"] == "incomplete" + assert {item["nodeid"]: item["outcome"] for item in receipt["node_outcomes"]} == { + expected[0]: "passed", + expected[1]: "missing", + } + + (artifact_dir / "selection.json").write_text(json.dumps({})) + (artifact_dir / "events.jsonl").write_text( + "\n".join( + json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) + for nodeid in expected + ) + + "\n" + ) + missing_selection = _finalize_testmon_seed_attempt( + prepared=prepared, + step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + assert missing_selection["status"] == "incomplete" + finally: + monkeypatch.undo() def test_testmon_database_state_reports_missing_and_failed_nodes( @@ -508,13 +775,21 @@ def test_testmon_database_state_reports_missing_and_failed_nodes( monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) with sqlite3.connect(TESTMON_DATA) as conn: + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") conn.execute( "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" ) + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") conn.executemany( "INSERT INTO test_execution(test_name, failed) VALUES (?, ?)", [("tests/test_a.py::test_ok", 0), ("tests/test_b.py::test_failed", 1)], ) + conn.executemany( + "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", + [(1, "a.py", "a"), (2, "b.py", "b")], + ) + conn.executemany("INSERT INTO test_execution_file_fp VALUES (?, ?)", [(1, 1), (2, 2)]) state = _testmon_database_state( ["tests/test_a.py::test_ok", "tests/test_b.py::test_failed", "tests/test_c.py::test_missing"] @@ -599,23 +874,42 @@ def test_seed_receipt_classifies_every_node_terminal_outcome( (artifact_dir / "events.jsonl").write_text("".join(json.dumps(event) + "\n" for event in events)) TESTMON_DATA.parent.mkdir(parents=True) with sqlite3.connect(TESTMON_DATA) as conn: + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") conn.execute( "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" ) + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") conn.executemany( "INSERT INTO test_execution(test_name, failed) VALUES (?, ?)", [(nodeid, int(nodeid != expected[0])) for nodeid in expected[:-1]], ) + conn.executemany( + "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", + [(index, f"file-{index}.py", f"sha-{index}") for index, _nodeid in enumerate(expected[:-1], start=1)], + ) + conn.executemany( + "INSERT INTO test_execution_file_fp VALUES (?, ?)", + [(index, index) for index, _nodeid in enumerate(expected[:-1], start=1)], + ) + _write_run_receipt(tmp_path, "run-mixed") receipt = _finalize_testmon_seed_attempt( prepared={ "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, "status": "running", - "identity": {"git_head": "head"}, + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + **_testmon_runtime_identity_fields(Path.cwd()), + }, "resume": False, "expected_nodeids": [], "run_id": "run-mixed", - "artifact_dir": str(tmp_path / "run-mixed"), + "artifact_dir": ".cache/verify/runs/run-mixed", }, step_results=[ { @@ -661,6 +955,65 @@ def test_seed_node_outcomes_preserve_interrupted_active_node(tmp_path: Path) -> assert outcomes[0]["outcome"] == "interrupted" +def test_seed_node_outcomes_accept_setup_skip_as_terminal_skip(tmp_path: Path) -> None: + events = tmp_path / "events.jsonl" + events.write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/test_a.py::test_setup_skip", + "when": "setup", + "outcome": "skipped", + } + ) + + "\n" + ) + + outcomes = _seed_node_outcomes_from_events( + events, + expected_nodeids=["tests/test_a.py::test_setup_skip"], + database={"node_outcomes": {"tests/test_a.py::test_setup_skip": "missing"}}, + pytest_step={}, + use_database_fallback=False, + ) + + assert outcomes == [ + { + "nodeid": "tests/test_a.py::test_setup_skip", + "outcome": "skipped", + "reason": "test setup or teardown skipped", + "started": False, + "finished": False, + "phases": [{"when": "setup", "outcome": "skipped", "duration_s": None}], + } + ] + + +def test_resumed_seed_carries_forward_prior_terminal_outcome(tmp_path: Path) -> None: + events = tmp_path / "events.jsonl" + events.write_text( + json.dumps( + {"event": "test_report", "nodeid": "tests/test_a.py::test_repaired", "when": "call", "outcome": "passed"} + ) + + "\n" + ) + outcomes = _seed_node_outcomes_from_events( + events, + expected_nodeids=["tests/test_a.py::test_repaired", "tests/test_b.py::test_prior"], + database={"node_outcomes": {"tests/test_b.py::test_prior": "passed"}}, + pytest_step={}, + use_database_fallback=False, + prior_node_outcomes={ + "tests/test_b.py::test_prior": {"nodeid": "tests/test_b.py::test_prior", "outcome": "passed"} + }, + ) + + assert {item["nodeid"]: item["outcome"] for item in outcomes} == { + "tests/test_a.py::test_repaired": "passed", + "tests/test_b.py::test_prior": "passed", + } + + def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) artifact_dir = tmp_path / "artifacts" @@ -676,26 +1029,52 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon } ) ) - (artifact_dir / "events.jsonl").write_text("") + (artifact_dir / "events.jsonl").write_text( + "".join( + json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) + "\n" + for nodeid in expected + ) + ) TESTMON_DATA.parent.mkdir(parents=True) with sqlite3.connect(TESTMON_DATA) as conn: + conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") + conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") conn.execute( "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" ) + conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") conn.executemany( "INSERT INTO test_execution(test_name, failed) VALUES (?, 0)", [(nodeid,) for nodeid in expected], ) + conn.executemany( + "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", + [(index, f"file-{index}.py", f"sha-{index}") for index, _nodeid in enumerate(expected, start=1)], + ) + conn.executemany( + "INSERT INTO test_execution_file_fp VALUES (?, ?)", + [(index, index) for index, _nodeid in enumerate(expected, start=1)], + ) + _write_run_receipt(tmp_path, "run-1") + _write_run_receipt(tmp_path, "run-stale-db") + _write_run_receipt(tmp_path, "run-orphaned") receipt = _finalize_testmon_seed_attempt( prepared={ "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, "status": "running", - "identity": {"git_head": "head"}, + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + **_testmon_runtime_identity_fields(Path.cwd()), + }, "resume": False, "expected_nodeids": [], "run_id": "run-1", - "artifact_dir": str(tmp_path / "run-1"), + "artifact_dir": ".cache/verify/runs/run-1", }, step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 0}], exit_code=0, @@ -704,8 +1083,176 @@ def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, mon assert receipt["status"] == "complete" assert receipt["expected_count"] == 2 stamp = json.loads((tmp_path / ".cache" / "testmon" / "seed.json").read_text()) - assert stamp["status"] == "complete" - assert stamp["expected_count"] == 2 + assert stamp["status"] == "usable" + assert stamp["collection"]["expected_count"] == 2 + + _write_run_receipt(tmp_path, "run-authorized") + authorized_receipt = _finalize_testmon_seed_attempt( + prepared={ + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "git_tree": "tree-hash", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + "terminal_authorization": "narrow-terminal", + **_testmon_runtime_identity_fields(Path.cwd()), + }, + "resume": False, + "expected_nodeids": [], + "run_id": "run-authorized", + "artifact_dir": ".cache/verify/runs/run-authorized", + }, + step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 0}], + exit_code=0, + ) + assert authorized_receipt["status"] == "complete" + assert authorized_receipt["release_baseline_allowed"] is True + + _write_run_receipt(tmp_path, "run-red") + with sqlite3.connect(TESTMON_DATA) as connection: + connection.execute("update test_execution set failed = 1 where test_name = ?", (expected[0],)) + red_receipt = _finalize_testmon_seed_attempt( + prepared={ + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + **_testmon_runtime_identity_fields(Path.cwd()), + }, + "resume": False, + "expected_nodeids": [], + "run_id": "run-red", + "artifact_dir": ".cache/verify/runs/run-red", + }, + step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 1}], + exit_code=1, + ) + assert red_receipt["status"] == "reusable" + assert red_receipt["release_baseline_allowed"] is False + persisted_attempt = json.loads((tmp_path / ".cache" / "testmon" / "seed-attempt.json").read_text()) + assert persisted_attempt["release_baseline_allowed"] is False + assert not (tmp_path / ".cache" / "testmon" / "seed.json").exists() + + (artifact_dir / "events.jsonl").write_text("") + stale_database = _finalize_testmon_seed_attempt( + prepared={ + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "resume": False, + "expected_nodeids": [], + "run_id": "run-stale-db", + "artifact_dir": ".cache/verify/runs/run-stale-db", + }, + step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + assert stale_database["status"] == "incomplete" + + with sqlite3.connect(TESTMON_DATA) as connection: + connection.execute("insert into test_execution_file_fp values (999, 1)") + orphaned = _finalize_testmon_seed_attempt( + prepared={ + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": True, + "lab": False, + }, + "resume": False, + "expected_nodeids": [], + "run_id": "run-orphaned", + "artifact_dir": ".cache/verify/runs/run-orphaned", + }, + step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + assert orphaned["status"] == "incomplete" + + +def test_resumed_seed_persists_full_selection_before_stamp_publication( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir() + (artifact_dir / "selection.json").write_text( + json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) + ) + (artifact_dir / "events.jsonl").write_text( + json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" + ) + TESTMON_DATA.parent.mkdir(parents=True) + with sqlite3.connect(TESTMON_DATA) as connection: + connection.execute("create table environment (id integer primary key, environment_name text)") + connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") + connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") + connection.execute("create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)") + connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) + connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) + connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) + _write_run_receipt(tmp_path, "resumed") + prepared = { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "git_tree": "tree-hash", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + "terminal_authorization": None, + **_testmon_runtime_identity_fields(Path.cwd()), + }, + "resume": True, + "expected_nodeids": expected, + "expected_count": len(expected), + "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), + "prior_node_outcomes": [{"nodeid": expected[1], "outcome": "passed"}], + "run_id": "resumed", + "artifact_dir": ".cache/verify/runs/resumed", + } + original_write = verify._atomic_write_json + + def crash_before_stamp(path: Path, payload: object) -> None: + if path == TESTMON_SEED_STAMP: + raise RuntimeError("simulated crash before seed publication") + assert isinstance(payload, dict) + original_write(path, payload) + + with patch("devtools.verify._atomic_write_json", side_effect=crash_before_stamp): + with pytest.raises(RuntimeError, match="before seed publication"): + _finalize_testmon_seed_attempt( + prepared=prepared, + step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + + persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) + assert persisted["status"] == "complete" + assert persisted["expected_count"] == len(expected) + assert persisted["selection"]["selected_count"] == len(expected) + assert persisted["selection"]["selected_nodeids_omitted"] == 0 + assert not TESTMON_SEED_STAMP.exists() def test_classify_late_sigterm_after_pytest_success_summary() -> None: @@ -1166,6 +1713,27 @@ def test_run_records_managed_basetemp_cleanup_metadata(tmp_path: Path) -> None: assert metadata["basetemp_cleanup"] == str(cleaned) +def test_run_receipt_uses_capped_pytest_command_concurrency() -> None: + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + + class UncappedPolicy: + workers = 12 + + def to_dict(self) -> dict[str, int]: + return {"workers": self.workers} + + with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, UncappedPolicy())), + patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), + patch("devtools.verify._read_pytest_report", return_value=None), + ): + rc, _elapsed, metadata = _run("pytest seed-testmon", ["pytest", "--testmon", "-n", "4"]) + + assert rc == 0 + assert metadata["pytest_runtime_policy"] == {"workers": 12} + assert metadata["workload_receipt"]["spec"]["concurrency"] == 4 + + def test_run_forces_subprocesses_to_current_checkout(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("POLYLOGUE_ROOT", "/stale/main") monkeypatch.setenv("POLYLOGUE_REPO_ROOT", "/stale/main") @@ -1579,6 +2147,36 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo assert '"exit_code": 1' in payload +@pytest.mark.parametrize( + ("argv", "expected_scope", "expected_permission"), + [ + (["--all", "--skip-slow"], "narrow-terminal", False), + (["--all", "--skip-slow", "--terminal-authorization", "narrow-terminal"], "narrow-terminal", True), + ], +) +def test_verify_main_types_skip_slow_terminal_authority( + capsys: pytest.CaptureFixture[str], argv: list[str], expected_scope: str, expected_permission: bool +) -> None: + def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: + del label, command, kwargs + return 0, 0.01, {} + + with ( + patch("devtools.verify._run", side_effect=fake_run), + patch("devtools.verify.build_verify_steps", return_value=[("pytest full", ["pytest"])]), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + ): + assert main([*argv, "--json"]) == 0 + + payload = json.loads(capsys.readouterr().out) + assert payload["verification_scope"] == expected_scope + assert payload["release_baseline_allowed"] is expected_permission + assert payload["terminal_authorization"] == ("narrow-terminal" if expected_permission else None) + + def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.CaptureFixture[str]) -> None: with ( patch("devtools.verify.build_verify_steps", side_effect=PytestResourceError("only 0.50 GiB available")), @@ -1593,6 +2191,16 @@ def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.Ca assert "only 0.50 GiB available" in capsys.readouterr().err +def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(ROOT / "devtools") + + _anchor_verification_paths() + + assert Path.cwd() == ROOT.resolve() + + def test_verify_rejects_zero_testmon_selection_for_executable_change( capsys: pytest.CaptureFixture[str], ) -> None: @@ -1646,19 +2254,8 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo def test_testmon_coverage_receipts_are_content_exact() -> None: paths = ("polylogue/example.py",) - TESTMON_SEED_STAMP.parent.mkdir(parents=True, exist_ok=True) - TESTMON_SEED_STAMP.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "complete", - "identity": {"worktree_fingerprint": "covered"}, - } - ) - ) - - with patch("devtools.verify._worktree_fingerprint", return_value="covered"): - assert _matching_testmon_coverage(paths) == "complete_seed" + _write_real_testmon_state() + assert _matching_testmon_coverage(paths) is None TESTMON_SEED_STAMP.unlink() with patch("devtools.verify._worktree_fingerprint", return_value="affected"): @@ -1674,6 +2271,10 @@ def test_testmon_coverage_receipts_are_content_exact() -> None: with patch("devtools.verify._worktree_fingerprint", return_value="changed"): assert _matching_testmon_coverage(paths) is None + TESTMON_AFFECTED_STAMP.write_text(json.dumps({"identity": {"worktree_fingerprint": "affected"}})) + with patch("devtools.verify._worktree_fingerprint", return_value="affected"): + assert _matching_testmon_coverage(paths) is None + def test_failed_step_stop_policy_distinguishes_cheap_and_heavy_steps() -> None: assert _stop_after_failed_step("ruff check") is False