diff --git a/TESTING.md b/TESTING.md index 1d5eda8c40..77eeee105a 100644 --- a/TESTING.md +++ b/TESTING.md @@ -125,18 +125,23 @@ stale per-run dirs from every known root (`/dev/shm`, `/realm/tmp/polylogue-pyte `/tmp/polylogue-pytest`, plus any explicit configured root) — never based on age alone: each managed basetemp carries a PID plus process-start identity, and a directory whose exact owner process is still alive is never removed -regardless of age. An owner that cannot be confirmed dead (no marker) gets a -multi-hour grace period rather than the normal ~30-minute one. The sweeper +regardless of age. A tree without a valid managed claim or whose owner cannot +be confirmed dead is never removed. The thirty-minute age threshold applies +only after a positive managed claim identifies a dead owner. The sweeper restores owner-write permission only after a tree is adjudicated stale, so published read-only fixture copies cannot leak tmpfs indefinitely. Shared `pytest-polylogue-*-seeded-*` caches are never touched by the sweep — they are shared, reused, and built once behind their own `.build.done` guard. Managed verification refuses to start below 1 GiB available memory instead of -falling back to the pathological disk lane. Passing-test roots are reclaimed at -teardown; the external supervisor and parent runner independently remove the -whole run root on completion or termination, with startup stale-root cleanup as -recovery after an uncatchable process kill or reboot. +falling back to the pathological disk lane. Every per-test `tmp_path` tree is +reclaimed in fixture teardown, including failures and interruptions; node +failure evidence remains in the managed event, longrepr, summary, and resource +receipts. The controller removes only the exact basetemp it created. An +explicit `--basetemp` is retained for targeted filesystem diagnosis. The +external supervisor and parent runner independently remove the whole run root +on completion or termination, with startup stale-root cleanup as recovery +after an uncatchable process kill or reboot. An affected run that selects zero tests is accepted only when no executable, test, dependency, or harness path changed. A zero selection after such a change @@ -163,6 +168,9 @@ and a postmortem diagnosis. The latest run is mirrored to - `.cache/verify/current-pytest-resources.jsonl` - `.cache/verify/current-pytest-postmortem.json` - `.cache/verify/current-pytest-containment.json` +- `.cache/verify/current-pytest-statistics.json` — derived phase + distributions, worker count, storage, resource peaks, and cleanup outcome; + the same file is retained under each run's `steps/*/statistics.json`. - `.cache/verify/current-pytest-output.log` The devtools process drains pytest output, prints periodic heartbeat lines, and @@ -195,9 +203,13 @@ IDs by default (`POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT`, default 500) so broad collection does not retain or write unbounded node-id lists in controller or worker processes. -`devtools workspace tasks recent` shows the run id, diagnosis, and peak pytest -RSS when the current run metadata is available. `devtools workspace tasks stats ---resources` aggregates recorded pytest memory peaks over time. +The detailed artifacts above are checkout-local and disposable. Each `devtools +verify` or `devtools test` invocation automatically appends its compact run +summary to `$XDG_STATE_HOME/polylogue/devtools/verify-history.jsonl` (or the +corresponding `~/.local/state` path), shared across linked worktrees without a +separate recording command. `devtools verify --history` prints the recent +cross-worktree runs. Setup, call, and teardown timings come only from pytest +reports in the event stream. `devtools test` uses the same pytest progress plugin and process supervisor for focused selections. During or after a run, inspect diff --git a/devtools/evidence_dashboard.py b/devtools/evidence_dashboard.py index 6959909d41..2f167bb49b 100644 --- a/devtools/evidence_dashboard.py +++ b/devtools/evidence_dashboard.py @@ -9,7 +9,7 @@ - pytest health from ``.cache/verify/last-pytest.json``; - coverage from ``.coverage`` / ``coverage.xml`` when present; - benchmark/SLO catalog rows and their required-artifact coverage; -- static gate status from ``.cache/verify-history.jsonl``; +- static gate status from the shared XDG verify history; - witness lifecycle counts; - mutation/benchmark campaign freshness. @@ -28,12 +28,13 @@ from typing import Any from devtools import repo_root as _get_root +from devtools.verify_runs import VERIFY_HISTORY_PATH, git_dirty, git_head +from devtools.verify_runs import worktree_fingerprint as _worktree_fingerprint ROOT = _get_root() # Artifact paths (relative to repo root). PYTEST_REPORT_REL = Path(".cache/verify/last-pytest.json") -VERIFY_HISTORY_REL = Path(".cache/verify-history.jsonl") LAST_VERIFY_RESULT_REL = Path(".cache/last-verify-result.json") COVERAGE_DATA_REL = Path(".coverage") COVERAGE_XML_REL = Path("coverage.xml") @@ -224,9 +225,37 @@ def _benchmark_slo(root: Path, *, now: datetime) -> dict[str, Any]: ) +def _static_evidence_is_bound( + entry: dict[str, Any], + *, + checkout_root: str, + checkout_head: str | None, + worktree_fingerprint: str, +) -> bool: + """Accept only evidence tied to the exact checkout contents being viewed.""" + steps = entry.get("steps") + stability_failed = isinstance(steps, list) and any( + isinstance(step, dict) and step.get("name") == "checkout stability" and step.get("exit") != 0 for step in steps + ) + return ( + not stability_failed + and entry.get("diagnosis") not in {"checkout_changed_during_verification", "checkout_fingerprint_unavailable"} + and entry.get("checkout_root") == checkout_root + and entry.get("git_head") == checkout_head + and entry.get("worktree_fingerprint") == worktree_fingerprint + and entry.get("final_worktree_fingerprint") == worktree_fingerprint + ) + + def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: - history_path = root / VERIFY_HISTORY_REL + history_path = VERIFY_HISTORY_PATH last_result_path = root / LAST_VERIFY_RESULT_REL + checkout_root = str(root.resolve()) + checkout_head = git_head(root) + checkout_dirty = git_dirty(root) + fingerprint = None if checkout_dirty or checkout_head is None else _worktree_fingerprint(root) + worktree_fingerprint = None if fingerprint == "unavailable" else fingerprint + identity_available = checkout_head is not None and worktree_fingerprint is not None # Prefer last-verify-result.json (the most recent run) then walk back through # history to find the last status for each gate. @@ -236,7 +265,16 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: try: data = json.loads(last_result_path.read_text()) result = data.get("result") if isinstance(data, dict) else None - if isinstance(result, dict): + if ( + isinstance(result, dict) + and worktree_fingerprint is not None + and _static_evidence_is_bound( + result, + checkout_root=checkout_root, + checkout_head=checkout_head, + worktree_fingerprint=worktree_fingerprint, + ) + ): for step in result.get("steps", []): if isinstance(step, dict) and isinstance(step.get("name"), str): last_steps[step["name"]] = step @@ -263,6 +301,13 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: # appearance in history. if history_entries: for entry in reversed(history_entries): + if worktree_fingerprint is None or not _static_evidence_is_bound( + entry, + checkout_root=checkout_root, + checkout_head=checkout_head, + worktree_fingerprint=worktree_fingerprint, + ): + continue steps = entry.get("steps", []) for step in steps: if not isinstance(step, dict): @@ -275,7 +320,14 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: for gate_name in _STATIC_GATE_NAMES: step = last_steps.get(gate_name) if step is None: - gates.append({"name": gate_name, "available": False, "reason": "no run observed in cached history"}) + reason = ( + "checkout has uncommitted changes" + if checkout_dirty + else "checkout Git identity is unavailable" + if not identity_available + else "no bound run observed in cached history" + ) + gates.append({"name": gate_name, "available": False, "reason": reason}) continue exit_code = step.get("exit", -1) gates.append( @@ -290,8 +342,8 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: ) failing = [g for g in gates if g.get("status") == "fail"] return { - "available": last_result_path.exists() or history_path.exists(), - "history_path": str(VERIFY_HISTORY_REL), + "available": bool(last_steps) and not checkout_dirty and identity_available, + "history_path": str(history_path), "last_result_path": str(LAST_VERIFY_RESULT_REL), "total_gates_tracked": len(_STATIC_GATE_NAMES), "gates_with_status": sum(1 for g in gates if g.get("available")), diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index aefce1e8e7..b7f4f90b6d 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -73,6 +73,7 @@ import sys import tempfile import time +import uuid from collections.abc import Iterator, Mapping from pathlib import Path from typing import Any @@ -608,7 +609,13 @@ def cmd_merge( return 1 head_sha = info["headRefOid"] - scope = merge_gate._scope_verdict(pr, info, head_sha=head_sha) + checkout_root = merge_gate._repository_root() + scope = merge_gate._scope_verdict( + pr, + info, + head_sha=head_sha, + checkout_root=checkout_root, + ) if not scope.ok: print(f"REFUSING to merge PR #{pr}: invalid structured pr-scope carrier:", file=sys.stderr) @@ -666,7 +673,12 @@ def cmd_merge( file=sys.stderr, ) return 1 - final_scope = merge_gate._scope_verdict(pr, final_info, head_sha=head_sha) + final_scope = merge_gate._scope_verdict( + pr, + final_info, + head_sha=head_sha, + checkout_root=checkout_root, + ) initial_attestation = pr_scope.attestation_payload( scope, head_sha=head_sha, base_sha=merge_gate._base_sha(info) ).get("attestation_digest") @@ -798,20 +810,30 @@ def cmd_record_full_verify( if execution_root is not None: argv = ["direnv", "exec", str(execution_root), *argv] started = verification_started_at - try: - result = subprocess.run(argv, capture_output=True, text=True, cwd=cwd) - except OSError as exc: - print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) - return 2 + checkout_root = Path(cwd) if cwd is not None else Path.cwd() + invocation_id = uuid.uuid4().hex + with tempfile.TemporaryDirectory(prefix="polylogue-terminal-verify-") as temp_dir: + receipt_path = Path(temp_dir) / "run.json" + env = dict(os.environ) + env[merge_gate.VERIFICATION_INVOCATION_ID_ENV] = invocation_id + env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV] = str(receipt_path) + try: + result = subprocess.run(argv, capture_output=True, text=True, cwd=checkout_root, env=env) + except OSError as exc: + print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) + return 2 + receipt = merge_gate._invocation_receipt( + path=receipt_path, + invocation_id=invocation_id, + head_sha=target_sha, + command_exit=result.returncode, + checkout_root=checkout_root, + ) duration_s = round(time.time() - started, 2) - release_allowed = merge_gate._release_baseline_permission(result.stdout) - verification_scope = merge_gate._verification_scope(result.stdout) - terminal_authorization = merge_gate._terminal_authorization(result.stdout) - try: - structured = json.loads(result.stdout) - except (TypeError, json.JSONDecodeError): - structured = None - verified_head = structured.get("git_head") if isinstance(structured, dict) else None + release_allowed = merge_gate._release_baseline_permission(receipt) + verification_scope = merge_gate._verification_scope(receipt) + terminal_authorization = merge_gate._terminal_authorization(receipt) + verified_head = receipt.get("git_head") if isinstance(receipt, dict) else None accepted = ( result.returncode == 0 and release_allowed is True diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index 198cf027b2..427038007b 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -70,16 +70,22 @@ import argparse import json +import os import shlex import subprocess import sys +import tempfile import time +import uuid +from collections.abc import Mapping from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any from devtools import pr_scope from devtools.testmon_state import TerminalAuthorization, VerificationScope +from devtools.verify_runs import VERIFICATION_INVOCATION_ID_ENV as VERIFICATION_INVOCATION_ID_ENV +from devtools.verify_runs import VERIFICATION_RECEIPT_PATH_ENV as VERIFICATION_RECEIPT_PATH_ENV _RECEIPT_DIR = Path(".cache/verify/merge-gate") _DEFAULT_MAX_AGE_S = 3600 @@ -192,6 +198,18 @@ def _git_is_clean() -> bool: return result.returncode == 0 and not result.stdout.strip() +def _repository_root() -> Path: + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + timeout=15, + ) + if result.returncode == 0 and result.stdout.strip(): + return Path(result.stdout.strip()).resolve(strict=False) + return Path.cwd().resolve(strict=False) + + @dataclass class GateVerdict: pr: int @@ -205,11 +223,11 @@ class GateVerdict: def _receipt_path(pr: int) -> Path: - return _RECEIPT_DIR / f"pr-{pr}.json" + return _repository_root() / _RECEIPT_DIR / f"pr-{pr}.json" def _ack_path(pr: int) -> Path: - return _RECEIPT_DIR / f"pr-{pr}-acks.json" + return _repository_root() / _RECEIPT_DIR / f"pr-{pr}-acks.json" def _command_skips_tests(command: str) -> bool: @@ -219,37 +237,62 @@ def _command_skips_tests(command: str) -> bool: return not any(marker in lowered for marker in _LOOKS_LIKE_TESTS_MARKERS) -def _release_baseline_permission(stdout: str) -> bool | None: - """Read the structured verify decision when the command emitted one.""" +def _invocation_receipt( + *, + path: Path, + invocation_id: str, + head_sha: str, + command_exit: int, + checkout_root: Path, +) -> dict[str, Any] | None: + """Load the exact run artifact bound to the launched verifier process.""" + try: - payload = json.loads(stdout) - except (TypeError, json.JSONDecodeError): + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): return None if not isinstance(payload, dict): return None + if ( + payload.get("invocation_id") != invocation_id + or payload.get("git_head") != head_sha + or payload.get("exit_code") != command_exit + ): + return None + recorded_root = payload.get("checkout_root") + if not isinstance(recorded_root, str) or Path(recorded_root).resolve(strict=False) != checkout_root.resolve( + strict=False + ): + return None + if _verification_scope(payload) is None or _release_baseline_permission(payload) is None: + return None + terminal_authorization = payload.get("terminal_authorization") + if terminal_authorization is not None and terminal_authorization not in { + authorization.value for authorization in TerminalAuthorization + }: + return None + return payload + + +def _release_baseline_permission(payload: Mapping[str, Any] | None) -> bool | None: + """Read the typed release decision from an invocation-bound receipt.""" + if payload is None: + return None value = payload.get("release_baseline_allowed") return value if isinstance(value, bool) else None -def _verification_scope(stdout: str) -> str | None: - """Read the typed verification scope from a structured verify receipt.""" - try: - payload = json.loads(stdout) - except (TypeError, json.JSONDecodeError): - return None - if not isinstance(payload, dict): +def _verification_scope(payload: Mapping[str, Any] | None) -> str | None: + """Read the typed verification scope from an invocation-bound receipt.""" + if payload is None: return None value = payload.get("verification_scope") return value if value in {scope.value for scope in VerificationScope} else None -def _terminal_authorization(stdout: str) -> str | None: - """Read the typed terminal authorization from a structured receipt.""" - try: - payload = json.loads(stdout) - except (TypeError, json.JSONDecodeError): - return None - if not isinstance(payload, dict): +def _terminal_authorization(payload: Mapping[str, Any] | None) -> str | None: + """Read terminal authorization from an invocation-bound receipt.""" + if payload is None: return None value = payload.get("terminal_authorization") return value if value in {authorization.value for authorization in TerminalAuthorization} else None @@ -261,7 +304,13 @@ def _base_sha(info: dict[str, Any]) -> str | None: return value if isinstance(value, str) else None -def _scope_verdict(pr: int, info: dict[str, Any], *, head_sha: str) -> pr_scope.ScopeVerdict: +def _scope_verdict( + pr: int, + info: dict[str, Any], + *, + head_sha: str, + checkout_root: Path, +) -> pr_scope.ScopeVerdict: """Use the same carrier or typed bot exception for record and check.""" author = info.get("author") files = info.get("files") @@ -284,6 +333,7 @@ def _scope_verdict(pr: int, info: dict[str, Any], *, head_sha: str) -> pr_scope. info.get("body") or "", head_sha=head_sha, is_draft=bool(info.get("isDraft")), + beads_path=checkout_root / ".beads" / "issues.jsonl", base_sha=_base_sha(info), ) @@ -291,6 +341,7 @@ def _scope_verdict(pr: int, info: dict[str, Any], *, head_sha: str) -> pr_scope. def cmd_record(pr: int, command: str) -> int: info = _gh_json(["pr", "view", str(pr), "--json", "headRefOid,headRefName,baseRefOid,body,isDraft,author,files"]) head_sha = info["headRefOid"] + checkout_root = _repository_root() local_head = _git_head_sha() if local_head != head_sha: @@ -310,7 +361,7 @@ def cmd_record(pr: int, command: str) -> int: ) return 2 - scope = _scope_verdict(pr, info, head_sha=head_sha) + scope = _scope_verdict(pr, info, head_sha=head_sha, checkout_root=checkout_root) if not scope.ok: print(f"REFUSING to record: PR #{pr} has an invalid structured pr-scope carrier:", file=sys.stderr) for reason in scope.reasons: @@ -321,12 +372,25 @@ def cmd_record(pr: int, command: str) -> int: if not argv: print("REFUSING to record: --command is empty after shell splitting.", file=sys.stderr) return 2 + invocation_id = uuid.uuid4().hex started = time.time() - try: - result = subprocess.run(argv, capture_output=True, text=True) - except OSError as exc: - print(f"REFUSING to record: could not run {command!r}: {exc}", file=sys.stderr) - return 2 + with tempfile.TemporaryDirectory(prefix="polylogue-merge-gate-") as temp_dir: + receipt_path = Path(temp_dir) / "run.json" + env = dict(os.environ) + env[VERIFICATION_INVOCATION_ID_ENV] = invocation_id + env[VERIFICATION_RECEIPT_PATH_ENV] = str(receipt_path) + try: + result = subprocess.run(argv, capture_output=True, text=True, cwd=checkout_root, env=env) + except OSError as exc: + print(f"REFUSING to record: could not run {command!r}: {exc}", file=sys.stderr) + return 2 + verification_receipt = _invocation_receipt( + path=receipt_path, + invocation_id=invocation_id, + head_sha=head_sha, + command_exit=result.returncode, + checkout_root=checkout_root, + ) duration_s = round(time.time() - started, 2) receipt = { @@ -344,17 +408,18 @@ def cmd_record(pr: int, command: str) -> int: "branch": info["headRefName"], "command": command, "skips_tests": _command_skips_tests(command), - "verification_scope": _verification_scope(result.stdout), - "release_baseline_allowed": _release_baseline_permission(result.stdout), - "terminal_authorization": _terminal_authorization(result.stdout), + "verification_scope": _verification_scope(verification_receipt), + "release_baseline_allowed": _release_baseline_permission(verification_receipt), + "terminal_authorization": _terminal_authorization(verification_receipt), "exit_code": result.returncode, "duration_s": duration_s, "recorded_at": time.time(), "stdout_tail": result.stdout[-4000:], "stderr_tail": result.stderr[-4000:], } - _RECEIPT_DIR.mkdir(parents=True, exist_ok=True) - _receipt_path(pr).write_text(json.dumps(receipt, indent=2)) + receipt_path = _receipt_path(pr) + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt, indent=2)) print(f"recorded receipt for PR #{pr} @ {head_sha[:8]}: exit={result.returncode} ({duration_s}s)") if receipt["skips_tests"]: @@ -385,7 +450,7 @@ def cmd_ack(pr: int, comment_id: int, *, reason: str) -> int: else: acks = {} acks[str(comment_id)] = {"head_sha": head_sha, "reason": reason, "acked_at": time.time()} - _RECEIPT_DIR.mkdir(parents=True, exist_ok=True) + ack_path.parent.mkdir(parents=True, exist_ok=True) ack_path.write_text(json.dumps(acks, indent=2)) print(f"acknowledged comment {comment_id} on PR #{pr} @ {head_sha[:8]}: {reason}") return 0 @@ -498,7 +563,7 @@ def cmd_check( "current checkout has uncommitted changes; merge-gate check requires committed PR content" ) - scope = _scope_verdict(pr, info, head_sha=head_sha) + scope = _scope_verdict(pr, info, head_sha=head_sha, checkout_root=_repository_root()) verdict.pr_scope = asdict(scope) if not scope.ok: verdict.ok = False diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index f1c07c6c85..6abb073d46 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -12,6 +12,8 @@ import json import os import time +import uuid +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -27,10 +29,97 @@ _DESELECTED_COUNT = 0 _SELECTED_COUNT = 0 _SLOWEST_REPORTS: list[dict[str, Any]] = [] +_RECORDED_REPORT_KEYS: set[tuple[int, str, str, str, float]] = set() _COLLECTION_STARTED_AT: float | None = None _COLLECTION_DURATION_S: float | None = None +_CONTROLLER_COLLECTION_PAYLOAD: dict[str, Any] | None = None _SLOW_REPORT_LIMIT = 20 _DEFAULT_SELECTION_NODEID_LIMIT = 500 +_COLLECTION_FACT_SUFFIX = ".collection.json" +_ARTIFACT_ENV_NAMES = (_EVENTS_ENV, _EVENTS_DIR_ENV, _SELECTION_ENV, _SUMMARY_ENV) + + +@dataclass +class _SessionState: + deselected_nodeids_sample: list[str] + deselected_count: int + selected_count: int + slowest_reports: list[dict[str, Any]] + recorded_report_keys: set[tuple[int, str, str, str, float]] + collection_started_at: float | None + collection_duration_s: float | None + controller_collection_payload: dict[str, Any] | None + artifact_environment: dict[str, str | None] + + +_SESSION_STATE_STACK: list[_SessionState] = [] + + +def _capture_session_state() -> _SessionState: + return _SessionState( + deselected_nodeids_sample=list(_DESELECTED_NODEIDS_SAMPLE), + deselected_count=_DESELECTED_COUNT, + selected_count=_SELECTED_COUNT, + slowest_reports=list(_SLOWEST_REPORTS), + recorded_report_keys=set(_RECORDED_REPORT_KEYS), + collection_started_at=_COLLECTION_STARTED_AT, + collection_duration_s=_COLLECTION_DURATION_S, + controller_collection_payload=( + dict(_CONTROLLER_COLLECTION_PAYLOAD) if _CONTROLLER_COLLECTION_PAYLOAD else None + ), + artifact_environment={name: os.environ.get(name) for name in _ARTIFACT_ENV_NAMES}, + ) + + +def _restore_session_state(state: _SessionState) -> None: + global _COLLECTION_STARTED_AT, _COLLECTION_DURATION_S, _CONTROLLER_COLLECTION_PAYLOAD + global _DESELECTED_COUNT, _SELECTED_COUNT + _DESELECTED_NODEIDS_SAMPLE[:] = state.deselected_nodeids_sample + _DESELECTED_COUNT = state.deselected_count + _SELECTED_COUNT = state.selected_count + _SLOWEST_REPORTS[:] = state.slowest_reports + _RECORDED_REPORT_KEYS.clear() + _RECORDED_REPORT_KEYS.update(state.recorded_report_keys) + _COLLECTION_STARTED_AT = state.collection_started_at + _COLLECTION_DURATION_S = state.collection_duration_s + _CONTROLLER_COLLECTION_PAYLOAD = state.controller_collection_payload + for name, value in state.artifact_environment.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def _reset_session_state() -> None: + global _COLLECTION_STARTED_AT, _COLLECTION_DURATION_S, _CONTROLLER_COLLECTION_PAYLOAD + global _DESELECTED_COUNT, _SELECTED_COUNT + _DESELECTED_NODEIDS_SAMPLE.clear() + _DESELECTED_COUNT = 0 + _SELECTED_COUNT = 0 + _SLOWEST_REPORTS.clear() + _RECORDED_REPORT_KEYS.clear() + _COLLECTION_STARTED_AT = None + _COLLECTION_DURATION_S = None + _CONTROLLER_COLLECTION_PAYLOAD = None + + +def _isolate_nested_artifact_destinations() -> None: + """Give an in-process nested pytest invocation its own durable evidence.""" + raw_candidates = [os.environ.get(name) for name in _ARTIFACT_ENV_NAMES] + base = next((Path(value).parent for value in raw_candidates if value), None) + if base is None: + return + root = base / f"nested-pytest-{os.getpid()}-{uuid.uuid4().hex}" + if os.environ.get(_EVENTS_DIR_ENV): + os.environ[_EVENTS_DIR_ENV] = str(root / "events") + os.environ.pop(_EVENTS_ENV, None) + elif os.environ.get(_EVENTS_ENV): + os.environ[_EVENTS_ENV] = str(root / "events.jsonl") + os.environ.pop(_EVENTS_DIR_ENV, None) + if os.environ.get(_SELECTION_ENV): + os.environ[_SELECTION_ENV] = str(root / "selection.json") + if os.environ.get(_SUMMARY_ENV): + os.environ[_SUMMARY_ENV] = str(root / "summary.json") def _selection_nodeid_limit() -> int: @@ -84,6 +173,70 @@ def _write_selection(payload: dict[str, Any]) -> None: tmp.replace(path) +def _write_worker_collection_fact(payload: dict[str, Any]) -> None: + """Publish one worker-local collection fact for controller aggregation.""" + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + raw_dir = os.environ.get(_EVENTS_DIR_ENV) + if not worker_id or not raw_dir: + return + path = Path(raw_dir) / f"{worker_id.replace('/', '-')}-{os.getpid()}{_COLLECTION_FACT_SUFFIX}" + payload = {"worker_id": worker_id, "pid": os.getpid(), **payload} + with contextlib.suppress(OSError): + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + tmp.replace(path) + + +def _worker_collection_payloads(events_dir: Path | None = None) -> list[dict[str, Any]]: + """Read worker collection facts in a stable order for the controller.""" + if events_dir is None: + raw_dir = os.environ.get(_EVENTS_DIR_ENV) + if not raw_dir: + return [] + events_dir = Path(raw_dir) + payloads: list[tuple[str, int, str, dict[str, Any]]] = [] + for path in events_dir.glob(f"*{_COLLECTION_FACT_SUFFIX}"): + with contextlib.suppress(OSError, json.JSONDecodeError): + payload = json.loads(path.read_text(encoding="utf-8")) + worker_id = payload.get("worker_id") + pid = payload.get("pid") + if isinstance(worker_id, str) and isinstance(pid, int): + payloads.append((worker_id, pid, path.name, payload)) + return [payload for _worker_id, _pid, _name, payload in sorted(payloads)] + + +def _collection_payload() -> dict[str, Any]: + """Return this process's complete collection fact.""" + limit = _selection_nodeid_limit() + payload: dict[str, Any] = { + "selected_count": _SELECTED_COUNT, + "deselected_count": _DESELECTED_COUNT, + "selected_nodeids": [], + "selected_node_markers": {}, + "selected_nodeids_omitted": _SELECTED_COUNT, + "deselected_nodeids": list(_DESELECTED_NODEIDS_SAMPLE), + "deselected_nodeids_omitted": max(0, _DESELECTED_COUNT - len(_DESELECTED_NODEIDS_SAMPLE)), + "nodeid_sample_limit": limit, + } + if _COLLECTION_DURATION_S is not None: + payload["collection_duration_s"] = _COLLECTION_DURATION_S + return payload + + +def merge_worker_collection_payloads(events_dir: Path | None = None) -> dict[str, Any] | None: + """Choose one canonical xdist collection set and the slowest wall time.""" + payloads = _worker_collection_payloads(events_dir) + if not payloads: + return None + merged = dict(payloads[0]) + durations = [payload.get("collection_duration_s") for payload in payloads] + numeric_durations = [duration for duration in durations if isinstance(duration, (int, float))] + if numeric_durations: + merged["collection_duration_s"] = max(numeric_durations) + return merged + + def _write_summary(payload: dict[str, Any]) -> None: raw_path = os.environ.get(_SUMMARY_ENV) if not raw_path: @@ -109,17 +262,25 @@ def _remember_report(payload: dict[str, Any]) -> None: del _SLOWEST_REPORTS[_SLOW_REPORT_LIMIT:] +def _durable_report_outcome(report: Any, outcome: str) -> str: + """Preserve pytest's xfail semantics in the append-only event ledger.""" + if not getattr(report, "wasxfail", None): + return outcome + if outcome == "skipped": + return "xfailed" + if outcome == "passed": + return "xpassed" + return outcome + + @pytest.hookimpl def pytest_sessionstart(session: Any) -> None: """Reset per-session ledgers when tests invoke pytest in-process.""" del session - global _COLLECTION_STARTED_AT, _COLLECTION_DURATION_S, _DESELECTED_COUNT, _SELECTED_COUNT - _DESELECTED_NODEIDS_SAMPLE.clear() - _DESELECTED_COUNT = 0 - _SELECTED_COUNT = 0 - _SLOWEST_REPORTS.clear() - _COLLECTION_STARTED_AT = None - _COLLECTION_DURATION_S = None + _SESSION_STATE_STACK.append(_capture_session_state()) + _reset_session_state() + if len(_SESSION_STATE_STACK) > 1: + _isolate_nested_artifact_destinations() # The worker environment is assigned after process exec, so it is not # reliably visible through /proc//environ. Emit the identity from # inside the worker for the supervisor's process-state sampler. @@ -150,7 +311,7 @@ def pytest_deselected(items: list[Any]) -> None: def pytest_collection_modifyitems(session: Any, config: Any, items: list[Any]) -> None: """Write the final selected test set after pytest/testmon deselection.""" del config - global _COLLECTION_DURATION_S, _SELECTED_COUNT + global _COLLECTION_DURATION_S, _CONTROLLER_COLLECTION_PAYLOAD, _SELECTED_COUNT if _COLLECTION_STARTED_AT is not None: _COLLECTION_DURATION_S = round(time.monotonic() - _COLLECTION_STARTED_AT, 4) _SELECTED_COUNT = len(items) @@ -165,19 +326,19 @@ def pytest_collection_modifyitems(session: Any, config: Any, items: list[Any]) - ) for item in items } - payload: dict[str, Any] = { - "selected_count": _SELECTED_COUNT, - "deselected_count": _DESELECTED_COUNT, - "selected_nodeids": selected_nodeids, - "selected_node_markers": selected_node_markers, - "selected_nodeids_omitted": max(0, _SELECTED_COUNT - len(selected_nodeids)), - "deselected_nodeids": list(_DESELECTED_NODEIDS_SAMPLE), - "deselected_nodeids_omitted": max(0, _DESELECTED_COUNT - len(_DESELECTED_NODEIDS_SAMPLE)), - "nodeid_sample_limit": limit, - } - if _COLLECTION_DURATION_S is not None: - payload["collection_duration_s"] = _COLLECTION_DURATION_S - _write_selection(payload) + payload = _collection_payload() + payload.update( + { + "selected_nodeids": selected_nodeids, + "selected_node_markers": selected_node_markers, + "selected_nodeids_omitted": max(0, _SELECTED_COUNT - len(selected_nodeids)), + } + ) + if os.environ.get("PYTEST_XDIST_WORKER"): + _write_worker_collection_fact(payload) + else: + _CONTROLLER_COLLECTION_PAYLOAD = dict(payload) + _write_selection(payload) _write_event( { "event": "collection_finished", @@ -213,10 +374,16 @@ def pytest_runtest_logfinish(nodeid: str, location: tuple[str, int | None, str]) @pytest.hookimpl -def pytest_runtest_logreport(report: Any) -> None: +def _record_phase_report(report: Any, *, write_event: bool = True) -> None: """Append one phase report so slow setup/call/teardown remains visible.""" when = str(getattr(report, "when", "")) - outcome = str(getattr(report, "outcome", "")) + nodeid = str(getattr(report, "nodeid", "")) + outcome = _durable_report_outcome(report, str(getattr(report, "outcome", ""))) + duration = float(getattr(report, "duration", 0.0) or 0.0) + report_key = (id(report), when, nodeid, outcome, duration) + if report_key in _RECORDED_REPORT_KEYS: + return + _RECORDED_REPORT_KEYS.add(report_key) if when not in {"setup", "call", "teardown"}: return payload = { @@ -224,24 +391,58 @@ def pytest_runtest_logreport(report: Any) -> None: "nodeid": str(getattr(report, "nodeid", "")), "when": when, "outcome": outcome, - "duration_s": round(float(getattr(report, "duration", 0.0) or 0.0), 4), + "duration_s": round(duration, 4), } if payload["outcome"] == "failed": payload["longrepr"] = str(getattr(report, "longrepr", "")) _remember_report(payload) - _write_event(payload) + if write_event: + _write_event(payload) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: Any, call: Any) -> Any: + """Capture the phase report before other reporting plugins transform it.""" + del item, call + outcome = yield + _record_phase_report(outcome.get_result()) + + +@pytest.hookimpl +def pytest_runtest_logreport(report: Any) -> None: + """Retain the direct/log-hook fallback used by older pytest plugins/tests.""" + # xdist forwards each worker's report to the controller. The worker has + # already written the authoritative shard event through makereport. Keep + # its timing in the controller's summary, but do not duplicate the ledger. + if not os.environ.get("PYTEST_XDIST_WORKER") and getattr(report, "worker_id", None): + _record_phase_report(report, write_event=False) + return + _record_phase_report(report) @pytest.hookimpl def pytest_sessionfinish(session: Any, exitstatus: int) -> None: """Write a compact post-run diagnosis artifact independent of pytest-json-report.""" del session - payload: dict[str, Any] = { - "exitstatus": int(exitstatus), - "selected_count": _SELECTED_COUNT, - "deselected_count": _DESELECTED_COUNT, - "slowest_reports": list(_SLOWEST_REPORTS), - } - if _COLLECTION_DURATION_S is not None: - payload["collection_duration_s"] = _COLLECTION_DURATION_S - _write_summary(payload) + try: + # Worker processes have their own in-memory slowest lists. The controller + # receives the forwarded timings and is the only writer for the shared + # summary path, so an empty worker summary cannot overwrite it. + if os.environ.get("PYTEST_XDIST_WORKER"): + return + collection_payload = ( + merge_worker_collection_payloads() or _CONTROLLER_COLLECTION_PAYLOAD or _collection_payload() + ) + _write_selection(collection_payload) + payload: dict[str, Any] = { + "exitstatus": int(exitstatus), + "selected_count": collection_payload["selected_count"], + "deselected_count": collection_payload["deselected_count"], + "slowest_reports": list(_SLOWEST_REPORTS), + } + if "collection_duration_s" in collection_payload: + payload["collection_duration_s"] = collection_payload["collection_duration_s"] + _write_summary(payload) + finally: + if _SESSION_STATE_STACK: + _restore_session_state(_SESSION_STATE_STACK.pop()) diff --git a/devtools/pytest_supervisor.py b/devtools/pytest_supervisor.py index 2cb1b5a02d..265f1a47de 100644 --- a/devtools/pytest_supervisor.py +++ b/devtools/pytest_supervisor.py @@ -931,7 +931,9 @@ def main(argv: Sequence[str] | None = None) -> int: runtime_cap_s=args.runtime_cap_s, ) finally: - cleanup_complete = cleanup_managed_tmpfs_path(args.cleanup_path) + receipt = read_receipt(args.receipt) + receipt_quiescent = receipt is not None and receipt.get("controller_group_alive") is False + cleanup_complete = cleanup_managed_tmpfs_path(args.cleanup_path) if receipt_quiescent else False if args.cleanup_path is not None: with contextlib.suppress(OSError): update_receipt( diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 59948062c7..5c1ac8d8d7 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -47,10 +47,150 @@ _clear_pytest_report, _run, ) -from devtools.verify_runs import VerifyRun, git_head +from devtools.verify_runs import ( + CheckoutMutationMonitor, + CheckoutMutationObservation, + VerifyRun, + append_verify_history, + finalize_checkout_mutation_monitors, + finish_checkout_mutation_monitor, + git_head, + pytest_command_worker_request, + start_checkout_mutation_monitor, + worktree_fingerprint, +) ROOT = Path(__file__).resolve().parent.parent _LOCK_PATH = ROOT / ".cache" / "test-run.lock" +_PATH_VALUE_OPTIONS = frozenset( + { + "-c", + "--basetemp", + "--config-file", + "--confcutdir", + "--debug", + "--ignore", + "--ignore-glob", + "--junit-xml", + "--junitxml", + "--log-file", + "--rootdir", + } +) +_ENV_EXPANDING_PATH_OPTIONS = frozenset({"--rootdir"}) +_NON_PATH_VALUE_OPTIONS = frozenset( + { + "-k", + "--keyword", + "-m", + "--mark", + "--deselect", + "--maxfail", + "--tb", + "--capture", + "--durations", + "--durations-min", + "--override-ini", + "-o", + } +) + + +def _absolute_option_path( + value: str, + *, + invocation_directory: Path, + expand_environment_variables: bool = False, +) -> str: + if expand_environment_variables: + value = os.path.expandvars(value) + path = Path(value) + # pytest deliberately uses ``os.path.abspath`` for command-line paths: + # resolving here would make ``-c config-link.ini`` select the linked + # target as its rootdir instead of preserving the caller's spelling. + return os.path.abspath(path if path.is_absolute() else invocation_directory / path) + + +def _normalize_selection_paths(selection: list[str], *, invocation_directory: Path) -> list[str]: + """Preserve path selections relative to the directory that invoked devtools.""" + normalized: list[str] = [] + pending_option: str | None = None + for argument in selection: + if pending_option is not None: + # pytest's --debug accepts an optional file name. A following + # option belongs to pytest, not to --debug's optional value. + if pending_option == "--debug" and argument.startswith("-"): + pending_option = None + elif pending_option in _PATH_VALUE_OPTIONS: + normalized.append( + _absolute_option_path( + argument, + invocation_directory=invocation_directory, + expand_environment_variables=pending_option in _ENV_EXPANDING_PATH_OPTIONS, + ) + ) + pending_option = None + continue + else: + normalized.append(argument) + pending_option = None + continue + if argument.startswith("-c="): + normalized.append( + "-c" + + _absolute_option_path( + argument[len("-c=") :], + invocation_directory=invocation_directory, + ) + ) + continue + option_name, equals, option_value = argument.partition("=") + if option_name in _PATH_VALUE_OPTIONS: + if equals: + normalized_value = _absolute_option_path( + option_value, + invocation_directory=invocation_directory, + expand_environment_variables=option_name in _ENV_EXPANDING_PATH_OPTIONS, + ) + normalized.append(f"{option_name}={normalized_value}") + else: + normalized.append(argument) + pending_option = option_name + continue + if option_name in _NON_PATH_VALUE_OPTIONS: + normalized.append(argument) + if not equals: + pending_option = option_name + continue + if argument.startswith("-c") and len(argument) > len("-c"): + normalized.append( + "-c" + + _absolute_option_path( + argument[len("-c") :], + invocation_directory=invocation_directory, + ) + ) + continue + if argument.startswith("-"): + normalized.append(argument) + continue + path_text, separator, node_suffix = argument.partition("::") + candidate = Path(path_text) + if candidate.is_absolute() or not (invocation_directory / candidate).exists(): + normalized.append(argument) + continue + resolved = (invocation_directory / candidate).resolve() + try: + anchored = resolved.relative_to(ROOT).as_posix() + except ValueError: + anchored = str(resolved) + normalized.append(f"{anchored}{separator}{node_suffix}") + return normalized + + +def _anchor_test_paths() -> None: + """Anchor focused-test execution and artifacts to this checkout.""" + os.chdir(ROOT) def _has_worker_flag(selection: list[str]) -> bool: @@ -66,8 +206,20 @@ def _worker_args(selection: list[str]) -> list[str]: return ["-n", workers] +def _xdist_distribution_args(selection: list[str], worker_args: list[str]) -> list[str]: + """Keep declared shared-state groups together whenever xdist is active.""" + if any(arg == "--dist" or arg.startswith("--dist=") for arg in selection): + return [] + command = [*selection, *worker_args] + request = pytest_command_worker_request(command) + if request in {None, "0"}: + return [] + return ["--dist=loadgroup"] + + def build_pytest_cmd(selection: list[str]) -> list[str]: """Compose the pytest command for a focused selection.""" + worker_args = _worker_args(selection) return [ sys.executable, "-m", @@ -78,7 +230,8 @@ def build_pytest_cmd(selection: list[str]) -> list[str]: "--json-report-omit=collectors,log,streams,warnings", f"--json-report-file={PYTEST_REPORT_PATH}", *selection, - *_worker_args(selection), + *worker_args, + *_xdist_distribution_args(selection, worker_args), ] @@ -111,7 +264,12 @@ def _run_lock(*, enabled: bool) -> Iterator[None]: handle.truncate() +@finalize_checkout_mutation_monitors def main(argv: list[str] | None = None) -> int: + invocation_directory = Path.cwd() + selection = list(sys.argv[1:] if argv is None else argv) + selection = _normalize_selection_paths(selection, invocation_directory=invocation_directory) + _anchor_test_paths() try: fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools test") except CheckoutImportMismatchError as exc: @@ -121,7 +279,6 @@ def main(argv: list[str] | None = None) -> int: environment_fingerprint = fingerprint.as_dict() sys.stderr.write(f"devtools test: polylogue package → {polylogue_import_path}\n") - selection = list(sys.argv[1:] if argv is None else argv) use_json = "--json" in selection # The control-plane dispatch may append a bare ``--json`` machine-readable # flag; it is meaningless for a streamed test run, so drop it before pytest. @@ -140,6 +297,9 @@ def main(argv: list[str] | None = None) -> int: no_lock = os.environ.get("POLYLOGUE_TEST_NO_LOCK") == "1" with _run_lock(enabled=not no_lock): _clear_pytest_report(cmd) + mutation_monitor = CheckoutMutationMonitor(ROOT) + start_checkout_mutation_monitor(mutation_monitor) + initial_worktree_fingerprint = worktree_fingerprint(ROOT) run = VerifyRun( tier="focused-test", argv=selection, @@ -147,16 +307,82 @@ def main(argv: list[str] | None = None) -> int: root=ROOT, polylogue_import_path=str(polylogue_import_path), environment_fingerprint=environment_fingerprint, + worktree_fingerprint=initial_worktree_fingerprint, ) started = time.monotonic() - rc, _elapsed, metadata = _run("pytest focused", cmd, cwd=str(ROOT), run=run) + final_worktree_fingerprint = "unavailable" + mutation_observation = CheckoutMutationObservation(changed=False, unavailable=True) + runner_exception = False + try: + rc, _elapsed, metadata = _run("pytest focused", cmd, cwd=str(ROOT), run=run) + except KeyboardInterrupt: + rc = 130 + metadata = {"diagnosis": "pytest_interrupted", "termination_reason": "operator_interrupt"} + run.finish_interrupted_steps(exit_code=rc, diagnosis=str(metadata["diagnosis"])) + except Exception as exc: + runner_exception = True + rc = 125 + metadata = { + "diagnosis": "focused_test_runner_exception", + "exception_type": type(exc).__name__, + "error": str(exc), + "termination_reason": "runner_exception", + } + run.finish_interrupted_steps( + exit_code=rc, + diagnosis=str(metadata["diagnosis"]), + termination_reason="runner_exception", + ) + try: + final_worktree_fingerprint = worktree_fingerprint(ROOT) + except Exception: + final_worktree_fingerprint = "unavailable" + try: + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) + except Exception: + mutation_observation = CheckoutMutationObservation(changed=False, unavailable=True) + sys.stderr.write(f"devtools test: unexpected runner exception: {exc}\n") + if not runner_exception: + final_worktree_fingerprint = worktree_fingerprint(ROOT) + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) + if ( + "unavailable" in {initial_worktree_fingerprint, final_worktree_fingerprint} + or mutation_observation.unavailable + ): + checkout_diagnosis = "checkout_fingerprint_unavailable" + if rc == 130: + metadata["checkout_diagnosis"] = checkout_diagnosis + else: + metadata["diagnosis"] = checkout_diagnosis + if rc == 0: + rc = 125 + sys.stderr.write("devtools test: checkout fingerprint unavailable; evidence is not exact-head.\n") + elif mutation_observation.changed or final_worktree_fingerprint != initial_worktree_fingerprint: + checkout_diagnosis = "checkout_changed_during_focused_test" + if rc == 130: + metadata["checkout_diagnosis"] = checkout_diagnosis + else: + metadata["diagnosis"] = checkout_diagnosis + metadata["transient_checkout_mutation"] = mutation_observation.changed + metadata["checkout_mutation_path"] = mutation_observation.observed_path + if rc == 0: + rc = 125 + sys.stderr.write( + "devtools test: checkout contents changed during pytest; evidence is not exact-head.\n" + ) payload = run.finish( exit_code=rc, duration_s=time.monotonic() - started, diagnosis=metadata.get("diagnosis"), verification_scope="affected", release_baseline_allowed=False, + final_worktree_fingerprint=final_worktree_fingerprint, + checkout_mutation_path=mutation_observation.observed_path, + checkout_diagnosis=( + metadata["checkout_diagnosis"] if isinstance(metadata.get("checkout_diagnosis"), str) else None + ), ) + append_verify_history(payload) if use_json: print(json.dumps(payload, indent=2, ensure_ascii=False)) sys.stderr.write( diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index fd3dc16e65..8ac4bdaa14 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -78,7 +78,12 @@ class TerminalAuthorization(StrEnum): NARROW_TERMINAL = "narrow-terminal" -_TERMINAL_NODE_OUTCOMES = frozenset({"passed", "failed", "error", "skipped"}) +# Pytest reports expected failures separately from ordinary skips/passes. They +# still finish the selected node and therefore make its dependency capture +# reusable. ``xpassed`` remains subject to pytest's configured strictness via +# the process exit code; it is not independently recast as a failure here. +TERMINAL_NODE_OUTCOMES = frozenset({"passed", "failed", "error", "skipped", "xfailed", "xpassed"}) +SUCCESSFUL_NODE_OUTCOMES = frozenset({"passed", "skipped", "xfailed", "xpassed"}) def seed_shard_plan( @@ -179,7 +184,7 @@ def validate_seed_shard_ledger( return None if status is SeedShardStatus.COMPLETE and ( set(outcome_by_node) != set(nodeids) - or any(item.get("outcome") not in _TERMINAL_NODE_OUTCOMES for item in outcome_by_node.values()) + or any(item.get("outcome") not in TERMINAL_NODE_OUTCOMES for item in outcome_by_node.values()) ): return None if ( @@ -771,7 +776,7 @@ def attempt_is_checkout_bound( nodeids = [item.get("nodeid") for item in outcomes if isinstance(item, Mapping)] if len(nodeids) != len(outcomes) or set(nodeids) != set(expected) or len(set(nodeids)) != len(nodeids): return False - if any(item.get("outcome") not in {"passed", "failed", "error", "skipped"} for item in outcomes): + if any(item.get("outcome") not in TERMINAL_NODE_OUTCOMES for item in outcomes): return False return True @@ -1048,7 +1053,7 @@ def stamp_from_attempt( not isinstance(nodeid, str) or not nodeid for nodeid in outcome_by_node ): return None - if any(outcome not in {"passed", "failed", "error", "skipped"} for outcome in outcome_by_node.values()): + if any(outcome not in TERMINAL_NODE_OUTCOMES for outcome in outcome_by_node.values()): return None exit_code = attempt.get("exit_code") if not isinstance(exit_code, int) or isinstance(exit_code, bool): @@ -1066,7 +1071,7 @@ def stamp_from_attempt( BaselineStatus.GREEN if attempt.get("status") == "complete" and exit_code == 0 - and all(outcome in {"passed", "skipped"} for outcome in outcome_by_node.values()) + and all(outcome in SUCCESSFUL_NODE_OUTCOMES for outcome in outcome_by_node.values()) and not graph.failed_nodeids else BaselineStatus.RED ) @@ -1133,6 +1138,8 @@ def stamp_from_attempt( "SeedAttemptOutcome", "TestmonBinding", "TestmonIdentity", + "SUCCESSFUL_NODE_OUTCOMES", + "TERMINAL_NODE_OUTCOMES", "TestmonSeedStamp", "TerminalAuthorization", "VerificationScope", diff --git a/devtools/verify.py b/devtools/verify.py index 65d3e2c47f..da4a268652 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -31,10 +31,10 @@ import shlex import shutil import signal -import stat import subprocess import sys import time +import uuid from collections.abc import Mapping, Sequence from datetime import datetime, timezone from pathlib import Path @@ -44,6 +44,7 @@ CheckoutImportMismatchError, assert_polylogue_matches_checkout, ) +from devtools.pytest_progress_plugin import merge_worker_collection_payloads from devtools.pytest_supervisor import ( SupervisorLaunch, build_supervisor_launch, @@ -59,6 +60,8 @@ ) from devtools.testmon_bootstrap import maybe_bootstrap_testmon_seed from devtools.testmon_state import ( + SUCCESSFUL_NODE_OUTCOMES, + TERMINAL_NODE_OUTCOMES, BindingMode, GraphStatus, SeedAttemptOutcome, @@ -81,23 +84,34 @@ CURRENT_EVENTS_DIR, CURRENT_POSTMORTEM_PATH, CURRENT_RESOURCES_PATH, + CURRENT_STATISTICS_PATH, + PYTEST_CANONICAL_REPORT_NAME, + PYTEST_EXPLICIT_BASETEMP_ENV, + VERIFY_HISTORY_PATH, + CheckoutMutationMonitor, PytestResourceError, PytestStepArtifacts, ResourceSampler, VerifyRun, adaptive_pytest_worker_count, + append_verify_history, apply_managed_pytest_runtime_policy, classify_pytest_result, cleanup_managed_pytest_basetemp, copy_current_pytest_artifacts, env_for_pytest_step, + finalize_checkout_mutation_monitors, + finish_checkout_mutation_monitor, force_managed_pytest_scratch, latest_event_from_paths, - merge_worker_events, normalize_pytest_basetemp_env, pytest_basetemp_path, + pytest_command_worker_request, + pytest_tmpfs_budget_exceeded, pytest_tmpfs_budget_kb, + start_checkout_mutation_monitor, utc_now, + worktree_fingerprint, xdist_uninterruptible_stall_reason, ) from polylogue.scenarios.workload import ( @@ -215,7 +229,7 @@ def _format_completion_notification( # ── history (JSONL) ──────────────────────────────────────────────── -HISTORY_PATH = Path(".cache/verify-history.jsonl") +HISTORY_PATH = VERIFY_HISTORY_PATH TESTMON_DATA = Path(".cache/testmon/testmondata") TESTMON_SEED_STAMP = Path(".cache/testmon/seed.json") TESTMON_SEED_ATTEMPT = Path(".cache/testmon/seed-attempt.json") @@ -263,9 +277,7 @@ def _load_history() -> list[dict[str, Any]]: def _save_history(entry: dict[str, Any]) -> None: - HISTORY_PATH.parent.mkdir(parents=True, exist_ok=True) - with open(HISTORY_PATH, "a") as f: - f.write(json.dumps(entry, ensure_ascii=False) + "\n") + append_verify_history(entry, path=HISTORY_PATH) def _print_history(file: Path | None = None) -> None: @@ -277,12 +289,35 @@ def _print_history(file: Path | None = None) -> None: print(f"{'time':<20} {'tier':<8} {'head':<10} {'dur':>7} {'exit':>4} steps") print("-" * 75) for entry in entries[-10:]: - ts = entry["timestamp"][5:19] # MM-DD HH:MM - tier = entry["tier"][:8] - head = entry["git_head"][:8] - dur = f"{entry['total_duration_s']:.0f}s" - ec = entry["exit_code"] - steps = ", ".join(f"{s['name']}({s['duration_s']:.0f}s{' FAIL' if s['exit'] else ''})" for s in entry["steps"]) + timestamp = str(entry.get("timestamp") or entry.get("finished_at") or entry.get("started_at") or "unknown") + ts = timestamp[5:19] if timestamp != "unknown" else timestamp + tier = str(entry.get("tier") or "unknown")[:8] + head = str(entry.get("git_head") or "unknown")[:8] + duration = entry.get("total_duration_s", entry.get("duration_s", 0.0)) + try: + dur = f"{float(duration or 0.0):.0f}s" + except (TypeError, ValueError): + dur = "0s" + raw_exit = entry.get("exit_code", 1) + try: + ec = int(raw_exit if raw_exit is not None else 1) + except (TypeError, ValueError): + ec = 1 + rendered_steps: list[str] = [] + for step in entry.get("steps", []): + if not isinstance(step, dict): + continue + try: + step_duration = float(step.get("duration_s") or 0.0) + except (TypeError, ValueError): + step_duration = 0.0 + raw_step_exit = step.get("exit", 1) + try: + step_exit = int(raw_step_exit if raw_step_exit is not None else 1) + except (TypeError, ValueError): + step_exit = 1 + rendered_steps.append(f"{step.get('name', 'unknown')}({step_duration:.0f}s{' FAIL' if step_exit else ''})") + steps = ", ".join(rendered_steps) print(f"{ts:<20} {tier:<8} {head:<10} {dur:>7} {ec:>4} {steps}") @@ -380,7 +415,7 @@ def _pytest_metadata_from_report(report: dict[str, Any], *, report_path: Path) - def _pytest_command_metadata(cmd: list[str]) -> dict[str, Any]: """Return verify metadata that explains the pytest worker policy.""" metadata: dict[str, Any] = {} - metadata["pytest_workers"] = _pytest_command_worker_request(cmd) or "unset" + metadata["pytest_workers"] = pytest_command_worker_request(cmd) or "unset" if "--testmon" in cmd: metadata["pytest_selection"] = "testmon-noselect" if "--testmon-noselect" in cmd else "testmon" else: @@ -456,7 +491,8 @@ def _pytest_workload_receipt( peak_swap_pss_kb = resource_summary.get("peak_tree_swap_pss_kb") read_bytes = resource_summary.get("tree_read_bytes_delta") write_bytes = resource_summary.get("tree_write_bytes_delta") - peak_basetemp_kb = resource_summary.get("peak_basetemp_size_kb") + peak_basetemp_kb = resource_summary.get("peak_basetemp_allocated_kb") + logical_basetemp_kb = resource_summary.get("peak_basetemp_size_kb") final_rss_kb = last_resource_sample.get("tree_rss_kb") if last_resource_sample is not None else None final_pss_kb = last_resource_sample.get("tree_pss_kb") if last_resource_sample is not None else None total_cpu_s = last_resource_sample.get("tree_cpu_s") if last_resource_sample is not None else None @@ -554,7 +590,14 @@ def _pytest_workload_receipt( ), cancellation_requested=termination_reason is not None, cleanup_complete=True if basetemp_cleanup is not None else None, - notes=("Managed pytest process-tree sampler adapter.",), + notes=( + "Managed pytest process-tree sampler adapter.", + ( + f"Logical basetemp peak retained as diagnostic evidence: {logical_basetemp_kb * 1024} bytes." + if isinstance(logical_basetemp_kb, int) + else "Logical basetemp peak unavailable." + ), + ), ) return dict(receipt.to_payload()) @@ -591,6 +634,7 @@ def _clear_pytest_report(cmd: Sequence[str] = ()) -> None: CURRENT_RESOURCES_PATH, CURRENT_POSTMORTEM_PATH, CURRENT_CONTAINMENT_PATH, + CURRENT_STATISTICS_PATH, ): with contextlib.suppress(FileNotFoundError): if path.is_dir(): @@ -605,6 +649,20 @@ def _write_pytest_output(stdout: str, stderr: str) -> None: PYTEST_OUTPUT_PATH.write_text(stdout + ("\n" if stdout and stderr else "") + stderr, encoding="utf-8") +def _persist_pytest_output(stdout: str, stderr: str, *, artifacts: PytestStepArtifacts | None) -> None: + """Persist drained pytest output on both ordinary and exceptional exits.""" + with contextlib.suppress(OSError): + _write_pytest_output(stdout, stderr) + if artifacts is not None: + for path, content in ( + (artifacts.stdout_path, stdout), + (artifacts.stderr_path, stderr), + (artifacts.output_path, stdout + stderr), + ): + with contextlib.suppress(OSError): + path.write_text(content, encoding="utf-8") + + def _write_pytest_progress( *, event: str, @@ -666,10 +724,12 @@ def _write_pytest_progress( } if latest_event.get("event") == "test_started" and isinstance(latest_event.get("nodeid"), str): payload["current_test_nodeid"] = latest_event["nodeid"] - PYTEST_PROGRESS_PATH.parent.mkdir(parents=True, exist_ok=True) - tmp = PYTEST_PROGRESS_PATH.with_name(f"{PYTEST_PROGRESS_PATH.name}.{os.getpid()}.{time.monotonic_ns()}.tmp") - tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n") - tmp.replace(PYTEST_PROGRESS_PATH) + targets = [PYTEST_PROGRESS_PATH] + if artifact_dir is not None: + targets.insert(0, Path(artifact_dir) / "progress.json") + for target in dict.fromkeys(targets): + with contextlib.suppress(OSError): + _atomic_write_json(target, payload) def _process_cpu_seconds(pid: int) -> float | None: @@ -774,6 +834,59 @@ def _request_supervisor_termination( process.send_signal(signal.SIGTERM) +class PytestContainmentError(RuntimeError): + """Raised when an interrupted pytest supervisor cannot be confirmed stopped.""" + + +def _await_interrupted_pytest_containment( + process: subprocess.Popen[bytes], + launch: SupervisorLaunch, + *, + term_grace_s: float, + preserved_runner_descendants: Sequence[tuple[int, int]], +) -> None: + """Wait for an interrupted pytest supervisor before its caller cleans up.""" + if process.poll() is None: + _request_supervisor_termination(process, launch, reason="pytest runner interrupted") + try: + process.wait(timeout=max(1.0, term_grace_s + 1.0)) + except subprocess.TimeoutExpired: + _force_kill_owned_run( + process, + launch, + preserved_runner_descendants=preserved_runner_descendants, + ) + try: + process.wait(timeout=1.0) + except subprocess.TimeoutExpired as exc: + raise PytestContainmentError( + "pytest containment did not quiesce after forced termination; leaving its basetemp intact" + ) from exc + reap_exited_children() + receipt = read_receipt(launch.receipt_path) + remaining_descendants = tuple( + identity + for identity in descendant_process_identities(os.getpid()) + if identity not in preserved_runner_descendants + ) + if ( + receipt is None + or receipt.get("status") not in {"finished", "terminated"} + or receipt.get("controller_group_alive") is not False + or remaining_descendants + ): + raise PytestContainmentError( + "pytest containment did not quiesce its owned process tree; leaving its basetemp intact" + ) + + +def _supervised_tmpfs_cleanup_path(*, root: Path, run_id: str, env: dict[str, str]) -> Path | None: + """Return only a supervisor-owned tmpfs path eligible for cleanup.""" + if env.get(PYTEST_EXPLICIT_BASETEMP_ENV) or pytest_tmpfs_budget_kb(env) is None: + return None + return pytest_basetemp_path(root=root, run_id=run_id, env=env) + + def _force_kill_owned_run( process: subprocess.Popen[bytes], launch: SupervisorLaunch, @@ -931,14 +1044,10 @@ def _run_pytest_with_heartbeat( else Path(env.get("POLYLOGUE_PYTEST_CONTAINMENT_PATH", str(Path.cwd() / PYTEST_CONTAINMENT_PATH))) ) pytest_run_id = run.run_id if run is not None else env.get("POLYLOGUE_PYTEST_RUN_ID", str(os.getpid())) - tmpfs_cleanup_path = ( - pytest_basetemp_path( - root=Path(cwd) if cwd is not None else Path.cwd(), - run_id=pytest_run_id, - env=env, - ) - if tmpfs_budget_kb is not None - else None + tmpfs_cleanup_path = _supervised_tmpfs_cleanup_path( + root=Path(cwd) if cwd is not None else Path.cwd(), + run_id=pytest_run_id, + env=env, ) launch = build_supervisor_launch( cmd, @@ -1388,15 +1497,14 @@ def _refresh_progress_marker(at: float, latest: dict[str, Any] | None = None) -> and sample_now - last_resource_sample >= resource_interval_s ): resource_sample = sampler.sample(event="sample") - basetemp_size_kb = resource_sample.get("basetemp_size_kb") if ( termination_reason is None and tmpfs_budget_kb is not None - and isinstance(basetemp_size_kb, int) - and basetemp_size_kb > tmpfs_budget_kb + and pytest_tmpfs_budget_exceeded(resource_sample, budget_kb=tmpfs_budget_kb) ): + basetemp_allocated_kb = int(resource_sample["basetemp_allocated_kb"]) termination_reason = ( - f"pytest tmpfs budget exceeded: {basetemp_size_kb / 1024:.1f} MiB " + f"pytest tmpfs budget exceeded: {basetemp_allocated_kb / 1024:.1f} MiB allocated " f"> {tmpfs_budget_kb / 1024:.0f} MiB" ) if resource_sample.get("all_xdist_workers_uninterruptible") is True: @@ -1415,8 +1523,19 @@ def _refresh_progress_marker(at: float, latest: dict[str, Any] | None = None) -> if process.poll() is not None and not selector.get_map(): break except BaseException: - if process.poll() is None: - _request_supervisor_termination(process, launch, reason="pytest runner interrupted") + try: + _await_interrupted_pytest_containment( + process, + launch, + term_grace_s=term_grace_s, + preserved_runner_descendants=preserved_runner_descendants, + ) + finally: + _persist_pytest_output( + b"".join(output["stdout"]).decode(errors="replace"), + b"".join(output["stderr"]).decode(errors="replace"), + artifacts=artifacts, + ) raise finally: selector.close() @@ -1485,11 +1604,7 @@ def _refresh_progress_marker(at: float, latest: dict[str, Any] | None = None) -> events_path=events_path, events_dir=events_dir, ) - _write_pytest_output(stdout, stderr) - if artifacts is not None: - artifacts.stdout_path.write_text(stdout, encoding="utf-8") - artifacts.stderr_path.write_text(stderr, encoding="utf-8") - artifacts.output_path.write_text(stdout + stderr, encoding="utf-8") + _persist_pytest_output(stdout, stderr, artifacts=artifacts) return subprocess.CompletedProcess(cmd, returncode, stdout, stderr) @@ -1507,10 +1622,16 @@ def _run( # ``bench slo`` starts pytest-benchmark itself, so it needs the same # bounded temp policy and run marker as a direct pytest step. has_managed_pytest_child = label == "bench slo" + if is_pytest and run is not None: + isolated_report = run.run_dir / f"pytest-report-{uuid.uuid4().hex}.json" + cmd = [f"--json-report-file={isolated_report}" if arg.startswith("--json-report-file=") else arg for arg in cmd] if is_pytest: _clear_pytest_report(cmd) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None env = _subprocess_env() + explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd, env=env) + if explicit_basetemp is not None: + env[PYTEST_EXPLICIT_BASETEMP_ENV] = str(explicit_basetemp) pytest_tmpfs = False pytest_tmpfs_budget_mb: float | None = None runtime_policy = None @@ -1543,10 +1664,14 @@ def _run( "release_baseline_allowed": False, } if run is not None and artifacts is not None: - run.finish_step( + finalized_step = run.finish_step( step_id=artifacts.step_id, result={"duration_s": round(elapsed, 2), "exit": 125, **refusal_metadata}, ) + if isinstance(finalized_step, dict): + for key in ("statistics", "statistics_path"): + if key in finalized_step: + refusal_metadata[key] = finalized_step[key] return 125, elapsed, refusal_metadata pytest_tmpfs = env.get("POLYLOGUE_PYTEST_TMPFS") == "1" budget_kb = pytest_tmpfs_budget_kb(env) @@ -1558,27 +1683,42 @@ def _run( env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" if run is not None and artifacts is not None: env = env_for_pytest_step(env, run=run, artifacts=artifacts) + interrupted = False + pytest_containment_quiescent = True + containment_error: str | None = None if is_pytest: try: - result = _run_pytest_with_heartbeat(cmd, cwd=cwd, env=env, t0=t0, run=run, artifacts=artifacts) + try: + result = _run_pytest_with_heartbeat(cmd, cwd=cwd, env=env, t0=t0, run=run, artifacts=artifacts) + except PytestContainmentError as exc: + pytest_containment_quiescent = False + containment_error = str(exc) + result = subprocess.CompletedProcess(args=cmd, returncode=125, stdout="", stderr=str(exc)) + except KeyboardInterrupt: + interrupted = True + result = subprocess.CompletedProcess(args=cmd, returncode=130, stdout="", stderr="") finally: - basetemp_cleanup = cleanup_managed_pytest_basetemp( - root=ROOT, - run_id=env.get("POLYLOGUE_PYTEST_RUN_ID", ""), - env=env, - ) - if artifacts is not None: - merge_worker_events(artifacts.events_dir, artifacts.events_merged_path) - with contextlib.suppress(FileNotFoundError): - shutil.copyfile(PYTEST_PROGRESS_PATH, artifacts.progress_path) + if pytest_containment_quiescent: + basetemp_cleanup = cleanup_managed_pytest_basetemp( + root=ROOT, + run_id=env.get("POLYLOGUE_PYTEST_RUN_ID", ""), + env=env, + ) else: - result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + try: + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + except KeyboardInterrupt: + interrupted = True + result = subprocess.CompletedProcess(args=cmd, returncode=130, stdout="", stderr="") elapsed = time.monotonic() - t0 metadata: dict[str, Any] = {} if artifacts is not None: metadata["run_id"] = run.run_id if run is not None else None metadata["artifact_dir"] = str(artifacts.step_dir.relative_to(Path.cwd())) if is_pytest: + if containment_error is not None: + metadata["diagnosis"] = "pytest_containment_unproven" + metadata["termination_reason"] = f"pytest containment did not quiesce: {containment_error}" metadata.update(_pytest_command_metadata(cmd)) metadata["heartbeat_s"] = _pytest_heartbeat_interval() metadata["timeout_s"] = _pytest_timeout_s() @@ -1608,6 +1748,19 @@ def _run( if report is not None: metadata.update(_pytest_metadata_from_report(report, report_path=report_path)) metadata["report_status"] = "present" + if artifacts is not None: + durable_report_path = artifacts.step_dir / PYTEST_CANONICAL_REPORT_NAME + try: + shutil.copyfile(report_path, durable_report_path) + except OSError: + metadata["report_path"] = None + else: + metadata["report_path"] = str( + durable_report_path.relative_to(run.root if run is not None else ROOT) + ) + if report_path != durable_report_path: + with contextlib.suppress(OSError): + report_path.unlink() else: # Fallback: terminal scraping when the structured report is # missing (pytest crashed before writing it, or the plugin is @@ -1631,6 +1784,15 @@ def _run( if isinstance(termination_reason, str): metadata["termination_reason"] = termination_reason selection_path = artifacts.selection_path if artifacts is not None else PYTEST_SELECTION_PATH + if interrupted or containment_error is not None: + _recover_worker_collection_facts( + events_dir=( + artifacts.events_dir + if artifacts is not None + else Path(env.get("POLYLOGUE_PYTEST_EVENTS_DIR", str(PYTEST_EVENTS_DIR))) + ), + selection_path=selection_path, + ) selection = _read_json_artifact(selection_path) if selection is not None: selected_count = selection.get("selected_count") @@ -1674,6 +1836,7 @@ def _run( peak_swap_pss: int | None = None peak_process_count = 0 peak_basetemp_size_kb: int | None = None + peak_basetemp_allocated_kb: int | None = None with artifacts.resources_path.open(encoding="utf-8") as resource_handle: for line in resource_handle: if not line.strip(): @@ -1703,6 +1866,11 @@ def _run( peak_basetemp_size_kb or 0, int(row["basetemp_size_kb"]), ) + if row.get("basetemp_allocated_kb") is not None: + peak_basetemp_allocated_kb = max( + peak_basetemp_allocated_kb or 0, + int(row["basetemp_allocated_kb"]), + ) if sample_count: resource_summary = { "resource_sample_count": sample_count, @@ -1741,6 +1909,10 @@ def _run( "peak_basetemp_size_mb": ( round(peak_basetemp_size_kb / 1024, 1) if peak_basetemp_size_kb is not None else None ), + "peak_basetemp_allocated_kb": peak_basetemp_allocated_kb, + "peak_basetemp_allocated_mb": ( + round(peak_basetemp_allocated_kb / 1024, 1) if peak_basetemp_allocated_kb is not None else None + ), } metadata.update(resource_summary) diagnosis = classify_pytest_result( @@ -1752,6 +1924,11 @@ def _run( summary=summary if isinstance(summary, dict) else None, progress_event=metadata.get("progress_event") if isinstance(metadata.get("progress_event"), str) else None, ) + if containment_error is not None: + diagnosis = "pytest_containment_unproven" + if interrupted: + diagnosis = "pytest_interrupted" + metadata["termination_reason"] = "operator_interrupt" metadata["diagnosis"] = diagnosis termination_reason = ( metadata.get("termination_reason") if isinstance(metadata.get("termination_reason"), str) else None @@ -1788,17 +1965,8 @@ def _run( **resource_summary, } artifacts.postmortem_path.write_text(json.dumps(postmortem, indent=2, ensure_ascii=False) + "\n") - copy_current_pytest_artifacts( - Path.cwd(), - artifacts, - legacy_paths={ - "progress_path": PYTEST_PROGRESS_PATH, - "events_merged_path": PYTEST_EVENTS_PATH, - "selection_path": PYTEST_SELECTION_PATH, - "summary_path": PYTEST_SUMMARY_PATH, - "output_path": PYTEST_OUTPUT_PATH, - }, - ) + elif interrupted: + metadata.update({"diagnosis": "verification_interrupted", "termination_reason": "operator_interrupt"}) if result.returncode == 0: sys.stderr.write(f"ok ({elapsed:.1f}s)\n") else: @@ -1808,14 +1976,56 @@ def _run( if result.stderr.strip(): sys.stderr.write(result.stderr + "\n") if run is not None and artifacts is not None: - run.finish_step( + finalized_step = run.finish_step( step_id=artifacts.step_id, result={"duration_s": round(elapsed, 2), "exit": result.returncode, **metadata} ) + if isinstance(finalized_step, dict): + for key in ("statistics", "statistics_path"): + if key in finalized_step: + metadata[key] = finalized_step[key] + if is_pytest and artifacts is not None: + copy_current_pytest_artifacts( + Path.cwd(), + artifacts, + legacy_paths={ + "progress_path": PYTEST_PROGRESS_PATH, + "events_merged_path": PYTEST_EVENTS_PATH, + "selection_path": PYTEST_SELECTION_PATH, + "summary_path": PYTEST_SUMMARY_PATH, + "output_path": PYTEST_OUTPUT_PATH, + }, + ) return result.returncode, elapsed, metadata +def _pytest_command_basetemp( + cmd: Sequence[str], *, cwd: str | None, env: Mapping[str, str] | None = None +) -> Path | None: + """Return the effective explicit pytest basetemp, if the command has one.""" + raw_path: str | None = None + addopts: list[str] = [] + if env is not None: + with contextlib.suppress(ValueError): + addopts = shlex.split(env.get("PYTEST_ADDOPTS", "")) + arguments = [*addopts, *cmd] + for index, argument in enumerate(arguments): + if argument.startswith("--basetemp="): + raw_path = argument.partition("=")[2] + elif argument == "--basetemp" and index + 1 < len(arguments): + raw_path = arguments[index + 1] + if not raw_path: + return None + path = Path(raw_path) + if path.is_absolute(): + return path + return (Path(cwd) if cwd is not None else Path.cwd()) / path + + def _subprocess_env() -> dict[str, str]: env = normalize_pytest_basetemp_env(os.environ) + # Tests and verification helpers may inspect Git, but observational reads + # must not refresh the index and invalidate the exact-head mutation watch. + env["GIT_OPTIONAL_LOCKS"] = "0" env["POLYLOGUE_ROOT"] = str(ROOT) env["POLYLOGUE_REPO_ROOT"] = str(ROOT) inherited_pythonpath = env.get("PYTHONPATH", "") @@ -2106,11 +2316,23 @@ def _compare_against_last(step_results: list[dict[str, Any]]) -> list[str]: entries = _load_history() if len(entries) < 1: return [] - last = entries[-1] - last_steps = {s["name"]: s["duration_s"] for s in last.get("steps", [])} flags: list[str] = [] for s in step_results: - prev = last_steps.get(s["name"]) + name = s.get("name") + if not isinstance(name, str): + continue + prev = next( + ( + prior.get("duration_s") + for entry in reversed(entries) + if entry.get("tier") != "focused-test" + for prior in entry.get("steps", []) + if isinstance(prior, dict) + and prior.get("name") == name + and isinstance(prior.get("duration_s"), (int, float)) + ), + None, + ) if prev is not None and prev > 0: delta = s["duration_s"] - prev pct = (delta / prev) * 100 @@ -2134,14 +2356,8 @@ def _print_json(result: dict[str, Any]) -> None: def _git_head() -> str | None: - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - capture_output=True, - text=True, - ) - if result.returncode == 0: - return result.stdout.strip() - return None + """Resolve HEAD through the bounded authority-sensitive Git probe.""" + return _git_commit("HEAD") def _git_committed_tree() -> str | None: @@ -2155,6 +2371,25 @@ def _git_committed_tree() -> str | None: return None +def _git_commit(ref: str) -> str | None: + """Resolve a mutable Git ref once for an authority-sensitive run.""" + try: + result = subprocess.run( + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], + capture_output=True, + text=True, + timeout=5, + cwd=ROOT, + env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if result.returncode != 0 or result.stderr.strip(): + return None + commit = result.stdout.strip() + return commit or None + + def _stamp_head() -> None: head = _git_head() if head is None: @@ -2182,7 +2417,7 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: workers = adaptive_pytest_worker_count(os.environ) if maximum is not None: workers = min(workers, maximum) - return ["-n", str(workers)] + return ["--dist=loadgroup", "-n", str(workers)] BROAD_PYTEST_STEP_LABELS = { @@ -2194,25 +2429,6 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: } -def _pytest_command_worker_request(cmd: Sequence[str]) -> str | None: - """Return the last xdist worker request from a final pytest command. - - ``devtools test`` forwards pytest arguments unchanged, so this accepts - both xdist spellings and their compact forms. The final occurrence wins, - matching pytest's normal option precedence. - """ - request: str | None = None - for index, arg in enumerate(cmd): - if arg in {"-n", "--numprocesses"}: - if index + 1 < len(cmd): - request = cmd[index + 1] - elif arg.startswith("--numprocesses="): - request = arg.removeprefix("--numprocesses=") - elif arg.startswith("-n") and len(arg) > 2: - request = arg[2:].removeprefix("=") - return request - - def _pytest_command_concurrency(cmd: Sequence[str], *, env: Mapping[str, str] | None = None) -> int: """Return a fail-closed reservation for the final pytest command. @@ -2220,7 +2436,7 @@ def _pytest_command_concurrency(cmd: Sequence[str], *, env: Mapping[str, str] | instead of guessing one worker; an unrecognised xdist value is treated the same way so malformed or future values cannot weaken admission. """ - request = _pytest_command_worker_request(cmd) + request = pytest_command_worker_request(cmd) if request is None: return 0 if request == "auto": @@ -2249,38 +2465,49 @@ def _pytest_uses_full_suite_basetemp(label: str) -> bool: } -def _changed_paths() -> set[str]: +def _changed_paths(base_commit: str, head_commit: str) -> set[str]: + """Return changes between immutable start-time Git authorities.""" changed: set[str] = set() commands = ( - ["git", "diff", "--name-only", "HEAD", "--"], - ["git", "diff", "--name-only", "origin/master...HEAD", "--"], + ["git", "diff", "--no-renames", "--name-only", "-z", head_commit, "--"], + ["git", "diff", "--no-renames", "--name-only", "-z", f"{base_commit}...{head_commit}", "--"], + ["git", "ls-files", "--others", "--exclude-standard", "-z", "--"], ) for command in commands: try: - result = subprocess.run(command, capture_output=True, text=True, timeout=5) - except (OSError, subprocess.TimeoutExpired): - continue - if result.returncode == 0: - changed.update(line.strip() for line in result.stdout.splitlines() if line.strip()) + result = subprocess.run( + command, + capture_output=True, + timeout=5, + cwd=ROOT, + env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise PytestResourceError("testmon changed-path authority is unavailable") from exc + if result.returncode != 0 or result.stderr.strip(): + raise PytestResourceError("testmon changed-path authority is unavailable") + changed.update(os.fsdecode(raw_path) for raw_path in result.stdout.split(b"\0") if raw_path) return changed -def _default_testmon_is_broad_change() -> bool: +def _default_testmon_is_broad_change(base_commit: str, head_commit: str) -> bool: """Return true when affected-test selection should be treated as broad.""" - return bool(_changed_paths() & _BROAD_TESTMON_CHANGED_PATHS) + return bool(_changed_paths(base_commit, head_commit) & _BROAD_TESTMON_CHANGED_PATHS) -def _changed_executable_paths() -> tuple[str, ...]: +def _changed_executable_paths(base_commit: str, head_commit: str) -> tuple[str, ...]: """Return changed paths whose behavior should select at least one test.""" roots = ("polylogue/", "devtools/", "tests/", "packaging/") exact = {"pyproject.toml", "uv.lock"} - return tuple(sorted(path for path in _changed_paths() if path in exact or path.startswith(roots))) + return tuple( + sorted(path for path in _changed_paths(base_commit, head_commit) if path in exact or path.startswith(roots)) + ) def _testmon_coverage_identity(executable_paths: Sequence[str]) -> dict[str, Any]: """Identify the exact worktree contents covered by an affected/full run.""" return { - "worktree_fingerprint": _worktree_fingerprint(), + "worktree_fingerprint": worktree_fingerprint(), "executable_paths": list(executable_paths), } @@ -2372,54 +2599,6 @@ def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: temporary.replace(path) -def _worktree_fingerprint() -> str: - """Fingerprint tracked changes plus exact non-ignored untracked content.""" - digest = hashlib.sha256() - for command in ( - ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"], - ["git", "diff", "--binary", "HEAD", "--"], - ): - try: - result = subprocess.run(command, capture_output=True, timeout=30) - except (OSError, subprocess.TimeoutExpired): - return "unavailable" - if result.returncode != 0: - return "unavailable" - digest.update(result.stdout) - digest.update(b"\0") - try: - untracked = subprocess.run( - ["git", "ls-files", "--others", "--exclude-standard", "-z"], - capture_output=True, - timeout=30, - ) - except (OSError, subprocess.TimeoutExpired): - return "unavailable" - if untracked.returncode != 0: - return "unavailable" - for raw_path in sorted(path for path in untracked.stdout.split(b"\0") if path): - try: - path_text = os.fsdecode(raw_path) - path = Path(path_text) - mode = path.lstat().st_mode - digest.update(raw_path) - digest.update(b"\0") - if stat.S_ISLNK(mode): - digest.update(b"symlink\0") - digest.update(os.fsencode(os.readlink(path))) - elif stat.S_ISREG(mode): - digest.update(b"file\0") - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - else: - digest.update(f"mode:{stat.S_IFMT(mode):o}".encode()) - digest.update(b"\0") - except OSError: - return "unavailable" - return digest.hexdigest() - - def _testmon_seed_identity( *, git_head: str | None, @@ -2435,7 +2614,7 @@ def _testmon_seed_identity( return { "git_head": git_head, "git_tree": git_tree, - "worktree_fingerprint": _worktree_fingerprint(), + "worktree_fingerprint": worktree_fingerprint(), "python": sys.version, "skip_slow": skip_slow, "lab": lab, @@ -2450,6 +2629,36 @@ def _read_testmon_seed_attempt() -> dict[str, Any] | None: return payload if isinstance(payload, dict) else None +def _recover_worker_collection_facts(*, events_dir: Path, selection_path: Path) -> bool: + """Publish xdist worker collection facts when its controller never finishes. + + The progress plugin normally merges these facts during controller + ``pytest_sessionfinish``. Interrupted containment bypasses that hook, so + the runner recovers the same canonical worker fact before it terminalizes + the durable step record. + """ + merged = merge_worker_collection_payloads(events_dir) + if merged is None: + return False + selection = dict(merged) + selection.update( + { + "updated_at": datetime.now(timezone.utc).isoformat(), + "worker_id": "runner", + "pid": os.getpid(), + "recovered_after_interruption": True, + } + ) + try: + selection_path.parent.mkdir(parents=True, exist_ok=True) + temporary = selection_path.with_name(f"{selection_path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(selection, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + temporary.replace(selection_path) + except OSError: + return False + return True + + def _flatten_seed_outcomes(attempt: Mapping[str, Any] | None) -> list[dict[str, Any]]: """Flatten outcomes from every interrupted attempt, newest result winning.""" if attempt is None: @@ -2715,7 +2924,6 @@ def _seed_shard_command( else: command.extend( [ - "--dist=loadgroup", *_pytest_worker_args(maximum=10), "--testmon", "--testmon-noselect", @@ -2779,7 +2987,7 @@ def _checkpoint_testmon_seed_shard( prior_node_outcomes=prior, use_database_fallback=False, ) - terminal = all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in outcomes) + terminal = all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in outcomes) selection_matches = selected == nodeids shard.update( { @@ -2886,6 +3094,10 @@ def _seed_node_outcomes_from_events( outcome, reason = "timeout", "pytest-timeout report" elif any(report.get("when") in {"setup", "teardown"} for report in failed_reports): outcome, reason = "error", "fixture setup/teardown failed" + elif any(report.get("outcome") == "xfailed" for report in node_reports): + outcome, reason = "xfailed", "pytest expected failure" + elif any(report.get("outcome") == "xpassed" for report in node_reports): + outcome, reason = "xpassed", "pytest unexpected pass" elif any(report.get("outcome") == "failed" for report in call_reports): outcome, reason = "failed", "test call failed" elif any(report.get("outcome") == "passed" for report in call_reports): @@ -2924,7 +3136,7 @@ def _seed_node_outcomes_from_events( elif prior_node_outcomes is not None and nodeid in prior_node_outcomes: prior = prior_node_outcomes[nodeid] prior_outcome = prior.get("outcome") - if prior_outcome in {"passed", "failed", "error", "skipped"}: + if prior_outcome in TERMINAL_NODE_OUTCOMES: outcome, reason = str(prior_outcome), "terminal outcome carried from the prior seed attempt" else: outcome, reason = "missing", "prior seed attempt has no terminal outcome" @@ -3059,7 +3271,7 @@ def _finalize_testmon_seed_attempt( }, ) unsuccessful_nodeids = [ - str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} + str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in SUCCESSFUL_NODE_OUTCOMES ] green_complete = ( exit_code == 0 @@ -3084,7 +3296,7 @@ def _finalize_testmon_seed_attempt( and not database["missing_nodeids"] and database["orphan_execution_edges"] == 0 and database["orphan_fingerprint_edges"] == 0 - and all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in node_outcomes) + and all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) ) outcome = _seed_attempt_outcome( release_eligible=release_eligible, @@ -3100,7 +3312,7 @@ def _finalize_testmon_seed_attempt( { "status": ( SeedShardStatus.COMPLETE.value - if all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in node_outcomes) + if all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) else SeedShardStatus.INCOMPLETE.value ), "node_outcomes": node_outcomes, @@ -3239,7 +3451,7 @@ def _refresh_testmon_selection_attempt( and database.get("orphan_execution_edges") == 0 and database.get("orphan_fingerprint_edges") == 0 ) - terminal = all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in node_outcomes) + terminal = all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) prior_selection = attempt.get("selection") payload = { **attempt, @@ -3270,7 +3482,7 @@ def _refresh_testmon_selection_attempt( ) ), "unsuccessful_nodeids": [ - str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} + str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in SUCCESSFUL_NODE_OUTCOMES ], "testmon_data": _file_fingerprint(TESTMON_DATA), "run_id": run.run_id, @@ -3285,10 +3497,43 @@ def _refresh_testmon_selection_attempt( _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) +def _discard_testmon_dependency_authority() -> None: + """Remove a dependency graph learned while checkout authority was unstable.""" + for path in ( + TESTMON_SEED_STAMP, + TESTMON_SEED_ATTEMPT, + TESTMON_DATA, + Path(f"{TESTMON_DATA}-wal"), + Path(f"{TESTMON_DATA}-shm"), + Path(f"{TESTMON_DATA}-journal"), + ): + path.unlink(missing_ok=True) + + # ── main ──────────────────────────────────────────────────────────── -def main(argv: list[str] | None = None) -> int: +_ACTIVE_VERIFY_RUN: tuple[VerifyRun, float, VerificationScope] | None = None + + +def _planned_verification_scope(args: argparse.Namespace, *, full_pytest: bool) -> VerificationScope: + """Return the immutable scope requested before the runner starts.""" + if args.quick or args.commit: + return VerificationScope.NON_TEST + if full_pytest or args.seed_testmon: + return VerificationScope.NARROW_TERMINAL if args.skip_slow else VerificationScope.RELEASE_BASELINE + return VerificationScope.AFFECTED + + +def _changed_paths_from_testmon_authority(base_commit: str | None, head_commit: str | None) -> tuple[str, ...]: + """Require immutable refs before deriving affected executable paths.""" + if base_commit is None or head_commit is None: + raise PytestResourceError("testmon changed-path authority is unavailable") + return _changed_executable_paths(base_commit, head_commit) + + +def _main(argv: list[str] | None = None) -> int: + global _ACTIVE_VERIFY_RUN parser = argparse.ArgumentParser(description="Run the local verification baseline.") parser.add_argument("--quick", action="store_true", help="Skip pytest and run only fast local gates.") parser.add_argument( @@ -3360,7 +3605,15 @@ def main(argv: list[str] | None = None) -> int: else: tier = "testmon" + head = _git_head() full_pytest = bool(args.all or args.full) + affected_testmon = not (args.quick or args.commit or args.seed_testmon or full_pytest) + planned_verification_scope = _planned_verification_scope(args, full_pytest=full_pytest) + testmon_base_commit = _git_commit("origin/master") if affected_testmon else None + testmon_head_commit = head if affected_testmon else None + if affected_testmon and (testmon_base_commit is None or testmon_head_commit is None): + sys.stderr.write("verify: cannot resolve immutable Git refs for affected-test authority.\n") + return 125 if args.terminal_authorization is not None and not ((full_pytest or args.seed_testmon) and args.skip_slow): parser.error("--terminal-authorization requires --all, --full, or --seed-testmon with --skip-slow") preflight_error = _testmon_preflight( @@ -3373,15 +3626,19 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.write(preflight_error) return 2 - head = _git_head() t0 = time.monotonic() + mutation_monitor = CheckoutMutationMonitor(ROOT) + start_checkout_mutation_monitor(mutation_monitor) + checkout_fingerprint = worktree_fingerprint(ROOT) verify_run = VerifyRun( tier=tier, argv=list(sys.argv[1:] if argv is None else argv), git_head=head, polylogue_import_path=str(polylogue_import_path), environment_fingerprint=environment_fingerprint, + worktree_fingerprint=checkout_fingerprint, ) + _ACTIVE_VERIFY_RUN = (verify_run, t0, planned_verification_scope) seed_identity: dict[str, Any] | None = None resume_testmon_seed = False prepared_seed_attempt: dict[str, Any] | None = None @@ -3395,12 +3652,14 @@ def main(argv: list[str] | None = None) -> int: terminal_authorization=args.terminal_authorization, ) except RuntimeError as exc: + finish_checkout_mutation_monitor(mutation_monitor) sys.stderr.write(f"verify: {exc}\n") - verify_run.finish( + early_payload = verify_run.finish( exit_code=125, duration_s=time.monotonic() - t0, diagnosis="testmon_environment_identity_unavailable", ) + _save_history(early_payload) return 125 resume_testmon_seed = _testmon_seed_can_resume(seed_identity) prepared_seed_attempt = _prepare_testmon_seed_attempt( @@ -3428,19 +3687,35 @@ def main(argv: list[str] | None = None) -> int: seed_testmon=bool(args.seed_testmon), resume_testmon_seed=resume_testmon_seed, full_pytest=full_pytest, - broad_testmon=_default_testmon_is_broad_change(), + broad_testmon=( + _default_testmon_is_broad_change(testmon_base_commit, testmon_head_commit) + if testmon_base_commit is not None and testmon_head_commit is not None + else False + ), ) except PytestResourceError as exc: + finish_checkout_mutation_monitor(mutation_monitor) sys.stderr.write(f"verify: {exc}\n") - verify_run.finish(exit_code=125, duration_s=time.monotonic() - t0, diagnosis="pytest_resource_preflight_failed") + early_payload = verify_run.finish( + exit_code=125, + duration_s=time.monotonic() - t0, + diagnosis="pytest_resource_preflight_failed", + ) + _save_history(early_payload) return 125 step_results: list[dict[str, Any]] = [] - + pending_testmon_stamp: TestmonSeedStamp | None = None + pending_affected_coverage: tuple[tuple[str, ...], int] | None = None + pending_selection_refresh: tuple[dict[str, Any], int] | None = None + testmon_graph_touched = False + changed_path_authority_failed = False for label, cmd in steps: if label.startswith("pytest"): _warn_low_memory() # check again right before the heavy step rc, elapsed, metadata = _run(label, cmd, run=verify_run) + if label in {"pytest testmon", "pytest testmon (broad)"} or label.startswith("pytest seed-testmon"): + testmon_graph_touched = True if rc == 0 and label in {"pytest testmon", "pytest testmon (broad)"}: raw_stamp = _read_json_artifact(TESTMON_SEED_STAMP) try: @@ -3454,8 +3729,20 @@ def main(argv: list[str] | None = None) -> int: if current_stamp is not None: refreshed_stamp = refresh_stamp(current_stamp, TESTMON_DATA) if refreshed_stamp is not None: - _atomic_write_json(TESTMON_SEED_STAMP, refreshed_stamp.as_dict()) - executable_paths = _changed_executable_paths() + pending_testmon_stamp = refreshed_stamp + try: + executable_paths = _changed_paths_from_testmon_authority(testmon_base_commit, testmon_head_commit) + except PytestResourceError as exc: + changed_path_authority_failed = True + executable_paths = () + rc = 125 + metadata["diagnosis"] = "testmon_changed_path_authority_unavailable" + metadata["error"] = str(exc) + pending_testmon_stamp = None + sys.stderr.write( + "verify: changed-path authority became unavailable after pytest; " + "discarding the affected dependency graph.\n" + ) selected_count = metadata.get("selected_count") if selected_count == 0 and executable_paths: coverage = _matching_testmon_coverage(executable_paths) @@ -3471,11 +3758,7 @@ def main(argv: list[str] | None = None) -> int: else: metadata["zero_selection_coverage"] = coverage elif isinstance(selected_count, int) and selected_count > 0: - _record_testmon_affected_coverage( - executable_paths=executable_paths, - selected_count=selected_count, - run_id=verify_run.run_id, - ) + pending_affected_coverage = (tuple(executable_paths), selected_count) step_result: dict[str, Any] = {"name": label, "duration_s": round(elapsed, 2), "exit": rc} step_result.update(metadata) step_results.append(step_result) @@ -3568,25 +3851,103 @@ def main(argv: list[str] | None = None) -> int: break continue if label in {"pytest testmon", "pytest testmon (broad)"} and not args.seed_testmon and not full_pytest: - _refresh_testmon_selection_attempt(step=step_result, run=verify_run, exit_code=rc) + pending_selection_refresh = (step_result, rc) if rc != 0: exit_code = rc - if _stop_after_failed_step(label): + if rc == 130 or _stop_after_failed_step(label): break - seed_receipt: dict[str, Any] | None = None - if prepared_seed_attempt is not None: - seed_receipt = _finalize_testmon_seed_attempt( - prepared=prepared_seed_attempt, - step_results=step_results, - exit_code=exit_code, + final_head = _git_head() + final_checkout_fingerprint = worktree_fingerprint(ROOT) + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) + checkout_stable = True + if ( + changed_path_authority_failed + or head is None + or final_head is None + or "unavailable" in {checkout_fingerprint, final_checkout_fingerprint} + or mutation_observation.unavailable + ): + checkout_stable = False + step_results.append( + { + "name": "checkout stability", + "duration_s": 0.0, + "exit": 125, + "diagnosis": ( + "testmon_changed_path_authority_unavailable" + if changed_path_authority_failed + else "checkout_fingerprint_unavailable" + ), + "initial_git_head": head, + "final_git_head": final_head, + "initial_worktree_fingerprint": checkout_fingerprint, + "final_worktree_fingerprint": final_checkout_fingerprint, + } ) - if exit_code == 0 and seed_receipt["status"] != "complete": - exit_code = 5 - sys.stderr.write( - "verify: pytest passed but the testmon dependency baseline is incomplete; " - f"inspect {TESTMON_SEED_ATTEMPT}.\n" + if exit_code == 0: + exit_code = 125 + sys.stderr.write("verify: checkout fingerprint unavailable; evidence is not exact-head.\n") + elif final_head != head or mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: + checkout_stable = False + step_results.append( + { + "name": "checkout stability", + "duration_s": 0.0, + "exit": 125, + "diagnosis": "checkout_changed_during_verification", + "initial_git_head": head, + "final_git_head": final_head, + "initial_worktree_fingerprint": checkout_fingerprint, + "final_worktree_fingerprint": final_checkout_fingerprint, + "transient_checkout_mutation": mutation_observation.changed, + "checkout_mutation_path": mutation_observation.observed_path, + } + ) + if exit_code == 0: + exit_code = 125 + sys.stderr.write("verify: checkout contents changed during verification; evidence is not exact-head.\n") + + seed_receipt: dict[str, Any] | None = None + if checkout_stable: + if pending_testmon_stamp is not None: + _atomic_write_json(TESTMON_SEED_STAMP, pending_testmon_stamp.as_dict()) + if pending_affected_coverage is not None: + executable_paths, selected_count = pending_affected_coverage + _record_testmon_affected_coverage( + executable_paths=executable_paths, + selected_count=selected_count, + run_id=verify_run.run_id, + ) + if pending_selection_refresh is not None: + step_result, selection_exit_code = pending_selection_refresh + _refresh_testmon_selection_attempt( + step=step_result, + run=verify_run, + exit_code=selection_exit_code, + ) + if prepared_seed_attempt is not None: + seed_receipt = _finalize_testmon_seed_attempt( + prepared=prepared_seed_attempt, + step_results=step_results, + exit_code=exit_code, ) + if exit_code == 0 and seed_receipt["status"] != "complete": + exit_code = 5 + sys.stderr.write( + "verify: pytest passed but the testmon dependency baseline is incomplete; " + f"inspect {TESTMON_SEED_ATTEMPT}.\n" + ) + elif testmon_graph_touched: + _discard_testmon_dependency_authority() + if prepared_seed_attempt is not None: + seed_receipt = { + "status": "discarded", + "outcome": SeedAttemptOutcome.INCOMPLETE.value, + "resume": False, + "expected_count": len(_testmon_seed_expected_nodeids(prepared_seed_attempt)), + "release_baseline_allowed": False, + } total_duration = round(time.monotonic() - t0, 2) @@ -3594,13 +3955,25 @@ def main(argv: list[str] | None = None) -> int: history_entry: dict[str, Any] = { "timestamp": datetime.now(timezone.utc).isoformat(), "git_head": head, + "final_git_head": final_head, "tier": tier, "run_id": verify_run.run_id, + "checkout_root": str(ROOT.resolve()), + "worktree_fingerprint": checkout_fingerprint, + "final_worktree_fingerprint": final_checkout_fingerprint, "artifact_dir": str(verify_run.relative_run_dir), "steps": step_results, "total_duration_s": total_duration, "exit_code": exit_code, } + checkout_stability_diagnosis = next( + ( + str(step["diagnosis"]) + for step in reversed(step_results) + if step.get("name") == "checkout stability" and "diagnosis" in step + ), + None, + ) fallback_pytest_diagnosis = next( ( str(step["diagnosis"]) @@ -3617,8 +3990,9 @@ def main(argv: list[str] | None = None) -> int: ), fallback_pytest_diagnosis, ) - if pytest_diagnosis is not None: - history_entry["diagnosis"] = pytest_diagnosis + run_diagnosis = checkout_stability_diagnosis or pytest_diagnosis + if run_diagnosis is not None: + history_entry["diagnosis"] = run_diagnosis if seed_receipt is not None: history_entry["testmon_seed"] = { "status": seed_receipt["status"], @@ -3630,8 +4004,8 @@ def main(argv: list[str] | None = None) -> int: "release_baseline_allowed": seed_receipt["release_baseline_allowed"], } + verification_scope = planned_verification_scope if args.quick or args.commit: - verification_scope = VerificationScope.NON_TEST # Non-test verification is intentionally not release authority, but it # is still a typed verification receipt. ``None`` made merge-gate # treat an explicit quick receipt as malformed instead of as a valid @@ -3640,9 +4014,6 @@ def main(argv: list[str] | None = None) -> int: elif full_pytest or args.seed_testmon: narrow_terminal = bool(args.skip_slow) authorized_narrow_terminal = args.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value - verification_scope = ( - VerificationScope.NARROW_TERMINAL if narrow_terminal else VerificationScope.RELEASE_BASELINE - ) if full_pytest: release_baseline_allowed = exit_code == 0 and (not narrow_terminal or authorized_narrow_terminal) else: @@ -3650,7 +4021,6 @@ def main(argv: list[str] | None = None) -> int: not narrow_terminal or authorized_narrow_terminal ) else: - verification_scope = VerificationScope.AFFECTED release_baseline_allowed = _testmon_release_baseline_permission() history_entry["verification_scope"] = verification_scope.value history_entry["release_baseline_allowed"] = release_baseline_allowed @@ -3680,7 +4050,16 @@ def main(argv: list[str] | None = None) -> int: # Persist history and stamp. _save_history(history_entry) - verify_run.finish(exit_code=exit_code, duration_s=total_duration, diagnosis=pytest_diagnosis) + verify_run.finish( + exit_code=exit_code, + duration_s=total_duration, + diagnosis=run_diagnosis, + verification_scope=verification_scope.value, + release_baseline_allowed=release_baseline_allowed, + terminal_authorization=args.terminal_authorization, + final_worktree_fingerprint=final_checkout_fingerprint, + checkout_mutation_path=mutation_observation.observed_path, + ) if exit_code == 0: _stamp_head() @@ -3697,3 +4076,61 @@ def main(argv: list[str] | None = None) -> int: ) return exit_code + + +def _finalize_verify_runner_exception( + run: VerifyRun, + exc: Exception, + *, + run_started: float, + verification_scope: VerificationScope, + use_json: bool, +) -> int: + """Leave typed, durable failed evidence when verification orchestration raises.""" + diagnosis = "verify_runner_exception" + run.finish_interrupted_steps( + exit_code=125, + diagnosis=diagnosis, + termination_reason="runner_exception", + ) + try: + final_worktree_fingerprint = worktree_fingerprint(ROOT) + except Exception: + final_worktree_fingerprint = "unavailable" + payload = run.finish( + exit_code=125, + duration_s=time.monotonic() - run_started, + diagnosis=diagnosis, + verification_scope=verification_scope.value, + release_baseline_allowed=False, + final_worktree_fingerprint=final_worktree_fingerprint, + ) + payload["exception_type"] = type(exc).__name__ + payload["error"] = str(exc) + _save_history(payload) + if use_json: + _print_json(payload) + sys.stderr.write(f"verify: unexpected runner exception: {exc}\n") + return 125 + + +@finalize_checkout_mutation_monitors +def main(argv: list[str] | None = None) -> int: + global _ACTIVE_VERIFY_RUN + _ACTIVE_VERIFY_RUN = None + try: + return _main(argv) + except Exception as exc: + if _ACTIVE_VERIFY_RUN is None: + raise + raw_argv = sys.argv[1:] if argv is None else argv + run, run_started, verification_scope = _ACTIVE_VERIFY_RUN + return _finalize_verify_runner_exception( + run, + exc, + run_started=run_started, + verification_scope=verification_scope, + use_json="--json" in raw_argv, + ) + finally: + _ACTIVE_VERIFY_RUN = None diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 95949457ac..30d46bfaf1 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -8,29 +8,44 @@ from __future__ import annotations import contextlib +import fcntl +import functools import hashlib import json import os +import platform import re import shutil import stat import subprocess +import threading import time import uuid -from collections.abc import Mapping +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, replace from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, ParamSpec, TextIO, TypeVar + +import watchfiles from polylogue.core.metrics import read_cgroup_memory_headroom_bytes VERIFY_CACHE = Path(".cache/verify") VERIFY_RUNS_DIR = VERIFY_CACHE / "runs" +_XDG_STATE_HOME = os.environ.get("XDG_STATE_HOME", "").strip() +DEVTOOLS_STATE_DIR = ( + (Path(_XDG_STATE_HOME) if _XDG_STATE_HOME else Path.home() / ".local" / "state") / "polylogue" / "devtools" +) +VERIFY_HISTORY_PATH = DEVTOOLS_STATE_DIR / "verify-history.jsonl" CURRENT_RUN_PATH = VERIFY_CACHE / "current-run.json" +VERIFICATION_INVOCATION_ID_ENV = "POLYLOGUE_VERIFICATION_INVOCATION_ID" +VERIFICATION_RECEIPT_PATH_ENV = "POLYLOGUE_VERIFICATION_RECEIPT_PATH" CURRENT_RESOURCES_PATH = VERIFY_CACHE / "current-pytest-resources.jsonl" CURRENT_POSTMORTEM_PATH = VERIFY_CACHE / "current-pytest-postmortem.json" CURRENT_CONTAINMENT_PATH = VERIFY_CACHE / "current-pytest-containment.json" +CURRENT_STATISTICS_PATH = VERIFY_CACHE / "current-pytest-statistics.json" +PYTEST_CANONICAL_REPORT_NAME = "pytest-report.json" CURRENT_EVENTS_DIR = VERIFY_CACHE / "current-pytest-events" DEFAULT_BASETEMP_SIZE_SAMPLE_INTERVAL_S = 15.0 DEFAULT_TMPFS_SIZE_SAMPLE_INTERVAL_S = 2.0 @@ -45,6 +60,7 @@ PYTEST_BASETEMP_MIN_FREE_MB_ENV = "POLYLOGUE_PYTEST_BASETEMP_MIN_FREE_MB" DEFAULT_PYTEST_BASETEMP_MIN_FREE_MB = 1024 PYTEST_BASETEMP_REQUIRED_MB_ENV = "POLYLOGUE_PYTEST_BASETEMP_REQUIRED_MB" +PYTEST_EXPLICIT_BASETEMP_ENV = "POLYLOGUE_PYTEST_EXPLICIT_BASETEMP" PYTEST_MEMORY_ENVELOPE_WORKERS = 4 PYTEST_MEMORY_ENVELOPE_PSS_KB = 4_353_168 PYTEST_MEMORY_ENVELOPE_TMPFS_KB = 1_472_636 @@ -68,6 +84,560 @@ class PytestResourceError(RuntimeError): """Raised when the host cannot safely start a managed pytest run.""" +def _trailing_history_record(descriptor: int, *, end: int) -> tuple[int, bytes]: + """Read only the final unterminated JSONL record and its start offset.""" + cursor = end + suffix: list[bytes] = [] + while cursor > 0: + start = max(0, cursor - 64 * 1024) + os.lseek(descriptor, start, os.SEEK_SET) + chunk = os.read(descriptor, cursor - start) + delimiter = chunk.rfind(b"\n") + if delimiter >= 0: + return start + delimiter + 1, chunk[delimiter + 1 :] + b"".join(reversed(suffix)) + suffix.append(chunk) + cursor = start + return 0, b"".join(reversed(suffix)) + + +def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTORY_PATH) -> None: + """Append one complete invocation to the cross-worktree run history. + + ``O_APPEND`` plus an advisory lock keeps concurrent worktrees from + overwriting or interleaving their records, including short writes. + Detailed artifacts remain checkout-local; this history is the compact + durable index used to find and compare them. + """ + path.parent.mkdir(parents=True, exist_ok=True) + payload = (json.dumps(dict(entry), ensure_ascii=False) + "\n").encode() + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_APPEND, 0o600) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + end = os.lseek(descriptor, 0, os.SEEK_END) + if end: + os.lseek(descriptor, end - 1, os.SEEK_SET) + if os.read(descriptor, 1) != b"\n": + trailing_start, trailing = _trailing_history_record(descriptor, end=end) + try: + json.loads(trailing) + except (UnicodeDecodeError, json.JSONDecodeError): + os.ftruncate(descriptor, trailing_start) + else: + # A complete JSON record can lose only its framing newline + # during an interrupted append. Preserve it before adding + # the next durable record. + os.lseek(descriptor, 0, os.SEEK_END) + os.write(descriptor, b"\n") + remaining = memoryview(payload) + while remaining: + written = os.write(descriptor, remaining) + if written <= 0: + raise OSError("verification history append made no progress") + remaining = remaining[written:] + finally: + os.close(descriptor) + + +def pytest_command_worker_request(cmd: Sequence[str]) -> str | None: + """Return the last xdist worker request from a final pytest command.""" + request: str | None = None + for index, argument in enumerate(cmd): + if argument in {"-n", "--numprocesses"}: + if index + 1 < len(cmd): + request = cmd[index + 1] + elif argument.startswith("--numprocesses="): + request = argument.removeprefix("--numprocesses=") + elif argument.startswith("-n") and len(argument) > 2: + request = argument[2:].removeprefix("=") + return request + + +def _read_only_git_env() -> dict[str, str]: + """Prevent observational Git commands from refreshing checkout authority.""" + return {**os.environ, "GIT_OPTIONAL_LOCKS": "0"} + + +def worktree_fingerprint(root: Path | None = None) -> str: + """Fingerprint tracked changes plus exact non-ignored untracked content.""" + checkout_root = (root or Path.cwd()).resolve() + digest = hashlib.sha256() + try: + tracked_flags = subprocess.run( + ["git", "ls-files", "-v", "-z"], + capture_output=True, + timeout=30, + cwd=checkout_root, + env=_read_only_git_env(), + ) + except (OSError, subprocess.TimeoutExpired): + return "unavailable" + if tracked_flags.returncode != 0 or tracked_flags.stderr.strip(): + return "unavailable" + for record in tracked_flags.stdout.split(b"\0"): + if not record: + continue + tag = record[:1] + if tag.islower() or tag == b"S": + # assume-unchanged and skip-worktree can hide worktree bytes from + # both status and diff, so Git cannot authorize exact evidence. + return "unavailable" + for command in ( + ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"], + ["git", "diff", "--binary", "HEAD", "--"], + ): + try: + result = subprocess.run( + command, + capture_output=True, + timeout=30, + cwd=checkout_root, + env=_read_only_git_env(), + ) + except (OSError, subprocess.TimeoutExpired): + return "unavailable" + if result.returncode != 0 or result.stderr.strip(): + return "unavailable" + digest.update(result.stdout) + digest.update(b"\0") + try: + untracked = subprocess.run( + ["git", "ls-files", "--others", "--exclude-standard", "-z"], + capture_output=True, + timeout=30, + cwd=checkout_root, + env=_read_only_git_env(), + ) + except (OSError, subprocess.TimeoutExpired): + return "unavailable" + if untracked.returncode != 0 or untracked.stderr.strip(): + return "unavailable" + for raw_path in sorted(path for path in untracked.stdout.split(b"\0") if path): + try: + path_text = os.fsdecode(raw_path) + path = checkout_root / path_text + mode = path.lstat().st_mode + digest.update(raw_path) + digest.update(b"\0") + if stat.S_ISLNK(mode): + digest.update(b"symlink\0") + digest.update(os.fsencode(os.readlink(path))) + elif stat.S_ISREG(mode): + digest.update(b"file\0") + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + else: + digest.update(f"mode:{stat.S_IFMT(mode):o}".encode()) + digest.update(b"\0") + except OSError: + return "unavailable" + return digest.hexdigest() + + +@dataclass(frozen=True) +class CheckoutMutationObservation: + """Whether an exact-head verification interval observed a checkout write.""" + + changed: bool + unavailable: bool + observed_path: str | None = None + + +class CheckoutMutationMonitor: + """Fail closed when watchfiles cannot observe the checkout interval. + + Endpoint hashes establish the state of the checkout, while this monitor + records writes that occur and are later reverted before the final sample. + Watches exclude verifier-owned disposable directories so receipts do not + invalidate themselves. + """ + + _IGNORED_TOP_LEVEL = frozenset( + { + ".cache", + ".git", + ".hypothesis", + ".local", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".venv", + "__pycache__", + } + ) + _WATCH_START_TIMEOUT_S = 1.0 + _WATCH_SETTLE_S = 0.2 + _WATCH_RUST_TIMEOUT_MS = 25 + _POLLING_DISABLED_VALUES = frozenset({"false", "disable", "disabled"}) + + def __init__(self, root: Path) -> None: + self.root = root.resolve() + self._changed = False + self._observed_path: str | None = None + self._unavailable = False + self._stop = threading.Event() + self._ready = threading.Event() + self._thread: threading.Thread | None = None + self._state_lock = threading.Lock() + self._tracked_paths: frozenset[Path] = frozenset() + self._tracked_directories: frozenset[Path] = frozenset() + self._ignored_roots: frozenset[Path] = frozenset() + self._git_index_path: Path | None = None + self._git_authority_paths: dict[Path, str] = {} + self._directory_topology_fingerprint: frozenset[str] | None = None + + def start(self) -> None: + """Start and prove the portable interval watcher before verification.""" + if self._polling_backend_requested(): + with self._state_lock: + self._unavailable = True + self._ready.set() + return + self._thread = threading.Thread(target=self._watch, name="checkout-mutation-monitor", daemon=True) + self._thread.start() + if not self._ready.wait(timeout=self._WATCH_START_TIMEOUT_S): + with self._state_lock: + self._unavailable = True + self._stop.set() + self._thread.join(timeout=self._WATCH_START_TIMEOUT_S) + + def finish(self) -> CheckoutMutationObservation: + """Stop monitoring only after the caller took its final fingerprint.""" + # The final fingerprint is already sampled. Give watchfiles one short + # backend turn to surface any event emitted before that sample, then + # stop the generator cleanly through its portable stop event. + self._stop.wait(self._WATCH_SETTLE_S) + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=1) + if self._thread.is_alive(): + with self._state_lock: + self._unavailable = True + with self._state_lock: + return CheckoutMutationObservation( + changed=self._changed, + unavailable=self._unavailable, + observed_path=self._observed_path, + ) + + def _watch(self) -> None: + try: + watched_directories = self._watched_directories() + if self._unavailable: + return + for changes in watchfiles.watch( + *watched_directories, + watch_filter=None, + debounce=0, + step=1, + stop_event=self._stop, + rust_timeout=self._WATCH_RUST_TIMEOUT_MS, + yield_on_timeout=True, + raise_interrupt=False, + force_polling=False, + recursive=False, + ): + # An empty timeout batch proves the backend initialized before + # a verification command starts, closing the startup race. + if not self._ready.is_set() and not self._directory_topology_is_stable(watched_directories): + with self._state_lock: + self._unavailable = True + return + self._ready.set() + for _change, raw_path in changes: + self._record_change(Path(raw_path)) + if self._changed or self._unavailable: + return + if not self._stop.is_set(): + with self._state_lock: + self._unavailable = True + except Exception: + with self._state_lock: + self._unavailable = True + finally: + self._ready.set() + + @classmethod + def _polling_backend_requested(cls) -> bool: + """Reject watchfiles modes that cannot witness every interval mutation.""" + forced = os.getenv("WATCHFILES_FORCE_POLLING") + if forced: + return forced.lower() not in cls._POLLING_DISABLED_VALUES + uname = platform.uname() + return uname.system.lower() == "linux" and "microsoft-standard" in uname.release.lower() + + def _watched_directories(self) -> list[Path]: + """Watch existing source directories shallowly and omit disposable trees.""" + self._tracked_paths = self._git_tracked_paths() + self._tracked_directories = frozenset( + parent for tracked_path in self._tracked_paths for parent in tracked_path.parents if parent != Path(".") + ) + self._ignored_roots = self._ignored_directory_roots() + directories: list[Path] = [] + + def walk_error(_error: OSError) -> None: + with self._state_lock: + self._unavailable = True + + for current, child_directories, _files in os.walk(self.root, onerror=walk_error): + current_path = Path(current) + relative_current = current_path.relative_to(self.root) + retained_children: list[str] = [] + for child in child_directories: + relative_child = relative_current / child + disposable = child in self._IGNORED_TOP_LEVEL or self._is_within_ignored_root( + relative_child, + self._ignored_roots, + ) + if disposable and relative_child not in self._tracked_directories: + continue + retained_children.append(child) + child_directories[:] = retained_children + directories.append(current_path) + self._git_index_path = self._resolve_git_index_path() + if self._git_index_path is not None: + self._git_authority_paths[self._git_index_path] = ".git/index" + self._git_authority_paths.update(self._resolve_git_head_paths()) + for authority_path in self._git_authority_paths: + watched_parent = authority_path.parent + while not watched_parent.exists() and watched_parent != watched_parent.parent: + watched_parent = watched_parent.parent + if watched_parent not in directories: + directories.append(watched_parent) + self._directory_topology_fingerprint = self._directory_topology(directories) + return directories + + def _directory_topology(self, directories: Sequence[Path]) -> frozenset[str]: + """Fingerprint source directory membership without trusting pre-watch state.""" + return frozenset( + relative.as_posix() + for directory in directories + if directory.exists() + and (relative := directory.resolve(strict=False)).is_relative_to(self.root) + and (relative := relative.relative_to(self.root)) is not None + ) + + def _directory_topology_is_stable(self, initial_directories: Sequence[Path]) -> bool: + """Reject source directories that changed while the watcher initialized.""" + initial = self._directory_topology_fingerprint or self._directory_topology(initial_directories) + current = self._directory_topology(self._watched_directories()) + return not self._unavailable and current == initial + + def _git_tracked_paths(self) -> frozenset[Path]: + """Snapshot index membership so tracked paths never inherit ignore rules.""" + result = self._git_command(["ls-files", "-z"]) + if result is None: + return frozenset() + return frozenset(Path(os.fsdecode(raw)) for raw in result.stdout.split(b"\0") if raw) + + def _resolve_git_index_path(self) -> Path | None: + """Resolve the worktree-specific index whose writes can change path authority.""" + result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", "index"]) + if result is None: + return None + raw_path = os.fsdecode(result.stdout).strip() + if not raw_path: + with self._state_lock: + self._unavailable = True + return None + return Path(raw_path) + + def _resolve_git_head_paths(self) -> dict[Path, str]: + """Resolve the worktree HEAD file and its current symbolic ref.""" + paths: dict[Path, str] = {} + head_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", "HEAD"]) + symbolic_result = self._git_command( + ["symbolic-ref", "--quiet", "HEAD"], + allowed_returncodes=frozenset({0, 1}), + ) + if head_result is None or symbolic_result is None: + return paths + raw_head_path = os.fsdecode(head_result.stdout).strip() + symbolic_ref = os.fsdecode(symbolic_result.stdout).strip() + if not raw_head_path or (symbolic_result.returncode == 0 and not symbolic_ref): + with self._state_lock: + self._unavailable = True + return paths + paths[Path(raw_head_path)] = ".git/HEAD" + packed_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", "packed-refs"]) + if packed_result is None: + return paths + raw_packed_path = os.fsdecode(packed_result.stdout).strip() + if not raw_packed_path: + with self._state_lock: + self._unavailable = True + return paths + paths[Path(raw_packed_path)] = ".git/packed-refs" + if symbolic_result.returncode == 0: + ref_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", symbolic_ref]) + if ref_result is None: + return paths + raw_ref_path = os.fsdecode(ref_result.stdout).strip() + if not raw_ref_path: + with self._state_lock: + self._unavailable = True + return paths + paths[Path(raw_ref_path)] = f".git/{symbolic_ref}" + return paths + + def _git_command( + self, + args: list[str], + *, + allowed_returncodes: frozenset[int] = frozenset({0}), + ) -> subprocess.CompletedProcess[bytes] | None: + try: + result = subprocess.run( + ["git", *args], + cwd=self.root, + capture_output=True, + timeout=2, + check=False, + env=_read_only_git_env(), + ) + except (OSError, subprocess.TimeoutExpired): + with self._state_lock: + self._unavailable = True + return None + if result.returncode not in allowed_returncodes or result.stderr.strip(): + with self._state_lock: + self._unavailable = True + return None + return result + + def _ignored_directory_roots(self) -> frozenset[Path]: + """Return existing ignored directory roots without traversing their contents.""" + result = self._git_command(["status", "--porcelain=v1", "-z", "--ignored=matching", "--untracked-files=normal"]) + if result is None: + return frozenset() + ignored: set[Path] = set() + for record in result.stdout.split(b"\0"): + if not record.startswith(b"!! "): + continue + relative = Path(os.fsdecode(record[3:]).rstrip("/")) + if relative.parts and (self.root / relative).is_dir(): + ignored.add(relative) + return frozenset(ignored) + + @staticmethod + def _is_within_ignored_root(relative: Path, ignored_roots: frozenset[Path]) -> bool: + return any(relative == root or relative.is_relative_to(root) for root in ignored_roots) + + def _record_change(self, candidate: Path) -> None: + if not candidate.is_absolute(): + candidate = self.root / candidate + for authority_path, label in self._git_authority_paths.items(): + if candidate != authority_path and authority_path.is_relative_to(candidate): + with self._state_lock: + self._changed = True + self._observed_path = label + return + if candidate.parent != authority_path.parent: + continue + if candidate.name == f"{authority_path.name}.lock": + # An uncommitted lock is not yet checkout authority. A + # completed transaction is witnessed when the lock replaces + # its authority file. + return + if candidate.name == authority_path.name: + with self._state_lock: + self._changed = True + self._observed_path = label + return + try: + relative = candidate.relative_to(self.root) + except ValueError: + return + if self._path_is_ignored(relative): + return + with self._state_lock: + self._changed = True + self._observed_path = relative.as_posix() + + def _path_is_ignored(self, relative: Path) -> bool: + if relative in self._tracked_paths: + return False + if any(part in self._IGNORED_TOP_LEVEL for part in relative.parts): + return True + if self._is_within_ignored_root(relative, self._ignored_roots): + return True + try: + result = subprocess.run( + ["git", "check-ignore", "--quiet", "--no-index", "--", relative.as_posix()], + cwd=self.root, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=1, + env=_read_only_git_env(), + ) + except (OSError, subprocess.TimeoutExpired): + with self._state_lock: + self._unavailable = True + return True + if result.returncode == 0: + return True + if result.returncode == 1: + return False + with self._state_lock: + self._unavailable = True + return True + + +_MonitorParams = ParamSpec("_MonitorParams") +_MonitorResult = TypeVar("_MonitorResult") +_MONITOR_STATE = threading.local() + + +def _checkout_monitor_stack() -> list[CheckoutMutationMonitor]: + stack = getattr(_MONITOR_STATE, "stack", None) + if stack is None: + stack = [] + _MONITOR_STATE.stack = stack + return stack + + +def start_checkout_mutation_monitor(monitor: CheckoutMutationMonitor) -> None: + """Start a runner-owned monitor and register it for exceptional cleanup.""" + stack = _checkout_monitor_stack() + stack.append(monitor) + try: + monitor.start() + except BaseException: + with contextlib.suppress(Exception): + finish_checkout_mutation_monitor(monitor) + raise + + +def finish_checkout_mutation_monitor(monitor: CheckoutMutationMonitor) -> CheckoutMutationObservation: + """Finish one monitor and retire its runner cleanup obligation.""" + stack = _checkout_monitor_stack() + try: + return monitor.finish() + finally: + for index in range(len(stack) - 1, -1, -1): + if stack[index] is monitor: + del stack[index] + break + + +def finalize_checkout_mutation_monitors( + function: Callable[_MonitorParams, _MonitorResult], +) -> Callable[_MonitorParams, _MonitorResult]: + """Guarantee that monitors started by a runner finish on every exit.""" + + @functools.wraps(function) + def wrapped(*args: _MonitorParams.args, **kwargs: _MonitorParams.kwargs) -> _MonitorResult: + stack = _checkout_monitor_stack() + baseline_depth = len(stack) + try: + return function(*args, **kwargs) + finally: + while len(stack) > baseline_depth: + finish_checkout_mutation_monitor(stack[-1]) + + return wrapped + + @dataclass(frozen=True) class PytestRuntimePolicy: """One start-time resource decision for a managed pytest run.""" @@ -142,9 +712,230 @@ def _slug(value: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-").lower() or "step" -def git_dirty() -> bool: +def _percentile(values: list[float], percentile: float) -> float | None: + if not values: + return None + ordered = sorted(values) + if len(ordered) == 1: + return round(ordered[0], 4) + position = (len(ordered) - 1) * percentile + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + fraction = position - lower + return round(ordered[lower] + (ordered[upper] - ordered[lower]) * fraction, 4) + + +def _distribution(values: list[float]) -> dict[str, float | int | None]: + return { + "count": len(values), + "p50_s": _percentile(values, 0.50), + "p95_s": _percentile(values, 0.95), + "p99_s": _percentile(values, 0.99), + "max_s": round(max(values), 4) if values else None, + "sum_s": round(sum(values), 4) if values else 0.0, + } + + +def aggregate_pytest_statistics( + step_dir: Path, + *, + command: list[Any] | tuple[Any, ...] = (), + step_result: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Reduce the append-only pytest evidence into one durable step summary. + + The raw event stream remains authoritative for forensics. This summary is + deliberately derived and replaceable: it makes repeated performance work + cheap without introducing a second source of truth for test outcomes. + """ + events: list[dict[str, Any]] = [] + events_path = step_dir / "events.jsonl" + if events_path.exists(): + for line in events_path.read_text(encoding="utf-8", errors="replace").splitlines(): + with contextlib.suppress(json.JSONDecodeError): + row = json.loads(line) + if isinstance(row, dict): + events.append(row) + + phases: dict[str, list[float]] = {"setup": [], "call": [], "teardown": []} + outcomes: dict[str, int] = {} + phase_outcomes: dict[str, dict[str, int]] = {"setup": {}, "call": {}, "teardown": {}} + nodes: set[str] = set() + workers: set[str] = set() + reports: dict[tuple[str, str], dict[str, Any]] = {} + for row in events: + worker = row.get("worker_id") + if isinstance(worker, str): + workers.add(worker) + nodeid = row.get("nodeid") + if isinstance(nodeid, str) and nodeid: + nodes.add(nodeid) + event = row.get("event") + if event != "test_report" or not isinstance(nodeid, str) or not nodeid: + continue + when = row.get("when") + if when not in phases: + continue + key = (nodeid, when) + prior = reports.get(key) + # xdist sends the worker's original report to the controller. Prefer + # the worker event when both arrive, while accepting old/controller-only + # artifacts produced before that forwarding copy was suppressed. + if prior is None or (prior.get("worker_id") == "controller" and row.get("worker_id") != "controller"): + reports[key] = row + + canonical_outcomes: dict[str, str] = {} + canonical_report_present = False + canonical_report_path = step_dir / PYTEST_CANONICAL_REPORT_NAME + if canonical_report_path.exists(): + with contextlib.suppress(OSError, json.JSONDecodeError): + canonical_report = json.loads(canonical_report_path.read_text(encoding="utf-8")) + canonical_tests = canonical_report.get("tests") if isinstance(canonical_report, dict) else None + if isinstance(canonical_tests, list): + canonical_report_present = True + for test in canonical_tests: + if not isinstance(test, dict): + continue + nodeid = test.get("nodeid") + outcome = test.get("outcome") + if not isinstance(nodeid, str) or not nodeid or not isinstance(outcome, str): + continue + nodes.add(nodeid) + canonical_outcomes[nodeid] = outcome + for when in phases: + phase = test.get(when) + if not isinstance(phase, dict) or (nodeid, when) in reports: + continue + phase_outcome = phase.get("outcome") + duration = phase.get("duration") + reports[(nodeid, when)] = { + "nodeid": nodeid, + "when": when, + "outcome": phase_outcome, + "duration_s": duration, + "worker_id": "canonical-report", + } + + reports_by_node: dict[str, dict[str, dict[str, Any]]] = {} + for (nodeid, when), row in reports.items(): + reports_by_node.setdefault(nodeid, {})[when] = row + duration = row.get("duration_s") + if isinstance(duration, (int, float)): + phases[when].append(float(duration)) + outcome = row.get("outcome") + if isinstance(outcome, str): + bucket = phase_outcomes[when] + bucket[outcome] = bucket.get(outcome, 0) + 1 + + for nodeid in nodes: + node_reports = reports_by_node.get(nodeid, {}) + setup = node_reports.get("setup", {}).get("outcome") + call = node_reports.get("call", {}).get("outcome") + teardown = node_reports.get("teardown", {}).get("outcome") + canonical_outcome = canonical_outcomes.get(nodeid) + if canonical_outcome is not None: + terminal = canonical_outcome + elif setup == "failed" or teardown == "failed": + terminal = "error" + elif isinstance(call, str): + terminal = call + elif setup in {"skipped", "xfailed", "xpassed"}: + terminal = str(setup) + elif teardown in {"skipped", "xfailed", "xpassed"}: + terminal = str(teardown) + else: + # A test may have emitted its start event just before an interrupt + # or forced containment cleanup. Keep that missing terminal phase + # visible so outcome totals still account for every started node. + terminal = "interrupted" + outcomes[terminal] = outcomes.get(terminal, 0) + 1 + + resources: list[dict[str, Any]] = [] + resources_path = step_dir / "resources.jsonl" + if resources_path.exists(): + for line in resources_path.read_text(encoding="utf-8", errors="replace").splitlines(): + with contextlib.suppress(json.JSONDecodeError): + row = json.loads(line) + if isinstance(row, dict): + resources.append(row) + explicit_worker_count: int | None = None + worker_request = pytest_command_worker_request([str(value) for value in command]) + if worker_request is not None: + with contextlib.suppress(ValueError): + explicit_worker_count = int(worker_request) + basetemp_sizes = [ + int(size_value) * 1024 for row in resources if isinstance((size_value := row.get("basetemp_size_kb")), int) + ] + basetemp_allocated = [ + int(allocated_value) * 1024 + for row in resources + if isinstance((allocated_value := row.get("basetemp_allocated_kb")), int) + ] + containment: dict[str, Any] = {} + containment_path = step_dir / "containment.json" + if containment_path.exists(): + with contextlib.suppress(json.JSONDecodeError): + raw = json.loads(containment_path.read_text(encoding="utf-8")) + if isinstance(raw, dict): + containment = raw + + parent_cleanup = (step_result or {}).get("basetemp_cleanup") + return { + "schema_version": 1, + "canonical_report_status": "present" if canonical_report_present else "missing", + "command": [str(value) for value in command], + "node_count": len(nodes), + "outcomes": outcomes, + "phase_outcomes": phase_outcomes, + "phases": {name: _distribution(values) for name, values in phases.items()}, + "xdist": { + "worker_ids": sorted(workers), + "worker_count": max( + 0, + len(workers) - (1 if "controller" in workers else 0), + explicit_worker_count or 0, + ), + }, + "storage": { + "basetemp_logical_bytes_max": max(basetemp_sizes, default=None), + "basetemp_allocated_bytes_max": max(basetemp_allocated, default=None), + "basetemp_root": next( + (row.get("basetemp") for row in reversed(resources) if isinstance(row.get("basetemp"), str)), + None, + ), + }, + "resources": { + "peak_tree_rss_kb": max( + (int(row["tree_rss_kb"]) for row in resources if isinstance(row.get("tree_rss_kb"), int)), + default=None, + ), + "peak_tree_pss_kb": max( + (int(row["tree_pss_kb"]) for row in resources if isinstance(row.get("tree_pss_kb"), int)), + default=None, + ), + "peak_cgroup_memory_bytes": max( + ( + int(row["cgroup_memory_peak_bytes"]) + for row in resources + if isinstance(row.get("cgroup_memory_peak_bytes"), int) + ), + default=None, + ), + }, + "cleanup": { + "complete": True + if isinstance(parent_cleanup, str) and parent_cleanup + else containment.get("tmpfs_cleanup_complete"), + "termination_reason": containment.get("termination_reason"), + "escalated_to_sigkill": containment.get("escalated_to_sigkill"), + "exit_code": containment.get("exit_code", (step_result or {}).get("exit")), + }, + } + + +def git_dirty(cwd: Path | None = None) -> bool: try: - result = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, timeout=5) + result = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, timeout=5, cwd=cwd) except (OSError, subprocess.TimeoutExpired): return True return bool(result.stdout.strip()) @@ -188,6 +979,7 @@ class PytestStepArtifacts: resources_path: Path postmortem_path: Path containment_path: Path + statistics_path: Path class VerifyRun: @@ -202,6 +994,7 @@ def __init__( root: Path | None = None, polylogue_import_path: str | None = None, environment_fingerprint: Mapping[str, Any] | None = None, + worktree_fingerprint: str | None = None, ) -> None: self.root = root or Path.cwd() self.run_id = make_run_id(tier=tier) @@ -211,7 +1004,7 @@ def __init__( "tier": tier, "argv": list(argv), "git_head": git_head, - "git_dirty": git_dirty(), + "git_dirty": git_dirty(self.root), # Receipt for the worktree-import hazard (devtools/checkout_guard.py): # the resolved `polylogue` package path this run actually used, so a # wrong-tree run is visible after the fact from the run artifact @@ -219,6 +1012,7 @@ def __init__( # caller and this fired for a different process boundary. "polylogue_import_path": polylogue_import_path, "environment_fingerprint": dict(environment_fingerprint) if environment_fingerprint is not None else None, + "worktree_fingerprint": worktree_fingerprint, # A VerifyRun can be constructed by maintenance/test helpers that # do not have a checkout fingerprint. Keep its current-run marker # attributable to this checkout either way. @@ -229,6 +1023,9 @@ def __init__( "steps": [], "artifact_dir": str(VERIFY_RUNS_DIR / self.run_id), } + invocation_id = os.environ.get(VERIFICATION_INVOCATION_ID_ENV) + if invocation_id: + self._payload["invocation_id"] = invocation_id self.run_dir.mkdir(parents=True, exist_ok=True) self.write() @@ -238,6 +1035,9 @@ def relative_run_dir(self) -> Path: def write(self) -> None: _write_json(self.run_dir / "run.json", self._payload) + invocation_receipt = os.environ.get(VERIFICATION_RECEIPT_PATH_ENV) + if invocation_receipt: + _write_json(Path(invocation_receipt), self._payload) current_path = self.root / CURRENT_RUN_PATH if not _current_owner_is_other_live_run(current_path): _write_json(current_path, self._payload) @@ -260,6 +1060,7 @@ def start_step(self, *, label: str, cmd: list[str]) -> PytestStepArtifacts: resources_path=step_dir / "resources.jsonl", postmortem_path=step_dir / "postmortem.json", containment_path=step_dir / "containment.json", + statistics_path=step_dir / "statistics.json", ) step_dir.mkdir(parents=True, exist_ok=True) self._payload["steps"].append( @@ -275,14 +1076,59 @@ def start_step(self, *, label: str, cmd: list[str]) -> PytestStepArtifacts: self.write() return artifacts - def finish_step(self, *, step_id: str, result: dict[str, Any]) -> None: + def finish_step(self, *, step_id: str, result: dict[str, Any]) -> dict[str, Any] | None: + """Finalize one step and return its durable compact representation.""" for step in self._payload["steps"]: if step.get("step_id") == step_id: step.update(result) step["finished_at"] = utc_now() step["status"] = "success" if result.get("exit") == 0 else "failed" + step_dir = self.run_dir / "steps" / step_id + if not str(step.get("name", "")).startswith("pytest"): + break + # An interrupted runner never returns through the normal + # post-subprocess merge. Fold shards here, before every + # aggregation path, so completed worker evidence survives. + with contextlib.suppress(OSError): + merge_worker_events(step_dir / "events", step_dir / "events.jsonl") + statistics_path = step_dir / "statistics.json" + with contextlib.suppress(OSError, ValueError): + statistics = aggregate_pytest_statistics( + step_dir, + command=step.get("cmd", []), + step_result=result, + ) + _write_json(statistics_path, statistics) + with contextlib.suppress(OSError): + shutil.copyfile(statistics_path, self.root / CURRENT_STATISTICS_PATH) + step["statistics_path"] = str(self.relative_run_dir / "steps" / step_id / "statistics.json") + # Keep the compact aggregate in the cross-worktree history + # itself. The detailed artifact path is checkout-local and + # may disappear when a merged lane is cleaned up. + step["statistics"] = statistics break self.write() + return next((dict(step) for step in self._payload["steps"] if step.get("step_id") == step_id), None) + + def finish_interrupted_steps( + self, + *, + exit_code: int, + diagnosis: str, + termination_reason: str = "operator_interrupt", + ) -> None: + """Close every open step when the outer runner cannot continue.""" + for step in self._payload["steps"]: + if step.get("status") == "running": + self.finish_step( + step_id=str(step["step_id"]), + result={ + "duration_s": None, + "exit": exit_code, + "diagnosis": diagnosis, + "termination_reason": termination_reason, + }, + ) def finish( self, @@ -293,6 +1139,9 @@ def finish( verification_scope: str | None = None, release_baseline_allowed: bool | None = None, terminal_authorization: str | None = None, + final_worktree_fingerprint: str | None = None, + checkout_mutation_path: str | None = None, + checkout_diagnosis: str | None = None, ) -> dict[str, Any]: self._payload["finished_at"] = utc_now() self._payload["duration_s"] = round(duration_s, 2) @@ -300,6 +1149,12 @@ def finish( self._payload["status"] = "success" if exit_code == 0 else "failed" if diagnosis: self._payload["diagnosis"] = diagnosis + if final_worktree_fingerprint is not None: + self._payload["final_worktree_fingerprint"] = final_worktree_fingerprint + if checkout_mutation_path is not None: + self._payload["checkout_mutation_path"] = checkout_mutation_path + if checkout_diagnosis is not None: + self._payload["checkout_diagnosis"] = checkout_diagnosis if verification_scope is not None: self._payload["verification_scope"] = verification_scope self._payload["release_baseline_allowed"] = release_baseline_allowed @@ -310,6 +1165,11 @@ def finish( def env_for_pytest_step(env: dict[str, str], *, run: VerifyRun, artifacts: PytestStepArtifacts) -> dict[str, str]: updated = dict(env) + # The merge-gate invocation receipt belongs to the top-level devtools + # process. Pytest and any nested harness commands must not inherit the + # token and overwrite that receipt with a child run. + updated.pop(VERIFICATION_INVOCATION_ID_ENV, None) + updated.pop(VERIFICATION_RECEIPT_PATH_ENV, None) updated["POLYLOGUE_VERIFY_RUN_ID"] = run.run_id updated["POLYLOGUE_PYTEST_RUN_ID"] = run.run_id updated["POLYLOGUE_PYTEST_EVENTS_DIR"] = str(artifacts.events_dir) @@ -338,6 +1198,8 @@ def copy_current_pytest_artifacts(root: Path, artifacts: PytestStepArtifacts, *, shutil.copyfile(artifacts.postmortem_path, root / CURRENT_POSTMORTEM_PATH) with contextlib.suppress(FileNotFoundError): shutil.copyfile(artifacts.containment_path, root / CURRENT_CONTAINMENT_PATH) + with contextlib.suppress(FileNotFoundError): + shutil.copyfile(artifacts.statistics_path, root / CURRENT_STATISTICS_PATH) def merge_worker_events(events_dir: Path, merged_path: Path) -> int: @@ -548,18 +1410,22 @@ def _fs_usage(path: Path) -> dict[str, int] | None: return None -def _dir_size_kb(path: Path) -> int | None: +def _dir_usage_kb(path: Path) -> tuple[int | None, int | None]: + """Measure apparent and allocated bytes owned by one basetemp tree.""" if not path.exists(): - return None - total = 0 + return None, None + logical_total = 0 + allocated_total = 0 try: for item in path.rglob("*"): with contextlib.suppress(OSError): - if item.is_file(): - total += item.stat().st_size + item_stat = item.lstat() + if not stat.S_ISDIR(item_stat.st_mode): + logical_total += item_stat.st_size + allocated_total += item_stat.st_blocks * 512 except OSError: - return None - return int(total / 1024) + return None, None + return int(logical_total / 1024), int(allocated_total / 1024) def checkout_hash(root: Path) -> str: @@ -569,6 +1435,7 @@ def checkout_hash(root: Path) -> str: DEFAULT_PYTEST_BASETEMP_ROOT = Path("/realm/tmp/polylogue-pytest") _CLOUD_PYTEST_BASETEMP_ROOT = Path("/tmp/polylogue-pytest") PYTEST_TMPFS_ROOT = Path("/dev/shm") +_PYTEST_BASETEMP_CLAIM_PREFIX = ".polylogue-pytest-claim-" def _is_beneath(path: Path, root: Path) -> bool: @@ -580,6 +1447,73 @@ def _is_beneath(path: Path, root: Path) -> bool: return True +def pytest_basetemp_claim_path(basetemp: Path, *, kind: str) -> Path: + """Return the durable, adjacent claim path for one pytest basetemp. + + Pytest lazily clears an explicit ``--basetemp`` before first use, so an + ownership record inside that tree cannot survive normal initialization. + Claims live beside the tree and are keyed by its canonical filesystem + path, not by a reusable basename. A configured symlink and an explicit + real-path spelling must therefore serialize through the same claim. + """ + try: + canonical = basetemp.resolve() + except OSError: + canonical = basetemp.absolute() + digest = hashlib.sha256(str(canonical).encode("utf-8")).hexdigest()[:20] + return canonical.parent / f"{_PYTEST_BASETEMP_CLAIM_PREFIX}{kind}-{digest}" + + +def clear_managed_pytest_basetemp_claim(basetemp: Path) -> None: + """Remove the durable claim after a managed run's tree is reclaimed.""" + with contextlib.suppress(OSError): + pytest_basetemp_claim_path(basetemp, kind="managed").unlink() + + +def _try_acquire_pytest_basetemp_claim_lock(basetemp: Path) -> TextIO | None: + """Acquire the adjacent claim lock without waiting on another pytest run.""" + lock_path = pytest_basetemp_claim_path(basetemp, kind="lock") + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+", encoding="utf-8") + except OSError: + return None + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except (BlockingIOError, OSError): + handle.close() + return None + return handle + + +def managed_pytest_basetemp_owner_alive(basetemp: Path) -> bool | None: + """Return whether a positive managed claim still names a live process.""" + try: + raw_identity = pytest_basetemp_claim_path(basetemp, kind="managed").read_text(encoding="utf-8").strip() + raw_pid, separator, raw_start_ticks = raw_identity.partition(":") + pid = int(raw_pid) + start_ticks = int(raw_start_ticks) if separator else None + except (OSError, ValueError): + return None + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1].split() + current_start_ticks = int(fields[19]) + except (OSError, ValueError, IndexError): + return False + return start_ticks is None or current_start_ticks == start_ticks + + +def pytest_tmpfs_budget_exceeded(sample: Mapping[str, Any], *, budget_kb: int) -> bool: + """Return whether a sampled basetemp exceeds its tmpfs allocation cap. + + ``st_size`` remains forensic evidence because sparse files can expose a + large logical extent. Tmpfs capacity is consumed by allocated blocks, so + the admission limit must use that physical measure. + """ + allocated_kb = sample.get("basetemp_allocated_kb") + return isinstance(allocated_kb, int) and allocated_kb > budget_kb + + def normalize_pytest_basetemp_env(env: Mapping[str, str]) -> dict[str, str]: """Keep cloud pytest defaults from escaping a workstation scratch volume. @@ -750,6 +1684,27 @@ def adaptive_pytest_runtime_policy( ) +def _tmpfs_admission_refusal( + *, + kind: str, + path: Path, + declared_demand_kb: int, + safe_budget_kb: int, + headroom_kb: int, +) -> PytestResourceError: + """Describe all failed tmpfs admission constraints in one refusal.""" + free_kb = _headroom_kb(path) + required_headroom_kb = headroom_kb + max(declared_demand_kb, safe_budget_kb) + available = f"{free_kb / 1024:.0f} MiB" if free_kb is not None else "unknown" + return PytestResourceError( + f"{kind} pytest basetemp declared demand exceeds its safe adaptive tmpfs budget " + f"({path}: declared demand={declared_demand_kb / 1024:.0f} MiB, " + f"safe tmpfs budget={safe_budget_kb / 1024:.0f} MiB, " + f"available filesystem space={available}, " + f"required filesystem headroom={required_headroom_kb / 1024:.0f} MiB)" + ) + + def apply_managed_pytest_runtime_policy( env: Mapping[str, str], *, worker_count: int | None = None, full_suite: bool = True ) -> tuple[dict[str, str], PytestRuntimePolicy | None]: @@ -762,19 +1717,27 @@ def apply_managed_pytest_runtime_policy( unrelated command minutes or hours later. """ normalized = normalize_pytest_basetemp_env(env) + explicit_basetemp = normalized.get(PYTEST_EXPLICIT_BASETEMP_ENV) default_full_suite_scratch = ( full_suite + and explicit_basetemp is None and not normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") and "POLYLOGUE_PYTEST_TMPFS" not in normalized ) configured_root = normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") configured_tmpfs = configured_root is not None and _is_beneath(Path(configured_root), PYTEST_TMPFS_ROOT) - manages_tmpfs = configured_tmpfs or (configured_root is None and normalized.get("POLYLOGUE_PYTEST_TMPFS") != "0") + explicit_tmpfs = explicit_basetemp is not None and _is_beneath(Path(explicit_basetemp), PYTEST_TMPFS_ROOT) + manages_tmpfs = ( + explicit_tmpfs + or configured_tmpfs + or (explicit_basetemp is None and configured_root is None and normalized.get("POLYLOGUE_PYTEST_TMPFS") != "0") + ) policy = adaptive_pytest_runtime_policy( worker_count=worker_count, shm_free_kb=None if manages_tmpfs else 0, full_suite=full_suite, ) + rejected_candidates: tuple[str, ...] = () if full_suite and policy.tmpfs_predicted_mb is not None: normalized.setdefault(PYTEST_BASETEMP_REQUIRED_MB_ENV, str(policy.tmpfs_predicted_mb)) if manages_tmpfs: @@ -791,23 +1754,69 @@ def apply_managed_pytest_runtime_policy( and effective_tmpfs_budget_kb is not None and effective_tmpfs_budget_kb < required_basetemp_kb ): - normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" + if explicit_tmpfs: + path = Path(explicit_basetemp or PYTEST_TMPFS_ROOT) + raise _tmpfs_admission_refusal( + kind="explicit", + path=path, + declared_demand_kb=required_basetemp_kb, + safe_budget_kb=effective_tmpfs_budget_kb, + headroom_kb=pytest_basetemp_min_free_kb(normalized), + ) if configured_tmpfs: # The configured tmpfs root has become unsafe for this run. # Leaving it in place would make the resolver select it even # though tmpfs has just been disabled, without its cap. + rejected_candidates = ( + str( + _tmpfs_admission_refusal( + kind="configured", + path=Path(configured_root or PYTEST_TMPFS_ROOT), + declared_demand_kb=required_basetemp_kb, + safe_budget_kb=effective_tmpfs_budget_kb, + headroom_kb=pytest_basetemp_min_free_kb(normalized), + ) + ), + ) normalized.pop("POLYLOGUE_PYTEST_BASETEMP_ROOT", None) + normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" if default_full_suite_scratch: # Broad-suite demand grows with the fixture universe and has exceeded # the supervised 2 GiB ceiling while tests were still progressing. # Keep that ceiling for explicit tmpfs runs; use NVMe for the default # broad route instead of guessing the next aggregate peak. normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" - selected_root, selected_label = resolve_pytest_basetemp_root(normalized) - if not normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") and selected_root != PYTEST_TMPFS_ROOT: + if explicit_basetemp is not None: + selected_root = Path(explicit_basetemp) + selected_label = "explicit" + free_kb = _headroom_kb(selected_root) + required_kb = pytest_basetemp_required_kb(normalized) + min_free_kb = max(pytest_basetemp_min_free_kb(normalized), required_kb or 0) + explicit_required_kb = min_free_kb + if _is_beneath(selected_root, PYTEST_TMPFS_ROOT) and normalized.get("POLYLOGUE_PYTEST_TMPFS") == "1": + explicit_required_kb = pytest_basetemp_min_free_kb(normalized) + max( + required_kb or 0, + pytest_tmpfs_budget_kb(normalized) or 0, + ) + if free_kb is None or free_kb < explicit_required_kb: + raise PytestResourceError( + "explicit pytest basetemp does not have enough free space " + f"({selected_root}: {free_kb / 1024:.0f} MiB free, need >= {explicit_required_kb / 1024:.0f} MiB)" + if free_kb is not None + else f"explicit pytest basetemp is unreachable: {selected_root}" + ) + else: + selected_root, selected_label = resolve_pytest_basetemp_root( + normalized, rejected_candidates=rejected_candidates + ) + free_kb = _headroom_kb(selected_root) + if ( + explicit_basetemp is None + and not normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") + and selected_root != PYTEST_TMPFS_ROOT + ): normalized["POLYLOGUE_PYTEST_BASETEMP_ROOT"] = str(selected_root) normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" - free_kb = _headroom_kb(selected_root) required_kb = pytest_basetemp_required_kb(normalized) policy = replace( policy, @@ -909,7 +1918,9 @@ def _basetemp_refusal(checked: list[str], min_free_kb: int) -> PytestResourceErr ) -def resolve_pytest_basetemp_root(env: Mapping[str, str]) -> tuple[Path, str]: +def resolve_pytest_basetemp_root( + env: Mapping[str, str], *, rejected_candidates: tuple[str, ...] = () +) -> tuple[Path, str]: """Pick the ONE basetemp root pytest will use this run. Single resolution order, shared by ``tests/conftest.py`` (direct pytest @@ -937,7 +1948,7 @@ def resolve_pytest_basetemp_root(env: Mapping[str, str]) -> tuple[Path, str]: required_kb = pytest_basetemp_required_kb(env) min_free_kb = max(pytest_basetemp_min_free_kb(env), required_kb or 0) normalized = normalize_pytest_basetemp_env(env) - checked: list[str] = [] + checked = list(rejected_candidates) configured = normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") if configured: @@ -995,6 +2006,9 @@ def pytest_basetemp_path(*, root: Path, run_id: str, env: dict[str, str]) -> Pat refusal here would just be noise for a monitoring/cleanup path. Fall back to the top placement candidate, ignoring headroom, rather than raising. """ + explicit = env.get(PYTEST_EXPLICIT_BASETEMP_ENV) + if explicit: + return Path(explicit) try: scratch_root, _label = resolve_pytest_basetemp_root(env) except PytestResourceError: @@ -1012,7 +2026,8 @@ def pytest_basetemp_path(*, root: Path, run_id: str, env: dict[str, str]) -> Pat def pytest_tmpfs_budget_kb(env: Mapping[str, str]) -> int | None: """Return the bounded per-run tmpfs budget shared by all pytest workers.""" - configured_root = env.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") + explicit = env.get(PYTEST_EXPLICIT_BASETEMP_ENV) + configured_root = explicit or env.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") configured_tmpfs = configured_root is not None and _is_beneath(Path(configured_root), PYTEST_TMPFS_ROOT) if env.get("POLYLOGUE_PYTEST_TMPFS") != "1" or (configured_root is not None and not configured_tmpfs): return None @@ -1033,14 +2048,40 @@ def cleanup_managed_pytest_basetemp(*, root: Path, run_id: str, env: dict[str, s basetemps immediately instead of waiting for the next pytest startup sweep. """ + if env.get(PYTEST_EXPLICIT_BASETEMP_ENV): + return None basetemp = pytest_basetemp_path(root=root, run_id=run_id, env=env) if not basetemp.name.startswith("pytest-polylogue-") or "-seeded-" in basetemp.name: return None - with contextlib.suppress(OSError): - if basetemp.exists(): - shutil.rmtree(basetemp) - if not basetemp.exists(): - return basetemp + claim_lock = _try_acquire_pytest_basetemp_claim_lock(basetemp) + if claim_lock is None: + # A successor with the same inherited run id owns this path. Leave + # both its claim and its fixture tree for that invocation to finish. + return None + try: + owner_alive = managed_pytest_basetemp_owner_alive(basetemp) + if owner_alive is True: + return None + # A serial pytest child may already have reclaimed this exact run-owned + # directory in sessionfinish. That is a completed cleanup, not an absent + # receipt for the durable summary to misclassify. + if not basetemp.exists(): + if owner_alive is False: + clear_managed_pytest_basetemp_claim(basetemp) + return basetemp + # Reclaim only a positively claimed tree whose owner is confirmed dead. + # An unknown claim/tree may be caller-owned or belong to a newer runner. + if owner_alive is not False: + return None + with contextlib.suppress(OSError): + if basetemp.exists(): + shutil.rmtree(basetemp) + if not basetemp.exists(): + clear_managed_pytest_basetemp_claim(basetemp) + return basetemp + finally: + with contextlib.suppress(OSError): + claim_lock.close() return None @@ -1091,11 +2132,12 @@ def __init__(self, *, root_pid: int, run_id: str, root: Path, env: dict[str, str self._basetemp_size_interval_s = _basetemp_size_sample_interval_s(env) self._last_basetemp_size_sample_at: float | None = None self._last_basetemp_size_kb: int | None = None + self._last_basetemp_allocated_kb: int | None = None - def _sample_basetemp_size_kb(self, *, event: str) -> int | None: + def _sample_basetemp_sizes(self, *, event: str) -> tuple[int | None, int | None]: """Return basetemp size without recursively walking it every sample.""" if self._basetemp_size_interval_s <= 0: - return None + return None, None now = time.monotonic() should_sample = ( self._last_basetemp_size_sample_at is None @@ -1104,9 +2146,9 @@ def _sample_basetemp_size_kb(self, *, event: str) -> int | None: or now - self._last_basetemp_size_sample_at >= self._basetemp_size_interval_s ) if should_sample: - self._last_basetemp_size_kb = _dir_size_kb(self._basetemp) + self._last_basetemp_size_kb, self._last_basetemp_allocated_kb = _dir_usage_kb(self._basetemp) self._last_basetemp_size_sample_at = now - return self._last_basetemp_size_kb + return self._last_basetemp_size_kb, self._last_basetemp_allocated_kb def sample(self, *, event: str) -> dict[str, Any]: pids = process_tree(self.root_pid) @@ -1180,6 +2222,7 @@ def sample(self, *, event: str) -> dict[str, Any]: meminfo = _meminfo() cgroup_path = _cgroup_path(self.root_pid) cgroup_io = _cgroup_io_bytes(cgroup_path) + basetemp_logical_kb, basetemp_allocated_kb = self._sample_basetemp_sizes(event=event) sample: dict[str, Any] = { "updated_at": utc_now(), "event": event, @@ -1214,7 +2257,8 @@ def sample(self, *, event: str) -> dict[str, Any]: "pressure_memory": _pressure("memory"), "shm": _fs_usage(Path("/dev/shm")), "basetemp": str(self._basetemp), - "basetemp_size_kb": self._sample_basetemp_size_kb(event=event), + "basetemp_size_kb": basetemp_logical_kb, + "basetemp_allocated_kb": basetemp_allocated_kb, "top_processes": sorted(processes, key=lambda row: int(row.get("rss_kb") or 0), reverse=True)[:8], } self.sample_count += 1 diff --git a/tests/conftest.py b/tests/conftest.py index 97a170327d..21bfd9b0cc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import fcntl import hashlib import json import os @@ -12,24 +13,19 @@ import threading import time import uuid -from collections.abc import AsyncIterator, Callable, Generator, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Iterator, Mapping from pathlib import Path from types import FrameType, ModuleType -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TextIO import pytest from hypothesis import HealthCheck, settings from hypothesis.configuration import set_hypothesis_home_dir from hypothesis.database import DirectoryBasedExampleDatabase -# --------------------------------------------------------------------------- -# Place pytest temp directories on the NVMe scratch area by default. -# Test SQLite databases are write-heavy; /tmp lives on the root SSD on the -# operator workstation, while /realm/tmp is the high-write-budget scratch -# volume. /dev/shm remains available for explicit performance lanes, but it -# must not be the default: interrupted full/xdist runs can otherwise leave -# multi-GiB RAM-backed basetemps resident until reboot. -# --------------------------------------------------------------------------- +# Basetemp placement is selected once by ``resolve_pytest_basetemp_root``. +# This conftest owns only pytest-side claims, stale-tree reclamation, and the +# optional btrfs no-CoW mark for a disk-backed selection. from devtools import verify_runs from devtools.checkout_guard import ( CheckoutImportMismatchError, @@ -37,7 +33,13 @@ resolved_polylogue_path, ) from devtools.pytest_supervisor import _process_start_ticks -from devtools.verify_runs import PytestResourceError, normalize_pytest_basetemp_env, resolve_pytest_basetemp_root +from devtools.verify_runs import ( + PytestResourceError, + clear_managed_pytest_basetemp_claim, + normalize_pytest_basetemp_env, + resolve_pytest_basetemp_root, +) +from devtools.verify_runs import pytest_basetemp_claim_path as _basetemp_claim_path # Resolve (but don't yet raise on) the polylogue-vs-checkout mismatch check # before the first `from polylogue...` import below: a shared/editable venv's @@ -67,6 +69,17 @@ "tests.infra.clock_guard", ) +# Pytest supports nested in-process ``pytest.main()`` calls. Every controller +# invocation occupies this process-local stack, including caller-owned scopes. +# A completed invocation therefore cannot authorize stale environment values, +# and an unmanaged middle scope cannot resurrect a managed outer identity. +_ACTIVE_PYTEST_SCOPES: list[tuple[str, str] | None] = [] +_ACTIVE_PYTEST_BASETEMPS: set[Path] = set() +_PYTEST_SCOPE_ATTR = "_polylogue_pytest_scope" +_PYTEST_SCOPE_BASETEMP_ATTR = "_polylogue_pytest_scope_basetemp" +_PYTEST_SCOPE_ENV_ATTR = "_polylogue_pytest_scope_environment" +_NESTED_BASETEMP_POLICY_ENV = ("POLYLOGUE_PYTEST_BASETEMP_ROOT", "POLYLOGUE_PYTEST_TMPFS") + if TYPE_CHECKING: from click.testing import CliRunner @@ -80,6 +93,53 @@ # --------------------------------------------------------------------------- +def _set_managed_pytest_identity(identity: tuple[str, str] | None) -> None: + """Expose only the managed identity owned by the active invocation.""" + if identity is None: + os.environ.pop("POLYLOGUE_PYTEST_RUN_ID", None) + os.environ.pop("POLYLOGUE_PYTEST_MANAGED_BASETEMP", None) + return + run_id, basetemp = identity + os.environ["POLYLOGUE_PYTEST_RUN_ID"] = run_id + os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = basetemp + + +def _push_pytest_scope( + config: pytest.Config, identity: tuple[str, str] | None, *, basetemp: Path | None = None +) -> None: + """Register one controller invocation, managed or caller-owned.""" + _ACTIVE_PYTEST_SCOPES.append(identity) + setattr(config, _PYTEST_SCOPE_ATTR, identity) + if basetemp is not None: + claim_path = _basetemp_claim_path(basetemp, kind="lock") + _ACTIVE_PYTEST_BASETEMPS.add(claim_path) + setattr(config, _PYTEST_SCOPE_BASETEMP_ATTR, claim_path) + + +def _force_nested_pytest_scratch(config: pytest.Config) -> None: + """Keep an unsupervised nested controller out of an outer tmpfs budget.""" + previous = {name: os.environ.get(name) for name in _NESTED_BASETEMP_POLICY_ENV} + setattr(config, _PYTEST_SCOPE_ENV_ATTR, previous) + forced = verify_runs.force_managed_pytest_scratch(os.environ) + for name in _NESTED_BASETEMP_POLICY_ENV: + value = forced.get(name) + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def _restore_nested_pytest_scratch(config: pytest.Config) -> None: + previous = getattr(config, _PYTEST_SCOPE_ENV_ATTR, None) + if previous is None: + return + for name, value in previous.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + def pytest_configure(config: pytest.Config) -> None: """Register custom markers and choose the managed test temp root.""" if _CHECKOUT_GUARD_ERROR is not None: @@ -103,11 +163,48 @@ def pytest_configure(config: pytest.Config) -> None: "scale_large: large-tier scale fixture (~10k convs / ~100k msgs); nightly CI / campaigns only (#1183)", ) + if config.option.basetemp is not None: + configured_basetemp = str(config.option.basetemp) + run_id = os.environ.get("POLYLOGUE_PYTEST_RUN_ID") + managed_basetemp = os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") + supervised_managed = ( + run_id is not None + and os.environ.get("POLYLOGUE_VERIFY_RUN_ID") == run_id + and managed_basetemp == configured_basetemp + ) + if supervised_managed and not hasattr(config, "workerinput"): + assert run_id is not None + if _basetemp_claim_path(Path(configured_basetemp), kind="lock") in _ACTIVE_PYTEST_BASETEMPS: + raise pytest.UsageError( + f"pytest: explicit basetemp is already active in this pytest process: {configured_basetemp}" + ) + identity = (run_id, configured_basetemp) + _push_pytest_scope(config, identity, basetemp=Path(configured_basetemp)) + return + # A second in-process pytest.main() inherits os.environ from the first + # run. Explicit basetemp ownership is per invocation, so stale managed + # markers must not turn the caller-owned diagnostic tree into cleanup + # fodder at session finish. + if not hasattr(config, "workerinput"): + _mark_caller_owned_basetemp(Path(configured_basetemp)) + _push_pytest_scope(config, None, basetemp=Path(configured_basetemp)) + _set_managed_pytest_identity(None) + return + if config.option.basetemp is None: + prior_scope = ( + _ACTIVE_PYTEST_SCOPES[-1] if _ACTIVE_PYTEST_SCOPES and not hasattr(config, "workerinput") else None + ) + if _ACTIVE_PYTEST_SCOPES and not hasattr(config, "workerinput"): + _set_managed_pytest_identity(None) + _force_nested_pytest_scratch(config) normalized_basetemp_env = normalize_pytest_basetemp_env(os.environ) - if ( - "POLYLOGUE_VERIFY_RUN_ID" not in os.environ - and "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in normalized_basetemp_env + configured_root = normalized_basetemp_env.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") + unmanaged_tmpfs_root = configured_root is not None and verify_runs._is_beneath( + Path(configured_root), verify_runs.PYTEST_TMPFS_ROOT + ) + if "POLYLOGUE_VERIFY_RUN_ID" not in os.environ and ( + "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in normalized_basetemp_env or unmanaged_tmpfs_root ): # Bare pytest has no devtools supervisor to enforce a tmpfs cap. # Keep its basetemp on scratch; managed devtools runs carry the @@ -124,18 +221,39 @@ def pytest_configure(config: pytest.Config) -> None: os.environ["POLYLOGUE_PYTEST_RUN_ID"] = run_id try: root, label = _managed_pytest_temp_root() + basetemp = root / f"pytest-polylogue-{checkout}-{run_id}" + if not hasattr(config, "workerinput"): + _mark_basetemp_owner(basetemp) except PytestResourceError as exc: + _restore_nested_pytest_scratch(config) + _set_managed_pytest_identity(prior_scope) # Fail loudly and early: refuse before pytest starts collecting, # rather than crashing an unrelated command later with a bare # OSError once the chosen basetemp fills up. raise pytest.UsageError(f"pytest: {exc}") from exc - basetemp = root / f"pytest-polylogue-{checkout}-{run_id}" config.option.basetemp = str(basetemp) + os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = str(basetemp) if not hasattr(config, "workerinput"): - _mark_basetemp_owner(basetemp) + identity = (run_id, str(basetemp)) + _push_pytest_scope(config, identity, basetemp=basetemp) sys.stderr.write(f"pytest: basetemp → {config.option.basetemp} ({label})\n") +def pytest_unconfigure(config: pytest.Config) -> None: + """Restore or retire process-local managed ownership after one invocation.""" + if not hasattr(config, _PYTEST_SCOPE_ATTR): + return + scope = getattr(config, _PYTEST_SCOPE_ATTR) + if not _ACTIVE_PYTEST_SCOPES or _ACTIVE_PYTEST_SCOPES[-1] != scope: + return + _ACTIVE_PYTEST_SCOPES.pop() + active_basetemp = getattr(config, _PYTEST_SCOPE_BASETEMP_ATTR, None) + if active_basetemp is not None: + _ACTIVE_PYTEST_BASETEMPS.discard(active_basetemp) + _restore_nested_pytest_scratch(config) + _set_managed_pytest_identity(_ACTIVE_PYTEST_SCOPES[-1] if _ACTIVE_PYTEST_SCOPES else None) + + # Per-run basetemps are freed on sessionfinish. A run killed before # sessionfinish (SIGKILL, OOM) leaks its basetemp, so the controller reclaims # clearly-dead orphans on startup. Seeded corpora (``pytest-polylogue-seeded-*``) @@ -145,8 +263,8 @@ def pytest_configure(config: pytest.Config) -> None: # ``devtools.verify_runs.resolve_pytest_basetemp_root`` — this module only # adds the mkdir/no-CoW-marking side effects and the stale-directory sweep. _STALE_BASETEMP_MAX_AGE_S = 30 * 60 -_STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S = 6 * 60 * 60 -_OWNER_PID_MARKER = ".owner-pid" +_BASE_TEMP_CLAIM_LOCKS: dict[Path, TextIO] = {} +_BASE_TEMP_CLAIM_THREAD_LOCKS: dict[Path, threading.Lock] = {} def _managed_pytest_temp_root() -> tuple[Path, str]: @@ -165,28 +283,66 @@ def _managed_pytest_temp_root() -> tuple[Path, str]: def _mark_basetemp_owner(basetemp: Path) -> None: - """Record the owning process identity so a sweep never races a live run.""" + """Claim a managed tree outside pytest's replaceable basetemp directory.""" + handle = _acquire_basetemp_claim_lock(basetemp, blocking=False) + if handle is None: + raise PytestResourceError(f"managed pytest basetemp is already claimed: {basetemp}") + pid = os.getpid() + start_ticks = _process_start_ticks(pid) + identity = f"{pid}:{start_ticks}" if start_ticks is not None else str(pid) + try: + _basetemp_claim_path(basetemp, kind="managed").write_text(identity, encoding="utf-8") + except OSError as exc: + _release_basetemp_claim_lock(basetemp) + raise PytestResourceError(f"cannot record managed pytest basetemp claim: {basetemp}") from exc + + +def _mark_caller_owned_basetemp(basetemp: Path) -> None: + """Claim an explicit ``--basetemp`` before pytest may replace its tree.""" + lock_path = _basetemp_claim_path(basetemp, kind="lock") + if lock_path in _ACTIVE_PYTEST_BASETEMPS: + raise pytest.UsageError(f"pytest: explicit basetemp is already active in this pytest process: {basetemp}") + handle = _acquire_basetemp_claim_lock(basetemp, blocking=True) + if handle is None: + raise pytest.UsageError(f"pytest: cannot claim the explicit basetemp: {basetemp}") with contextlib.suppress(OSError): basetemp.mkdir(parents=True, exist_ok=True) - pid = os.getpid() - start_ticks = _process_start_ticks(pid) - identity = f"{pid}:{start_ticks}" if start_ticks is not None else str(pid) - (basetemp / _OWNER_PID_MARKER).write_text(identity, encoding="utf-8") + clear_managed_pytest_basetemp_claim(basetemp) + _basetemp_claim_path(basetemp, kind="caller-owned").write_text("explicit\n", encoding="utf-8") -def _basetemp_owner_alive(entry: Path) -> bool | None: - """True/False when the owner marker resolves a live/dead process, else None.""" - marker = entry / _OWNER_PID_MARKER +def _acquire_basetemp_claim_lock(basetemp: Path, *, blocking: bool) -> TextIO | None: + """Serialize a claim against a stale sweep for the same basetemp path.""" + lock_path = _basetemp_claim_path(basetemp, kind="lock") + thread_lock = _BASE_TEMP_CLAIM_THREAD_LOCKS.setdefault(lock_path, threading.Lock()) + if not thread_lock.acquire(blocking=blocking): + return None try: - raw_identity = marker.read_text(encoding="utf-8").strip() - raw_pid, separator, raw_start_ticks = raw_identity.partition(":") - pid = int(raw_pid) - start_ticks = int(raw_start_ticks) if separator else None - except (OSError, ValueError): + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+", encoding="utf-8") + except OSError: + thread_lock.release() return None - if not Path(f"/proc/{pid}").exists(): - return False - return start_ticks is None or _process_start_ticks(pid) == start_ticks + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)) + except OSError: + handle.close() + thread_lock.release() + return None + _BASE_TEMP_CLAIM_LOCKS[lock_path] = handle + return handle + + +def _release_basetemp_claim_lock(basetemp: Path) -> None: + """Release this pytest process's claim lock after its session ends.""" + lock_path = _basetemp_claim_path(basetemp, kind="lock") + handle = _BASE_TEMP_CLAIM_LOCKS.pop(lock_path, None) + if handle is not None: + with contextlib.suppress(OSError): + handle.close() + thread_lock = _BASE_TEMP_CLAIM_THREAD_LOCKS.get(lock_path) + if thread_lock is not None and thread_lock.locked(): + thread_lock.release() def _remove_stale_basetemp(entry: Path) -> None: @@ -257,20 +413,18 @@ def _sweep_stale_polylogue_basetemps( ) -> None: """Best-effort reclaim of per-run basetemps left by crashed runs. - Safety invariant: never delete a basetemp whose owning process is still - alive, regardless of age. Age alone is not a liveness proxy — a - long-running scale/lab test can legitimately outlive the stale-age - threshold. Each managed basetemp carries a ``.owner-pid`` marker - (written in ``pytest_configure``); a confirmed-dead owner uses the normal - threshold, an unconfirmable owner (no marker — e.g. a directory from - before this mechanism existed, or a startup race) uses a much longer - threshold before being reclaimed at all. Seeded corpora + Safety invariant: reclamation requires a durable, positive managed claim + plus a confirmed-dead owner. Unknown paths are never deleted: they may be + an explicit caller path racing a startup sweep. The claim lock makes the + decision and a caller's claim mutually exclusive, while the claim itself + survives pytest's lazy replacement of the basetemp directory. Explicit + caller-owned paths carry their own durable claim and are excluded. + Seeded corpora (``pytest-polylogue-*-seeded-*``) are never touched here — they are shared, reusable, and built once behind their own ``.build.done`` guard. """ cutoff = time.time() - max_age_s - unknown_owner_cutoff = time.time() - _STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S for root in roots or _polylogue_basetemp_roots(): for entry in root.glob("pytest-polylogue-*"): if "-seeded-" in entry.name: @@ -278,15 +432,26 @@ def _sweep_stale_polylogue_basetemps( try: if not entry.is_dir(): continue - owner_alive = _basetemp_owner_alive(entry) - if owner_alive: + if _basetemp_claim_path(entry, kind="caller-owned").is_file(): + continue + if not _basetemp_claim_path(entry, kind="managed").is_file(): continue - mtime = entry.stat().st_mtime - if owner_alive is False: - if mtime < cutoff: + handle = _acquire_basetemp_claim_lock(entry, blocking=False) + if handle is None: + continue + try: + if _basetemp_claim_path(entry, kind="caller-owned").is_file(): + continue + if not _basetemp_claim_path(entry, kind="managed").is_file(): + continue + owner_alive = verify_runs.managed_pytest_basetemp_owner_alive(entry) + if owner_alive is not False: + continue + if entry.stat().st_mtime < cutoff: _remove_stale_basetemp(entry) - elif mtime < unknown_owner_cutoff: - _remove_stale_basetemp(entry) + clear_managed_pytest_basetemp_claim(entry) + finally: + _release_basetemp_claim_lock(entry) except OSError: pass @@ -302,41 +467,48 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: teardown, where xdist/json-report may still be flushing controller/worker artifacts. """ - if os.environ.get("PYTEST_XDIST_WORKER"): - return - if not os.environ.get("POLYLOGUE_PYTEST_RUN_ID"): - return - numprocesses = getattr(session.config.option, "numprocesses", None) - if numprocesses not in (None, 0, "0"): - return basetemp = session.config.option.basetemp if not basetemp: return basetemp_path = Path(str(basetemp)) - if basetemp_path.name.startswith("pytest-polylogue-") and "-seeded-" not in basetemp_path.name: - shutil.rmtree(basetemp_path, ignore_errors=True) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport( - item: pytest.Item, - call: pytest.CallInfo[None], -) -> Generator[None, Any, None]: - """Retain the call outcome so passing test temp trees can be reclaimed.""" - outcome = yield - report = outcome.get_result() - setattr(item, f"rep_{report.when}", report) + try: + if os.environ.get("PYTEST_XDIST_WORKER"): + return + if not os.environ.get("POLYLOGUE_PYTEST_RUN_ID"): + return + numprocesses = getattr(session.config.option, "numprocesses", None) + if numprocesses not in (None, 0, "0"): + return + if str(basetemp_path) == os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP"): + shutil.rmtree(basetemp_path, ignore_errors=True) + if not basetemp_path.exists(): + clear_managed_pytest_basetemp_claim(basetemp_path) + finally: + _release_basetemp_claim_lock(basetemp_path) @pytest.fixture(autouse=True) -def _reclaim_passing_test_tmp_path( - request: pytest.FixtureRequest, +def _reclaim_test_tmp_path( tmp_path: Path, + request: pytest.FixtureRequest, ) -> Iterator[None]: - """Bound broad-run temp growth while preserving failed-test evidence.""" - yield - report: pytest.TestReport | None = getattr(request.node, "rep_call", None) - if report is not None and report.passed: + """Release each test's private tree as soon as its teardown finishes. + + Failure evidence belongs in the managed event/longrepr/resource receipts, + not in an unbounded filesystem witness. Retaining every failed tree made + full-suite tmpfs usage proportional to the number of failures and caused + a calm 8-worker run to exceed 2 GiB before completing. A failing node can + still be rerun with an explicit basetemp when its files matter. That + explicit diagnostic path retains its per-test trees for inspection. + """ + configured = getattr(request.config.option, "basetemp", None) + managed = os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") + if configured and str(configured) != managed: + yield + return + try: + yield + finally: shutil.rmtree(tmp_path, ignore_errors=True) @@ -406,9 +578,11 @@ def _reclaim_passing_test_tmp_path( "POLYLOGUE_VERIFY_RUN_ID", "POLYLOGUE_PYTEST_EVENTS_DIR", "POLYLOGUE_PYTEST_EVENTS_PATH", + "POLYLOGUE_PYTEST_RUN_ID", "POLYLOGUE_PYTEST_SELECTION_PATH", "POLYLOGUE_PYTEST_SUMMARY_PATH", "POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", } ) @@ -416,7 +590,7 @@ def _reclaim_passing_test_tmp_path( @pytest.fixture(autouse=True) def _close_test_opened_sqlite_connections( monkeypatch: pytest.MonkeyPatch, - _reclaim_passing_test_tmp_path: None, + _reclaim_test_tmp_path: None, ) -> Iterator[None]: """Close sync ``sqlite3`` connections that *test code* opened but never closed. @@ -538,7 +712,7 @@ async def _close_async() -> None: def _clear_polylogue_env( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, - _reclaim_passing_test_tmp_path: None, + _reclaim_test_tmp_path: None, request: pytest.FixtureRequest, ) -> None: # Close any cached SQLite connections to prevent WAL sidecar corruption diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py index 74699dd315..5968d77279 100644 --- a/tests/integration/devtools/test_testmon_seed_recovery.py +++ b/tests/integration/devtools/test_testmon_seed_recovery.py @@ -11,7 +11,8 @@ import pytest from devtools import testmon_bootstrap, testmon_state, verify -from devtools.testmon_state import file_fingerprint, inspect_testmon_database +from devtools.testmon_state import file_fingerprint, inspect_testmon_database, seed_shard_plan +from devtools.verify_runs import CheckoutMutationObservation def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( @@ -19,6 +20,16 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: + class _StableMutationMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + source = tmp_path / "source" source.mkdir() (source / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") @@ -44,6 +55,12 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( runtime_identity = testmon_state.testmon_runtime_identity(source) assert runtime_identity is not None dependency_environment, pytest_harness = runtime_identity + shards = seed_shard_plan(expected, shard_size=len(expected)) + shards[0]["status"] = "complete" + shards[0]["node_outcomes"] = [ + {"nodeid": expected[0], "outcome": "passed"}, + {"nodeid": expected[1], "outcome": "failed"}, + ] attempt = { "protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, "status": "reusable", @@ -65,6 +82,7 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( {"nodeid": expected[0], "outcome": "passed"}, {"nodeid": expected[1], "outcome": "failed"}, ], + "shards": shards, "exit_code": 1, "run_id": "real-testmon", "artifact_dir": ".cache/verify/runs/real-testmon", @@ -128,8 +146,13 @@ def fake_run(*_args: object, **_kwargs: object) -> tuple[int, float, dict[str, o return 0, 0.01, {"selected_count": 1} monkeypatch.setattr(verify, "_run", fake_run) - monkeypatch.setattr(verify, "_changed_executable_paths", lambda: ()) + monkeypatch.setattr(verify, "_git_head", lambda: "head") + monkeypatch.setattr(verify, "_git_commit", lambda _ref: "base") + monkeypatch.setattr(verify, "_default_testmon_is_broad_change", lambda _base_commit, _head_commit: False) + monkeypatch.setattr(verify, "_changed_executable_paths", lambda _base_commit, _head_commit: ()) monkeypatch.setattr(verify, "_stamp_head", lambda: None) + monkeypatch.setattr(verify, "worktree_fingerprint", lambda *_args: "stable") + monkeypatch.setattr(verify, "CheckoutMutationMonitor", _StableMutationMonitor) assert verify.main([]) == 1 result = json.loads(capsys.readouterr().out) diff --git a/tests/unit/devtools/test_evidence_dashboard.py b/tests/unit/devtools/test_evidence_dashboard.py new file mode 100644 index 0000000000..6b643d9d3f --- /dev/null +++ b/tests/unit/devtools/test_evidence_dashboard.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from devtools import evidence_dashboard + + +def test_static_gates_read_shared_verify_history(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + history = tmp_path / "xdg-state" / "polylogue" / "devtools" / "verify-history.jsonl" + history.parent.mkdir(parents=True) + history.write_text( + json.dumps( + { + "timestamp": "2026-08-12T00:00:00+00:00", + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "final_worktree_fingerprint": "current-fingerprint", + "steps": [{"name": "ruff check", "duration_s": 1.0, "exit": 0}], + } + ) + + "\n" + + json.dumps( + { + "timestamp": "2026-08-12T00:01:00+00:00", + "checkout_root": str((tmp_path / "other-worktree").resolve()), + "git_head": "other-head", + "steps": [{"name": "mypy", "duration_s": 1.0, "exit": 0}], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + ruff = next(gate for gate in gates["gates"] if gate["name"] == "ruff check") + assert gates["history_path"] == str(history) + assert ruff["status"] == "ok" + mypy = next(gate for gate in gates["gates"] if gate["name"] == "mypy") + assert mypy["available"] is False + + +def test_static_gates_accept_exactly_bound_last_verify_result(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + result_path = tmp_path / evidence_dashboard.LAST_VERIFY_RESULT_REL + result_path.parent.mkdir(parents=True) + result_path.write_text( + json.dumps( + { + "result": { + "timestamp": "2026-08-12T00:00:00+00:00", + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "final_worktree_fingerprint": "current-fingerprint", + "steps": [{"name": "ruff check", "duration_s": 1.0, "exit": 0}], + } + } + ) + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", tmp_path / "history.jsonl") + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is True + assert next(gate for gate in gates["gates"] if gate["name"] == "ruff check")["status"] == "ok" + + +def test_static_gates_withhold_evidence_for_dirty_checkout(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + history = tmp_path / "history.jsonl" + history.write_text( + json.dumps( + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "steps": [{"name": "ruff check", "exit": 0}], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: True) + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["reason"] == "checkout has uncommitted changes" for gate in gates["gates"]) + + +@pytest.mark.parametrize( + ("checkout_head", "fingerprint"), + [(None, "unavailable"), ("current-head", "unavailable")], +) +def test_static_gates_withhold_evidence_when_git_identity_is_unavailable( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + checkout_head: str | None, + fingerprint: str, +) -> None: + history = tmp_path / "history.jsonl" + history.write_text( + json.dumps( + { + "checkout_root": str(tmp_path.resolve()), + "git_head": checkout_head, + "worktree_fingerprint": fingerprint, + "steps": [{"name": "ruff check", "exit": 0}], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: checkout_head) + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: fingerprint) + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["reason"] == "checkout Git identity is unavailable" for gate in gates["gates"]) + + +def test_static_gates_reject_wrong_checkout_fingerprint_and_legacy_evidence( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + history = tmp_path / "history.jsonl" + legacy_result = tmp_path / evidence_dashboard.LAST_VERIFY_RESULT_REL + legacy_result.parent.mkdir(parents=True) + legacy_result.write_text(json.dumps({"result": {"steps": [{"name": "ruff check", "exit": 0}]}})) + history.write_text( + "\n".join( + json.dumps(entry) + for entry in ( + { + "checkout_root": str(tmp_path / "other"), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "steps": [{"name": "ruff check", "exit": 0}], + }, + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "other-fingerprint", + "steps": [{"name": "mypy", "exit": 0}], + }, + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "steps": [{"name": "render all", "exit": 0}], + }, + ) + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["available"] is False for gate in gates["gates"]) + + +def test_static_gates_reject_a_run_whose_checkout_changed_mid_verification( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + history = tmp_path / "history.jsonl" + history.write_text( + json.dumps( + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "final_worktree_fingerprint": "changed-during-run", + "steps": [{"name": "ruff check", "exit": 0}], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["available"] is False for gate in gates["gates"]) + + +def test_static_gates_reject_transient_checkout_mutation_with_matching_endpoint_fingerprints( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + history = tmp_path / "history.jsonl" + history.write_text( + json.dumps( + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "final_worktree_fingerprint": "current-fingerprint", + "diagnosis": "checkout_changed_during_verification", + "steps": [ + {"name": "ruff check", "exit": 0}, + { + "name": "checkout stability", + "exit": 125, + "diagnosis": "checkout_changed_during_verification", + }, + ], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["available"] is False for gate in gates["gates"]) diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index 9ad800ff38..b4c41f1a0d 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -4,7 +4,7 @@ import os import subprocess import threading -from collections.abc import Callable +from collections.abc import Callable, Mapping from pathlib import Path from typing import Any from unittest.mock import MagicMock @@ -88,10 +88,29 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return MagicMock(returncode=0, stdout=json.dumps([[]]), stderr="") if "/pulls/" in joined and "/comments" in joined: return MagicMock(returncode=0, stdout=json.dumps([comments]), stderr="") + if cmd[:3] == ["git", "rev-parse", "--show-toplevel"]: + return MagicMock(returncode=0, stdout=str(Path.cwd()) + "\n", stderr="") if cmd[:2] == ["git", "rev-parse"]: return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: return MagicMock(returncode=0, stdout="", stderr="") + env = kwargs.get("env") + if isinstance(env, dict) and merge_gate.VERIFICATION_RECEIPT_PATH_ENV in env: + cwd = kwargs.get("cwd") + checkout_root = Path(cwd) if cwd is not None else Path.cwd() + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": local_head_sha, + "checkout_root": str(checkout_root), + "exit_code": local_exit, + "verification_scope": "affected", + "release_baseline_allowed": False, + "terminal_authorization": None, + } + ) + ) return MagicMock( returncode=local_exit, stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), @@ -101,6 +120,32 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return _run +def _write_terminal_receipt( + kwargs: Mapping[str, Any], + *, + head: str = "merged-master", + scope: str = "release-baseline", + release_allowed: bool = True, + terminal_authorization: str | None = None, +) -> None: + env = kwargs["env"] + assert isinstance(env, dict) + cwd = Path(kwargs["cwd"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": head, + "checkout_root": str(cwd), + "exit_code": 0, + "verification_scope": scope, + "release_baseline_allowed": release_allowed, + "terminal_authorization": terminal_authorization, + } + ) + ) + + # --------------------------------------------------------------------------- # clean_merge_title # --------------------------------------------------------------------------- @@ -484,6 +529,7 @@ def test_merge_with_verify_records_terminal_full_verify(monkeypatch: pytest.Monk def run(cmd: list[str], **kwargs: Any) -> MagicMock: if cmd[:3] == ["devtools", "verify", "--all"]: + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( @@ -577,7 +623,7 @@ def test_post_merge_terminal_verify_uses_target_checkout_devshell( monkeypatch.chdir(tmp_path) commands: list[list[str]] = [] - def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + def run(cmd: list[str], **kwargs: Any) -> MagicMock: commands.append(cmd) if cmd[:3] == ["git", "worktree", "add"]: return MagicMock(returncode=0, stdout="", stderr="") @@ -596,6 +642,7 @@ def run(cmd: list[str], **_kwargs: Any) -> MagicMock: python_executable=target / ".venv" / "bin" / "python", ) assert fingerprint.clean + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( @@ -803,6 +850,7 @@ def test_record_full_verify_clears_pending_prs(monkeypatch: pytest.MonkeyPatch, merge_boundary._append_merge_entry(1, "sha1", "some title") def _run(cmd: list[str], **kwargs: Any) -> MagicMock: + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( @@ -823,6 +871,29 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: assert merge_boundary.cmd_train_status(as_json=False) == 0 +def test_record_full_verify_rejects_plausible_unbound_stdout(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 1 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False + + def test_record_full_verify_propagates_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) merge_boundary._append_merge_entry(1, "sha1", "some title") @@ -884,22 +955,16 @@ def test_record_full_verify_accepts_explicit_typed_narrow_terminal_authorization ) -> None: monkeypatch.chdir(tmp_path) merge_boundary._append_merge_entry(1, "sha1", "some title") - monkeypatch.setattr( - subprocess, - "run", - lambda _cmd, **_kwargs: MagicMock( - returncode=0, - stdout=json.dumps( - { - "git_head": "merged-master", - "verification_scope": "narrow-terminal", - "terminal_authorization": "narrow-terminal", - "release_baseline_allowed": True, - } - ), - stderr="", - ), - ) + + def run(_cmd: list[str], **kwargs: Any) -> MagicMock: + _write_terminal_receipt( + kwargs, + scope="narrow-terminal", + terminal_authorization="narrow-terminal", + ) + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", run) assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 0 assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is True @@ -932,11 +997,12 @@ def test_concurrent_merge_during_terminal_verify_remains_pending( monkeypatch.chdir(tmp_path) inserted = False - def run(_cmd: list[str], **_kwargs: Any) -> MagicMock: + def run(_cmd: list[str], **kwargs: Any) -> MagicMock: nonlocal inserted if not inserted: inserted = True merge_boundary._append_merge_entry(99, "concurrent-sha", "concurrent merge") + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( @@ -963,7 +1029,7 @@ def test_concurrent_ledger_writer_cannot_lose_merge_entry(monkeypatch: pytest.Mo started = threading.Event() writer: threading.Thread | None = None - def run(_cmd: list[str], **_kwargs: Any) -> MagicMock: + def run(_cmd: list[str], **kwargs: Any) -> MagicMock: nonlocal writer def append() -> None: @@ -973,6 +1039,7 @@ def append() -> None: writer = threading.Thread(target=append) writer.start() assert started.wait(timeout=1) + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index ca1a98adf7..a619f53c30 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -78,9 +78,26 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: call_count["round"] += 1 return MagicMock(returncode=0, stdout=json.dumps([comment_rounds[round_index]]), stderr="") if cmd[:2] == ["git", "rev-parse"]: + if "--show-toplevel" in cmd: + return MagicMock(returncode=0, stdout=str(Path.cwd()) + "\n", stderr="") return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: return MagicMock(returncode=0, stdout=" M dirty.py\n" if dirty else "", stderr="") + env = kwargs.get("env") + if isinstance(env, dict) and merge_gate.VERIFICATION_RECEIPT_PATH_ENV in env: + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": local_head_sha, + "checkout_root": str(Path.cwd()), + "exit_code": local_exit, + "verification_scope": "affected", + "release_baseline_allowed": False, + "terminal_authorization": None, + } + ) + ) return MagicMock( returncode=local_exit, stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), @@ -143,7 +160,9 @@ def test_check_blocks_full_receipt_without_release_baseline_permission( assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 1 -def test_record_consumes_structured_verify_release_permission(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +def test_record_rejects_stale_plausible_stdout_without_bound_receipt( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: monkeypatch.chdir(tmp_path) pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) @@ -155,14 +174,205 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: return base(cmd, **kwargs) return MagicMock( returncode=0, - stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), + stdout=json.dumps( + { + "invocation_id": "stale-invocation", + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + "terminal_authorization": None, + } + ), stderr="", ) monkeypatch.setattr(subprocess, "run", _run) assert merge_gate.cmd_record(42, "devtools verify") == 0 receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] is None + assert receipt["release_baseline_allowed"] is None + + +def test_record_consumes_receipt_after_streamed_verifier_progress( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + payload = { + "verification_scope": "narrow-terminal", + "release_baseline_allowed": False, + "terminal_authorization": "narrow-terminal", + } + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + env = cast(dict[str, str], kwargs["env"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + **payload, + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + } + ) + ) + return MagicMock(returncode=0, stdout=f"pytest progress\n{json.dumps(payload, indent=2)}\n", stderr="") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] == "narrow-terminal" assert receipt["release_baseline_allowed"] is False + assert receipt["terminal_authorization"] == "narrow-terminal" + + +def test_record_consumes_exact_invocation_receipt_when_verifier_writes_only_stderr( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + env = cast(dict[str, str], kwargs["env"]) + receipt_path = Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]) + receipt_path.write_text( + json.dumps( + { + "run_id": "new", + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "affected", + "release_baseline_allowed": False, + "terminal_authorization": None, + } + ) + ) + return MagicMock(returncode=0, stdout="", stderr="pytest progress") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] == "affected" + assert receipt["release_baseline_allowed"] is False + + +def test_record_rejects_invocation_receipt_from_another_head(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + env = cast(dict[str, str], kwargs["env"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": "different", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "affected", + "release_baseline_allowed": False, + } + ) + ) + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] is None + assert receipt["release_baseline_allowed"] is None + + +def test_record_rejects_unrelated_invocation_receipt_with_same_head_and_exit( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + env = cast(dict[str, str], kwargs["env"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": "another-command", + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "affected", + "release_baseline_allowed": False, + } + ) + ) + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] is None + assert receipt["release_baseline_allowed"] is None + + +def test_record_anchors_invocation_to_repository_root_from_subdirectory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + subdirectory = tmp_path / "devtools" + subdirectory.mkdir() + monkeypatch.chdir(subdirectory) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd == ["git", "rev-parse", "--show-toplevel"]: + return MagicMock(returncode=0, stdout=str(tmp_path) + "\n", stderr="") + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + assert kwargs["cwd"] == tmp_path + env = cast(dict[str, str], kwargs["env"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "affected", + "release_baseline_allowed": False, + } + ) + ) + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] == "affected" def test_record_accepts_authoritative_dependabot_dependency_only_pr_without_carrier( diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index 0f9c3d0dde..76c07aeaa7 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -2,7 +2,6 @@ import json import os -import shutil import subprocess import sys from collections.abc import Iterator @@ -12,10 +11,11 @@ import pytest from devtools import pytest_progress_plugin +from devtools.verify_runs import aggregate_pytest_statistics @pytest.fixture(autouse=True) -def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[None]: +def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: # Unit tests own their event destinations; do not let a surrounding # managed verify invocation redirect them into its step artifacts. for name in ( @@ -23,6 +23,7 @@ def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> It "POLYLOGUE_PYTEST_EVENTS_PATH", "POLYLOGUE_PYTEST_SELECTION_PATH", "POLYLOGUE_PYTEST_SUMMARY_PATH", + "PYTEST_XDIST_WORKER", ): monkeypatch.delenv(name, raising=False) selected_count = pytest_progress_plugin._SELECTED_COUNT @@ -31,16 +32,22 @@ def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> It slowest_reports = list(pytest_progress_plugin._SLOWEST_REPORTS) collection_started_at = pytest_progress_plugin._COLLECTION_STARTED_AT collection_duration_s = pytest_progress_plugin._COLLECTION_DURATION_S + controller_collection_payload = pytest_progress_plugin._CONTROLLER_COLLECTION_PAYLOAD + recorded_report_keys = set(pytest_progress_plugin._RECORDED_REPORT_KEYS) + session_state_stack = list(pytest_progress_plugin._SESSION_STATE_STACK) + pytest_progress_plugin._RECORDED_REPORT_KEYS.clear() + pytest_progress_plugin._SESSION_STATE_STACK.clear() yield - checkout_cache = Path(__file__).resolve().parents[3] / ".cache" / "testmon" - if checkout_cache.exists(): - shutil.move(str(checkout_cache), str(tmp_path / "checkout-testmon-generated")) pytest_progress_plugin._SELECTED_COUNT = selected_count pytest_progress_plugin._DESELECTED_COUNT = deselected_count pytest_progress_plugin._DESELECTED_NODEIDS_SAMPLE[:] = deselected_nodeids pytest_progress_plugin._SLOWEST_REPORTS[:] = slowest_reports pytest_progress_plugin._COLLECTION_STARTED_AT = collection_started_at pytest_progress_plugin._COLLECTION_DURATION_S = collection_duration_s + pytest_progress_plugin._CONTROLLER_COLLECTION_PAYLOAD = controller_collection_payload + pytest_progress_plugin._RECORDED_REPORT_KEYS.clear() + pytest_progress_plugin._RECORDED_REPORT_KEYS.update(recorded_report_keys) + pytest_progress_plugin._SESSION_STATE_STACK[:] = session_state_stack @dataclass(frozen=True) @@ -50,6 +57,8 @@ class _Report: outcome: str duration: float = 0.0 longrepr: str = "" + worker_id: str | None = None + wasxfail: str | None = None def test_progress_plugin_records_call_and_setup_failures( @@ -75,23 +84,142 @@ def test_progress_plugin_records_call_and_setup_failures( assert events[2]["longrepr"] == "fixture exploded" +def test_progress_plugin_preserves_xfail_and_xpass_in_durable_statistics( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + step = tmp_path / "step" + step.mkdir() + events_path = step / "events.jsonl" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_PATH", str(events_path)) + pytest_progress_plugin.pytest_sessionstart(object()) + + pytest_progress_plugin.pytest_runtest_logstart("test_xfailed", ("tests/a.py", 1, "test_xfailed")) + pytest_progress_plugin.pytest_runtest_logreport( + _Report("test_xfailed", "call", "skipped", wasxfail="known failure") + ) + pytest_progress_plugin.pytest_runtest_logstart("test_setup_xfailed", ("tests/a.py", 2, "test_setup_xfailed")) + pytest_progress_plugin.pytest_runtest_logreport( + _Report("test_setup_xfailed", "setup", "skipped", wasxfail="fixture calls pytest.xfail()") + ) + pytest_progress_plugin.pytest_runtest_logstart("test_xpassed", ("tests/a.py", 3, "test_xpassed")) + pytest_progress_plugin.pytest_runtest_logreport(_Report("test_xpassed", "call", "passed", wasxfail="known failure")) + + statistics = aggregate_pytest_statistics(step) + + assert statistics["outcomes"] == {"xfailed": 2, "xpassed": 1} + + +def test_progress_plugin_observes_real_pytest_xfail_outcome(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + test_path = tmp_path / "test_xfail.py" + test_path.write_text( + "import pytest\n\n@pytest.mark.xfail(reason='known failure')\ndef test_expected_failure():\n assert False\n" + ) + env = os.environ.copy() + env["POLYLOGUE_PYTEST_EVENTS_PATH"] = str(events_path) + checkout_root = Path(__file__).resolve().parents[3] + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "devtools.pytest_progress_plugin", + "-p", + "no:testmon", + str(test_path), + ], + cwd=checkout_root, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + reports = [ + json.loads(line) + for line in events_path.read_text().splitlines() + if json.loads(line).get("event") == "test_report" + ] + assert any(report["when"] == "call" and report["outcome"] == "xfailed" for report in reports) + + +def test_progress_plugin_skips_xdist_controller_forwarding_copy( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + events_path = tmp_path / "events.jsonl" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_PATH", str(events_path)) + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw0") + pytest_progress_plugin.pytest_runtest_logreport(_Report("test_one", "call", "passed", worker_id="gw0")) + + monkeypatch.delenv("PYTEST_XDIST_WORKER") + pytest_progress_plugin.pytest_runtest_logreport(_Report("test_one", "call", "passed", worker_id="gw0")) + + events = [json.loads(line) for line in events_path.read_text().splitlines()] + assert [(event["nodeid"], event["when"], event["worker_id"]) for event in events] == [("test_one", "call", "gw0")] + + +def test_progress_plugin_keeps_xdist_worker_timings_in_controller_summary( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + events_path = tmp_path / "events.jsonl" + summary_path = tmp_path / "summary.json" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_PATH", str(events_path)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SUMMARY_PATH", str(summary_path)) + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw0") + pytest_progress_plugin.pytest_runtest_logreport( + _Report("test_slow", "call", "passed", duration=1.5, worker_id="gw0") + ) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + monkeypatch.delenv("PYTEST_XDIST_WORKER") + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_runtest_logreport( + _Report("test_slow", "call", "passed", duration=1.5, worker_id="gw0") + ) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + events = [ + json.loads(line) + for line in events_path.read_text().splitlines() + if json.loads(line).get("event") == "test_report" + ] + summary = json.loads(summary_path.read_text()) + assert [(event["nodeid"], event["worker_id"]) for event in events] == [("test_slow", "gw0")] + assert [report["nodeid"] for report in summary["slowest_reports"]] == ["test_slow"] + + def test_managed_event_ledger_survives_test_host_environment_scrub(tmp_path: Path) -> None: events_dir = tmp_path / "events" checkout_root = Path(__file__).resolve().parents[3] - # The real testmon plugin receives no TESTMON_DATAFILE here by design: - # this regression test models a child process after the host scrub. Give - # its default relative path a parent directory without permitting the - # resulting cache to leak into later tests. - (checkout_root / ".cache" / "testmon").mkdir(parents=True, exist_ok=True) env = os.environ.copy() + for name in ( + "POLYLOGUE_PYTEST_BASETEMP_ROOT", + "POLYLOGUE_PYTEST_TMPFS", + "POLYLOGUE_PYTEST_RUN_ID", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", + ): + env.pop(name, None) env.update( { + # Keep the nested real pytest away from the host-only scratch + # fallback while preserving the scrubbed event/testmon scenario. + "POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path / "pytest-basetemp"), + "POLYLOGUE_PYTEST_TMPFS": "0", "POLYLOGUE_PYTEST_EVENTS_DIR": str(events_dir), "POLYLOGUE_PYTEST_SELECTION_PATH": str(tmp_path / "selection.json"), "POLYLOGUE_PYTEST_SUMMARY_PATH": str(tmp_path / "summary.json"), "POLYLOGUE_VERIFY_RUN_ID": "subprocess-regression", + "TESTMON_DATAFILE": str(tmp_path / "testmon" / "testmon.sqlite"), } ) + Path(env["TESTMON_DATAFILE"]).parent.mkdir(parents=True) result = subprocess.run( [ sys.executable, @@ -274,3 +402,105 @@ def test_progress_plugin_records_collection_duration_and_summary( "collection_finished", ] assert events[2]["duration_s"] == 2.5 + + +def test_progress_plugin_retains_controller_selection_through_session_finish( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + selection_path = tmp_path / "selection.json" + monkeypatch.setenv("POLYLOGUE_PYTEST_SELECTION_PATH", str(selection_path)) + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_collection_modifyitems( + _Session(["tests/a.py::test_keep"]), + object(), + [_Item("tests/a.py::test_keep")], + ) + + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + selection = json.loads(selection_path.read_text()) + assert selection["selected_nodeids"] == ["tests/a.py::test_keep"] + assert selection["selected_nodeids_omitted"] == 0 + + +def test_nested_pytest_session_keeps_outer_progress_and_artifacts_isolated( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + outer_events = tmp_path / "outer-events.jsonl" + outer_selection = tmp_path / "outer-selection.json" + outer_summary = tmp_path / "outer-summary.json" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_PATH", str(outer_events)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SELECTION_PATH", str(outer_selection)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SUMMARY_PATH", str(outer_summary)) + + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_collection_modifyitems( + _Session(["tests/outer.py::test_outer"]), object(), [_Item("tests/outer.py::test_outer")] + ) + pytest_progress_plugin.pytest_runtest_logreport(_Report("tests/outer.py::test_outer", "call", "passed")) + + pytest_progress_plugin.pytest_sessionstart(object()) + nested_selection = Path(os.environ["POLYLOGUE_PYTEST_SELECTION_PATH"]) + nested_summary = Path(os.environ["POLYLOGUE_PYTEST_SUMMARY_PATH"]) + nested_events = Path(os.environ["POLYLOGUE_PYTEST_EVENTS_PATH"]) + assert nested_selection != outer_selection + assert nested_summary != outer_summary + assert nested_events != outer_events + pytest_progress_plugin.pytest_collection_modifyitems( + _Session(["tests/inner.py::test_inner"]), object(), [_Item("tests/inner.py::test_inner")] + ) + pytest_progress_plugin.pytest_runtest_logreport(_Report("tests/inner.py::test_inner", "call", "passed")) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + assert os.environ["POLYLOGUE_PYTEST_SELECTION_PATH"] == str(outer_selection) + assert json.loads(outer_selection.read_text())["selected_nodeids"] == ["tests/outer.py::test_outer"] + assert json.loads(nested_selection.read_text())["selected_nodeids"] == ["tests/inner.py::test_inner"] + assert [json.loads(line)["nodeid"] for line in nested_events.read_text().splitlines() if "nodeid" in line] == [ + "tests/inner.py::test_inner" + ] + + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + outer_payload = json.loads(outer_summary.read_text()) + nested_payload = json.loads(nested_summary.read_text()) + assert [report["nodeid"] for report in outer_payload["slowest_reports"]] == ["tests/outer.py::test_outer"] + assert [report["nodeid"] for report in nested_payload["slowest_reports"]] == ["tests/inner.py::test_inner"] + + +def test_progress_plugin_merges_xdist_collection_facts_without_double_counting( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + events_dir = tmp_path / "events" + selection_path = tmp_path / "selection.json" + summary_path = tmp_path / "summary.json" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_DIR", str(events_dir)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SELECTION_PATH", str(selection_path)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SUMMARY_PATH", str(summary_path)) + + for worker_id, duration in (("gw1", 1.5), ("gw0", 2.5)): + ticks = iter([10.0, 10.0 + duration]) + monkeypatch.setattr("devtools.pytest_progress_plugin.time.monotonic", lambda ticks=ticks: next(ticks)) + monkeypatch.setenv("PYTEST_XDIST_WORKER", worker_id) + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_collection(object()) + pytest_progress_plugin.pytest_deselected([_Item("tests/a.py::test_skip")]) + pytest_progress_plugin.pytest_collection_modifyitems( + _Session(["tests/a.py::test_keep"]), object(), [_Item("tests/a.py::test_keep")] + ) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + monkeypatch.delenv("PYTEST_XDIST_WORKER") + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + selection = json.loads(selection_path.read_text()) + summary = json.loads(summary_path.read_text()) + assert selection["selected_count"] == 1 + assert selection["deselected_count"] == 1 + assert selection["selected_nodeids"] == ["tests/a.py::test_keep"] + assert summary["selected_count"] == 1 + assert summary["deselected_count"] == 1 + assert summary["collection_duration_s"] == 2.5 diff --git a/tests/unit/devtools/test_pytest_supervisor.py b/tests/unit/devtools/test_pytest_supervisor.py index 35c1bb13b1..c374651719 100644 --- a/tests/unit/devtools/test_pytest_supervisor.py +++ b/tests/unit/devtools/test_pytest_supervisor.py @@ -12,6 +12,7 @@ import time from collections.abc import Callable, Mapping, Sequence from pathlib import Path +from unittest.mock import patch import pytest import tomllib @@ -441,6 +442,45 @@ def test_surviving_owned_process_forces_nonzero_containment_result( assert receipt["termination_reason"] == "owned pytest processes survived cleanup" +def test_supervisor_main_retains_tmpfs_tree_when_receipt_is_not_quiescent( + tmp_path: Path, +) -> None: + receipt_path = tmp_path / "containment.json" + cleanup_path = Path("/dev/shm") / "pytest-polylogue-retained" + + def fake_supervise(*_args: object, **_kwargs: object) -> int: + receipt_path.write_text(json.dumps({"controller_group_alive": True}), encoding="utf-8") + return 125 + + with ( + patch("devtools.pytest_supervisor.supervise", side_effect=fake_supervise), + patch("devtools.pytest_supervisor.cleanup_managed_tmpfs_path") as cleanup, + ): + rc = pytest_supervisor.main( + [ + "--receipt", + str(receipt_path), + "--owner-pid", + str(os.getpid()), + "--timeout-s", + "1", + "--term-grace-s", + "1", + "--mode", + "process-group", + "--cleanup-path", + str(cleanup_path), + "--", + "pytest", + ] + ) + + assert rc == 125 + cleanup.assert_not_called() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["tmpfs_cleanup_complete"] is False + + def test_managed_runner_retains_responsible_node_for_per_test_timeout( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index 0d126bd449..c57c78538a 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -2,15 +2,36 @@ from __future__ import annotations +import json import subprocess import sys from pathlib import Path -from typing import Any +from types import SimpleNamespace +from typing import Any, cast import pytest from devtools import run_tests, verify -from devtools.verify_runs import git_head +from devtools.verify_runs import ( + CURRENT_RUN_PATH, + CURRENT_STATISTICS_PATH, + VERIFICATION_INVOCATION_ID_ENV, + VERIFICATION_RECEIPT_PATH_ENV, + CheckoutMutationObservation, + git_head, + pytest_command_worker_request, +) + + +class _NoMutationMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) def test_build_pytest_cmd_defaults_to_single_process() -> None: @@ -30,7 +51,7 @@ def test_build_pytest_cmd_respects_explicit_worker_flag() -> None: cmd = run_tests.build_pytest_cmd(["tests/unit", "-n", "4"]) # No injected -n when the caller already chose one. assert cmd.count("-n") == 1 - assert cmd[-2:] == ["-n", "4"] + assert cmd[-3:] == ["-n", "4", "--dist=loadgroup"] @pytest.mark.parametrize( @@ -53,13 +74,26 @@ def test_build_pytest_cmd_forwards_exactly_one_xdist_worker_request( arg for arg in command if arg in {"-n", "--numprocesses"} or arg.startswith(("-n", "--numprocesses=")) ] assert len(worker_flags) == 1 - assert verify._pytest_command_worker_request(command) == expected_request + assert pytest_command_worker_request(command) == expected_request def test_build_pytest_cmd_honors_workers_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("POLYLOGUE_PYTEST_WORKERS", "8") cmd = run_tests.build_pytest_cmd(["tests/unit"]) - assert cmd[-2:] == ["-n", "8"] + assert cmd[-3:] == ["-n", "8", "--dist=loadgroup"] + + +def test_build_pytest_cmd_preserves_explicit_xdist_distribution() -> None: + cmd = run_tests.build_pytest_cmd(["tests/unit", "-n", "4", "--dist=worksteal"]) + + assert cmd.count("--dist=worksteal") == 1 + assert "--dist=loadgroup" not in cmd + + +def test_build_pytest_cmd_does_not_add_distribution_for_serial_run() -> None: + cmd = run_tests.build_pytest_cmd(["tests/unit", "-n", "0"]) + + assert not any(arg.startswith("--dist") for arg in cmd) def test_subprocess_env_anchors_pytest_artifacts_to_checkout(monkeypatch: pytest.MonkeyPatch) -> None: @@ -93,6 +127,7 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di monkeypatch.setattr("devtools.run_tests._clear_pytest_report", lambda _cmd: None) monkeypatch.setattr("devtools.run_tests._run", _fake_run) monkeypatch.setattr("devtools.run_tests.git_head", lambda _root: "abc123") + monkeypatch.setattr("devtools.run_tests.append_verify_history", lambda payload: captured.update(history=payload)) assert run_tests.main(["tests/unit/pipeline", "--json"]) == 0 assert "--json" not in captured["cmd"] assert "tests/unit/pipeline" in captured["cmd"] @@ -102,6 +137,378 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di assert isinstance(captured["run"]._payload["git_dirty"], bool) assert captured["run"]._payload["verification_scope"] == "affected" assert captured["run"]._payload["release_baseline_allowed"] is False + assert captured["history"]["run_id"] == captured["run"].run_id + assert captured["history"]["status"] == "success" + + +def test_main_preserves_relative_selection_from_subdirectory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, Any] = {} + + def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + captured["cmd"] = cmd + return 0, 0.01, {"diagnosis": "pytest_passed"} + + monkeypatch.chdir(run_tests.ROOT / "tests" / "unit") + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr("devtools.run_tests._clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr("devtools.run_tests._run", _fake_run) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "stable") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr("devtools.run_tests.append_verify_history", lambda _payload: None) + + assert run_tests.main(["core/test_identity_law.py::test_session_id_is_origin_native_id"]) == 0 + + assert "tests/unit/core/test_identity_law.py::test_session_id_is_origin_native_id" in captured["cmd"] + + +def test_main_preserves_path_valued_options_from_subdirectory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + captured: dict[str, Any] = {} + + def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + captured["cmd"] = cmd + return 0, 0.01, {"diagnosis": "pytest_passed"} + + invocation = tmp_path / "nested" + invocation.mkdir() + (invocation / "fixtures").mkdir() + monkeypatch.chdir(invocation) + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", _fake_run) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "stable") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) + + assert ( + run_tests.main( + [ + "-k", + "proof", + "--basetemp=diagnostic", + "--rootdir", + ".", + "--ignore", + "fixtures", + "--ignore-glob=fixtures/*.json", + "--junit-xml", + "reports/results.xml", + ] + ) + == 0 + ) + + command = cast(list[str], captured["cmd"]) + assert f"--basetemp={invocation / 'diagnostic'}" in command + assert command[command.index("--rootdir") + 1] == str(invocation) + assert command[command.index("--ignore") + 1] == str(invocation / "fixtures") + assert f"--ignore-glob={invocation / 'fixtures' / '*.json'}" in command + assert command[command.index("--junit-xml") + 1] == str(invocation / "reports" / "results.xml") + + +def test_main_keeps_interruption_diagnosis_when_checkout_verification_finds_a_change( + monkeypatch: pytest.MonkeyPatch, +) -> None: + history: dict[str, Any] = {} + fingerprints = iter(("before", "after")) + + def interrupt(*_args: Any, **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + raise KeyboardInterrupt + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", interrupt) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprints)) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: history.update(payload)) + + assert run_tests.main(["tests/unit/example.py"]) == 130 + assert history["diagnosis"] == "pytest_interrupted" + assert history["checkout_diagnosis"] == "checkout_changed_during_focused_test" + + +def test_main_persists_interrupted_checkout_diagnosis_to_all_run_artifacts( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + history: dict[str, Any] = {} + fingerprints = iter(("before", "after")) + receipt = tmp_path / "receipt.json" + + def interrupt(*_args: Any, **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + raise KeyboardInterrupt + + monkeypatch.setattr(run_tests, "ROOT", tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + run_tests, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setenv(VERIFICATION_INVOCATION_ID_ENV, "focused-interrupt") + monkeypatch.setenv(VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", interrupt) + monkeypatch.setattr(run_tests, "git_head", lambda _root: "head") + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprints)) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: history.update(payload)) + + assert run_tests.main(["tests/unit/example.py"]) == 130 + + run_payload = json.loads((tmp_path / history["artifact_dir"] / "run.json").read_text()) + current_payload = json.loads((tmp_path / CURRENT_RUN_PATH).read_text()) + receipt_payload = json.loads(receipt.read_text()) + for payload in (history, run_payload, current_payload, receipt_payload): + assert payload["diagnosis"] == "pytest_interrupted" + assert payload["checkout_diagnosis"] == "checkout_changed_during_focused_test" + + +def test_normalize_selection_paths_preserves_pytest_path_option_semantics( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + invocation = tmp_path / "invocation" + expanded_root = tmp_path / "expanded-root" + invocation.mkdir() + expanded_root.mkdir() + absolute_config = tmp_path / "absolute.ini" + monkeypatch.setenv("PYTEST_ROOT", str(expanded_root)) + + normalized = run_tests._normalize_selection_paths( + [ + "-cconfig/pytest.ini", + "-c", + "separate/pytest.ini", + "--config-file=other/pytest.ini", + "--config-file", + "separate-config/pytest.ini", + "--log-file", + "logs/test.log", + "--log-file=logs/joined.log", + "--debug", + "logs/debug-separated.log", + "--debug=logs/debug.log", + "--rootdir", + "$PYTEST_ROOT/relative", + "--rootdir=$PYTEST_ROOT/joined", + "--junitxml=reports/junit.xml", + "--junit-xml", + "reports/junit-alias.xml", + "--ignore-glob=fixtures/*.json", + "--basetemp", + str(absolute_config), + "--config-file", + "$PYTEST_ROOT/literal.ini", + ], + invocation_directory=invocation, + ) + + assert f"-c{invocation / 'config' / 'pytest.ini'}" in normalized + assert normalized[normalized.index("-c") + 1] == str(invocation / "separate" / "pytest.ini") + assert f"--config-file={invocation / 'other' / 'pytest.ini'}" in normalized + assert normalized[normalized.index("--config-file") + 1] == str(invocation / "separate-config" / "pytest.ini") + assert normalized[normalized.index("--log-file") + 1] == str(invocation / "logs" / "test.log") + assert f"--log-file={invocation / 'logs' / 'joined.log'}" in normalized + assert normalized[normalized.index("--debug") + 1] == str(invocation / "logs" / "debug-separated.log") + assert f"--debug={invocation / 'logs' / 'debug.log'}" in normalized + assert normalized[normalized.index("--rootdir") + 1] == str(expanded_root / "relative") + assert f"--rootdir={expanded_root / 'joined'}" in normalized + assert f"--junitxml={invocation / 'reports' / 'junit.xml'}" in normalized + assert normalized[normalized.index("--junit-xml") + 1] == str(invocation / "reports" / "junit-alias.xml") + assert f"--ignore-glob={invocation / 'fixtures' / '*.json'}" in normalized + assert normalized[normalized.index("--basetemp") + 1] == str(absolute_config) + assert normalized[-1] == str(invocation / "$PYTEST_ROOT" / "literal.ini") + + +def test_normalize_selection_paths_preserves_pytest_symlinks_and_optional_debug( + tmp_path: Path, +) -> None: + invocation = tmp_path / "invocation" + invocation.mkdir() + target = invocation / "target.ini" + target.write_text("[pytest]\n", encoding="utf-8") + config_link = invocation / "config-link.ini" + config_link.symlink_to(target.name) + + normalized = run_tests._normalize_selection_paths( + ["-c", "config-link.ini", "-c=config-link.ini", "--debug", "-k", "focused"], + invocation_directory=invocation, + ) + + lexical_link = str(invocation / "config-link.ini") + assert normalized[:2] == ["-c", lexical_link] + assert normalized[2] == f"-c{lexical_link}" + assert normalized[3:] == ["--debug", "-k", "focused"] + assert str(target) not in normalized + + +def test_main_preserves_keyword_and_marker_values_from_tests_directory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[str] = [] + monkeypatch.chdir(run_tests.ROOT / "tests") + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + + def capture(_label: str, command: list[str], **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + captured.extend(command) + return 0, 0.01, {"diagnosis": "pytest_passed"} + + monkeypatch.setattr( + run_tests, + "_run", + capture, + ) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "stable") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) + + assert run_tests.main(["-k", "unit", "-m", "unit"]) == 0 + + keyword_index = captured.index("-k") + marker_index = next(index for index in range(keyword_index + 1, len(captured)) if captured[index] == "-m") + assert captured[keyword_index + 1] == "unit" + assert captured[marker_index + 1] == "unit" + + +def test_main_finalizes_runner_exception_after_open_step( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + history: dict[str, Any] = {} + monkeypatch.setattr(run_tests, "ROOT", tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + run_tests, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "git_head", lambda _root: "head") + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "stable") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: history.update(payload)) + + def explode(_label: str, command: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: + run = kwargs["run"] + run.start_step(label="pytest focused", cmd=command) + raise RuntimeError("focused runner exploded") + + monkeypatch.setattr(run_tests, "_run", explode) + + assert run_tests.main(["focused-selector", "--json"]) == 125 + assert history["exit_code"] == 125 + assert history["diagnosis"] == "focused_test_runner_exception" + assert history["steps"][0]["status"] == "failed" + assert history["steps"][0]["exit"] == 125 + + +def test_main_withholds_success_when_checkout_changes_during_pytest( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, Any] = {} + fingerprints = iter(("initial", "changed")) + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", lambda *_args, **_kwargs: (0, 0.01, {"diagnosis": "pytest_passed"})) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprints)) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: captured.update(payload)) + + assert run_tests.main(["tests/unit/example.py"]) == 125 + assert captured["status"] == "failed" + assert captured["diagnosis"] == "checkout_changed_during_focused_test" + assert captured["worktree_fingerprint"] == "initial" + assert captured["final_worktree_fingerprint"] == "changed" + + +def test_main_starts_checkout_monitor_before_initial_fingerprint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[str] = [] + + class _OrderingMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("monitor-started") + + def finish(self) -> CheckoutMutationObservation: + events.append("monitor-finished") + return CheckoutMutationObservation(changed=False, unavailable=False) + + def fingerprint(_root: Path) -> str: + assert events[0] == "monitor-started" + events.append("fingerprinted") + return "stable" + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", lambda *_args, **_kwargs: (0, 0.01, {"diagnosis": "pytest_passed"})) + monkeypatch.setattr(run_tests, "worktree_fingerprint", fingerprint) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _OrderingMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) + + assert run_tests.main(["tests/unit/example.py"]) == 0 + assert events == ["monitor-started", "fingerprinted", "fingerprinted", "monitor-finished"] + + +def test_main_finalizes_checkout_monitor_when_initial_fingerprint_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[str] = [] + + class _ExceptionalExitMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("monitor-started") + + def finish(self) -> CheckoutMutationObservation: + events.append("monitor-finished") + return CheckoutMutationObservation(changed=False, unavailable=False) + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _ExceptionalExitMonitor) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: (_ for _ in ()).throw(RuntimeError("boom"))) + + with pytest.raises(RuntimeError, match="boom"): + run_tests.main(["tests/unit/example.py"]) + + assert events == ["monitor-started", "monitor-finished"] + + +@pytest.mark.parametrize("fingerprints", [("unavailable", "stable"), ("stable", "unavailable")]) +def test_main_withholds_success_when_checkout_fingerprint_is_unavailable( + monkeypatch: pytest.MonkeyPatch, + fingerprints: tuple[str, str], +) -> None: + captured: dict[str, Any] = {} + fingerprint_values = iter(fingerprints) + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", lambda *_args, **_kwargs: (0, 0.01, {"diagnosis": "pytest_passed"})) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprint_values)) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: captured.update(payload)) + + assert run_tests.main(["tests/unit/example.py"]) == 125 + assert captured["status"] == "failed" + assert captured["diagnosis"] == "checkout_fingerprint_unavailable" + assert captured["worktree_fingerprint"] == fingerprints[0] + assert captured["final_worktree_fingerprint"] == fingerprints[1] def test_main_returns_pytest_exit_code(monkeypatch: pytest.MonkeyPatch) -> None: @@ -114,6 +521,52 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di assert run_tests.main(["tests/unit/does_not_exist"]) == 5 +@pytest.mark.parametrize("invocation_location", ["inside", "external"]) +def test_main_anchors_and_refreshes_root_artifacts_from_any_invocation_directory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, invocation_location: str +) -> None: + root = tmp_path / "checkout" + subdirectory = root / "devtools" + external_directory = tmp_path / "unrelated" + subdirectory.mkdir(parents=True) + external_directory.mkdir() + stale_report = root / verify.PYTEST_REPORT_PATH + stale_statistics = root / CURRENT_STATISTICS_PATH + stale_report.parent.mkdir(parents=True) + stale_statistics.parent.mkdir(parents=True, exist_ok=True) + stale_report.write_text('{"stale": true}') + stale_statistics.write_text('{"stale": true}') + captured: dict[str, object] = {} + + def fake_run(_label: str, _cmd: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: + captured["cwd"] = kwargs["cwd"] + Path(verify.PYTEST_REPORT_PATH).write_text('{"fresh": true}') + return 0, 0.01, {"diagnosis": "pytest_passed"} + + monkeypatch.setattr(run_tests, "ROOT", root) + monkeypatch.setattr(run_tests, "_LOCK_PATH", root / ".cache" / "test-run.lock") + monkeypatch.setattr( + run_tests, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=root / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setattr(run_tests, "git_head", lambda _root: "head") + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "fingerprint") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "_run", fake_run) + monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + invocation_directory = subdirectory if invocation_location == "inside" else external_directory + monkeypatch.chdir(invocation_directory) + + assert run_tests.main(["tests/unit/example.py"]) == 0 + + assert captured["cwd"] == str(root) + assert stale_report.read_text() == '{"fresh": true}' + assert not stale_statistics.exists() + assert not (invocation_directory / ".cache").exists() + + def test_git_head_records_checkout_head(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: def _fake_run(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: assert cmd == ["git", "rev-parse", "HEAD"] diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index 0a01f3c9a2..3df6ce4440 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -122,6 +122,49 @@ def test_seed_shard_ledger_rejects_duplicate_nodes_across_shards() -> None: assert testmon_state.validate_seed_shard_ledger([shard, duplicate], expected_nodeids=[NODEIDS[0]]) is None +def test_seed_shard_ledger_accepts_expected_and_unexpected_xfail_outcomes() -> None: + nodes = sorted(NODEIDS) + shard = { + "index": 1, + "nodeids": nodes, + "nodeid_count": len(nodes), + "nodeid_digest": hashlib.sha256("\n".join(nodes).encode()).hexdigest(), + "status": "complete", + "node_outcomes": [ + {"nodeid": nodes[0], "outcome": "xfailed"}, + {"nodeid": nodes[1], "outcome": "xpassed"}, + ], + } + + assert testmon_state.validate_seed_shard_ledger([shard], expected_nodeids=nodes) == [shard] + + +def test_expected_failure_and_non_strict_xpass_preserve_green_baseline(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data, outcomes=("xfailed", "xpassed")) + attempt.update(status="complete", exit_code=0, release_baseline_allowed=True, verification_scope="release-baseline") + identity = attempt["identity"] + assert isinstance(identity, dict) + identity.update(skip_slow=False, terminal_authorization=None) + + green = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert green is not None + assert green.baseline_status is BaselineStatus.GREEN + serialized = green.as_dict() + assert _TestmonSeedStamp.from_mapping(serialized, protocol_version=PROTOCOL).baseline_status is BaselineStatus.GREEN + + # pytest controls strict-xpass behavior through its process exit code. A + # strict xpass therefore remains reusable graph evidence but is red. + attempt.update(status="reusable", exit_code=1, release_baseline_allowed=False) + strict = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert strict is not None + assert strict.baseline_status is BaselineStatus.RED + assert strict.affected_selection_allowed + + def test_testmon_database_canonicalizes_xdist_group_names(tmp_path: Path) -> None: data = tmp_path / "testmondata" _write_graph(data) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 2d1208e15a..829793d5d5 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1,16 +1,24 @@ from __future__ import annotations +import fcntl import hashlib import json import os +import platform import shutil import sqlite3 import subprocess import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from unittest.mock import patch +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import MagicMock, patch import pytest +import watchfiles from devtools import run_tests, verify, verify_runs from devtools.testmon_state import ( @@ -69,39 +77,39 @@ _testmon_database_state, _testmon_preflight, _testmon_seed_can_resume, - _worktree_fingerprint, build_verify_steps, main, ) from devtools.verify_runs import ( + CheckoutMutationMonitor, + CheckoutMutationObservation, PytestResourceError, + PytestStepArtifacts, ResourceSampler, VerifyRun, adaptive_pytest_runtime_policy, adaptive_pytest_worker_count, + aggregate_pytest_statistics, + append_verify_history, apply_managed_pytest_runtime_policy, classify_pytest_result, cleanup_managed_pytest_basetemp, pytest_basetemp_known_roots, pytest_basetemp_path, + pytest_tmpfs_budget_exceeded, pytest_tmpfs_budget_kb, resolve_pytest_basetemp_root, xdist_uninterruptible_stall_reason, ) +from devtools.verify_runs import ( + worktree_fingerprint as _worktree_fingerprint, +) @pytest.fixture(autouse=True) def _isolate_verify_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Keep supervisor and testmon receipts private to each test. - - These tests exercise the real checkout guard, so leaving a synthetic - ``.cache/testmon`` behind makes a later guard test observe a fixture - artifact as if it were a developer's checkout state. - """ + """Keep supervisor and testmon receipts private to each test.""" monkeypatch.chdir(tmp_path) - checkout_cache = ROOT / ".cache" / "testmon" - if checkout_cache.exists(): - shutil.move(str(checkout_cache), str(tmp_path / "checkout-testmon-generated")) for name in ( "TESTMON_DATA", "TESTMON_SEED_STAMP", @@ -113,33 +121,6 @@ def _isolate_verify_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) - monkeypatch.setattr(sys.modules[__name__], name, isolated) -@pytest.fixture(scope="session", autouse=True) -def _quarantine_checkout_testmon(tmp_path_factory: pytest.TempPathFactory) -> object: - """Prevent subprocess-backed verify tests from contaminating the checkout. - - A few tests intentionally re-anchor verification to ``ROOT``. Their child - pytest process therefore uses the real checkout's relative testmon path, - even though the parent test has a private working directory. Keep any - pre-existing state safe for restoration and quarantine only state created - during this test module. - """ - checkout_cache = ROOT / ".cache" / "testmon" - quarantine = tmp_path_factory.mktemp("checkout-testmon") - original: Path | None = None - if checkout_cache.exists(): - original = quarantine / "original" - original.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(checkout_cache), str(original)) - try: - yield - finally: - if checkout_cache.exists(): - shutil.move(str(checkout_cache), str(quarantine / "generated")) - if original is not None and not checkout_cache.exists(): - checkout_cache.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(original), str(checkout_cache)) - - def _pytest_marker_expr(command: list[str]) -> str: marker_indexes = [idx for idx, item in enumerate(command) if item == "-m"] assert marker_indexes @@ -263,6 +244,7 @@ def test_default_verify_uses_adaptive_pytest_testmon(monkeypatch: pytest.MonkeyP assert "--testmon-forceselect" in command assert "-n" in command assert command[command.index("-n") + 1] == "8" + assert "--dist=loadgroup" in command def test_broad_default_verify_uses_parallel_testmon(monkeypatch: pytest.MonkeyPatch) -> None: @@ -436,7 +418,7 @@ def test_seed_shard_checkpoint_preserves_completed_shards_for_resume( "event": "test_report", "nodeid": f"{ordered[0]}@web-reader", "when": "call", - "outcome": "passed", + "outcome": "xfailed", } ) + "\n" @@ -450,7 +432,7 @@ def test_seed_shard_checkpoint_preserves_completed_shards_for_resume( assert checkpointed["shards"][0]["status"] == "complete" assert checkpointed["shards"][1]["status"] == "pending" - assert json.loads(TESTMON_SEED_ATTEMPT.read_text())["shards"][0]["node_outcomes"][0]["outcome"] == "passed" + assert json.loads(TESTMON_SEED_ATTEMPT.read_text())["shards"][0]["node_outcomes"][0]["outcome"] == "xfailed" resumed = _prepare_testmon_seed_attempt( identity={ "git_head": "head", @@ -628,6 +610,7 @@ def test_full_verify_includes_full_pytest_without_testmon(monkeypatch: pytest.Mo assert "--testmon" not in bulk_command assert "-n" in bulk_command assert bulk_command[bulk_command.index("-n") + 1] == "8" + assert "--dist=loadgroup" in bulk_command isolated_label, isolated_command = steps[-1] assert isolated_label == "pytest load-sensitive (isolated)" @@ -952,170 +935,826 @@ def test_focused_run_can_record_typed_affected_scope(tmp_path: Path) -> None: assert json.loads((tmp_path / ".cache" / "verify" / "current-run.json").read_text()) == payload -def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - artifact_dir = tmp_path / ".cache" / "verify" / "runs" / "interrupted" - step_dir = artifact_dir / "steps" / "17-pytest-seed-testmon" - step_dir.mkdir(parents=True) - expected = ["tests/unit/test_example.py::test_one"] - (step_dir / "selection.json").write_text( - json.dumps({"selected_nodeids": expected, "selected_nodeids_omitted": 0, "selected_count": 1}) +def test_verify_run_writes_invocation_receipt_without_leaking_token_to_pytest( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + receipt = tmp_path / "invocation" / "run.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "invocation-1") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="pytest focused", cmd=["pytest", "tests/unit/example.py"]) + + payload = run.finish( + exit_code=0, + duration_s=0.1, + verification_scope="affected", + release_baseline_allowed=False, ) - identity = { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": True, - "lab": False, - "terminal_authorization": None, - } - TESTMON_SEED_ATTEMPT.write_text( + child_env = verify_runs.env_for_pytest_step(dict(os.environ), run=run, artifacts=artifacts) + + assert json.loads(receipt.read_text()) == payload + assert payload["invocation_id"] == "invocation-1" + assert verify_runs.VERIFICATION_INVOCATION_ID_ENV not in child_env + assert verify_runs.VERIFICATION_RECEIPT_PATH_ENV not in child_env + + +def test_aggregate_pytest_statistics_reduces_phases_fixtures_and_resources(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + (step / "events.jsonl").write_text( + "\n".join( + json.dumps(row) + for row in ( + { + "event": "test_report", + "nodeid": "a", + "when": "setup", + "duration_s": 1.0, + "outcome": "passed", + "worker_id": "controller", + }, + { + "event": "test_report", + "nodeid": "a", + "when": "call", + "duration_s": 2.0, + "outcome": "passed", + "worker_id": "gw0", + }, + { + "event": "test_report", + "nodeid": "a", + "when": "teardown", + "duration_s": 0.5, + "outcome": "passed", + "worker_id": "gw0", + }, + ) + ) + + "\n" + ) + (step / "resources.jsonl").write_text( json.dumps( { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": identity, - "expected_nodeids": [], - "artifact_dir": str(artifact_dir.relative_to(tmp_path)), + "basetemp": "/dev/shm/run", + "basetemp_size_kb": 12, + "tree_rss_kb": 100, + "tree_pss_kb": 80, + "cgroup_memory_peak_bytes": 200, + "xdist_worker_count": 1, } ) + + "\n" ) + (step / "containment.json").write_text(json.dumps({"tmpfs_cleanup_complete": False, "exit_code": 0})) - assert _testmon_seed_can_resume({**identity, "git_head": "fixed", "git_tree": "tree-hash"}) is True - - run = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="fixed") - prepared = _prepare_testmon_seed_attempt( - identity={**identity, "git_head": "fixed", "git_tree": "tree-hash"}, run=run, resume=True + result = aggregate_pytest_statistics( + step, + command=["pytest"], + step_result={"exit": 0, "basetemp_cleanup": "/realm/tmp/polylogue-pytest/pytest-polylogue-run"}, ) - assert prepared["expected_nodeids"] == expected - assert prepared["expected_count"] == 1 - assert prepared["expected_digest"] == hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() - persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) - assert persisted["expected_digest"] == prepared["expected_digest"] + assert result["node_count"] == 1 + assert result["phases"]["call"]["p50_s"] == 2.0 + assert result["phases"]["setup"]["count"] == 1 + assert result["storage"]["basetemp_logical_bytes_max"] == 12 * 1024 + assert result["resources"]["peak_tree_pss_kb"] == 80 + assert result["cleanup"]["complete"] is True -def test_seed_resume_rejects_selection_artifact_outside_checkout( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - outside = tmp_path.parent / "outside-testmon-artifacts" - step_dir = outside / "steps" / "17-pytest-seed-testmon" - step_dir.mkdir(parents=True) - (step_dir / "selection.json").write_text( +def test_aggregate_pytest_statistics_deduplicates_xdist_reports_and_terminal_failures(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + rows = [ + { + "event": "test_report", + "nodeid": "test_setup", + "when": "setup", + "outcome": "failed", + "duration_s": 1.0, + "worker_id": "gw0", + }, + { + "event": "test_report", + "nodeid": "test_setup", + "when": "setup", + "outcome": "failed", + "duration_s": 1.0, + "worker_id": "controller", + }, + { + "event": "test_report", + "nodeid": "test_teardown", + "when": "call", + "outcome": "passed", + "duration_s": 0.2, + "worker_id": "gw1", + }, + { + "event": "test_report", + "nodeid": "test_teardown", + "when": "teardown", + "outcome": "failed", + "duration_s": 0.3, + "worker_id": "gw1", + }, + ] + (step / "events.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) + + result = aggregate_pytest_statistics(step, command=["pytest", "-n", "2"]) + + assert result["phases"]["setup"]["count"] == 1 + assert result["xdist"]["worker_count"] == 2 + assert result["outcomes"] == {"error": 2} + + compact_result = aggregate_pytest_statistics(step, command=["pytest", "--numprocesses=3"]) + assert compact_result["xdist"]["worker_count"] == 3 + + +def test_aggregate_pytest_statistics_accounts_for_started_node_without_a_phase(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + rows = [ + {"event": "test_started", "nodeid": "tests/a.py::test_completed", "worker_id": "gw0"}, + { + "event": "test_report", + "nodeid": "tests/a.py::test_completed", + "when": "call", + "outcome": "passed", + "duration_s": 0.1, + "worker_id": "gw0", + }, + {"event": "test_started", "nodeid": "tests/a.py::test_interrupted", "worker_id": "gw1"}, + ] + (step / "events.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) + + result = aggregate_pytest_statistics(step) + + assert result["node_count"] == 2 + assert result["outcomes"] == {"passed": 1, "interrupted": 1} + assert sum(result["outcomes"].values()) == result["node_count"] + + +def test_aggregate_pytest_statistics_uses_completed_report_to_fill_event_gaps(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + (step / "events.jsonl").write_text( json.dumps( { - "selected_nodeids": ["tests/unit/test_example.py::test_one"], - "selected_nodeids_omitted": 0, - "selected_count": 1, + "event": "test_report", + "nodeid": "tests/a.py::test_event", + "when": "call", + "outcome": "passed", + "duration_s": 0.1, + "worker_id": "gw0", } ) + + "\n" ) - identity = { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": True, - "lab": False, - } - TESTMON_SEED_ATTEMPT.write_text( + (step / "pytest-report.json").write_text( json.dumps( { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": identity, - "expected_nodeids": [], - "artifact_dir": str(outside), + "tests": [ + { + "nodeid": "tests/a.py::test_event", + "outcome": "passed", + "call": {"outcome": "passed", "duration": 0.1}, + }, + { + "nodeid": "tests/a.py::test_redirected", + "outcome": "xfailed", + "setup": {"outcome": "passed", "duration": 0.2}, + "call": {"outcome": "skipped", "duration": 0.3}, + "teardown": {"outcome": "passed", "duration": 0.1}, + }, + ] } ) ) - assert _testmon_seed_can_resume(identity) is False + result = aggregate_pytest_statistics(step) + assert result["canonical_report_status"] == "present" + assert result["node_count"] == 2 + assert result["outcomes"] == {"passed": 1, "xfailed": 1} + assert result["phases"]["setup"]["count"] == 1 + assert result["phases"]["call"]["count"] == 2 -def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) -> None: - monkeypatch = pytest.MonkeyPatch() - monkeypatch.chdir(tmp_path) - try: - expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) - ) - (artifact_dir / "events.jsonl").write_text( - json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" + +def test_aggregate_pytest_statistics_recognizes_completed_empty_report(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + (step / "pytest-report.json").write_text(json.dumps({"tests": []})) + + result = aggregate_pytest_statistics(step) + + assert result["canonical_report_status"] == "present" + assert result["node_count"] == 0 + assert result["outcomes"] == {} + + +def test_verify_run_statistics_only_cover_pytest_steps(tmp_path: Path) -> None: + run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="ruff check", cmd=["ruff", "check"]) + + run.finish_step(step_id=artifacts.step_id, result={"exit": 0, "duration_s": 0.1}) + + step = run._payload["steps"][0] + assert "statistics" not in step + assert not artifacts.statistics_path.exists() + + +def test_verify_run_embeds_compact_statistics_before_worktree_cleanup(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="pytest focused", cmd=["pytest", "tests/unit/example.py"]) + artifacts.events_merged_path.write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/unit/example.py::test_one", + "when": "call", + "duration_s": 0.25, + "outcome": "passed", + "worker_id": "controller", + } ) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("create table environment (id integer primary key, environment_name text)") - connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") - connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") - connection.execute( - "create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)" - ) - connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) - connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) - connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) - prepared = { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": True, - "expected_nodeids": expected, - "run_id": "resume", - "artifact_dir": ".cache/verify/runs/resume", - } - _write_run_receipt(tmp_path, "resume") + + "\n" + ) - receipt = _finalize_testmon_seed_attempt( - prepared=prepared, - step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], - exit_code=0, + run.finish_step(step_id=artifacts.step_id, result={"exit": 0, "duration_s": 0.25}) + payload = run.finish(exit_code=0, duration_s=0.25) + shutil.rmtree(run.run_dir) + + statistics = payload["steps"][0]["statistics"] + assert statistics["node_count"] == 1 + assert statistics["phases"]["call"]["p50_s"] == 0.25 + + +def test_interrupted_run_merges_worker_events_before_statistics(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="pytest focused", cmd=["pytest", "tests/unit/example.py"]) + artifacts.events_dir.mkdir() + (artifacts.events_dir / "gw0-1.jsonl").write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/unit/example.py::test_one", + "when": "call", + "outcome": "passed", + "duration_s": 0.2, + "worker_id": "gw0", + } ) + + "\n" + ) - assert receipt["status"] == "incomplete" - assert {item["nodeid"]: item["outcome"] for item in receipt["node_outcomes"]} == { - expected[0]: "passed", - expected[1]: "missing", - } + run.finish_interrupted_steps(exit_code=130, diagnosis="pytest_interrupted") - (artifact_dir / "selection.json").write_text(json.dumps({})) - (artifact_dir / "events.jsonl").write_text( - "\n".join( - json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) - for nodeid in expected + assert artifacts.events_merged_path.exists() + assert run._payload["steps"][0]["statistics"]["node_count"] == 1 + + +def test_run_returns_finalized_statistics_for_verify_history(tmp_path: Path) -> None: + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) + history_path = tmp_path / "state" / "verify-history.jsonl" + + def _complete_with_evidence( + *_args: object, artifacts: object, **_kwargs: object + ) -> subprocess.CompletedProcess[str]: + assert isinstance(artifacts, verify_runs.PytestStepArtifacts) + artifacts.events_merged_path.write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/unit/example.py::test_one", + "when": "call", + "duration_s": 0.25, + "outcome": "passed", + "worker_id": "controller", + } ) + "\n" ) - missing_selection = _finalize_testmon_seed_attempt( - prepared=prepared, - step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], - exit_code=0, + artifacts.resources_path.write_text(json.dumps({"tree_rss_kb": 512}) + "\n") + return completed + + with ( + patch("devtools.verify._run_pytest_with_heartbeat", side_effect=_complete_with_evidence), + patch("devtools.verify._read_pytest_report", return_value=None), + patch("devtools.verify.copy_current_pytest_artifacts"), + ): + rc, _elapsed, metadata = _run("pytest testmon", ["pytest", "-n", "0"], run=run) + + assert rc == 0 + append_verify_history( + {"tier": "quick", "steps": [{"name": "pytest testmon", "exit": rc, **metadata}]}, path=history_path + ) + shutil.rmtree(run.run_dir) + + durable_row = json.loads(history_path.read_text(encoding="utf-8")) + assert durable_row["steps"][0]["statistics"]["node_count"] == 1 + assert durable_row["steps"][0]["statistics"]["resources"]["peak_tree_rss_kb"] == 512 + assert metadata["statistics_path"].endswith("statistics.json") + + +def test_interrupted_pytest_waits_for_forced_containment_quiescence() -> None: + process = MagicMock() + process.poll.return_value = None + process.wait.side_effect = [subprocess.TimeoutExpired(cmd="pytest", timeout=2.0), None] + launch = MagicMock() + + with ( + patch("devtools.verify._request_supervisor_termination") as request_termination, + patch("devtools.verify._force_kill_owned_run") as force_kill, + patch("devtools.verify.reap_exited_children") as reap, + patch( + "devtools.verify.read_receipt", + return_value={"status": "terminated", "controller_group_alive": False}, + ), + patch("devtools.verify.descendant_process_identities", return_value=()), + ): + verify._await_interrupted_pytest_containment( + process, + launch, + term_grace_s=1.0, + preserved_runner_descendants=(), ) - assert missing_selection["status"] == "incomplete" - finally: - monkeypatch.undo() + request_termination.assert_called_once() + force_kill.assert_called_once_with(process, launch, preserved_runner_descendants=()) + assert process.wait.call_count == 2 + reap.assert_called_once() -def test_testmon_database_state_reports_missing_and_failed_nodes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as conn: + +def test_interrupted_pytest_refuses_cleanup_without_containment_quiescence() -> None: + process = MagicMock() + process.poll.return_value = None + process.wait.side_effect = [ + subprocess.TimeoutExpired(cmd="pytest", timeout=2.0), + subprocess.TimeoutExpired(cmd="pytest", timeout=1.0), + ] + launch = MagicMock() + + with ( + patch("devtools.verify._request_supervisor_termination"), + patch("devtools.verify._force_kill_owned_run") as force_kill, + patch("devtools.verify.reap_exited_children") as reap, + pytest.raises(verify.PytestContainmentError, match="did not quiesce"), + ): + verify._await_interrupted_pytest_containment( + process, + launch, + term_grace_s=1.0, + preserved_runner_descendants=(), + ) + + force_kill.assert_called_once_with(process, launch, preserved_runner_descendants=()) + reap.assert_not_called() + + +def test_interrupted_pytest_refuses_cleanup_when_controller_group_survives() -> None: + process = MagicMock() + process.poll.return_value = None + process.wait.return_value = None + launch = MagicMock() + + with ( + patch("devtools.verify._request_supervisor_termination"), + patch("devtools.verify.reap_exited_children"), + patch( + "devtools.verify.read_receipt", + return_value={"status": "terminated", "controller_group_alive": True}, + ), + patch("devtools.verify.descendant_process_identities", return_value=()), + pytest.raises(verify.PytestContainmentError, match="owned process tree"), + ): + verify._await_interrupted_pytest_containment( + process, + launch, + term_grace_s=1.0, + preserved_runner_descendants=(), + ) + + +def test_run_cleans_and_finalizes_only_after_contained_interrupt(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + order: list[str] = [] + original_finish_step = run.finish_step + + def _contained_interrupt(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + order.append("contained") + raise KeyboardInterrupt + + def _cleanup(**_kwargs: object) -> None: + order.append("cleanup") + return None + + def _finish_step(*, step_id: str, result: dict[str, Any]) -> dict[str, Any] | None: + order.append("finalize") + return original_finish_step(step_id=step_id, result=result) + + with ( + patch("devtools.verify._run_pytest_with_heartbeat", side_effect=_contained_interrupt), + patch("devtools.verify.cleanup_managed_pytest_basetemp", side_effect=_cleanup), + patch.object(run, "finish_step", side_effect=_finish_step), + ): + rc, _elapsed, _metadata = _run("pytest focused", ["pytest", "-n", "0"], run=run) + + assert rc == 130 + assert order == ["contained", "cleanup", "finalize"] + + +def test_run_terminalizes_containment_failure_without_cleaning_basetemp(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + + with ( + patch( + "devtools.verify._run_pytest_with_heartbeat", + side_effect=verify.PytestContainmentError("still running"), + ), + patch("devtools.verify.cleanup_managed_pytest_basetemp") as cleanup, + ): + rc, _elapsed, metadata = _run("pytest focused", ["pytest", "-n", "0"], run=run) + + cleanup.assert_not_called() + assert rc == 125 + assert metadata["diagnosis"] == "pytest_containment_unproven" + assert run._payload["steps"][0]["status"] == "failed" + assert run._payload["steps"][0]["termination_reason"].startswith("pytest containment did not quiesce") + + +def test_run_recovers_xdist_collection_facts_after_containment_failure(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + + def _write_worker_facts(*_args: object, artifacts: PytestStepArtifacts, **_kwargs: object) -> None: + artifacts.events_dir.mkdir(parents=True, exist_ok=True) + for worker_id, pid, duration in (("gw1", 11, 1.5), ("gw0", 10, 2.5)): + (artifacts.events_dir / f"{worker_id}-{pid}.collection.json").write_text( + json.dumps( + { + "worker_id": worker_id, + "pid": pid, + "selected_count": 3, + "deselected_count": 2, + "selected_nodeids": ["tests/unit/example.py::test_selected"], + "collection_duration_s": duration, + } + ), + encoding="utf-8", + ) + raise verify.PytestContainmentError("controller interrupted") + + with patch("devtools.verify._run_pytest_with_heartbeat", side_effect=_write_worker_facts): + rc, _elapsed, metadata = _run("pytest focused", ["pytest", "-n", "2"], run=run) + + assert rc == 125 + assert metadata["selected_count"] == 3 + assert metadata["deselected_count"] == 2 + assert metadata["collection_duration_s"] == 2.5 + selection = json.loads((run.run_dir / "steps" / "01-pytest-focused" / "selection.json").read_text()) + assert selection["recovered_after_interruption"] is True + assert selection["worker_id"] == "runner" + + +def test_verify_main_records_containment_failure_as_terminal_history( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + history_path = tmp_path / "verify-history.jsonl" + monkeypatch.setattr(verify, "HISTORY_PATH", history_path) + + with ( + patch("devtools.verify._anchor_verification_paths"), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), + patch("devtools.verify._testmon_preflight", return_value=None), + patch("devtools.verify.build_verify_steps", return_value=[("pytest containment", ["pytest", "-n", "0"])]), + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), + patch( + "devtools.verify._run_pytest_with_heartbeat", + side_effect=verify.PytestContainmentError("owned child still running"), + ), + patch("devtools.verify.cleanup_managed_pytest_basetemp") as cleanup, + patch("devtools.verify._notify"), + ): + rc = main(["--json"]) + + history = json.loads(history_path.read_text(encoding="utf-8")) + run_json = next((tmp_path / ".cache" / "verify" / "runs").glob("*/run.json")) + run_payload = json.loads(run_json.read_text(encoding="utf-8")) + payload = json.loads(capsys.readouterr().out) + + assert rc == 125 + cleanup.assert_not_called() + assert payload["diagnosis"] == "pytest_containment_unproven" + assert history["exit_code"] == 125 + assert history["diagnosis"] == "pytest_containment_unproven" + assert run_payload["status"] == "failed" + assert run_payload["steps"][0]["status"] == "failed" + + +def test_print_history_accepts_verify_and_focused_run_records( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + verify, + "_load_history", + lambda: [ + { + "timestamp": "2026-08-12T20:00:00+00:00", + "tier": "quick", + "git_head": "a" * 40, + "total_duration_s": 2.0, + "exit_code": 0, + "steps": [{"name": "ruff", "duration_s": 1.0, "exit": 0}], + }, + { + "finished_at": "2026-08-12T20:01:00+00:00", + "tier": "focused-test", + "git_head": "b" * 40, + "duration_s": 3.0, + "exit_code": 1, + "steps": [{"name": "pytest focused", "duration_s": None, "exit": 1}], + }, + { + "finished_at": "2026-08-12T20:02:00+00:00", + "tier": "focused-test", + "git_head": "c" * 40, + "duration_s": "invalid", + "exit_code": None, + "steps": [{"name": "pytest interrupted", "duration_s": "invalid", "exit": None}], + }, + ], + ) + + verify._print_history() + + output = capsys.readouterr().out + assert "quick" in output + assert "focused-" in output + assert "pytest focused(0s FAIL)" in output + assert "pytest interrupted(0s FAIL)" in output + + +def test_verify_history_appends_concurrent_records_without_interleaving(tmp_path: Path) -> None: + history = tmp_path / "state" / "verify-history.jsonl" + + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda sequence: append_verify_history({"sequence": sequence}, path=history), range(64))) + + rows = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] + assert sorted(row["sequence"] for row in rows) == list(range(64)) + + +def test_verify_history_repairs_or_frames_an_incomplete_trailing_record(tmp_path: Path) -> None: + history = tmp_path / "state" / "verify-history.jsonl" + history.parent.mkdir(parents=True) + history.write_text('{"sequence": 0}', encoding="utf-8") + + append_verify_history({"sequence": 1}, path=history) + + history.write_text(history.read_text(encoding="utf-8") + '{"interrupted":', encoding="utf-8") + append_verify_history({"sequence": 2}, path=history) + + rows = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] + assert rows == [{"sequence": 0}, {"sequence": 1}, {"sequence": 2}] + + +def test_verify_history_append_reads_only_the_trailing_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + history = tmp_path / "state" / "verify-history.jsonl" + history.parent.mkdir(parents=True) + history.write_bytes(b'{"padding":"' + (b"x" * (5 * 1024 * 1024)) + b'"}\n{"interrupted":') + bytes_read = 0 + real_read = os.read + + def measured_read(descriptor: int, count: int) -> bytes: + nonlocal bytes_read + payload = real_read(descriptor, count) + bytes_read += len(payload) + return payload + + monkeypatch.setattr(os, "read", measured_read) + + append_verify_history({"sequence": 1}, path=history) + + assert bytes_read < 128 * 1024 + assert json.loads(history.read_text(encoding="utf-8").splitlines()[-1]) == {"sequence": 1} + + +def test_compare_against_last_skips_intervening_focused_history(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + verify, + "_load_history", + lambda: [ + {"tier": "quick", "steps": [{"name": "ruff check", "duration_s": 1.0}]}, + {"tier": "focused-test", "steps": [{"name": "pytest focused", "duration_s": 999.0}]}, + ], + ) + + flags = verify._compare_against_last([{"name": "ruff check", "duration_s": 7.0}]) + + assert flags and "ruff check" in flags[0] + + +def test_compare_against_last_selects_prior_run_independently_per_step(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + verify, + "_load_history", + lambda: [ + { + "tier": "default", + "steps": [ + {"name": "ruff check", "duration_s": 1.0}, + {"name": "pytest testmon", "duration_s": 2.0}, + ], + }, + {"tier": "quick", "steps": [{"name": "ruff check", "duration_s": 1.0}]}, + ], + ) + + flags = verify._compare_against_last( + [ + {"name": "ruff check", "duration_s": 1.1}, + {"name": "pytest testmon", "duration_s": 8.0}, + ] + ) + + assert len(flags) == 1 + assert "pytest testmon" in flags[0] + + +def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + TESTMON_DATA.parent.mkdir(parents=True) + TESTMON_DATA.write_text("partial") + artifact_dir = tmp_path / ".cache" / "verify" / "runs" / "interrupted" + step_dir = artifact_dir / "steps" / "17-pytest-seed-testmon" + step_dir.mkdir(parents=True) + expected = ["tests/unit/test_example.py::test_one"] + (step_dir / "selection.json").write_text( + json.dumps({"selected_nodeids": expected, "selected_nodeids_omitted": 0, "selected_count": 1}) + ) + identity = { + "git_head": "head", + "git_tree": "tree-hash", + "worktree_fingerprint": "tree", + "python": "3.13", + "skip_slow": True, + "lab": False, + "terminal_authorization": None, + } + TESTMON_SEED_ATTEMPT.write_text( + json.dumps( + { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": identity, + "expected_nodeids": [], + "artifact_dir": str(artifact_dir.relative_to(tmp_path)), + } + ) + ) + + assert _testmon_seed_can_resume({**identity, "git_head": "fixed", "git_tree": "tree-hash"}) is True + + run = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="fixed") + prepared = _prepare_testmon_seed_attempt( + identity={**identity, "git_head": "fixed", "git_tree": "tree-hash"}, run=run, resume=True + ) + + assert prepared["expected_nodeids"] == expected + assert prepared["expected_count"] == 1 + assert prepared["expected_digest"] == hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() + persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) + assert persisted["expected_digest"] == prepared["expected_digest"] + + +def test_seed_resume_rejects_selection_artifact_outside_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + TESTMON_DATA.parent.mkdir(parents=True) + TESTMON_DATA.write_text("partial") + outside = tmp_path.parent / "outside-testmon-artifacts" + step_dir = outside / "steps" / "17-pytest-seed-testmon" + step_dir.mkdir(parents=True) + (step_dir / "selection.json").write_text( + json.dumps( + { + "selected_nodeids": ["tests/unit/test_example.py::test_one"], + "selected_nodeids_omitted": 0, + "selected_count": 1, + } + ) + ) + identity = { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "3.13", + "skip_slow": True, + "lab": False, + } + TESTMON_SEED_ATTEMPT.write_text( + json.dumps( + { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": identity, + "expected_nodeids": [], + "artifact_dir": str(outside), + } + ) + ) + + assert _testmon_seed_can_resume(identity) is False + + +def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) -> None: + monkeypatch = pytest.MonkeyPatch() + monkeypatch.chdir(tmp_path) + try: + expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir() + (artifact_dir / "selection.json").write_text( + json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) + ) + (artifact_dir / "events.jsonl").write_text( + json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" + ) + TESTMON_DATA.parent.mkdir(parents=True) + with sqlite3.connect(TESTMON_DATA) as connection: + connection.execute("create table environment (id integer primary key, environment_name text)") + connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") + connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") + connection.execute( + "create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)" + ) + connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) + connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) + connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) + prepared = { + "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, + "status": "running", + "identity": { + "git_head": "head", + "worktree_fingerprint": "tree", + "python": "python", + "skip_slow": False, + "lab": False, + **_testmon_runtime_identity_fields(Path.cwd()), + }, + "resume": True, + "expected_nodeids": expected, + "run_id": "resume", + "artifact_dir": ".cache/verify/runs/resume", + } + _write_run_receipt(tmp_path, "resume") + + receipt = _finalize_testmon_seed_attempt( + prepared=prepared, + step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + + assert receipt["status"] == "incomplete" + assert {item["nodeid"]: item["outcome"] for item in receipt["node_outcomes"]} == { + expected[0]: "passed", + expected[1]: "missing", + } + + (artifact_dir / "selection.json").write_text(json.dumps({})) + (artifact_dir / "events.jsonl").write_text( + "\n".join( + json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) + for nodeid in expected + ) + + "\n" + ) + missing_selection = _finalize_testmon_seed_attempt( + prepared=prepared, + step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], + exit_code=0, + ) + assert missing_selection["status"] == "incomplete" + finally: + monkeypatch.undo() + + +def test_testmon_database_state_reports_missing_and_failed_nodes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + TESTMON_DATA.parent.mkdir(parents=True) + with sqlite3.connect(TESTMON_DATA) as conn: conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") conn.execute( @@ -1164,6 +1803,602 @@ def test_worktree_fingerprint_hashes_untracked_file_contents(tmp_path: Path) -> assert before != after +def test_worktree_fingerprint_rejects_partial_git_output( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + real_run = subprocess.run + + def warning_run(*args: Any, **kwargs: Any) -> subprocess.CompletedProcess[bytes]: + result = cast(subprocess.CompletedProcess[bytes], real_run(*args, **kwargs)) + command = args[0] + if isinstance(command, list) and command[:2] == ["git", "diff"]: + return subprocess.CompletedProcess(command, 0, result.stdout, b"warning: partial enumeration\n") + return result + + monkeypatch.setattr(subprocess, "run", warning_run) + + assert _worktree_fingerprint(tmp_path) == "unavailable" + + +def test_changed_paths_keep_start_time_base_when_remote_ref_advances( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + source = tmp_path / "polylogue" / "example.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "polylogue/example.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + base = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + subprocess.run(["git", "switch", "-qc", "feature"], cwd=tmp_path, check=True) + source.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "commit", "-qam", "feature"], cwd=tmp_path, check=True) + feature_head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + subprocess.run(["git", "update-ref", "refs/remotes/origin/master", base], cwd=tmp_path, check=True) + monkeypatch.setattr(verify, "ROOT", tmp_path) + + pinned_base = verify._git_commit("origin/master") + assert pinned_base == base + subprocess.run(["git", "update-ref", "refs/remotes/origin/master", "HEAD"], cwd=tmp_path, check=True) + + assert verify._changed_executable_paths(pinned_base, feature_head) == ("polylogue/example.py",) + + +def test_changed_paths_include_untracked_executable_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "README.md" + tracked.write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "README.md"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + untracked = tmp_path / "devtools" / "new_command.py" + untracked.parent.mkdir() + untracked.write_text("value = 1\n", encoding="utf-8") + monkeypatch.setattr(verify, "ROOT", tmp_path) + + assert verify._changed_executable_paths(head, head) == ("devtools/new_command.py",) + + +def test_changed_paths_include_executable_rename_sources( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + source = tmp_path / "polylogue" / "example.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "polylogue/example.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + base = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + docs = tmp_path / "docs" + docs.mkdir() + subprocess.run(["git", "mv", "polylogue/example.py", "docs/example.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "move module"], cwd=tmp_path, check=True) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + monkeypatch.setattr(verify, "ROOT", tmp_path) + + assert verify._changed_executable_paths(base, head) == ("polylogue/example.py",) + + +def test_changed_paths_parse_non_ascii_names_without_git_quoting( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + source = tmp_path / "polylogue" / "café.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "polylogue/café.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + base = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + source.write_text("value = 2\n", encoding="utf-8") + monkeypatch.setattr(verify, "ROOT", tmp_path) + + assert verify._changed_executable_paths(base, base) == ("polylogue/café.py",) + + +def test_git_head_uses_bounded_authoritative_probe() -> None: + with patch("devtools.verify._git_commit", return_value="resolved-head") as resolve: + assert verify._git_head() == "resolved-head" + + resolve.assert_called_once_with("HEAD") + + +def test_checkout_mutation_monitor_detects_a_change_that_reverts_before_the_final_fingerprint( + tmp_path: Path, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + original = "VALUE = 1\n" + tracked.write_text(original, encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + tracked.write_text("VALUE = 2\n", encoding="utf-8") + tracked.write_text(original, encoding="utf-8") + observation = monitor.finish() + + assert observation.changed is True + assert observation.unavailable is False + assert observation.observed_path == "tracked.py" + + +def test_checkout_mutation_monitor_ignores_nested_disposable_cache_writes(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + package = tmp_path / "package" + package.mkdir() + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + cache_file = package / "__pycache__" / "module.pyc" + cache_file.parent.mkdir(parents=True) + cache_file.write_bytes(b"cache") + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + + +def test_checkout_mutation_monitor_observes_tracked_file_inside_disposable_cache(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "package" / "__pycache__" / "authority.py" + tracked.parent.mkdir(parents=True) + tracked.write_text("before\n", encoding="utf-8") + subprocess.run(["git", "add", "-f", "package/__pycache__/authority.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed tracked cache path"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + tracked.write_text("during\n", encoding="utf-8") + tracked.write_text("before\n", encoding="utf-8") + observation = monitor.finish() + + assert observation == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path="package/__pycache__/authority.py", + ) + + +def test_checkout_mutation_monitor_uses_gitignore_for_verifier_task_history( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / ".gitignore").write_text(".agent/*\n", encoding="utf-8") + history = tmp_path / ".agent" / "task-history" / "tasks.jsonl" + history.parent.mkdir(parents=True) + + def portable_watch(*_paths: Path, **kwargs: object) -> object: + yield set() + yield {(watchfiles.Change.modified, str(history))} + stop_event = kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait() + + monkeypatch.setattr(watchfiles, "watch", portable_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + + +def test_checkout_mutation_monitor_observes_tracked_file_that_matches_gitignore(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / ".gitignore").write_text(".agent/*\n", encoding="utf-8") + tracked = tmp_path / ".agent" / "script.py" + tracked.parent.mkdir() + tracked.write_text("before\n", encoding="utf-8") + subprocess.run(["git", "add", "-f", ".agent/script.py"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + tracked.write_text("during\n", encoding="utf-8") + tracked.write_text("before\n", encoding="utf-8") + observation = monitor.finish() + + assert observation == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path=".agent/script.py", + ) + + +def test_checkout_mutation_monitor_uses_portable_watchfiles_events_without_linux_kernel( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("VALUE = 1\n", encoding="utf-8") + calls: dict[str, object] = {} + event_emitted = threading.Event() + + def portable_watch(*paths: Path, **kwargs: object) -> object: + calls["paths"] = paths + calls["kwargs"] = kwargs + yield set() + event_emitted.set() + yield {(watchfiles.Change.modified, str(tracked))} + + monkeypatch.setattr(watchfiles, "watch", portable_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + assert event_emitted.wait(timeout=1) + observation = monitor.finish() + + assert calls["paths"] == ( + tmp_path.resolve(), + (tmp_path / ".git").resolve(), + (tmp_path / ".git" / "refs" / "heads").resolve(), + ) + assert calls["kwargs"] == { + "watch_filter": None, + "debounce": 0, + "step": 1, + "stop_event": monitor._stop, + "rust_timeout": monitor._WATCH_RUST_TIMEOUT_MS, + "yield_on_timeout": True, + "raise_interrupt": False, + "force_polling": False, + "recursive": False, + } + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="tracked.py") + + +def test_checkout_mutation_monitor_rejects_source_topology_changed_during_startup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + source = tmp_path / "source" / "package" + source.mkdir(parents=True) + shutil.rmtree(source) + + def portable_watch(*paths: Path, **_kwargs: object) -> object: + assert source not in paths + source.mkdir(parents=True) + yield set() + stop_event = _kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait(timeout=1) + + monkeypatch.setattr(watchfiles, "watch", portable_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + + assert monitor.finish() == CheckoutMutationObservation(changed=False, unavailable=True) + + +def test_checkout_mutation_monitor_rejects_forced_polling_backend( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("WATCHFILES_FORCE_POLLING", "1") + + def unexpected_watch(*_paths: Path, **_kwargs: object) -> object: + raise AssertionError("polling mode must fail before watchfiles starts") + yield set() + + monkeypatch.setattr(watchfiles, "watch", unexpected_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + +def test_checkout_mutation_monitor_rejects_wsl_auto_polling_backend( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("WATCHFILES_FORCE_POLLING", raising=False) + monkeypatch.setattr( + platform, + "uname", + lambda: SimpleNamespace(system="Linux", release="6.6.0-microsoft-standard-WSL2"), + ) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + +def test_checkout_mutation_monitor_prunes_disposable_trees_and_observes_new_source_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / ".gitignore").write_text( + "browser-extension/node_modules/\ncustom/generated-output/\n", + encoding="utf-8", + ) + source = tmp_path / "src" / "package" + source.mkdir(parents=True) + for disposable in (".venv", ".git", ".cache"): + (tmp_path / disposable / "nested").mkdir(parents=True, exist_ok=True) + ignored_dependency = tmp_path / "browser-extension" / "node_modules" / "dependency" + ignored_dependency.mkdir(parents=True) + ignored_build = tmp_path / "custom" / "generated-output" / "deep" / "tree" + ignored_build.mkdir(parents=True) + calls: dict[str, object] = {} + allow_event = threading.Event() + new_source = tmp_path / "new_source" + + def portable_watch(*paths: Path, **kwargs: object) -> object: + calls["paths"] = paths + calls["kwargs"] = kwargs + yield set() + assert allow_event.wait(timeout=1) + yield {(watchfiles.Change.added, str(new_source))} + + monkeypatch.setattr(watchfiles, "watch", portable_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + new_source.mkdir() + allow_event.set() + observation = monitor.finish() + + raw_paths = calls["paths"] + raw_kwargs = calls["kwargs"] + assert isinstance(raw_paths, tuple) + assert isinstance(raw_kwargs, dict) + watched = {Path(path) for path in raw_paths} + assert tmp_path.resolve() in watched + assert source in watched + git_dir = (tmp_path / ".git").resolve() + assert all( + path.is_relative_to(git_dir) or not any(part in {".venv", ".git", ".cache"} for part in path.parts) + for path in watched + ) + assert all("node_modules" not in path.parts for path in watched) + assert all("generated-output" not in path.parts for path in watched) + assert tmp_path / "browser-extension" in watched + assert tmp_path / "custom" in watched + assert raw_kwargs["recursive"] is False + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="new_source") + + +def test_checkout_mutation_monitor_remembers_deleted_ignored_root(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / ".gitignore").write_text("browser-extension/node_modules/\n", encoding="utf-8") + ignored_root = tmp_path / "browser-extension" / "node_modules" + ignored_root.mkdir(parents=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + shutil.rmtree(ignored_root) + monitor._record_change(ignored_root) + + assert monitor.finish() == CheckoutMutationObservation(changed=False, unavailable=False) + + +@pytest.mark.uses_real_clock("waits for the real filesystem watcher to witness an index replacement") +def test_checkout_mutation_monitor_observes_transient_index_authority_change(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / ".gitignore").write_text("ignored/\n", encoding="utf-8") + baseline = tmp_path / "baseline.py" + baseline.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", ".gitignore", "baseline.py"], cwd=tmp_path, check=True) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + hidden = tmp_path / "ignored" / "hidden.py" + hidden.parent.mkdir() + hidden.write_text("secret authority\n", encoding="utf-8") + subprocess.run(["git", "add", "-f", "ignored/hidden.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "reset", "-q", "--", "ignored/hidden.py"], cwd=tmp_path, check=True) + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") + + +@pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a branch-ref replacement") +def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "first"], cwd=tmp_path, check=True) + first = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + tracked.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "commit", "-qam", "second"], cwd=tmp_path, check=True) + second = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + branch = subprocess.run( + ["git", "symbolic-ref", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + subprocess.run(["git", "update-ref", branch, first], cwd=tmp_path, check=True) + subprocess.run(["git", "update-ref", branch, second], cwd=tmp_path, check=True) + observation = monitor.finish() + + assert observation == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path=f".git/{branch}", + ) + + +@pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a loose ref created from packed authority") +def test_checkout_mutation_monitor_observes_packed_nested_branch_ref_change(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "first"], cwd=tmp_path, check=True) + first = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + subprocess.run(["git", "switch", "-qc", "feature/nested"], cwd=tmp_path, check=True) + tracked.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "commit", "-qam", "second"], cwd=tmp_path, check=True) + subprocess.run(["git", "pack-refs", "--all", "--prune"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + subprocess.run(["git", "update-ref", "refs/heads/feature/nested", first], cwd=tmp_path, check=True) + observation = monitor.finish() + + assert observation == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path=".git/refs/heads/feature/nested", + ) + + +def test_worktree_fingerprint_rejects_assume_unchanged_tracked_content(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + source = tmp_path / "polylogue" / "hidden.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "polylogue/hidden.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + subprocess.run(["git", "update-index", "--assume-unchanged", "polylogue/hidden.py"], cwd=tmp_path, check=True) + source.write_text("value = 2\n", encoding="utf-8") + + assert _worktree_fingerprint(tmp_path) == "unavailable" + + +def test_checkout_mutation_monitor_ignores_uncommitted_git_index_lock(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + monitor._record_change(tmp_path / ".git" / "index.lock") + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + + +def test_checkout_mutation_monitor_treats_every_ready_index_event_as_authority_change( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / "tracked.py").write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + index = tmp_path / ".git" / "index" + + def portable_watch(*_paths: Path, **_kwargs: object) -> object: + yield set() + yield {(watchfiles.Change.modified, str(index))} + stop_event = _kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait(timeout=1) + + monkeypatch.setattr(watchfiles, "watch", portable_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") + + +def test_checkout_mutation_monitor_rejects_partial_git_enumeration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + real_run = subprocess.run + + def warning_run(*args: Any, **kwargs: Any) -> subprocess.CompletedProcess[bytes]: + result = cast(subprocess.CompletedProcess[bytes], real_run(*args, **kwargs)) + command = args[0] + if isinstance(command, list) and command[:2] == ["git", "status"]: + return subprocess.CompletedProcess(command, 0, result.stdout, b"warning: partial enumeration\n") + return result + + monkeypatch.setattr(subprocess, "run", warning_run) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + +def test_checkout_mutation_monitor_rejects_filesystem_walk_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + real_walk = os.walk + + def failing_walk(top: Path, *, onerror: object = None) -> object: + assert callable(onerror) + onerror(PermissionError("unreadable source directory")) + yield from real_walk(top) + + monkeypatch.setattr(os, "walk", failing_walk) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + +def test_checkout_mutation_monitor_fails_closed_when_portable_watcher_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def broken_watch(*_paths: Path, **_kwargs: object) -> object: + raise OSError("watcher unavailable") + yield set() + + monkeypatch.setattr(watchfiles, "watch", broken_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + def test_seed_receipt_classifies_every_node_terminal_outcome( tmp_path: Path, ) -> None: @@ -1423,6 +2658,39 @@ def test_seed_node_outcomes_accept_setup_skip_as_terminal_skip(tmp_path: Path) - ] +def test_seed_node_outcomes_preserve_call_and_fixture_xfail_xpass(tmp_path: Path) -> None: + """Durable pytest reports, including fixture ``pytest.xfail()``, finish seed nodes.""" + events = tmp_path / "events.jsonl" + nodes = [ + "tests/test_a.py::test_call_xfailed", + "tests/test_a.py::test_call_xpassed", + "tests/test_a.py::test_setup_xfailed", + ] + events.write_text( + "\n".join( + json.dumps(event) + for event in ( + {"event": "test_report", "nodeid": nodes[0], "when": "call", "outcome": "xfailed"}, + {"event": "test_report", "nodeid": nodes[1], "when": "call", "outcome": "xpassed"}, + {"event": "test_report", "nodeid": nodes[2], "when": "setup", "outcome": "xfailed"}, + ) + ) + + "\n" + ) + + outcomes = _seed_node_outcomes_from_events( + events, + expected_nodeids=nodes, + database={"node_outcomes": {}}, + pytest_step={}, + use_database_fallback=False, + ) + + assert {item["nodeid"]: item["outcome"] for item in outcomes} == dict( + zip(nodes, ("xfailed", "xpassed", "xfailed"), strict=True) + ) + + def test_resumed_seed_carries_forward_prior_terminal_outcome(tmp_path: Path) -> None: events = tmp_path / "events.jsonl" events.write_text( @@ -1433,18 +2701,27 @@ def test_resumed_seed_carries_forward_prior_terminal_outcome(tmp_path: Path) -> ) outcomes = _seed_node_outcomes_from_events( events, - expected_nodeids=["tests/test_a.py::test_repaired", "tests/test_b.py::test_prior"], + expected_nodeids=[ + "tests/test_a.py::test_repaired", + "tests/test_b.py::test_prior", + "tests/test_c.py::test_expected_failure", + ], database={"node_outcomes": {"tests/test_b.py::test_prior": "passed"}}, pytest_step={}, use_database_fallback=False, prior_node_outcomes={ - "tests/test_b.py::test_prior": {"nodeid": "tests/test_b.py::test_prior", "outcome": "passed"} + "tests/test_b.py::test_prior": {"nodeid": "tests/test_b.py::test_prior", "outcome": "passed"}, + "tests/test_c.py::test_expected_failure": { + "nodeid": "tests/test_c.py::test_expected_failure", + "outcome": "xfailed", + }, }, ) assert {item["nodeid"]: item["outcome"] for item in outcomes} == { "tests/test_a.py::test_repaired": "passed", "tests/test_b.py::test_prior": "passed", + "tests/test_c.py::test_expected_failure": "xfailed", } @@ -1887,12 +3164,12 @@ def test_resource_sampler_throttles_basetemp_size_walk(tmp_path: Path, monkeypat (basetemp / "artifact.txt").write_text("payload") calls = 0 - def counted_size(_path: Path) -> int: + def counted_usage(_path: Path) -> tuple[int, int]: nonlocal calls calls += 1 - return calls + return calls, calls + 1 - monkeypatch.setattr("devtools.verify_runs._dir_size_kb", counted_size) + monkeypatch.setattr("devtools.verify_runs._dir_usage_kb", counted_usage) sampler = ResourceSampler( root_pid=os.getpid(), run_id="test-run", @@ -1905,10 +3182,56 @@ def counted_size(_path: Path) -> int: second = sampler.sample(event="sample") assert first["basetemp_size_kb"] == 1 + assert first["basetemp_allocated_kb"] == 2 assert second["basetemp_size_kb"] == 1 assert calls == 1 +def test_sparse_basetemp_enforces_allocated_tmpfs_bytes_and_retains_logical_evidence(tmp_path: Path) -> None: + env = { + "POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path), + "POLYLOGUE_PYTEST_TMPFS": "1", + "POLYLOGUE_VERIFY_BASETEMP_SIZE_INTERVAL_S": "1", + } + run_id = "sparse-physical-accounting" + basetemp = pytest_basetemp_path(root=tmp_path, run_id=run_id, env=env) + basetemp.mkdir(parents=True) + with (basetemp / "sparse.bin").open("wb") as handle: + handle.seek(64 * 1024 * 1024) + handle.write(b"x") + sampler = ResourceSampler( + root_pid=os.getpid(), run_id=run_id, root=tmp_path, env=env, output_path=tmp_path / "resources.jsonl" + ) + + sample = sampler.sample(event="sample") + logical_kb = sample["basetemp_size_kb"] + allocated_kb = sample["basetemp_allocated_kb"] + + assert isinstance(logical_kb, int) + assert isinstance(allocated_kb, int) + assert logical_kb > allocated_kb + assert not pytest_tmpfs_budget_exceeded(sample, budget_kb=allocated_kb + 1) + assert pytest_tmpfs_budget_exceeded(sample, budget_kb=allocated_kb - 1) + + +def test_resource_sampler_does_not_charge_symlink_targets_to_managed_basetemp(tmp_path: Path) -> None: + env = {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)} + run_id = "symlink-accounting" + basetemp = pytest_basetemp_path(root=tmp_path, run_id=run_id, env=env) + basetemp.mkdir(parents=True) + outside_target = tmp_path / "outside-target.bin" + outside_target.write_bytes(b"x" * (4 * 1024 * 1024)) + (basetemp / "external-link").symlink_to(outside_target) + sampler = ResourceSampler( + root_pid=os.getpid(), run_id=run_id, root=tmp_path, env=env, output_path=tmp_path / "resources.jsonl" + ) + + sample = sampler.sample(event="sample") + + assert sample["basetemp_size_kb"] < 512 + assert sample["basetemp_allocated_kb"] < 512 + + def test_pytest_basetemp_path_tracks_tmpfs_opt_in(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: shm, _scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) path = pytest_basetemp_path(root=tmp_path, run_id="run-1", env={"POLYLOGUE_PYTEST_TMPFS": "1"}) @@ -2177,6 +3500,10 @@ def test_managed_pytest_policy_preserves_headroom_for_explicit_tmpfs_root( def test_full_suite_explicit_root_requires_measured_basetemp_space( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: + # The managed test harness itself uses /dev/shm, so keep this custom-root + # regression on a distinct disk route rather than accidentally admitting + # it as a configured tmpfs path. + monkeypatch.setattr(verify_runs, "PYTEST_TMPFS_ROOT", tmp_path / "other-shm") monkeypatch.setattr(verify_runs, "_meminfo", lambda: {"MemAvailable": 8 * 1024 * 1024}) monkeypatch.setattr(verify_runs, "read_cgroup_memory_headroom_bytes", lambda: None) monkeypatch.setattr(verify_runs, "_pressure", lambda _kind: {"full_avg10": 0.0}) @@ -2319,7 +3646,7 @@ def test_inherited_512_mib_tmpfs_cap_reroutes_measured_demand_to_scratch( assert env["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(scratch) -def test_explicit_tmpfs_root_reroutes_to_scratch_when_its_cap_is_too_small( +def test_configured_tmpfs_root_reroutes_to_scratch_when_its_cap_is_too_small( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) @@ -2341,6 +3668,36 @@ def test_explicit_tmpfs_root_reroutes_to_scratch_when_its_cap_is_too_small( assert env["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(scratch) +def test_configured_tmpfs_reroute_keeps_admission_evidence_when_scratch_refuses( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=15_190) + configured = shm / "configured" + configured.mkdir() + + def constrained_headroom(path: Path) -> int | None: + if path == scratch: + return 1 * 1024 + return 8 * 1024 * 1024 + + monkeypatch.setattr(verify_runs, "_headroom_kb", constrained_headroom) + + with pytest.raises(PytestResourceError) as excinfo: + apply_managed_pytest_runtime_policy( + { + "POLYLOGUE_PYTEST_BASETEMP_ROOT": str(configured), + "POLYLOGUE_PYTEST_TMPFS_MAX_MB": "512", + }, + worker_count=4, + ) + + message = str(excinfo.value) + assert f"configured pytest basetemp declared demand exceeds its safe adaptive tmpfs budget ({configured}" in message + assert "safe tmpfs budget=512 MiB" in message + assert f"{scratch} (scratch): 1 MiB free" in message + + def test_focused_policy_keeps_full_suite_basetemp_demand_out_of_scratch_preflight( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -2608,6 +3965,7 @@ def test_cleanup_managed_pytest_basetemp_removes_run_root(tmp_path: Path) -> Non env = {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)} basetemp = pytest_basetemp_path(root=tmp_path, run_id="run-1", env=env) (basetemp / "worker-output").mkdir(parents=True) + verify_runs.pytest_basetemp_claim_path(basetemp, kind="managed").write_text("999999:1", encoding="utf-8") cleaned = cleanup_managed_pytest_basetemp(root=tmp_path, run_id="run-1", env=env) @@ -2615,6 +3973,65 @@ def test_cleanup_managed_pytest_basetemp_removes_run_root(tmp_path: Path) -> Non assert not basetemp.exists() +def test_pytest_basetemp_claim_path_canonicalizes_symlink_aliases(tmp_path: Path) -> None: + real_root = tmp_path / "real-root" + real_root.mkdir() + linked_root = tmp_path / "linked-root" + linked_root.symlink_to(real_root, target_is_directory=True) + + real_basetemp = real_root / "pytest-polylogue-run" + linked_basetemp = linked_root / "pytest-polylogue-run" + + assert verify_runs.pytest_basetemp_claim_path(real_basetemp, kind="lock") == verify_runs.pytest_basetemp_claim_path( + linked_basetemp, kind="lock" + ) + + +def test_cleanup_managed_pytest_basetemp_leaves_successor_claim_while_locked(tmp_path: Path) -> None: + env = {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)} + basetemp = pytest_basetemp_path(root=tmp_path, run_id="reused-run", env=env) + basetemp.mkdir(parents=True) + (basetemp / "successor-fixture").write_text("live", encoding="utf-8") + claim_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="managed") + claim_path.write_text("999999:1", encoding="utf-8") + lock_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="lock") + + with lock_path.open("a+", encoding="utf-8") as lock_handle: + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) + assert cleanup_managed_pytest_basetemp(root=tmp_path, run_id="reused-run", env=env) is None + + assert basetemp.exists() + assert claim_path.exists() + + +def test_pytest_workload_receipt_uses_allocated_basetemp_peak() -> None: + receipt = verify._pytest_workload_receipt( + label="pytest sparse", + cmd=["pytest", "tests/unit/example.py"], + elapsed_s=1.0, + returncode=0, + termination_reason=None, + resource_summary={"peak_basetemp_size_kb": 64 * 1024, "peak_basetemp_allocated_kb": 8}, + last_resource_sample={"tree_rss_kb": 0, "tree_pss_kb": 0, "tree_cpu_s": 0.0}, + tmpfs_budget_mb=1, + basetemp_cleanup=None, + concurrency=1, + ) + + execute = next(phase for phase in receipt["phases"] if phase["name"] == "execute") + assert execute["temp_storage_bytes"] == 8 * 1024 + assert "Logical basetemp peak retained as diagnostic evidence: 67108864 bytes." in receipt["notes"] + + +def test_cleanup_managed_pytest_basetemp_recognizes_child_cleanup(tmp_path: Path) -> None: + env = {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)} + basetemp = pytest_basetemp_path(root=tmp_path, run_id="run-cleaned-by-child", env=env) + + cleaned = cleanup_managed_pytest_basetemp(root=tmp_path, run_id="run-cleaned-by-child", env=env) + + assert cleaned == basetemp + + def test_cleanup_managed_pytest_basetemp_does_not_receipt_residual_tree( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -2671,6 +4088,7 @@ def test_run_records_pytest_count_metadata_from_terminal_fallback() -> None: ) with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), patch("devtools.verify._read_pytest_report", return_value=None), ): @@ -2693,15 +4111,93 @@ def test_run_records_managed_basetemp_cleanup_metadata(tmp_path: Path) -> None: cleaned = tmp_path / "pytest-polylogue-run-1" with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), patch("devtools.verify._read_pytest_report", return_value=None), patch("devtools.verify.cleanup_managed_pytest_basetemp", return_value=cleaned) as cleanup, ): rc, _elapsed, metadata = _run("pytest testmon", ["pytest", "--testmon", "-n", "4"]) - assert rc == 0 - cleanup.assert_called_once() - assert metadata["basetemp_cleanup"] == str(cleaned) + assert rc == 0 + cleanup.assert_called_once() + assert metadata["basetemp_cleanup"] == str(cleaned) + + +@pytest.mark.uses_real_clock("the heartbeat loop computes elapsed containment time") +def test_heartbeat_persists_drained_output_before_interrupting(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="pytest buffered", cmd=["pytest"]) + stdout_read, stdout_write = os.pipe() + stderr_read, stderr_write = os.pipe() + os.write(stdout_write, b"buffered stdout before interrupt\n") + os.close(stdout_write) + os.close(stderr_write) + stdout_pipe = os.fdopen(stdout_read, "rb", closefd=True) + stderr_pipe = os.fdopen(stderr_read, "rb", closefd=True) + + class _Process: + pid = os.getpid() + stdout = stdout_pipe + stderr = stderr_pipe + returncode = None + + def poll(self) -> None: + return None + + class _InterruptingSelector: + def __init__(self) -> None: + self.calls = 0 + + def register(self, _fileobj: object, _events: int, _data: str) -> None: + return None + + def get_map(self) -> dict[int, object]: + return {1: object()} + + def select(self, timeout: float | None = None) -> list[tuple[SimpleNamespace, int]]: + del timeout + if self.calls == 0: + self.calls += 1 + return [(SimpleNamespace(fd=stdout_pipe.fileno(), data="stdout", fileobj=stdout_pipe), 1)] + raise KeyboardInterrupt + + def close(self) -> None: + return None + + launch = SimpleNamespace( + argv=["pytest"], + receipt_path=tmp_path / "containment.json", + request_path=tmp_path / "request.json", + mode="process-group", + unit=None, + cgroup_path=None, + fallback_argv=None, + runtime_cap_s=0.0, + ) + try: + with ( + patch("devtools.verify.enable_child_subreaper", return_value=True), + patch("devtools.verify.descendant_process_identities", return_value=()), + patch("devtools.verify.build_supervisor_launch", return_value=launch), + patch("devtools.verify.subprocess.Popen", return_value=_Process()), + patch("devtools.verify._wait_for_supervisor_start", return_value={"status": "started"}), + patch("devtools.verify.selectors.DefaultSelector", _InterruptingSelector), + patch("devtools.verify._await_interrupted_pytest_containment"), + patch("devtools.verify._write_pytest_progress"), + patch("devtools.verify.ResourceSampler", return_value=MagicMock()), + ): + with pytest.raises(KeyboardInterrupt): + verify._run_pytest_with_heartbeat( + ["pytest"], cwd=str(tmp_path), env={}, t0=time.monotonic(), run=run, artifacts=artifacts + ) + finally: + stdout_pipe.close() + stderr_pipe.close() + + assert artifacts.stdout_path.read_text(encoding="utf-8") == "buffered stdout before interrupt\n" + assert artifacts.stderr_path.read_text(encoding="utf-8") == "" + assert artifacts.output_path.read_text(encoding="utf-8") == "buffered stdout before interrupt\n" + assert (tmp_path / PYTEST_OUTPUT_PATH).read_text(encoding="utf-8") == "buffered stdout before interrupt\n" def test_explicit_basetemp_root_retains_managed_resource_monitoring( @@ -2731,6 +4227,144 @@ def test_explicit_basetemp_root_retains_managed_resource_monitoring( assert metadata["resource_sample_count"] >= 1 +def test_run_propagates_explicit_basetemp_to_resource_policy(tmp_path: Path) -> None: + explicit = tmp_path / "diagnostic-basetemp" + captured: dict[str, str] = {} + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + + def apply_policy(env: dict[str, str], **_kwargs: object) -> tuple[dict[str, str], None]: + captured.update(env) + return env, None + + with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", side_effect=apply_policy), + patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), + patch("devtools.verify._read_pytest_report", return_value=None), + ): + rc, _elapsed, _metadata = _run("pytest focused", ["pytest", "--basetemp", str(explicit)]) + + assert rc == 0 + assert captured["POLYLOGUE_PYTEST_EXPLICIT_BASETEMP"] == str(explicit) + + +def test_run_propagates_pytest_addopts_basetemp_to_resource_policy( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + explicit = tmp_path / "diagnostic-basetemp" + captured: dict[str, str] = {} + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + monkeypatch.setenv("PYTEST_ADDOPTS", f"--basetemp {explicit}") + + def apply_policy(env: dict[str, str], **_kwargs: object) -> tuple[dict[str, str], None]: + captured.update(env) + return env, None + + with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", side_effect=apply_policy), + patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), + patch("devtools.verify._read_pytest_report", return_value=None), + ): + rc, _elapsed, _metadata = _run("pytest focused", ["pytest"]) + + assert rc == 0 + assert captured["POLYLOGUE_PYTEST_EXPLICIT_BASETEMP"] == str(explicit) + + +def test_run_clears_stale_current_statistics_before_an_interrupted_pytest_step(tmp_path: Path) -> None: + stale_statistics = tmp_path / verify_runs.CURRENT_STATISTICS_PATH + stale_statistics.parent.mkdir(parents=True) + stale_statistics.write_text('{"node_count": 99}\n', encoding="utf-8") + + with patch("devtools.verify._run_pytest_with_heartbeat", side_effect=KeyboardInterrupt): + rc, _elapsed, metadata = _run("pytest focused", ["pytest"]) + + assert rc == 130 + assert metadata["diagnosis"] == "pytest_interrupted" + assert not stale_statistics.exists() + + +def test_explicit_basetemp_policy_uses_actual_path_for_admission( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=15_190) + explicit = tmp_path / "diagnostic-basetemp" + explicit.mkdir() + monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 32 * 1024 * 1024) + + _env, policy = apply_managed_pytest_runtime_policy( + {"POLYLOGUE_PYTEST_EXPLICIT_BASETEMP": str(explicit)}, worker_count=0, full_suite=False + ) + + assert policy is not None + assert policy.basetemp_root == str(explicit) + assert policy.basetemp_label == "explicit" + + +def test_explicit_tmpfs_basetemp_requires_declared_demand_and_headroom( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=15_190) + explicit = shm / "pytest-polylogue-diagnostic" + monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 2500 * 1024) + + with pytest.raises(PytestResourceError, match="need >= 3072 MiB"): + apply_managed_pytest_runtime_policy( + {verify_runs.PYTEST_EXPLICIT_BASETEMP_ENV: str(explicit)}, worker_count=4, full_suite=True + ) + + +def test_explicit_tmpfs_basetemp_reports_adaptive_and_filesystem_refusals_together( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=3072) + explicit = shm / "pytest-polylogue-diagnostic" + monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 2 * 1024 * 1024) + + with pytest.raises(PytestResourceError) as excinfo: + apply_managed_pytest_runtime_policy( + {verify_runs.PYTEST_EXPLICIT_BASETEMP_ENV: str(explicit)}, worker_count=0, full_suite=True + ) + message = str(excinfo.value) + + assert "declared demand=1522 MiB" in message + assert "safe tmpfs budget=1082 MiB" in message + assert "available filesystem space=2048 MiB" in message + assert "required filesystem headroom=2546 MiB" in message + + +def test_supervisor_never_cleans_an_explicit_tmpfs_basetemp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(verify_runs, "PYTEST_TMPFS_ROOT", tmp_path / "dev-shm") + explicit = verify_runs.PYTEST_TMPFS_ROOT / "pytest-polylogue-diagnostic" + + cleanup_path = verify._supervised_tmpfs_cleanup_path( + root=tmp_path, + run_id="run-1", + env={verify_runs.PYTEST_EXPLICIT_BASETEMP_ENV: str(explicit), "POLYLOGUE_PYTEST_TMPFS": "1"}, + ) + + assert cleanup_path is None + + +def test_run_resource_refusal_returns_finalized_compact_statistics(tmp_path: Path) -> None: + run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) + + with patch( + "devtools.verify.apply_managed_pytest_runtime_policy", + side_effect=PytestResourceError("starved basetemp"), + ): + rc, _elapsed, metadata = _run("pytest testmon", ["pytest", "-n", "0"], run=run) + + assert rc == 125 + assert metadata["statistics"]["node_count"] == 0 + assert metadata["statistics_path"].endswith("statistics.json") + + def test_run_receipt_uses_capped_pytest_command_concurrency() -> None: completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") @@ -2872,6 +4506,7 @@ def test_run_reads_structured_pytest_report() -> None: with ( patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), patch("devtools.verify._read_pytest_report", return_value=report), + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), ): rc, _elapsed, metadata = _run("pytest testmon", ["pytest", "--testmon", "-n", "8"]) @@ -3014,6 +4649,55 @@ def test_pytest_run_preserves_other_lane_reports( assert metadata["junitxml_path"] == str(isolated_junit) +def test_managed_pytest_run_reads_only_its_invocation_report( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + run = VerifyRun(tier="focused-test", argv=[], git_head="head", root=tmp_path) + seen_report: Path | None = None + + def fake_pytest(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + nonlocal seen_report + seen_report = verify._pytest_json_report_path(cmd) + assert seen_report is not None + assert seen_report.parent == run.run_dir + seen_report.write_text('{"summary":{"passed":1,"total":1}}', encoding="utf-8") + return subprocess.CompletedProcess(cmd, 0, "1 passed in 0.01s\n", "") + + with patch("devtools.verify._run_pytest_with_heartbeat", side_effect=fake_pytest): + rc, _elapsed, metadata = _run( + "pytest focused", + [sys.executable, "-m", "pytest", f"--json-report-file={PYTEST_REPORT_PATH}"], + run=run, + ) + + assert rc == 0 + assert seen_report is not None and not seen_report.exists() + assert metadata["report_path"].endswith("/pytest-report.json") + + +def test_pytest_progress_is_durable_per_step_and_mirrored_current( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + artifact_dir = tmp_path / ".cache" / "verify" / "runs" / "run" / "steps" / "01-pytest" + + verify._write_pytest_progress( + event="running", + cmd=["pytest"], + started_at=0.0, + elapsed_s=0.0, + artifact_dir=str(artifact_dir), + ) + + durable = json.loads((artifact_dir / "progress.json").read_text(encoding="utf-8")) + current = json.loads((tmp_path / PYTEST_PROGRESS_PATH).read_text(encoding="utf-8")) + assert durable == current + assert durable["event"] == "running" + + def test_pytest_run_terminates_after_runtime_budget( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -3231,6 +4915,220 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo assert payload["release_baseline_allowed"] is False +@pytest.mark.parametrize("fingerprints", [("unavailable", "stable"), ("stable", "unavailable")]) +def test_verify_withholds_success_when_checkout_fingerprint_is_unavailable( + capsys: pytest.CaptureFixture[str], + fingerprints: tuple[str, str], +) -> None: + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {})), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.worktree_fingerprint", side_effect=fingerprints), + ): + rc = main(["--quick", "--json"]) + + assert rc == 125 + payload = json.loads(capsys.readouterr().out) + assert payload["exit_code"] == 125 + checkout_step = next(step for step in payload["steps"] if step["name"] == "checkout stability") + assert checkout_step["diagnosis"] == "checkout_fingerprint_unavailable" + assert checkout_step["initial_worktree_fingerprint"] == fingerprints[0] + assert checkout_step["final_worktree_fingerprint"] == fingerprints[1] + + +def test_verify_rejects_git_head_change_with_matching_worktree_fingerprints( + capsys: pytest.CaptureFixture[str], +) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {})), + patch("devtools.verify._git_head", side_effect=("start-head", "different-head")), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + ): + assert main(["--quick", "--json"]) == 125 + + payload = json.loads(capsys.readouterr().out) + checkout_step = next(step for step in payload["steps"] if step["name"] == "checkout stability") + assert checkout_step["diagnosis"] == "checkout_changed_during_verification" + assert checkout_step["initial_git_head"] == "start-head" + assert checkout_step["final_git_head"] == "different-head" + + +@pytest.mark.parametrize( + ("fingerprints", "expected_diagnosis"), + [ + (("unavailable", "stable"), "checkout_fingerprint_unavailable"), + (("stable", "changed"), "checkout_changed_during_verification"), + ], +) +def test_checkout_stability_failure_controls_every_broad_run_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + fingerprints: tuple[str, str], + expected_diagnosis: str, +) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + + history: dict[str, Any] = {} + receipt = tmp_path / "invocation" / "run.json" + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.setattr( + verify, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "broad-invocation") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {"diagnosis": "pytest_passed"})), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", side_effect=fingerprints), + ): + assert main(["--quick", "--json"]) == 125 + + payload = json.loads(capsys.readouterr().out) + run_payload = json.loads(next((tmp_path / ".cache" / "verify" / "runs").glob("*/run.json")).read_text()) + current_payload = json.loads((tmp_path / ".cache" / "verify" / "current-run.json").read_text()) + receipt_payload = json.loads(receipt.read_text()) + + for durable_payload in (history, payload, run_payload, current_payload, receipt_payload): + assert durable_payload["diagnosis"] == expected_diagnosis + assert durable_payload["final_worktree_fingerprint"] == fingerprints[1] + + +def test_transient_checkout_mutation_controls_every_broad_run_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class _ChangedMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=True, unavailable=False) + + history: dict[str, Any] = {} + receipt = tmp_path / "invocation" / "run.json" + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.setattr( + verify, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setattr(verify, "CheckoutMutationMonitor", _ChangedMonitor) + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "broad-invocation") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {"diagnosis": "pytest_passed"})), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + ): + assert main(["--quick", "--json"]) == 125 + + payload = json.loads(capsys.readouterr().out) + run_payload = json.loads(next((tmp_path / ".cache" / "verify" / "runs").glob("*/run.json")).read_text()) + current_payload = json.loads((tmp_path / ".cache" / "verify" / "current-run.json").read_text()) + receipt_payload = json.loads(receipt.read_text()) + for durable_payload in (history, payload, run_payload, current_payload, receipt_payload): + assert durable_payload["diagnosis"] == "checkout_changed_during_verification" + assert durable_payload["final_worktree_fingerprint"] == "stable" + + +def test_transient_checkout_mutation_discards_testmon_graph_before_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class _ChangedMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=True, unavailable=False, observed_path="polylogue/example.py") + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.setattr(verify, "CheckoutMutationMonitor", _ChangedMonitor) + monkeypatch.setattr( + verify, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + TESTMON_DATA.parent.mkdir(parents=True) + TESTMON_DATA.write_bytes(b"transient dependency graph") + TESTMON_SEED_STAMP.write_text("{}", encoding="utf-8") + affected_publish = MagicMock() + selection_publish = MagicMock() + + with ( + patch("devtools.verify._anchor_verification_paths"), + patch("devtools.verify.maybe_bootstrap_testmon_seed", return_value=None), + patch("devtools.verify._testmon_preflight", return_value=None), + patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest", "--testmon"])]), + patch("devtools.verify._run", return_value=(0, 0.01, {"selected_count": 1})), + patch("devtools.verify._changed_executable_paths", return_value=("polylogue/example.py",)), + patch("devtools.verify._record_testmon_affected_coverage", affected_publish), + patch("devtools.verify._refresh_testmon_selection_attempt", selection_publish), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), + patch("devtools.verify._testmon_release_baseline_permission", return_value=False), + patch("devtools.verify._warn_low_memory"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + ): + assert main(["--json"]) == 125 + + assert not TESTMON_DATA.exists() + assert not TESTMON_SEED_STAMP.exists() + affected_publish.assert_not_called() + selection_publish.assert_not_called() + assert json.loads(capsys.readouterr().out)["diagnosis"] == "checkout_changed_during_verification" + + def test_verify_stops_after_failed_heavy_step(capsys: pytest.CaptureFixture[str]) -> None: calls: list[str] = [] @@ -3240,7 +5138,10 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo with ( patch("devtools.verify._run", side_effect=fake_run), + patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._save_history"), patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), @@ -3403,7 +5304,7 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo patch("devtools.verify._run", side_effect=fake_run), patch("devtools.verify.build_verify_steps", return_value=[("pytest full", ["pytest"])]), patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._save_history"), + patch("devtools.verify._save_history") as save_history, patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), ): @@ -3413,22 +5314,131 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo assert payload["verification_scope"] == expected_scope assert payload["release_baseline_allowed"] is expected_permission assert payload["terminal_authorization"] == ("narrow-terminal" if expected_permission else None) + assert save_history.call_args.args[0]["checkout_root"] == str(ROOT.resolve()) def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.CaptureFixture[str]) -> None: with ( patch("devtools.verify.build_verify_steps", side_effect=PytestResourceError("only 0.50 GiB available")), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._testmon_preflight", return_value=None), patch("devtools.verify._run") as run, + patch("devtools.verify._save_history") as save_history, ): rc = main(["--json"]) assert rc == 125 run.assert_not_called() + assert save_history.call_args.args[0]["diagnosis"] == "pytest_resource_preflight_failed" assert "only 0.50 GiB available" in capsys.readouterr().err +def test_verify_starts_checkout_monitor_before_broad_change_classification( + capsys: pytest.CaptureFixture[str], +) -> None: + events: list[str] = [] + + class _OrderingMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("monitor-started") + + def finish(self) -> CheckoutMutationObservation: + events.append("monitor-finished") + return CheckoutMutationObservation(changed=False, unavailable=False) + + def classify(_base: str, _head: str) -> bool: + assert events == ["monitor-started"] + events.append("classified") + return False + + with ( + patch("devtools.verify.CheckoutMutationMonitor", _OrderingMonitor), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._default_testmon_is_broad_change", side_effect=classify), + patch("devtools.verify.build_verify_steps", return_value=[]), + patch("devtools.verify._testmon_preflight", return_value=None), + patch("devtools.verify._testmon_release_baseline_permission", return_value=False), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + ): + assert main(["--json"]) == 0 + + assert events == ["monitor-started", "classified", "monitor-finished"] + assert json.loads(capsys.readouterr().out)["exit_code"] == 0 + + +def test_verify_finalizes_checkout_monitor_when_startup_fingerprint_raises() -> None: + events: list[str] = [] + + class _ExceptionalExitMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("monitor-started") + + def finish(self) -> CheckoutMutationObservation: + events.append("monitor-finished") + return CheckoutMutationObservation(changed=False, unavailable=False) + + with ( + patch("devtools.verify.CheckoutMutationMonitor", _ExceptionalExitMonitor), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify.worktree_fingerprint", side_effect=RuntimeError("fingerprint failed")), + ): + with pytest.raises(RuntimeError, match="fingerprint failed"): + main(["--quick", "--json"]) + + assert events == ["monitor-started", "monitor-finished"] + + +def test_verify_finalizes_runner_exception_after_open_step( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + history: dict[str, Any] = {} + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + verify, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setattr(verify, "maybe_bootstrap_testmon_seed", lambda *_args, **_kwargs: None) + monkeypatch.setattr(verify, "_git_head", lambda: "head") + monkeypatch.setattr(verify, "worktree_fingerprint", lambda _root: "stable") + monitor = MagicMock() + monitor.finish.return_value = CheckoutMutationObservation(changed=False, unavailable=False) + monkeypatch.setattr(verify, "CheckoutMutationMonitor", lambda _root: monitor) + monkeypatch.setattr(verify, "_save_history", lambda payload: history.update(payload)) + monkeypatch.setattr(verify, "build_verify_steps", lambda **_kwargs: [("ruff check", ["ruff", "check"])]) + + def explode(_label: str, command: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: + run = kwargs["run"] + run.start_step(label="ruff check", cmd=command) + raise RuntimeError("verification runner exploded") + + monkeypatch.setattr(verify, "_run", explode) + monotonic_values = iter((100.0, 107.5)) + monkeypatch.setattr("devtools.verify.time.monotonic", lambda: next(monotonic_values)) + + assert verify.main(["--quick", "--json"]) == 125 + assert history["exit_code"] == 125 + assert history["diagnosis"] == "verify_runner_exception" + assert history["duration_s"] == 7.5 + assert history["verification_scope"] == "non-test" + assert history["steps"][0]["status"] == "failed" + assert history["steps"][0]["exit"] == 125 + + def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -3442,6 +5452,8 @@ def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirector def test_verify_rejects_zero_testmon_selection_for_executable_change( capsys: pytest.CaptureFixture[str], ) -> None: + changed_executable_paths = MagicMock(return_value=("polylogue/example.py",)) + def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: del command, kwargs return 0, 0.01, ({"selected_count": 0} if label.startswith("pytest") else {}) @@ -3449,11 +5461,13 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo with ( patch("devtools.verify._run", side_effect=fake_run), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="pinned-base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._save_history"), patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._changed_executable_paths", return_value=("polylogue/example.py",)), + patch("devtools.verify._changed_executable_paths", changed_executable_paths), patch("devtools.verify._matching_testmon_coverage", return_value=None), ): rc = main(["--json"]) @@ -3463,6 +5477,48 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo pytest_step = next(step for step in payload["steps"] if step["name"].startswith("pytest")) assert pytest_step["diagnosis"] == "zero_testmon_selection_for_executable_change" assert pytest_step["zero_selection_changed_paths"] == ["polylogue/example.py"] + changed_executable_paths.assert_called_once_with("pinned-base", "head") + + +def test_verify_finalizes_and_discards_graph_when_post_pytest_path_authority_fails( + capsys: pytest.CaptureFixture[str], +) -> None: + monitor = MagicMock() + monitor.finish.return_value = CheckoutMutationObservation(changed=False, unavailable=False) + discard = MagicMock() + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {"selected_count": 1})), + patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), + patch("devtools.verify._changed_executable_paths", side_effect=PytestResourceError("git unavailable")), + patch("devtools.verify._discard_testmon_dependency_authority", discard), + patch("devtools.verify.CheckoutMutationMonitor", return_value=monitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify._testmon_preflight", return_value=None), + ): + rc = main(["--json"]) + + assert rc == 125 + monitor.finish.assert_called_once_with() + discard.assert_called_once_with() + payload = json.loads(capsys.readouterr().out) + assert payload["diagnosis"] == "testmon_changed_path_authority_unavailable" + + +def test_testmon_changed_path_authority_refuses_missing_commit_binding() -> None: + changed_paths = MagicMock() + + with patch("devtools.verify._changed_executable_paths", changed_paths): + with pytest.raises(PytestResourceError, match="changed-path authority is unavailable"): + verify._changed_paths_from_testmon_authority(None, "head") + + changed_paths.assert_not_called() def test_verify_accepts_zero_testmon_selection_after_matching_coverage( @@ -3474,7 +5530,10 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo with ( patch("devtools.verify._run", side_effect=fake_run), + patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._save_history"), patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), @@ -3496,7 +5555,7 @@ def test_testmon_coverage_receipts_are_content_exact() -> None: assert _matching_testmon_coverage(paths) is None TESTMON_SEED_STAMP.unlink() - with patch("devtools.verify._worktree_fingerprint", return_value="affected"): + with patch("devtools.verify.worktree_fingerprint", return_value="affected"): _record_testmon_affected_coverage( executable_paths=paths, selected_count=3, @@ -3506,11 +5565,11 @@ def test_testmon_coverage_receipts_are_content_exact() -> None: assert _matching_testmon_coverage(paths) == "successful_affected_run" assert _matching_testmon_coverage(("polylogue/other.py",)) is None - with patch("devtools.verify._worktree_fingerprint", return_value="changed"): + with patch("devtools.verify.worktree_fingerprint", return_value="changed"): assert _matching_testmon_coverage(paths) is None TESTMON_AFFECTED_STAMP.write_text(json.dumps({"identity": {"worktree_fingerprint": "affected"}})) - with patch("devtools.verify._worktree_fingerprint", return_value="affected"): + with patch("devtools.verify.worktree_fingerprint", return_value="affected"): assert _matching_testmon_coverage(paths) is None diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index a8f91eb534..9b4242851e 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -1,6 +1,13 @@ from __future__ import annotations +import fcntl import os +import shutil +import subprocess +import sys +import threading +from collections.abc import Generator +from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -12,6 +19,18 @@ from tests.infra.frozen_clock import FrozenClock +@contextmanager +def _configured_pytest(config: Any) -> Generator[None, None, None]: + """Run the configure hook directly without leaking its claim lock.""" + conftest.pytest_configure(cast("pytest.Config", config)) + basetemp = Path(str(config.option.basetemp)) + try: + yield + finally: + conftest.pytest_unconfigure(cast("pytest.Config", config)) + conftest._release_basetemp_claim_lock(basetemp) + + def _make_real_candidates( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, realm_mounted: bool = True ) -> tuple[Path, Path]: @@ -49,18 +68,41 @@ def _make_real_candidates( return shm, scratch -def test_runtest_makereport_wrapper_preserves_each_phase_report() -> None: - item = SimpleNamespace() - reports = [SimpleNamespace(when=phase) for phase in ("setup", "call", "teardown")] +@pytest.mark.parametrize("exception", [KeyboardInterrupt(), RuntimeError("teardown failure")]) +def test_test_tmp_path_reclamation_runs_after_failure_or_interrupt( + tmp_path: Path, + exception: BaseException, +) -> None: + tree = tmp_path / "test-private" + tree.mkdir() + fixture_generator = cast(Any, conftest._reclaim_test_tmp_path).__wrapped__ + request = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=None))) + cleanup = cast("Generator[None, BaseException, None]", fixture_generator(tree, request)) - for report in reports: - wrapper = conftest.pytest_runtest_makereport( - cast("pytest.Item", item), cast("pytest.CallInfo[None]", SimpleNamespace()) - ) - assert next(wrapper) is None - with pytest.raises(StopIteration): - wrapper.send(cast("Any", SimpleNamespace(get_result=lambda report=report: report))) - assert getattr(item, f"rep_{report.when}") is report + assert next(cleanup) is None + with pytest.raises(type(exception)): + cleanup.throw(exception) + + assert not tree.exists() + + +def test_test_tmp_path_reclamation_keeps_explicit_diagnostic_basetemp( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + explicit = tmp_path / "diagnostic" + tree = explicit / "test-private" + tree.mkdir(parents=True) + fixture_generator = cast(Any, conftest._reclaim_test_tmp_path).__wrapped__ + request = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(explicit)))) + monkeypatch.delenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", raising=False) + cleanup = cast("Generator[None, BaseException, None]", fixture_generator(tree, request)) + + assert next(cleanup) is None + with pytest.raises(RuntimeError): + cleanup.throw(RuntimeError("failed diagnostic rerun")) + + assert tree.exists() def test_managed_pytest_temp_root_defaults_to_scratch( @@ -172,6 +214,7 @@ def test_pytest_configure_reports_low_space_as_usage_error( # teardown reverts the leak regardless of what the call under test does. monkeypatch.delenv("POLYLOGUE_PYTEST_RUN_ID", raising=False) monkeypatch.delenv("POLYLOGUE_PYTEST_CHECKOUT", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", raising=False) config = SimpleNamespace( option=SimpleNamespace(basetemp=None), addinivalue_line=lambda *args, **kwargs: None, @@ -193,6 +236,7 @@ def test_bare_pytest_configure_defaults_to_scratch_without_a_supervisor( "POLYLOGUE_PYTEST_TMPFS", "POLYLOGUE_PYTEST_RUN_ID", "POLYLOGUE_PYTEST_CHECKOUT", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", ): monkeypatch.delenv(name, raising=False) config = SimpleNamespace( @@ -201,10 +245,9 @@ def test_bare_pytest_configure_defaults_to_scratch_without_a_supervisor( rootpath=tmp_path, ) - conftest.pytest_configure(cast("pytest.Config", config)) - - assert Path(str(config.option.basetemp)).parent == scratch - assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" + with _configured_pytest(config): + assert Path(str(config.option.basetemp)).parent == scratch + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" def test_bare_pytest_ignores_leaked_cloud_basetemp_on_workstation( @@ -217,20 +260,43 @@ def test_bare_pytest_ignores_leaked_cloud_basetemp_on_workstation( monkeypatch.delenv("POLYLOGUE_PYTEST_TMPFS", raising=False) monkeypatch.delenv("POLYLOGUE_PYTEST_RUN_ID", raising=False) monkeypatch.delenv("POLYLOGUE_PYTEST_CHECKOUT", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", raising=False) config = SimpleNamespace( option=SimpleNamespace(basetemp=None), addinivalue_line=lambda *args, **kwargs: None, rootpath=tmp_path, ) - conftest.pytest_configure(cast("pytest.Config", config)) + with _configured_pytest(config): + assert Path(str(config.option.basetemp)).parent == scratch + assert "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in os.environ + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" - assert Path(str(config.option.basetemp)).parent == scratch - assert "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in os.environ - assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" +def test_bare_pytest_routes_an_environment_configured_tmpfs_root_to_scratch( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _make_real_candidates(monkeypatch, tmp_path) + monkeypatch.setenv("POLYLOGUE_PYTEST_BASETEMP_ROOT", str(shm / "configured")) + monkeypatch.delenv("POLYLOGUE_VERIFY_RUN_ID", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_TMPFS", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_RUN_ID", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_CHECKOUT", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", raising=False) + config = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(config): + assert Path(str(config.option.basetemp)).parent == scratch + assert "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in os.environ + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" -def test_sweep_stale_polylogue_basetemps_preserves_seeded_and_recent( + +def test_sweep_stale_polylogue_basetemps_preserves_unknown_seeded_and_recent( tmp_path: Path, frozen_clock: FrozenClock, ) -> None: @@ -241,18 +307,415 @@ def test_sweep_stale_polylogue_basetemps_preserves_seeded_and_recent( for path in (stale, seeded, recent, unrelated): path.mkdir() - old = frozen_clock.time() - conftest._STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S - 1 + old = frozen_clock.time() - 24 * 60 * 60 os.utime(stale, (old, old)) os.utime(seeded, (old, old)) conftest._sweep_stale_polylogue_basetemps(max_age_s=60, roots=(tmp_path,)) - assert not stale.exists() + assert stale.exists() assert seeded.exists() assert recent.exists() assert unrelated.exists() +def test_explicit_basetemp_remains_outside_a_later_startup_stale_sweep( + tmp_path: Path, + frozen_clock: FrozenClock, +) -> None: + explicit = tmp_path / "pytest-polylogue-debug" + config = SimpleNamespace( + option=SimpleNamespace(basetemp=str(explicit)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(config): + assert verify_runs.pytest_basetemp_claim_path(explicit, kind="caller-owned").is_file() + old = frozen_clock.time() - 24 * 60 * 60 + os.utime(explicit, (old, old)) + + conftest._sweep_stale_polylogue_basetemps(roots=(tmp_path,)) + + assert explicit.exists() + + +def test_explicit_basetemp_clears_stale_managed_identity_from_prior_in_process_run( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + explicit = tmp_path / "pytest-polylogue-debug" + config = SimpleNamespace( + option=SimpleNamespace(basetemp=str(explicit)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "prior-run") + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(explicit)) + + with _configured_pytest(config): + assert "POLYLOGUE_PYTEST_RUN_ID" not in os.environ + assert "POLYLOGUE_PYTEST_MANAGED_BASETEMP" not in os.environ + explicit.mkdir(exist_ok=True) + + assert explicit.exists() + + +def test_nested_explicit_basetemp_restores_active_outer_managed_identity( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + ambient_identity = ( + os.environ.get("POLYLOGUE_PYTEST_RUN_ID"), + os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP"), + ) + _shm, _scratch = _make_real_candidates(monkeypatch, tmp_path) + for name in ( + "POLYLOGUE_VERIFY_RUN_ID", + "POLYLOGUE_PYTEST_BASETEMP_ROOT", + "POLYLOGUE_PYTEST_TMPFS", + "POLYLOGUE_PYTEST_RUN_ID", + "POLYLOGUE_PYTEST_CHECKOUT", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", + ): + monkeypatch.delenv(name, raising=False) + outer = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + explicit = tmp_path / "nested-diagnostic" + nested = SimpleNamespace( + option=SimpleNamespace(basetemp=str(explicit)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(outer): + outer_identity = ( + os.environ["POLYLOGUE_PYTEST_RUN_ID"], + os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"], + ) + with _configured_pytest(nested): + assert "POLYLOGUE_PYTEST_RUN_ID" not in os.environ + assert "POLYLOGUE_PYTEST_MANAGED_BASETEMP" not in os.environ + assert os.environ["POLYLOGUE_PYTEST_RUN_ID"] == outer_identity[0] + assert os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] == outer_identity[1] + + assert os.environ.get("POLYLOGUE_PYTEST_RUN_ID") == ambient_identity[0] + assert os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") == ambient_identity[1] + + +@pytest.mark.parametrize("alias", [False, True]) +def test_nested_explicit_basetemp_reuse_is_rejected_before_the_nonreentrant_claim_lock( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + alias: bool, +) -> None: + active = tmp_path / "active-basetemp" + requested = active + if alias: + active.mkdir() + requested = tmp_path / "active-basetemp-alias" + requested.symlink_to(active, target_is_directory=True) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_BASETEMPS", set()) + outer = SimpleNamespace( + option=SimpleNamespace(basetemp=str(active)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + nested = SimpleNamespace( + option=SimpleNamespace(basetemp=str(requested)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(outer): + with pytest.raises(pytest.UsageError, match="already active in this pytest process"): + conftest.pytest_configure(cast("pytest.Config", nested)) + + +def test_nested_supervised_explicit_basetemp_reuse_is_rejected_before_claiming( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + active = tmp_path / "active-basetemp" + active.mkdir() + run_id = "supervised-run" + config = SimpleNamespace( + option=SimpleNamespace(basetemp=str(active)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_BASETEMPS", set()) + monkeypatch.setenv("POLYLOGUE_VERIFY_RUN_ID", run_id) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", run_id) + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(active)) + conftest._mark_basetemp_owner(active) + try: + with _configured_pytest(config): + with pytest.raises(pytest.UsageError, match="already active in this pytest process"): + conftest.pytest_configure(cast("pytest.Config", config)) + finally: + conftest._release_basetemp_claim_lock(active) + + +def test_nested_managed_pytest_forces_scratch_outside_the_outer_tmpfs_budget( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _make_real_candidates(monkeypatch, tmp_path) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) + monkeypatch.setenv("POLYLOGUE_VERIFY_RUN_ID", "outer-supervisor") + monkeypatch.setenv("POLYLOGUE_PYTEST_BASETEMP_ROOT", str(shm)) + monkeypatch.setenv("POLYLOGUE_PYTEST_TMPFS", "1") + outer = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + nested = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(outer): + assert Path(str(outer.option.basetemp)).parent == shm + with _configured_pytest(nested): + assert Path(str(nested.option.basetemp)).parent == scratch + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" + assert os.environ["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(shm) + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "1" + + +def test_nested_unmanaged_scopes_do_not_reclaim_the_live_outer_tree( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _shm, _scratch = _make_real_candidates(monkeypatch, tmp_path) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) + for name in ( + "POLYLOGUE_VERIFY_RUN_ID", + "POLYLOGUE_PYTEST_BASETEMP_ROOT", + "POLYLOGUE_PYTEST_TMPFS", + "POLYLOGUE_PYTEST_RUN_ID", + "POLYLOGUE_PYTEST_CHECKOUT", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", + ): + monkeypatch.delenv(name, raising=False) + outer = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + middle = SimpleNamespace( + option=SimpleNamespace(basetemp=str(tmp_path / "middle-diagnostic")), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + inner = SimpleNamespace( + option=SimpleNamespace(basetemp=str(tmp_path / "inner-diagnostic")), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(outer): + outer_basetemp = Path(str(outer.option.basetemp)) + live_outer_tree = outer_basetemp / "still-live" + live_outer_tree.mkdir(parents=True) + with _configured_pytest(middle): + with _configured_pytest(inner): + assert "POLYLOGUE_PYTEST_MANAGED_BASETEMP" not in os.environ + assert "POLYLOGUE_PYTEST_MANAGED_BASETEMP" not in os.environ + fixture_generator = cast(Any, conftest._reclaim_test_tmp_path).__wrapped__ + request = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(outer_basetemp)))) + cleanup = cast("Generator[None, BaseException, None]", fixture_generator(live_outer_tree, request)) + assert next(cleanup) is None + cleanup.close() + assert live_outer_tree.exists() + + +def test_explicit_basetemp_claim_survives_real_pytest_basetemp_replacement(tmp_path: Path) -> None: + """Exercise pytest's lazy TempPathFactory clearing against our real conftest.""" + explicit = tmp_path / "pytest-polylogue-diagnostic" + explicit.mkdir() + cleared_by_pytest = explicit / "cleared-by-temp-path-factory" + cleared_by_pytest.write_text("old", encoding="utf-8") + repo_root = Path(__file__).resolve().parents[2] + env = {key: value for key, value in os.environ.items() if not key.startswith("POLYLOGUE_PYTEST_")} + env.pop("POLYLOGUE_VERIFY_RUN_ID", None) + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--basetemp", + str(explicit), + "tests/unit/test_pytest_temp_policy.py::test_archive_template_clone_is_private", + ], + cwd=repo_root, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert not cleared_by_pytest.exists() + assert verify_runs.pytest_basetemp_claim_path(explicit, kind="caller-owned").is_file() + + +def test_claim_lock_inode_stays_contended_after_managed_claim_clear(tmp_path: Path) -> None: + """A second process cannot lock a replacement inode for this basetemp.""" + basetemp = tmp_path / "pytest-polylogue-lock-inode" + conftest._mark_caller_owned_basetemp(basetemp) + lock_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="lock") + try: + verify_runs.clear_managed_pytest_basetemp_claim(basetemp) + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import fcntl, sys\n" + "with open(sys.argv[1], 'a+', encoding='utf-8') as handle:\n" + " try:\n" + " fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)\n" + " except BlockingIOError:\n" + " raise SystemExit(0)\n" + "raise SystemExit(1)\n" + ), + str(lock_path), + ], + capture_output=True, + text=True, + timeout=10, + ) + finally: + conftest._release_basetemp_claim_lock(basetemp) + + assert result.returncode == 0, result.stdout + result.stderr + assert lock_path.is_file() + + +def test_claim_lock_failure_closes_handle_and_releases_thread_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + basetemp = tmp_path / "pytest-polylogue-lock-failure" + lock_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="lock") + + with monkeypatch.context() as scoped: + + def fail_lock(*_args: object) -> None: + raise OSError("lock failed") + + scoped.setattr(fcntl, "flock", fail_lock) + assert conftest._acquire_basetemp_claim_lock(basetemp, blocking=True) is None + + assert not conftest._BASE_TEMP_CLAIM_THREAD_LOCKS[lock_path].locked() + handle = conftest._acquire_basetemp_claim_lock(basetemp, blocking=True) + assert handle is not None + conftest._release_basetemp_claim_lock(basetemp) + + +def test_explicit_basetemp_claim_failure_is_a_usage_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + basetemp = tmp_path / "unclaimable" + monkeypatch.setattr(conftest, "_acquire_basetemp_claim_lock", lambda *_args, **_kwargs: None) + + with pytest.raises(pytest.UsageError, match="cannot claim the explicit basetemp"): + conftest._mark_caller_owned_basetemp(basetemp) + + +def test_managed_basetemp_claim_collision_is_rejected_across_processes(tmp_path: Path) -> None: + basetemp = tmp_path / "pytest-polylogue-managed-collision" + conftest._mark_basetemp_owner(basetemp) + try: + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "from pathlib import Path\n" + "import sys\n" + "import tests.conftest as conftest\n" + "from devtools.verify_runs import PytestResourceError\n" + "try:\n" + " conftest._mark_basetemp_owner(Path(sys.argv[1]))\n" + "except PytestResourceError:\n" + " raise SystemExit(0)\n" + "raise SystemExit(1)\n" + ), + str(basetemp), + ], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + timeout=10, + ) + finally: + conftest._release_basetemp_claim_lock(basetemp) + + assert result.returncode == 0, result.stdout + result.stderr + + +def test_stale_sweep_and_explicit_claim_are_atomic_for_one_path( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + frozen_clock: FrozenClock, +) -> None: + basetemp = tmp_path / "pytest-polylogue-race-123" + basetemp.mkdir() + conftest._mark_basetemp_owner(basetemp) + verify_runs.pytest_basetemp_claim_path(basetemp, kind="managed").write_text("999999999", encoding="utf-8") + conftest._release_basetemp_claim_lock(basetemp) + old = frozen_clock.time() - 120 + os.utime(basetemp, (old, old)) + sweep_checked = threading.Event() + allow_sweep = threading.Event() + caller_claimed = threading.Event() + original_owner_alive = verify_runs.managed_pytest_basetemp_owner_alive + + def pause_after_admission(entry: Path) -> bool | None: + sweep_checked.set() + assert allow_sweep.wait(timeout=2) + return original_owner_alive(entry) + + monkeypatch.setattr(verify_runs, "managed_pytest_basetemp_owner_alive", pause_after_admission) + sweeper = threading.Thread( + target=conftest._sweep_stale_polylogue_basetemps, + kwargs={"max_age_s": 60, "roots": (tmp_path,)}, + ) + sweeper.start() + assert sweep_checked.wait(timeout=2) + + def claim_and_use() -> None: + conftest._mark_caller_owned_basetemp(basetemp) + basetemp.mkdir(exist_ok=True) + caller_claimed.set() + + caller = threading.Thread(target=claim_and_use) + caller.start() + assert not caller_claimed.wait(timeout=0.1) + allow_sweep.set() + sweeper.join(timeout=2) + caller.join(timeout=2) + + assert not sweeper.is_alive() + assert not caller.is_alive() + assert caller_claimed.is_set() + assert basetemp.exists() + assert verify_runs.pytest_basetemp_claim_path(basetemp, kind="caller-owned").is_file() + + def test_sweep_stale_polylogue_basetemps_never_deletes_a_live_owner( tmp_path: Path, frozen_clock: FrozenClock, @@ -280,7 +743,7 @@ def test_sweep_stale_polylogue_basetemps_reclaims_a_confirmed_dead_owner( dead.mkdir() # A pid that is guaranteed not to be alive right now (max pid + 1 territory # would flake on hosts near pid rollover; /proc simply never has this one). - (dead / conftest._OWNER_PID_MARKER).write_text("999999999", encoding="utf-8") + verify_runs.pytest_basetemp_claim_path(dead, kind="managed").write_text("999999999", encoding="utf-8") old = frozen_clock.time() - 120 os.utime(dead, (old, old)) @@ -296,7 +759,7 @@ def test_sweep_stale_polylogue_basetemps_reclaims_reused_pid_identity( ) -> None: stale = tmp_path / "pytest-polylogue-reused-pid-123" stale.mkdir() - (stale / conftest._OWNER_PID_MARKER).write_text(f"{os.getpid()}:100", encoding="utf-8") + verify_runs.pytest_basetemp_claim_path(stale, kind="managed").write_text(f"{os.getpid()}:100", encoding="utf-8") monkeypatch.setattr(conftest, "_process_start_ticks", lambda _pid: 200) old = frozen_clock.time() - 120 os.utime(stale, (old, old)) @@ -318,7 +781,8 @@ def test_sweep_stale_polylogue_basetemps_reclaims_read_only_fixture_tree( payload.chmod(0o400) nested.chmod(0o500) (stale / "published").chmod(0o500) - old = frozen_clock.time() - conftest._STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S - 1 + verify_runs.pytest_basetemp_claim_path(stale, kind="managed").write_text("999999999", encoding="utf-8") + old = frozen_clock.time() - 120 os.utime(stale, (old, old)) conftest._sweep_stale_polylogue_basetemps(max_age_s=60, roots=(tmp_path,)) @@ -338,7 +802,7 @@ def test_sweep_stale_polylogue_basetemps_does_not_follow_top_level_symlink( payload.chmod(0o400) nested.chmod(0o500) target.chmod(0o500) - old = frozen_clock.time() - conftest._STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S - 1 + old = frozen_clock.time() - 24 * 60 * 60 os.utime(target, (old, old)) link = tmp_path / "pytest-polylogue-stale-symlink-123" @@ -353,18 +817,15 @@ def test_sweep_stale_polylogue_basetemps_does_not_follow_top_level_symlink( assert (target.stat().st_mode, nested.stat().st_mode, payload.stat().st_mode) == before_modes -def test_sweep_stale_polylogue_basetemps_gives_unknown_owner_a_long_grace_period( +def test_sweep_stale_polylogue_basetemps_never_deletes_an_unknown_owner( tmp_path: Path, ) -> None: - """A directory with no owner marker (pre-fix leftover, or a startup - race) cannot be confirmed dead, so it gets a much longer grace period - rather than the normal stale-age cutoff.""" + """Unknown paths may be an explicit caller racing a sweep, so retain them.""" unmarked = tmp_path / "pytest-polylogue-unmarked-123" unmarked.mkdir() - # Past the normal (60s, for this test) stale-age cutoff, but nowhere near - # the multi-hour unknown-owner grace period. Derive "now" from the - # directory's own just-created mtime (filesystem metadata) rather than a - # direct host-clock read, which test code may not perform (clock_guard). + # Derive "now" from the directory's own just-created mtime (filesystem + # metadata) rather than a direct host-clock read, which test code may not + # perform (clock_guard). now = unmarked.stat().st_mtime past_normal_cutoff = now - 120 os.utime(unmarked, (past_normal_cutoff, past_normal_cutoff)) @@ -389,6 +850,79 @@ def test_sessionfinish_leaves_xdist_basetemp_for_supervisor_cleanup( assert basetemp.exists() +@pytest.mark.parametrize("worker_id", [None, "gw0"]) +def test_sessionfinish_releases_explicit_claim_without_managed_run( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + worker_id: str | None, +) -> None: + basetemp = tmp_path / "pytest-polylogue-explicit" + conftest._mark_caller_owned_basetemp(basetemp) + lock_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="lock") + session = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(basetemp), numprocesses=0))) + monkeypatch.delenv("POLYLOGUE_PYTEST_RUN_ID", raising=False) + if worker_id is None: + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + else: + monkeypatch.setenv("PYTEST_XDIST_WORKER", worker_id) + + conftest.pytest_sessionfinish(cast("pytest.Session", session), 0) + + assert not conftest._BASE_TEMP_CLAIM_THREAD_LOCKS[lock_path].locked() + + +def test_sessionfinish_reclaims_only_its_managed_basetemp( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + basetemp = tmp_path / "pytest-polylogue-run-123" + basetemp.mkdir() + session = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(basetemp), numprocesses=0))) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "run-123") + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(basetemp)) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + + conftest.pytest_sessionfinish(cast("pytest.Session", session), 1) + + assert not basetemp.exists() + + +def test_sessionfinish_retains_managed_claim_after_failed_rmtree( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + basetemp = tmp_path / "pytest-polylogue-rmtree-failure" + basetemp.mkdir() + managed_claim = verify_runs.pytest_basetemp_claim_path(basetemp, kind="managed") + managed_claim.write_text("999999999", encoding="utf-8") + session = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(basetemp), numprocesses=0))) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "run-123") + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(basetemp)) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + monkeypatch.setattr(shutil, "rmtree", lambda _path, **_kwargs: None) + + conftest.pytest_sessionfinish(cast("pytest.Session", session), 1) + + assert basetemp.exists() + assert managed_claim.is_file() + + +def test_sessionfinish_retains_explicit_diagnostic_basetemp( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + explicit = tmp_path / "pytest-polylogue-diagnostic" + explicit.mkdir() + session = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(explicit), numprocesses=0))) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "run-123") + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(tmp_path / "pytest-polylogue-other-run")) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + + conftest.pytest_sessionfinish(cast("pytest.Session", session), 1) + + assert explicit.exists() + + def test_archive_template_clone_is_private(tmp_path: Path) -> None: source = tmp_path / "source" destination = tmp_path / "destination"