From d35c3cdd17b5f84cc320eefded6bbe36226bcace Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 18:00:51 +0200 Subject: [PATCH 01/53] perf: record durable pytest run statistics Problem: full-suite performance work lacked a compact, durable summary of phase timing, storage allocation, worker policy, and cleanup outcomes. The raw event and resource streams existed, but repeated analysis had to rebuild those facts manually, and interrupted test commands could leave stale running receipts. What changed: derive per-step pytest statistics with phase distributions, outcomes, fixture clone timing, apparent and allocated basetemp usage, worker counts, resource peaks, and cleanup status. Capture archive-clone metadata, finalize interrupted test runs, and retain the compact aggregate in task history. Use the structured pytest report to complete normal-run phase data when incremental hooks are incomplete. Compatibility/migration: existing raw event/resource artifacts remain authoritative for forensics; the new statistics JSON is derived and replaceable. No test selection or runtime coverage policy changes. Verification: devtools test --json -q tests/unit/devtools/test_pytest_progress_plugin.py tests/unit/devtools/test_verify.py (152 passed); ruff check; git diff --check. --- TESTING.md | 3 + devtools/pytest_progress_plugin.py | 45 ++++- devtools/run_tests.py | 7 +- devtools/task_history.py | 35 ++++ devtools/verify_runs.py | 254 ++++++++++++++++++++++++++++- tests/conftest.py | 31 ++++ tests/unit/devtools/test_verify.py | 62 +++++++ 7 files changed, 430 insertions(+), 7 deletions(-) diff --git a/TESTING.md b/TESTING.md index 1d5eda8c40..237647fa51 100644 --- a/TESTING.md +++ b/TESTING.md @@ -163,6 +163,9 @@ and a postmortem diagnosis. The latest run is mirrored to - `.cache/verify/current-pytest-resources.jsonl` - `.cache/verify/current-pytest-postmortem.json` - `.cache/verify/current-pytest-containment.json` +- `.cache/verify/current-pytest-statistics.json` — derived phase/fixture + distributions, worker count, storage, resource peaks, and cleanup outcome; + the same file is retained under each run's `steps/*/statistics.json`. - `.cache/verify/current-pytest-output.log` The devtools process drains pytest output, prints periodic heartbeat lines, and diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index f1c07c6c85..4755f16664 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -27,12 +27,32 @@ _DESELECTED_COUNT = 0 _SELECTED_COUNT = 0 _SLOWEST_REPORTS: list[dict[str, Any]] = [] +_RECORDED_REPORT_KEYS: set[tuple[int, str, str, str, float]] = set() _COLLECTION_STARTED_AT: float | None = None _COLLECTION_DURATION_S: float | None = None _SLOW_REPORT_LIMIT = 20 _DEFAULT_SELECTION_NODEID_LIMIT = 500 +def record_fixture_timing(name: str, duration_s: float, **metadata: Any) -> None: + """Record an explicitly instrumented fixture operation in the run stream. + + Pytest reports setup as one number per node, which cannot distinguish a + shared archive clone from ordinary fixture construction. Small, named + instrumentation points use this public helper so the run aggregator can + retain that distinction without making fixture code depend on the + aggregator's implementation. + """ + _write_event( + { + "event": "fixture_timing", + "name": name, + "duration_s": round(float(duration_s), 4), + **metadata, + } + ) + + def _selection_nodeid_limit() -> int: raw = os.environ.get(_SELECTION_NODEID_LIMIT_ENV) if raw is None: @@ -118,6 +138,7 @@ def pytest_sessionstart(session: Any) -> None: _DESELECTED_COUNT = 0 _SELECTED_COUNT = 0 _SLOWEST_REPORTS.clear() + _RECORDED_REPORT_KEYS.clear() _COLLECTION_STARTED_AT = None _COLLECTION_DURATION_S = None # The worker environment is assigned after process exec, so it is not @@ -213,10 +234,16 @@ def pytest_runtest_logfinish(nodeid: str, location: tuple[str, int | None, str]) @pytest.hookimpl -def pytest_runtest_logreport(report: Any) -> None: +def _record_phase_report(report: Any) -> None: """Append one phase report so slow setup/call/teardown remains visible.""" when = str(getattr(report, "when", "")) + nodeid = str(getattr(report, "nodeid", "")) outcome = str(getattr(report, "outcome", "")) + duration = float(getattr(report, "duration", 0.0) or 0.0) + report_key = (id(report), when, nodeid, outcome, duration) + if report_key in _RECORDED_REPORT_KEYS: + return + _RECORDED_REPORT_KEYS.add(report_key) if when not in {"setup", "call", "teardown"}: return payload = { @@ -224,7 +251,7 @@ def pytest_runtest_logreport(report: Any) -> None: "nodeid": str(getattr(report, "nodeid", "")), "when": when, "outcome": outcome, - "duration_s": round(float(getattr(report, "duration", 0.0) or 0.0), 4), + "duration_s": round(duration, 4), } if payload["outcome"] == "failed": payload["longrepr"] = str(getattr(report, "longrepr", "")) @@ -232,6 +259,20 @@ def pytest_runtest_logreport(report: Any) -> None: _write_event(payload) +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: Any, call: Any) -> Any: + """Capture the phase report before other reporting plugins transform it.""" + del item, call + outcome = yield + _record_phase_report(outcome.get_result()) + + +@pytest.hookimpl +def pytest_runtest_logreport(report: Any) -> None: + """Retain the direct/log-hook fallback used by older pytest plugins/tests.""" + _record_phase_report(report) + + @pytest.hookimpl def pytest_sessionfinish(session: Any, exitstatus: int) -> None: """Write a compact post-run diagnosis artifact independent of pytest-json-report.""" diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 59948062c7..27c5dda0ac 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -149,7 +149,12 @@ def main(argv: list[str] | None = None) -> int: environment_fingerprint=environment_fingerprint, ) started = time.monotonic() - rc, _elapsed, metadata = _run("pytest focused", cmd, cwd=str(ROOT), run=run) + try: + rc, _elapsed, metadata = _run("pytest focused", cmd, cwd=str(ROOT), run=run) + except KeyboardInterrupt: + rc = 130 + metadata = {"diagnosis": "pytest_interrupted", "termination_reason": "operator_interrupt"} + run.finish_interrupted_steps(exit_code=rc, diagnosis=str(metadata["diagnosis"])) payload = run.finish( exit_code=rc, duration_s=time.monotonic() - started, diff --git a/devtools/task_history.py b/devtools/task_history.py index 77a205a360..37e8708208 100644 --- a/devtools/task_history.py +++ b/devtools/task_history.py @@ -113,6 +113,16 @@ def _latest_verify_run_metadata(command: str) -> dict[str, Any]: ): if key in latest_pytest: metadata[f"pytest_{key}"] = latest_pytest[key] + statistics_path = _get_root() / ".cache" / "verify" / "current-pytest-statistics.json" + try: + statistics = json.loads(statistics_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + statistics = None + if isinstance(statistics, dict): + # Keep the derived aggregate in the append-only invocation record so + # Lynchpin can consume one stable row without crawling disposable + # .cache/verify directories. Raw events remain in the run artifact. + metadata["pytest_statistics"] = statistics return {key: value for key, value in metadata.items() if value is not None} @@ -412,6 +422,31 @@ def _cmd_stats(args: argparse.Namespace) -> int: "peak_rss_mb_max": max(peaks), "peak_rss_mb_p95": _percentile(peaks, 95), } + pytest_statistics = [task["pytest_statistics"] for task in tasks if isinstance(task.get("pytest_statistics"), dict)] + if args.resources and pytest_statistics: + pss = [ + float(item["resources"]["peak_tree_pss_kb"]) + for item in pytest_statistics + if isinstance(item.get("resources"), dict) + and isinstance(item["resources"].get("peak_tree_pss_kb"), (int, float)) + ] + temp = [ + int(item["storage"]["basetemp_logical_bytes_max"]) + for item in pytest_statistics + if isinstance(item.get("storage"), dict) + and isinstance(item["storage"].get("basetemp_logical_bytes_max"), int) + ] + workers = [ + int(item["xdist"]["worker_count"]) + for item in pytest_statistics + if isinstance(item.get("xdist"), dict) and isinstance(item["xdist"].get("worker_count"), int) + ] + stats["pytest_runs"] = { + "count": len(pytest_statistics), + "peak_pss_kb_max": max(pss, default=None), + "basetemp_logical_bytes_max": max(temp, default=None), + "xdist_worker_counts": sorted(set(workers)), + } slow_tests = _latest_pytest_slow_tests(args.slow_tests) if slow_tests: stats["slow_tests"] = slow_tests diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 95949457ac..06808a7d7d 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -31,6 +31,7 @@ CURRENT_RESOURCES_PATH = VERIFY_CACHE / "current-pytest-resources.jsonl" CURRENT_POSTMORTEM_PATH = VERIFY_CACHE / "current-pytest-postmortem.json" CURRENT_CONTAINMENT_PATH = VERIFY_CACHE / "current-pytest-containment.json" +CURRENT_STATISTICS_PATH = VERIFY_CACHE / "current-pytest-statistics.json" CURRENT_EVENTS_DIR = VERIFY_CACHE / "current-pytest-events" DEFAULT_BASETEMP_SIZE_SAMPLE_INTERVAL_S = 15.0 DEFAULT_TMPFS_SIZE_SAMPLE_INTERVAL_S = 2.0 @@ -142,6 +143,191 @@ def _slug(value: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-").lower() or "step" +def _percentile(values: list[float], percentile: float) -> float | None: + if not values: + return None + ordered = sorted(values) + if len(ordered) == 1: + return round(ordered[0], 4) + position = (len(ordered) - 1) * percentile + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + fraction = position - lower + return round(ordered[lower] + (ordered[upper] - ordered[lower]) * fraction, 4) + + +def _distribution(values: list[float]) -> dict[str, float | int | None]: + return { + "count": len(values), + "p50_s": _percentile(values, 0.50), + "p95_s": _percentile(values, 0.95), + "p99_s": _percentile(values, 0.99), + "max_s": round(max(values), 4) if values else None, + "sum_s": round(sum(values), 4) if values else 0.0, + } + + +def aggregate_pytest_statistics( + step_dir: Path, + *, + command: list[Any] | tuple[Any, ...] = (), + step_result: Mapping[str, Any] | None = None, + report_path: Path | None = None, +) -> dict[str, Any]: + """Reduce the append-only pytest evidence into one durable step summary. + + The raw event stream remains authoritative for forensics. This summary is + deliberately derived and replaceable: it makes repeated performance work + cheap without introducing a second source of truth for test outcomes. + """ + events: list[dict[str, Any]] = [] + events_path = step_dir / "events.jsonl" + if events_path.exists(): + for line in events_path.read_text(encoding="utf-8", errors="replace").splitlines(): + with contextlib.suppress(json.JSONDecodeError): + row = json.loads(line) + if isinstance(row, dict): + events.append(row) + + phases: dict[str, list[float]] = {"setup": [], "call": [], "teardown": []} + outcomes: dict[str, int] = {} + phase_outcomes: dict[str, dict[str, int]] = {"setup": {}, "call": {}, "teardown": {}} + nodes: set[str] = set() + fixture_timings: dict[str, list[float]] = {} + workers: set[str] = set() + for row in events: + worker = row.get("worker_id") + if isinstance(worker, str): + workers.add(worker) + nodeid = row.get("nodeid") + if isinstance(nodeid, str) and nodeid: + nodes.add(nodeid) + event = row.get("event") + if event == "test_report": + when = row.get("when") + duration = row.get("duration_s") + if when in phases and isinstance(duration, (int, float)): + phases[when].append(float(duration)) + outcome = row.get("outcome") + if isinstance(outcome, str) and when in phase_outcomes: + bucket = phase_outcomes[when] + bucket[outcome] = bucket.get(outcome, 0) + 1 + if when == "call": + outcomes[outcome] = outcomes.get(outcome, 0) + 1 + elif event == "fixture_timing": + name = row.get("name") + duration = row.get("duration_s") + if isinstance(name, str) and isinstance(duration, (int, float)): + fixture_timings.setdefault(name, []).append(float(duration)) + + # Some pytest/plugin combinations expose only the final teardown report to + # pytest_runtest_logreport. The structured report has complete per-phase + # data when the run reaches a normal pytest exit, so use it to complete the + # aggregate instead of publishing a deceptively partial distribution. + if report_path is not None and report_path.exists(): + with contextlib.suppress(OSError, json.JSONDecodeError): + report = json.loads(report_path.read_text(encoding="utf-8")) + report_tests = report.get("tests", []) if isinstance(report, dict) else [] + if isinstance(report_tests, list): + for test in report_tests: + if not isinstance(test, Mapping): + continue + nodeid = test.get("nodeid") + if isinstance(nodeid, str) and nodeid: + nodes.add(nodeid) + outcome = test.get("outcome") + if isinstance(outcome, str): + outcomes[outcome] = outcomes.get(outcome, 0) + 1 + for when in phases: + phase = test.get(when) + if not isinstance(phase, Mapping): + continue + duration = phase.get("duration") + phase_outcome = phase.get("outcome") + if isinstance(duration, (int, float)): + phases[when].append(float(duration)) + if isinstance(phase_outcome, str): + bucket = phase_outcomes[when] + bucket[phase_outcome] = bucket.get(phase_outcome, 0) + 1 + + resources: list[dict[str, Any]] = [] + resources_path = step_dir / "resources.jsonl" + if resources_path.exists(): + for line in resources_path.read_text(encoding="utf-8", errors="replace").splitlines(): + with contextlib.suppress(json.JSONDecodeError): + row = json.loads(line) + if isinstance(row, dict): + resources.append(row) + explicit_worker_count: int | None = None + command_values = [str(value) for value in command] + for index, value in enumerate(command_values[:-1]): + if value in {"-n", "--numprocesses"}: + with contextlib.suppress(ValueError): + explicit_worker_count = int(command_values[index + 1]) + break + basetemp_sizes = [row.get("basetemp_size_kb") for row in resources] + basetemp_sizes = [int(value) * 1024 for value in basetemp_sizes if isinstance(value, int)] + basetemp_allocated = [row.get("basetemp_allocated_kb") for row in resources] + basetemp_allocated = [int(value) * 1024 for value in basetemp_allocated if isinstance(value, int)] + containment: dict[str, Any] = {} + containment_path = step_dir / "containment.json" + if containment_path.exists(): + with contextlib.suppress(json.JSONDecodeError): + raw = json.loads(containment_path.read_text(encoding="utf-8")) + if isinstance(raw, dict): + containment = raw + + return { + "schema_version": 1, + "command": [str(value) for value in command], + "node_count": len(nodes), + "outcomes": outcomes, + "phase_outcomes": phase_outcomes, + "phases": {name: _distribution(values) for name, values in phases.items()}, + "fixtures": {name: _distribution(values) for name, values in fixture_timings.items()}, + "xdist": { + "worker_ids": sorted(workers), + "worker_count": max( + 0, + len(workers) - (1 if "controller" in workers else 0), + explicit_worker_count or 0, + ), + }, + "storage": { + "basetemp_logical_bytes_max": max(basetemp_sizes, default=None), + "basetemp_allocated_bytes_max": max(basetemp_allocated, default=None), + "basetemp_root": next( + (row.get("basetemp") for row in reversed(resources) if isinstance(row.get("basetemp"), str)), + None, + ), + }, + "resources": { + "peak_tree_rss_kb": max( + (int(row["tree_rss_kb"]) for row in resources if isinstance(row.get("tree_rss_kb"), int)), + default=None, + ), + "peak_tree_pss_kb": max( + (int(row["tree_pss_kb"]) for row in resources if isinstance(row.get("tree_pss_kb"), int)), + default=None, + ), + "peak_cgroup_memory_bytes": max( + ( + int(row["cgroup_memory_peak_bytes"]) + for row in resources + if isinstance(row.get("cgroup_memory_peak_bytes"), int) + ), + default=None, + ), + }, + "cleanup": { + "complete": containment.get("tmpfs_cleanup_complete"), + "termination_reason": containment.get("termination_reason"), + "escalated_to_sigkill": containment.get("escalated_to_sigkill"), + "exit_code": containment.get("exit_code", (step_result or {}).get("exit")), + }, + } + + def git_dirty() -> bool: try: result = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, timeout=5) @@ -188,6 +374,7 @@ class PytestStepArtifacts: resources_path: Path postmortem_path: Path containment_path: Path + statistics_path: Path class VerifyRun: @@ -260,6 +447,7 @@ def start_step(self, *, label: str, cmd: list[str]) -> PytestStepArtifacts: resources_path=step_dir / "resources.jsonl", postmortem_path=step_dir / "postmortem.json", containment_path=step_dir / "containment.json", + statistics_path=step_dir / "statistics.json", ) step_dir.mkdir(parents=True, exist_ok=True) self._payload["steps"].append( @@ -281,9 +469,39 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> None: step.update(result) step["finished_at"] = utc_now() step["status"] = "success" if result.get("exit") == 0 else "failed" + statistics_path = self.run_dir / "steps" / step_id / "statistics.json" + with contextlib.suppress(OSError, ValueError): + statistics = aggregate_pytest_statistics( + self.run_dir / "steps" / step_id, + command=step.get("cmd", []), + step_result=result, + report_path=( + self.root / str(result["report_path"]) + if isinstance(result.get("report_path"), str) + else None + ), + ) + _write_json(statistics_path, statistics) + with contextlib.suppress(OSError): + shutil.copyfile(statistics_path, self.root / CURRENT_STATISTICS_PATH) + step["statistics_path"] = str(self.relative_run_dir / "steps" / step_id / "statistics.json") break self.write() + def finish_interrupted_steps(self, *, exit_code: int, diagnosis: str) -> None: + """Close any open step when the outer runner receives Ctrl-C.""" + for step in self._payload["steps"]: + if step.get("status") == "running": + self.finish_step( + step_id=str(step["step_id"]), + result={ + "duration_s": None, + "exit": exit_code, + "diagnosis": diagnosis, + "termination_reason": "operator_interrupt", + }, + ) + def finish( self, *, @@ -338,6 +556,8 @@ def copy_current_pytest_artifacts(root: Path, artifacts: PytestStepArtifacts, *, shutil.copyfile(artifacts.postmortem_path, root / CURRENT_POSTMORTEM_PATH) with contextlib.suppress(FileNotFoundError): shutil.copyfile(artifacts.containment_path, root / CURRENT_CONTAINMENT_PATH) + with contextlib.suppress(FileNotFoundError): + shutil.copyfile(artifacts.statistics_path, root / CURRENT_STATISTICS_PATH) def merge_worker_events(events_dir: Path, merged_path: Path) -> int: @@ -562,6 +782,28 @@ def _dir_size_kb(path: Path) -> int | None: return int(total / 1024) +def _dir_allocated_kb(path: Path) -> int | None: + """Return filesystem blocks charged to files beneath *path*. + + This is deliberately reported alongside apparent bytes. On btrfs it is + the filesystem's per-file block charge (and may differ from compressed or + shared physical allocation); on tmpfs it is the RAM-backed block charge. + It is still the useful apples-to-apples signal available without requiring + filesystem-specific ioctl tooling in the test harness. + """ + if not path.exists(): + return None + total = 0 + try: + for item in path.rglob("*"): + with contextlib.suppress(OSError): + if item.is_file(): + total += item.stat().st_blocks * 512 + except OSError: + return None + return int(total / 1024) + + def checkout_hash(root: Path) -> str: return hashlib.sha1(str(root).encode("utf-8"), usedforsecurity=False).hexdigest()[:8] @@ -1091,11 +1333,12 @@ def __init__(self, *, root_pid: int, run_id: str, root: Path, env: dict[str, str self._basetemp_size_interval_s = _basetemp_size_sample_interval_s(env) self._last_basetemp_size_sample_at: float | None = None self._last_basetemp_size_kb: int | None = None + self._last_basetemp_allocated_kb: int | None = None - def _sample_basetemp_size_kb(self, *, event: str) -> int | None: + def _sample_basetemp_sizes(self, *, event: str) -> tuple[int | None, int | None]: """Return basetemp size without recursively walking it every sample.""" if self._basetemp_size_interval_s <= 0: - return None + return None, None now = time.monotonic() should_sample = ( self._last_basetemp_size_sample_at is None @@ -1105,8 +1348,9 @@ def _sample_basetemp_size_kb(self, *, event: str) -> int | None: ) if should_sample: self._last_basetemp_size_kb = _dir_size_kb(self._basetemp) + self._last_basetemp_allocated_kb = _dir_allocated_kb(self._basetemp) self._last_basetemp_size_sample_at = now - return self._last_basetemp_size_kb + return self._last_basetemp_size_kb, self._last_basetemp_allocated_kb def sample(self, *, event: str) -> dict[str, Any]: pids = process_tree(self.root_pid) @@ -1180,6 +1424,7 @@ def sample(self, *, event: str) -> dict[str, Any]: meminfo = _meminfo() cgroup_path = _cgroup_path(self.root_pid) cgroup_io = _cgroup_io_bytes(cgroup_path) + basetemp_logical_kb, basetemp_allocated_kb = self._sample_basetemp_sizes(event=event) sample: dict[str, Any] = { "updated_at": utc_now(), "event": event, @@ -1214,7 +1459,8 @@ def sample(self, *, event: str) -> dict[str, Any]: "pressure_memory": _pressure("memory"), "shm": _fs_usage(Path("/dev/shm")), "basetemp": str(self._basetemp), - "basetemp_size_kb": self._sample_basetemp_size_kb(event=event), + "basetemp_size_kb": basetemp_logical_kb, + "basetemp_allocated_kb": basetemp_allocated_kb, "top_processes": sorted(processes, key=lambda row: int(row.get("rss_kb") or 0), reverse=True)[:8], } self.sample_count += 1 diff --git a/tests/conftest.py b/tests/conftest.py index 97a170327d..5fb53e2b2c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -783,8 +783,20 @@ def cli_workspace( } +def _tree_bytes(path: Path, *, allocated: bool) -> int: + total = 0 + for item in path.rglob("*"): + with contextlib.suppress(OSError): + if item.is_file(): + stat_result = item.stat() + total += stat_result.st_blocks * 512 if allocated else stat_result.st_size + return total + + def _clone_archive_template(source: Path, destination: Path) -> None: """Clone one immutable empty archive into a test-private workspace.""" + started = time.perf_counter() + method = "reflink-auto" destination.mkdir(parents=True, exist_ok=True) try: subprocess.run( @@ -795,8 +807,27 @@ def _clone_archive_template(source: Path, destination: Path) -> None: timeout=10, ) except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + method = "copytree" shutil.copytree(source, destination, dirs_exist_ok=True) + # The managed pytest plugin turns this into a durable fixture-cost record; + # ordinary pytest runs remain unaffected because the event sink is absent. + try: + from devtools.pytest_progress_plugin import record_fixture_timing + + record_fixture_timing( + "archive_clone", + time.perf_counter() - started, + method=method, + source=str(source), + destination=str(destination), + source_apparent_bytes=_tree_bytes(source, allocated=False), + destination_apparent_bytes=_tree_bytes(destination, allocated=False), + destination_allocated_bytes=_tree_bytes(destination, allocated=True), + ) + except (ImportError, OSError): + pass + bootstrap_marker = destination / ".maintenance-state" / "durable-change-trains" / ".bootstrap" if bootstrap_marker.is_file(): from polylogue.storage.sqlite.durable_change_train import _record_fresh_durable_bootstrap diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 2d1208e15a..054eaba58e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -79,6 +79,7 @@ VerifyRun, adaptive_pytest_runtime_policy, adaptive_pytest_worker_count, + aggregate_pytest_statistics, apply_managed_pytest_runtime_policy, classify_pytest_result, cleanup_managed_pytest_basetemp, @@ -952,6 +953,67 @@ def test_focused_run_can_record_typed_affected_scope(tmp_path: Path) -> None: assert json.loads((tmp_path / ".cache" / "verify" / "current-run.json").read_text()) == payload +def test_aggregate_pytest_statistics_reduces_phases_fixtures_and_resources(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + (step / "events.jsonl").write_text( + "\n".join( + json.dumps(row) + for row in ( + { + "event": "test_report", + "nodeid": "a", + "when": "setup", + "duration_s": 1.0, + "outcome": "passed", + "worker_id": "controller", + }, + { + "event": "test_report", + "nodeid": "a", + "when": "call", + "duration_s": 2.0, + "outcome": "passed", + "worker_id": "gw0", + }, + { + "event": "test_report", + "nodeid": "a", + "when": "teardown", + "duration_s": 0.5, + "outcome": "passed", + "worker_id": "gw0", + }, + {"event": "fixture_timing", "name": "archive_clone", "duration_s": 0.25, "method": "reflink-auto"}, + ) + ) + + "\n" + ) + (step / "resources.jsonl").write_text( + json.dumps( + { + "basetemp": "/dev/shm/run", + "basetemp_size_kb": 12, + "tree_rss_kb": 100, + "tree_pss_kb": 80, + "cgroup_memory_peak_bytes": 200, + "xdist_worker_count": 1, + } + ) + + "\n" + ) + (step / "containment.json").write_text(json.dumps({"tmpfs_cleanup_complete": True, "exit_code": 0})) + + result = aggregate_pytest_statistics(step, command=["pytest"], step_result={"exit": 0}) + + assert result["node_count"] == 1 + assert result["phases"]["call"]["p50_s"] == 2.0 + assert result["fixtures"]["archive_clone"]["sum_s"] == 0.25 + assert result["storage"]["basetemp_logical_bytes_max"] == 12 * 1024 + assert result["resources"]["peak_tree_pss_kb"] == 80 + assert result["cleanup"]["complete"] is True + + def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) From c54028a7ed05383dd2f42b7268ef235d03fa45f7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 18:18:32 +0200 Subject: [PATCH 02/53] perf: reclaim pytest temp trees eagerly Problem: the calm 8-worker full selection retained every failed test tmp_path and exceeded the 2 GiB tmpfs budget after 14 minutes, even though passing trees were already reclaimed. What changed: remove each test private tmp_path at fixture teardown regardless of outcome. Failure diagnostics remain in the managed event, longrepr, selection, summary, and resource receipts; a failing node can be rerun with an explicit basetemp when filesystem evidence is needed. Remove the now-unused report-retention hook. Compatibility/migration: tests that need a persistent artifact already write it to an explicit path or can rerun the node with a chosen basetemp. No production behavior changes. Verification: devtools test --json -q tests/unit/test_pytest_temp_policy.py tests/unit/devtools/test_pytest_progress_plugin.py tests/unit/devtools/test_verify.py (171 passed); ruff check. --- tests/conftest.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5fb53e2b2c..4d919f22c8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ import threading import time import uuid -from collections.abc import AsyncIterator, Callable, Generator, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Iterator, Mapping from pathlib import Path from types import FrameType, ModuleType from typing import TYPE_CHECKING, Any @@ -317,26 +317,21 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: shutil.rmtree(basetemp_path, ignore_errors=True) -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport( - item: pytest.Item, - call: pytest.CallInfo[None], -) -> Generator[None, Any, None]: - """Retain the call outcome so passing test temp trees can be reclaimed.""" - outcome = yield - report = outcome.get_result() - setattr(item, f"rep_{report.when}", report) - - @pytest.fixture(autouse=True) -def _reclaim_passing_test_tmp_path( - request: pytest.FixtureRequest, +def _reclaim_test_tmp_path( tmp_path: Path, ) -> Iterator[None]: - """Bound broad-run temp growth while preserving failed-test evidence.""" - yield - report: pytest.TestReport | None = getattr(request.node, "rep_call", None) - if report is not None and report.passed: + """Release each test's private tree as soon as its teardown finishes. + + Failure evidence belongs in the managed event/longrepr/resource receipts, + not in an unbounded filesystem witness. Retaining every failed tree made + full-suite tmpfs usage proportional to the number of failures and caused + a calm 8-worker run to exceed 2 GiB before completing. A failing node can + still be rerun with an explicit basetemp when its files matter. + """ + try: + yield + finally: shutil.rmtree(tmp_path, ignore_errors=True) @@ -416,7 +411,7 @@ def _reclaim_passing_test_tmp_path( @pytest.fixture(autouse=True) def _close_test_opened_sqlite_connections( monkeypatch: pytest.MonkeyPatch, - _reclaim_passing_test_tmp_path: None, + _reclaim_test_tmp_path: None, ) -> Iterator[None]: """Close sync ``sqlite3`` connections that *test code* opened but never closed. @@ -538,7 +533,7 @@ async def _close_async() -> None: def _clear_polylogue_env( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, - _reclaim_passing_test_tmp_path: None, + _reclaim_test_tmp_path: None, request: pytest.FixtureRequest, ) -> None: # Close any cached SQLite connections to prevent WAL sidecar corruption From 20a660b4ad8f39824b061228939ac3acab4f18ec Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 18:21:34 +0200 Subject: [PATCH 03/53] docs: describe eager pytest temp cleanup Document that per-test temporary trees are reclaimed for both passing and failing nodes, while structured failure receipts remain available and explicit basetemps can preserve a filesystem witness for a targeted rerun. --- TESTING.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/TESTING.md b/TESTING.md index 237647fa51..3111658a2a 100644 --- a/TESTING.md +++ b/TESTING.md @@ -133,10 +133,13 @@ published read-only fixture copies cannot leak tmpfs indefinitely. Shared are shared, reused, and built once behind their own `.build.done` guard. Managed verification refuses to start below 1 GiB available memory instead of -falling back to the pathological disk lane. Passing-test roots are reclaimed at -teardown; the external supervisor and parent runner independently remove the -whole run root on completion or termination, with startup stale-root cleanup as -recovery after an uncatchable process kill or reboot. +falling back to the pathological disk lane. Every per-test `tmp_path` tree is +reclaimed at teardown, including failed tests; node failure evidence remains +in the managed event, longrepr, summary, and resource receipts. Rerun a node +with an explicit basetemp when its filesystem witness is needed. The external +supervisor and parent runner independently remove the whole run root on +completion or termination, with startup stale-root cleanup as recovery after +an uncatchable process kill or reboot. An affected run that selects zero tests is accepted only when no executable, test, dependency, or harness path changed. A zero selection after such a change From 744db1722b3e451ef960f7d6172b448c1b856a32 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 18:51:55 +0200 Subject: [PATCH 04/53] perf: measure fixture setup without duplicate scans Preserve the managed pytest evidence channel through the per-test configuration scrub so function-scoped fixture timing is recorded. Measure seeded archive clones, cache immutable template size lookups, and keep direct plugin tests isolated from the managed sink. Tighten the new statistics aggregation types so mypy remains green. --- devtools/pytest_progress_plugin.py | 31 ++++++++++++++++++++++++++++++ devtools/task_history.py | 6 ++++-- devtools/verify_runs.py | 12 ++++++++---- tests/conftest.py | 12 +++++++++--- tests/infra/workload_artifacts.py | 16 ++++++++++++++- 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index 4755f16664..2051984539 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -32,6 +32,18 @@ _COLLECTION_DURATION_S: float | None = None _SLOW_REPORT_LIMIT = 20 _DEFAULT_SELECTION_NODEID_LIMIT = 500 +_MEASURED_FIXTURES = frozenset( + { + "empty_archive_template", + "workspace_env", + "cli_workspace", + "seeded_archive", + "seeded_archive_writable", + "named_seeded_archive", + "corpus_seeded_db", + "seeded_db", + } +) def record_fixture_timing(name: str, duration_s: float, **metadata: Any) -> None: @@ -53,6 +65,25 @@ def record_fixture_timing(name: str, duration_s: float, **metadata: Any) -> None ) +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef: Any, request: Any) -> Any: + """Measure high-cost fixture setup without tracing every fixture call.""" + name = str(getattr(fixturedef, "argname", "")) + if name not in _MEASURED_FIXTURES: + yield + return + started = time.perf_counter() + outcome = yield + del outcome + record_fixture_timing( + f"fixture:{name}", + time.perf_counter() - started, + fixture=name, + scope=str(getattr(fixturedef, "scope", "function")), + test_nodeid=str(getattr(request.node, "nodeid", "")), + ) + + def _selection_nodeid_limit() -> int: raw = os.environ.get(_SELECTION_NODEID_LIMIT_ENV) if raw is None: diff --git a/devtools/task_history.py b/devtools/task_history.py index 37e8708208..4b5e2f599a 100644 --- a/devtools/task_history.py +++ b/devtools/task_history.py @@ -22,7 +22,7 @@ import sys from datetime import datetime, timezone from pathlib import Path -from typing import Any +from typing import Any, cast from devtools import repo_root as _get_root from devtools.verify_runs import CURRENT_RUN_PATH @@ -422,7 +422,9 @@ def _cmd_stats(args: argparse.Namespace) -> int: "peak_rss_mb_max": max(peaks), "peak_rss_mb_p95": _percentile(peaks, 95), } - pytest_statistics = [task["pytest_statistics"] for task in tasks if isinstance(task.get("pytest_statistics"), dict)] + pytest_statistics = [ + cast(dict[str, Any], value) for task in tasks if isinstance((value := task.get("pytest_statistics")), dict) + ] if args.resources and pytest_statistics: pss = [ float(item["resources"]["peak_tree_pss_kb"]) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 06808a7d7d..af61535a86 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -265,10 +265,14 @@ def aggregate_pytest_statistics( with contextlib.suppress(ValueError): explicit_worker_count = int(command_values[index + 1]) break - basetemp_sizes = [row.get("basetemp_size_kb") for row in resources] - basetemp_sizes = [int(value) * 1024 for value in basetemp_sizes if isinstance(value, int)] - basetemp_allocated = [row.get("basetemp_allocated_kb") for row in resources] - basetemp_allocated = [int(value) * 1024 for value in basetemp_allocated if isinstance(value, int)] + basetemp_sizes = [ + int(size_value) * 1024 for row in resources if isinstance((size_value := row.get("basetemp_size_kb")), int) + ] + basetemp_allocated = [ + int(allocated_value) * 1024 + for row in resources + if isinstance((allocated_value := row.get("basetemp_allocated_kb")), int) + ] containment: dict[str, Any] = {} containment_path = step_dir / "containment.json" if containment_path.exists(): diff --git a/tests/conftest.py b/tests/conftest.py index 4d919f22c8..1fa0b7d427 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ import time import uuid from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from functools import lru_cache from pathlib import Path from types import FrameType, ModuleType from typing import TYPE_CHECKING, Any @@ -788,6 +789,12 @@ def _tree_bytes(path: Path, *, allocated: bool) -> int: return total +@lru_cache(maxsize=8) +def _archive_template_apparent_bytes(source: Path) -> int: + """Cache the immutable template size instead of rescanning it per test.""" + return _tree_bytes(source, allocated=False) + + def _clone_archive_template(source: Path, destination: Path) -> None: """Clone one immutable empty archive into a test-private workspace.""" started = time.perf_counter() @@ -816,9 +823,8 @@ def _clone_archive_template(source: Path, destination: Path) -> None: method=method, source=str(source), destination=str(destination), - source_apparent_bytes=_tree_bytes(source, allocated=False), - destination_apparent_bytes=_tree_bytes(destination, allocated=False), - destination_allocated_bytes=_tree_bytes(destination, allocated=True), + source_apparent_bytes=_archive_template_apparent_bytes(source), + destination_apparent_bytes=_archive_template_apparent_bytes(source), ) except (ImportError, OSError): pass diff --git a/tests/infra/workload_artifacts.py b/tests/infra/workload_artifacts.py index 28d4eea68b..3f8a84e95c 100644 --- a/tests/infra/workload_artifacts.py +++ b/tests/infra/workload_artifacts.py @@ -551,6 +551,7 @@ def build_seeded_archive( def clone_seeded_archive(artifact: SeededArchiveArtifact, destination: Path) -> SeededArchiveClone: """Create a complete private writable archive clone, recording its method.""" + started = time.perf_counter() if destination.exists(): _remove_tree(destination) destination.parent.mkdir(parents=True, exist_ok=True) @@ -576,11 +577,24 @@ def clone_seeded_archive(artifact: SeededArchiveArtifact, destination: Path) -> bootstrap_marker.unlink() _record_fresh_durable_bootstrap(destination) - return SeededArchiveClone( + clone = SeededArchiveClone( root=destination, source_manifest_id=artifact.manifest.manifest_id, clone_method=method, ) + try: + from devtools.pytest_progress_plugin import record_fixture_timing + + record_fixture_timing( + "seeded_archive_clone", + time.perf_counter() - started, + fixture="seeded_archive_clone", + method=method, + source_manifest_id=artifact.manifest.manifest_id, + ) + except (ImportError, OSError): + pass + return clone __all__ = [ From 2256a91e2b5df15b5a438281db35280512536598 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 22:20:52 +0200 Subject: [PATCH 05/53] fix(test): keep pytest run evidence durable Problem: checkout-local verification artifacts and duplicate timing paths made future-run comparisons unreliable and could remove an explicit diagnostic basetemp. What changed: retain compact history in XDG state, aggregate only pytest phase reports, and reclaim only the exact basetemp minted for this run. --- TESTING.md | 25 ++++++---- devtools/pytest_progress_plugin.py | 50 ------------------- devtools/task_history.py | 11 +++-- devtools/verify.py | 3 +- devtools/verify_runs.py | 45 +---------------- tests/conftest.py | 41 ++-------------- tests/infra/workload_artifacts.py | 16 +------ tests/unit/devtools/test_task_history.py | 14 ++++++ tests/unit/devtools/test_verify.py | 3 +- tests/unit/test_pytest_temp_policy.py | 61 +++++++++++++++++++----- 10 files changed, 94 insertions(+), 175 deletions(-) diff --git a/TESTING.md b/TESTING.md index 3111658a2a..9f1d138a02 100644 --- a/TESTING.md +++ b/TESTING.md @@ -134,12 +134,13 @@ are shared, reused, and built once behind their own `.build.done` guard. Managed verification refuses to start below 1 GiB available memory instead of falling back to the pathological disk lane. Every per-test `tmp_path` tree is -reclaimed at teardown, including failed tests; node failure evidence remains -in the managed event, longrepr, summary, and resource receipts. Rerun a node -with an explicit basetemp when its filesystem witness is needed. The external -supervisor and parent runner independently remove the whole run root on -completion or termination, with startup stale-root cleanup as recovery after -an uncatchable process kill or reboot. +reclaimed in fixture teardown, including failures and interruptions; node +failure evidence remains in the managed event, longrepr, summary, and resource +receipts. The controller removes only the exact basetemp it created. An +explicit `--basetemp` is retained for targeted filesystem diagnosis. The +external supervisor and parent runner independently remove the whole run root +on completion or termination, with startup stale-root cleanup as recovery +after an uncatchable process kill or reboot. An affected run that selects zero tests is accepted only when no executable, test, dependency, or harness path changed. A zero selection after such a change @@ -166,7 +167,7 @@ and a postmortem diagnosis. The latest run is mirrored to - `.cache/verify/current-pytest-resources.jsonl` - `.cache/verify/current-pytest-postmortem.json` - `.cache/verify/current-pytest-containment.json` -- `.cache/verify/current-pytest-statistics.json` — derived phase/fixture +- `.cache/verify/current-pytest-statistics.json` — derived phase distributions, worker count, storage, resource peaks, and cleanup outcome; the same file is retained under each run's `steps/*/statistics.json`. - `.cache/verify/current-pytest-output.log` @@ -201,9 +202,13 @@ IDs by default (`POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT`, default 500) so broad collection does not retain or write unbounded node-id lists in controller or worker processes. -`devtools workspace tasks recent` shows the run id, diagnosis, and peak pytest -RSS when the current run metadata is available. `devtools workspace tasks stats ---resources` aggregates recorded pytest memory peaks over time. +The detailed artifacts above are checkout-local and disposable. Each `devtools +verify` or `devtools test` invocation automatically appends its compact run +summary to `$XDG_STATE_HOME/polylogue/devtools/` (or +`~/.local/state/polylogue/devtools/`), so `devtools workspace tasks recent` and +`devtools workspace tasks stats --resources` compare future runs across linked +worktrees without a separate recording command. Setup, call, and teardown +timings come only from pytest reports in the event stream. `devtools test` uses the same pytest progress plugin and process supervisor for focused selections. During or after a run, inspect diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index 2051984539..a792a6499a 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -32,56 +32,6 @@ _COLLECTION_DURATION_S: float | None = None _SLOW_REPORT_LIMIT = 20 _DEFAULT_SELECTION_NODEID_LIMIT = 500 -_MEASURED_FIXTURES = frozenset( - { - "empty_archive_template", - "workspace_env", - "cli_workspace", - "seeded_archive", - "seeded_archive_writable", - "named_seeded_archive", - "corpus_seeded_db", - "seeded_db", - } -) - - -def record_fixture_timing(name: str, duration_s: float, **metadata: Any) -> None: - """Record an explicitly instrumented fixture operation in the run stream. - - Pytest reports setup as one number per node, which cannot distinguish a - shared archive clone from ordinary fixture construction. Small, named - instrumentation points use this public helper so the run aggregator can - retain that distinction without making fixture code depend on the - aggregator's implementation. - """ - _write_event( - { - "event": "fixture_timing", - "name": name, - "duration_s": round(float(duration_s), 4), - **metadata, - } - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_fixture_setup(fixturedef: Any, request: Any) -> Any: - """Measure high-cost fixture setup without tracing every fixture call.""" - name = str(getattr(fixturedef, "argname", "")) - if name not in _MEASURED_FIXTURES: - yield - return - started = time.perf_counter() - outcome = yield - del outcome - record_fixture_timing( - f"fixture:{name}", - time.perf_counter() - started, - fixture=name, - scope=str(getattr(fixturedef, "scope", "function")), - test_nodeid=str(getattr(request.node, "nodeid", "")), - ) def _selection_nodeid_limit() -> int: diff --git a/devtools/task_history.py b/devtools/task_history.py index 4b5e2f599a..07fa0dcc04 100644 --- a/devtools/task_history.py +++ b/devtools/task_history.py @@ -1,7 +1,7 @@ """Agent-visible task execution history. -Maintains an append-only JSONL log of task executions under -``.agent/task-history/tasks.jsonl`` for use by agents and operators. +Maintains an append-only JSONL log of task executions in user state for use by +agents and operators across linked worktrees. Subcommands: @@ -25,7 +25,7 @@ from typing import Any, cast from devtools import repo_root as _get_root -from devtools.verify_runs import CURRENT_RUN_PATH +from devtools.verify_runs import CURRENT_RUN_PATH, DEVTOOLS_STATE_DIR from polylogue.core.json import JSONDocument TaskRecord = JSONDocument @@ -40,12 +40,13 @@ def task_history_file_path() -> Path: """Return the active task-history JSONL path. Honors ``POLYLOGUE_TASK_HISTORY_FILE`` (used by tests and one-off overrides); - otherwise defaults to ``/.agent/task-history/tasks.jsonl``. + otherwise defaults to the user's XDG state directory, shared across + worktrees so automatic verify records remain comparable. """ override = os.environ.get("POLYLOGUE_TASK_HISTORY_FILE") if override: return Path(override) - return _get_root() / ".agent" / "task-history" / "tasks.jsonl" + return DEVTOOLS_STATE_DIR / "task-history.jsonl" def _ensure_file(path: Path) -> None: diff --git a/devtools/verify.py b/devtools/verify.py index 65d3e2c47f..eaaf9d8c72 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -81,6 +81,7 @@ CURRENT_EVENTS_DIR, CURRENT_POSTMORTEM_PATH, CURRENT_RESOURCES_PATH, + VERIFY_HISTORY_PATH, PytestResourceError, PytestStepArtifacts, ResourceSampler, @@ -215,7 +216,7 @@ def _format_completion_notification( # ── history (JSONL) ──────────────────────────────────────────────── -HISTORY_PATH = Path(".cache/verify-history.jsonl") +HISTORY_PATH = VERIFY_HISTORY_PATH TESTMON_DATA = Path(".cache/testmon/testmondata") TESTMON_SEED_STAMP = Path(".cache/testmon/seed.json") TESTMON_SEED_ATTEMPT = Path(".cache/testmon/seed-attempt.json") diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index af61535a86..0407e2573a 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -27,6 +27,8 @@ VERIFY_CACHE = Path(".cache/verify") VERIFY_RUNS_DIR = VERIFY_CACHE / "runs" +DEVTOOLS_STATE_DIR = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state")) / "polylogue" / "devtools" +VERIFY_HISTORY_PATH = DEVTOOLS_STATE_DIR / "verify-history.jsonl" CURRENT_RUN_PATH = VERIFY_CACHE / "current-run.json" CURRENT_RESOURCES_PATH = VERIFY_CACHE / "current-pytest-resources.jsonl" CURRENT_POSTMORTEM_PATH = VERIFY_CACHE / "current-pytest-postmortem.json" @@ -172,7 +174,6 @@ def aggregate_pytest_statistics( *, command: list[Any] | tuple[Any, ...] = (), step_result: Mapping[str, Any] | None = None, - report_path: Path | None = None, ) -> dict[str, Any]: """Reduce the append-only pytest evidence into one durable step summary. @@ -193,7 +194,6 @@ def aggregate_pytest_statistics( outcomes: dict[str, int] = {} phase_outcomes: dict[str, dict[str, int]] = {"setup": {}, "call": {}, "teardown": {}} nodes: set[str] = set() - fixture_timings: dict[str, list[float]] = {} workers: set[str] = set() for row in events: worker = row.get("worker_id") @@ -214,41 +214,6 @@ def aggregate_pytest_statistics( bucket[outcome] = bucket.get(outcome, 0) + 1 if when == "call": outcomes[outcome] = outcomes.get(outcome, 0) + 1 - elif event == "fixture_timing": - name = row.get("name") - duration = row.get("duration_s") - if isinstance(name, str) and isinstance(duration, (int, float)): - fixture_timings.setdefault(name, []).append(float(duration)) - - # Some pytest/plugin combinations expose only the final teardown report to - # pytest_runtest_logreport. The structured report has complete per-phase - # data when the run reaches a normal pytest exit, so use it to complete the - # aggregate instead of publishing a deceptively partial distribution. - if report_path is not None and report_path.exists(): - with contextlib.suppress(OSError, json.JSONDecodeError): - report = json.loads(report_path.read_text(encoding="utf-8")) - report_tests = report.get("tests", []) if isinstance(report, dict) else [] - if isinstance(report_tests, list): - for test in report_tests: - if not isinstance(test, Mapping): - continue - nodeid = test.get("nodeid") - if isinstance(nodeid, str) and nodeid: - nodes.add(nodeid) - outcome = test.get("outcome") - if isinstance(outcome, str): - outcomes[outcome] = outcomes.get(outcome, 0) + 1 - for when in phases: - phase = test.get(when) - if not isinstance(phase, Mapping): - continue - duration = phase.get("duration") - phase_outcome = phase.get("outcome") - if isinstance(duration, (int, float)): - phases[when].append(float(duration)) - if isinstance(phase_outcome, str): - bucket = phase_outcomes[when] - bucket[phase_outcome] = bucket.get(phase_outcome, 0) + 1 resources: list[dict[str, Any]] = [] resources_path = step_dir / "resources.jsonl" @@ -288,7 +253,6 @@ def aggregate_pytest_statistics( "outcomes": outcomes, "phase_outcomes": phase_outcomes, "phases": {name: _distribution(values) for name, values in phases.items()}, - "fixtures": {name: _distribution(values) for name, values in fixture_timings.items()}, "xdist": { "worker_ids": sorted(workers), "worker_count": max( @@ -479,11 +443,6 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> None: self.run_dir / "steps" / step_id, command=step.get("cmd", []), step_result=result, - report_path=( - self.root / str(result["report_path"]) - if isinstance(result.get("report_path"), str) - else None - ), ) _write_json(statistics_path, statistics) with contextlib.suppress(OSError): diff --git a/tests/conftest.py b/tests/conftest.py index 1fa0b7d427..6c27fa58e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,6 @@ import time import uuid from collections.abc import AsyncIterator, Callable, Iterator, Mapping -from functools import lru_cache from pathlib import Path from types import FrameType, ModuleType from typing import TYPE_CHECKING, Any @@ -132,6 +131,7 @@ def pytest_configure(config: pytest.Config) -> None: raise pytest.UsageError(f"pytest: {exc}") from exc basetemp = root / f"pytest-polylogue-{checkout}-{run_id}" config.option.basetemp = str(basetemp) + os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = str(basetemp) if not hasattr(config, "workerinput"): _mark_basetemp_owner(basetemp) sys.stderr.write(f"pytest: basetemp → {config.option.basetemp} ({label})\n") @@ -314,7 +314,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if not basetemp: return basetemp_path = Path(str(basetemp)) - if basetemp_path.name.startswith("pytest-polylogue-") and "-seeded-" not in basetemp_path.name: + if str(basetemp_path) == os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP"): shutil.rmtree(basetemp_path, ignore_errors=True) @@ -405,6 +405,7 @@ def _reclaim_test_tmp_path( "POLYLOGUE_PYTEST_SELECTION_PATH", "POLYLOGUE_PYTEST_SUMMARY_PATH", "POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", } ) @@ -779,26 +780,8 @@ def cli_workspace( } -def _tree_bytes(path: Path, *, allocated: bool) -> int: - total = 0 - for item in path.rglob("*"): - with contextlib.suppress(OSError): - if item.is_file(): - stat_result = item.stat() - total += stat_result.st_blocks * 512 if allocated else stat_result.st_size - return total - - -@lru_cache(maxsize=8) -def _archive_template_apparent_bytes(source: Path) -> int: - """Cache the immutable template size instead of rescanning it per test.""" - return _tree_bytes(source, allocated=False) - - def _clone_archive_template(source: Path, destination: Path) -> None: """Clone one immutable empty archive into a test-private workspace.""" - started = time.perf_counter() - method = "reflink-auto" destination.mkdir(parents=True, exist_ok=True) try: subprocess.run( @@ -809,26 +792,8 @@ def _clone_archive_template(source: Path, destination: Path) -> None: timeout=10, ) except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): - method = "copytree" shutil.copytree(source, destination, dirs_exist_ok=True) - # The managed pytest plugin turns this into a durable fixture-cost record; - # ordinary pytest runs remain unaffected because the event sink is absent. - try: - from devtools.pytest_progress_plugin import record_fixture_timing - - record_fixture_timing( - "archive_clone", - time.perf_counter() - started, - method=method, - source=str(source), - destination=str(destination), - source_apparent_bytes=_archive_template_apparent_bytes(source), - destination_apparent_bytes=_archive_template_apparent_bytes(source), - ) - except (ImportError, OSError): - pass - bootstrap_marker = destination / ".maintenance-state" / "durable-change-trains" / ".bootstrap" if bootstrap_marker.is_file(): from polylogue.storage.sqlite.durable_change_train import _record_fresh_durable_bootstrap diff --git a/tests/infra/workload_artifacts.py b/tests/infra/workload_artifacts.py index 3f8a84e95c..28d4eea68b 100644 --- a/tests/infra/workload_artifacts.py +++ b/tests/infra/workload_artifacts.py @@ -551,7 +551,6 @@ def build_seeded_archive( def clone_seeded_archive(artifact: SeededArchiveArtifact, destination: Path) -> SeededArchiveClone: """Create a complete private writable archive clone, recording its method.""" - started = time.perf_counter() if destination.exists(): _remove_tree(destination) destination.parent.mkdir(parents=True, exist_ok=True) @@ -577,24 +576,11 @@ def clone_seeded_archive(artifact: SeededArchiveArtifact, destination: Path) -> bootstrap_marker.unlink() _record_fresh_durable_bootstrap(destination) - clone = SeededArchiveClone( + return SeededArchiveClone( root=destination, source_manifest_id=artifact.manifest.manifest_id, clone_method=method, ) - try: - from devtools.pytest_progress_plugin import record_fixture_timing - - record_fixture_timing( - "seeded_archive_clone", - time.perf_counter() - started, - fixture="seeded_archive_clone", - method=method, - source_manifest_id=artifact.manifest.manifest_id, - ) - except (ImportError, OSError): - pass - return clone __all__ = [ diff --git a/tests/unit/devtools/test_task_history.py b/tests/unit/devtools/test_task_history.py index 69d16a2c53..87fc917881 100644 --- a/tests/unit/devtools/test_task_history.py +++ b/tests/unit/devtools/test_task_history.py @@ -83,6 +83,20 @@ def test_log_and_recent_round_trip(isolated_task_history_file: Path, capsys: pyt assert entry["exit_code"] == 0 +def test_default_history_path_uses_user_state_across_worktrees( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.delenv("POLYLOGUE_TASK_HISTORY_FILE", raising=False) + monkeypatch.setattr(task_history, "DEVTOOLS_STATE_DIR", tmp_path / "state" / "polylogue" / "devtools") + + path = task_history.task_history_file_path() + task_history.record_invocation(command="verify", args=[], duration_ms=1.0, exit_code=0) + + assert path == tmp_path / "state" / "polylogue" / "devtools" / "task-history.jsonl" + assert json.loads(path.read_text(encoding="utf-8"))["command"] == "verify" + + def test_stats_by_class_and_slowest(isolated_task_history_file: Path, capsys: pytest.CaptureFixture[str]) -> None: samples = [ ("verify", 1000.0, 0), diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 054eaba58e..6e14ce1023 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -984,7 +984,6 @@ def test_aggregate_pytest_statistics_reduces_phases_fixtures_and_resources(tmp_p "outcome": "passed", "worker_id": "gw0", }, - {"event": "fixture_timing", "name": "archive_clone", "duration_s": 0.25, "method": "reflink-auto"}, ) ) + "\n" @@ -1008,7 +1007,7 @@ def test_aggregate_pytest_statistics_reduces_phases_fixtures_and_resources(tmp_p assert result["node_count"] == 1 assert result["phases"]["call"]["p50_s"] == 2.0 - assert result["fixtures"]["archive_clone"]["sum_s"] == 0.25 + assert result["phases"]["setup"]["count"] == 1 assert result["storage"]["basetemp_logical_bytes_max"] == 12 * 1024 assert result["resources"]["peak_tree_pss_kb"] == 80 assert result["cleanup"]["complete"] is True diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index a8f91eb534..ccc1c9389a 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from collections.abc import Generator from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -49,18 +50,21 @@ def _make_real_candidates( return shm, scratch -def test_runtest_makereport_wrapper_preserves_each_phase_report() -> None: - item = SimpleNamespace() - reports = [SimpleNamespace(when=phase) for phase in ("setup", "call", "teardown")] +@pytest.mark.parametrize("exception", [KeyboardInterrupt(), RuntimeError("teardown failure")]) +def test_test_tmp_path_reclamation_runs_after_failure_or_interrupt( + tmp_path: Path, + exception: BaseException, +) -> None: + tree = tmp_path / "test-private" + tree.mkdir() + fixture_generator = cast(Any, conftest._reclaim_test_tmp_path).__wrapped__ + cleanup = cast("Generator[None, BaseException, None]", fixture_generator(tree)) + + assert next(cleanup) is None + with pytest.raises(type(exception)): + cleanup.throw(exception) - for report in reports: - wrapper = conftest.pytest_runtest_makereport( - cast("pytest.Item", item), cast("pytest.CallInfo[None]", SimpleNamespace()) - ) - assert next(wrapper) is None - with pytest.raises(StopIteration): - wrapper.send(cast("Any", SimpleNamespace(get_result=lambda report=report: report))) - assert getattr(item, f"rep_{report.when}") is report + assert not tree.exists() def test_managed_pytest_temp_root_defaults_to_scratch( @@ -172,6 +176,7 @@ def test_pytest_configure_reports_low_space_as_usage_error( # teardown reverts the leak regardless of what the call under test does. monkeypatch.delenv("POLYLOGUE_PYTEST_RUN_ID", raising=False) monkeypatch.delenv("POLYLOGUE_PYTEST_CHECKOUT", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", raising=False) config = SimpleNamespace( option=SimpleNamespace(basetemp=None), addinivalue_line=lambda *args, **kwargs: None, @@ -193,6 +198,7 @@ def test_bare_pytest_configure_defaults_to_scratch_without_a_supervisor( "POLYLOGUE_PYTEST_TMPFS", "POLYLOGUE_PYTEST_RUN_ID", "POLYLOGUE_PYTEST_CHECKOUT", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", ): monkeypatch.delenv(name, raising=False) config = SimpleNamespace( @@ -217,6 +223,7 @@ def test_bare_pytest_ignores_leaked_cloud_basetemp_on_workstation( monkeypatch.delenv("POLYLOGUE_PYTEST_TMPFS", raising=False) monkeypatch.delenv("POLYLOGUE_PYTEST_RUN_ID", raising=False) monkeypatch.delenv("POLYLOGUE_PYTEST_CHECKOUT", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", raising=False) config = SimpleNamespace( option=SimpleNamespace(basetemp=None), addinivalue_line=lambda *args, **kwargs: None, @@ -389,6 +396,38 @@ def test_sessionfinish_leaves_xdist_basetemp_for_supervisor_cleanup( assert basetemp.exists() +def test_sessionfinish_reclaims_only_its_managed_basetemp( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + basetemp = tmp_path / "pytest-polylogue-run-123" + basetemp.mkdir() + session = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(basetemp), numprocesses=0))) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "run-123") + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(basetemp)) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + + conftest.pytest_sessionfinish(cast("pytest.Session", session), 1) + + assert not basetemp.exists() + + +def test_sessionfinish_retains_explicit_diagnostic_basetemp( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + explicit = tmp_path / "pytest-polylogue-diagnostic" + explicit.mkdir() + session = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(explicit), numprocesses=0))) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "run-123") + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(tmp_path / "pytest-polylogue-other-run")) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + + conftest.pytest_sessionfinish(cast("pytest.Session", session), 1) + + assert explicit.exists() + + def test_archive_template_clone_is_private(tmp_path: Path) -> None: source = tmp_path / "source" destination = tmp_path / "destination" From c0f5978e36c1a3f69c1a5b42765f29aff2c12293 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 22:41:23 +0200 Subject: [PATCH 06/53] fix(test): unify cross-worktree run history --- TESTING.md | 10 ++--- devtools/run_tests.py | 3 +- devtools/task_history.py | 50 +++--------------------- devtools/verify.py | 26 +++++++----- devtools/verify_runs.py | 16 ++++++++ tests/unit/devtools/test_run_tests.py | 3 ++ tests/unit/devtools/test_task_history.py | 14 ------- tests/unit/devtools/test_verify.py | 35 +++++++++++++++++ 8 files changed, 84 insertions(+), 73 deletions(-) diff --git a/TESTING.md b/TESTING.md index 9f1d138a02..86a00aff9b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -204,11 +204,11 @@ or worker processes. The detailed artifacts above are checkout-local and disposable. Each `devtools verify` or `devtools test` invocation automatically appends its compact run -summary to `$XDG_STATE_HOME/polylogue/devtools/` (or -`~/.local/state/polylogue/devtools/`), so `devtools workspace tasks recent` and -`devtools workspace tasks stats --resources` compare future runs across linked -worktrees without a separate recording command. Setup, call, and teardown -timings come only from pytest reports in the event stream. +summary to `$XDG_STATE_HOME/polylogue/devtools/verify-history.jsonl` (or the +corresponding `~/.local/state` path), shared across linked worktrees without a +separate recording command. `devtools verify --history` prints the recent +cross-worktree runs. Setup, call, and teardown timings come only from pytest +reports in the event stream. `devtools test` uses the same pytest progress plugin and process supervisor for focused selections. During or after a run, inspect diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 27c5dda0ac..da4886abcb 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -47,7 +47,7 @@ _clear_pytest_report, _run, ) -from devtools.verify_runs import VerifyRun, git_head +from devtools.verify_runs import VerifyRun, append_verify_history, git_head ROOT = Path(__file__).resolve().parent.parent _LOCK_PATH = ROOT / ".cache" / "test-run.lock" @@ -162,6 +162,7 @@ def main(argv: list[str] | None = None) -> int: verification_scope="affected", release_baseline_allowed=False, ) + append_verify_history(payload) if use_json: print(json.dumps(payload, indent=2, ensure_ascii=False)) sys.stderr.write( diff --git a/devtools/task_history.py b/devtools/task_history.py index 07fa0dcc04..77a205a360 100644 --- a/devtools/task_history.py +++ b/devtools/task_history.py @@ -1,7 +1,7 @@ """Agent-visible task execution history. -Maintains an append-only JSONL log of task executions in user state for use by -agents and operators across linked worktrees. +Maintains an append-only JSONL log of task executions under +``.agent/task-history/tasks.jsonl`` for use by agents and operators. Subcommands: @@ -22,10 +22,10 @@ import sys from datetime import datetime, timezone from pathlib import Path -from typing import Any, cast +from typing import Any from devtools import repo_root as _get_root -from devtools.verify_runs import CURRENT_RUN_PATH, DEVTOOLS_STATE_DIR +from devtools.verify_runs import CURRENT_RUN_PATH from polylogue.core.json import JSONDocument TaskRecord = JSONDocument @@ -40,13 +40,12 @@ def task_history_file_path() -> Path: """Return the active task-history JSONL path. Honors ``POLYLOGUE_TASK_HISTORY_FILE`` (used by tests and one-off overrides); - otherwise defaults to the user's XDG state directory, shared across - worktrees so automatic verify records remain comparable. + otherwise defaults to ``/.agent/task-history/tasks.jsonl``. """ override = os.environ.get("POLYLOGUE_TASK_HISTORY_FILE") if override: return Path(override) - return DEVTOOLS_STATE_DIR / "task-history.jsonl" + return _get_root() / ".agent" / "task-history" / "tasks.jsonl" def _ensure_file(path: Path) -> None: @@ -114,16 +113,6 @@ def _latest_verify_run_metadata(command: str) -> dict[str, Any]: ): if key in latest_pytest: metadata[f"pytest_{key}"] = latest_pytest[key] - statistics_path = _get_root() / ".cache" / "verify" / "current-pytest-statistics.json" - try: - statistics = json.loads(statistics_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - statistics = None - if isinstance(statistics, dict): - # Keep the derived aggregate in the append-only invocation record so - # Lynchpin can consume one stable row without crawling disposable - # .cache/verify directories. Raw events remain in the run artifact. - metadata["pytest_statistics"] = statistics return {key: value for key, value in metadata.items() if value is not None} @@ -423,33 +412,6 @@ def _cmd_stats(args: argparse.Namespace) -> int: "peak_rss_mb_max": max(peaks), "peak_rss_mb_p95": _percentile(peaks, 95), } - pytest_statistics = [ - cast(dict[str, Any], value) for task in tasks if isinstance((value := task.get("pytest_statistics")), dict) - ] - if args.resources and pytest_statistics: - pss = [ - float(item["resources"]["peak_tree_pss_kb"]) - for item in pytest_statistics - if isinstance(item.get("resources"), dict) - and isinstance(item["resources"].get("peak_tree_pss_kb"), (int, float)) - ] - temp = [ - int(item["storage"]["basetemp_logical_bytes_max"]) - for item in pytest_statistics - if isinstance(item.get("storage"), dict) - and isinstance(item["storage"].get("basetemp_logical_bytes_max"), int) - ] - workers = [ - int(item["xdist"]["worker_count"]) - for item in pytest_statistics - if isinstance(item.get("xdist"), dict) and isinstance(item["xdist"].get("worker_count"), int) - ] - stats["pytest_runs"] = { - "count": len(pytest_statistics), - "peak_pss_kb_max": max(pss, default=None), - "basetemp_logical_bytes_max": max(temp, default=None), - "xdist_worker_counts": sorted(set(workers)), - } slow_tests = _latest_pytest_slow_tests(args.slow_tests) if slow_tests: stats["slow_tests"] = slow_tests diff --git a/devtools/verify.py b/devtools/verify.py index eaaf9d8c72..b55420bb64 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -87,6 +87,7 @@ ResourceSampler, VerifyRun, adaptive_pytest_worker_count, + append_verify_history, apply_managed_pytest_runtime_policy, classify_pytest_result, cleanup_managed_pytest_basetemp, @@ -264,9 +265,7 @@ def _load_history() -> list[dict[str, Any]]: def _save_history(entry: dict[str, Any]) -> None: - HISTORY_PATH.parent.mkdir(parents=True, exist_ok=True) - with open(HISTORY_PATH, "a") as f: - f.write(json.dumps(entry, ensure_ascii=False) + "\n") + append_verify_history(entry, path=HISTORY_PATH) def _print_history(file: Path | None = None) -> None: @@ -278,12 +277,21 @@ def _print_history(file: Path | None = None) -> None: print(f"{'time':<20} {'tier':<8} {'head':<10} {'dur':>7} {'exit':>4} steps") print("-" * 75) for entry in entries[-10:]: - ts = entry["timestamp"][5:19] # MM-DD HH:MM - tier = entry["tier"][:8] - head = entry["git_head"][:8] - dur = f"{entry['total_duration_s']:.0f}s" - ec = entry["exit_code"] - steps = ", ".join(f"{s['name']}({s['duration_s']:.0f}s{' FAIL' if s['exit'] else ''})" for s in entry["steps"]) + timestamp = str(entry.get("timestamp") or entry.get("finished_at") or entry.get("started_at") or "unknown") + ts = timestamp[5:19] if timestamp != "unknown" else timestamp + tier = str(entry.get("tier") or "unknown")[:8] + head = str(entry.get("git_head") or "unknown")[:8] + duration = entry.get("total_duration_s", entry.get("duration_s", 0.0)) + dur = f"{float(duration or 0.0):.0f}s" + ec = int(entry.get("exit_code", 1)) + rendered_steps: list[str] = [] + for step in entry.get("steps", []): + if not isinstance(step, dict): + continue + step_duration = float(step.get("duration_s") or 0.0) + step_exit = int(step.get("exit", 1)) + rendered_steps.append(f"{step.get('name', 'unknown')}({step_duration:.0f}s{' FAIL' if step_exit else ''})") + steps = ", ".join(rendered_steps) print(f"{ts:<20} {tier:<8} {head:<10} {dur:>7} {ec:>4} {steps}") diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 0407e2573a..b0f5ac35c4 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -71,6 +71,22 @@ class PytestResourceError(RuntimeError): """Raised when the host cannot safely start a managed pytest run.""" +def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTORY_PATH) -> None: + """Append one complete invocation to the cross-worktree run history. + + A single ``O_APPEND`` write keeps concurrent worktrees from overwriting or + interleaving their records. Detailed artifacts remain checkout-local; this + history is the compact durable index used to find and compare them. + """ + path.parent.mkdir(parents=True, exist_ok=True) + payload = (json.dumps(dict(entry), ensure_ascii=False) + "\n").encode() + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + try: + os.write(descriptor, payload) + finally: + os.close(descriptor) + + @dataclass(frozen=True) class PytestRuntimePolicy: """One start-time resource decision for a managed pytest run.""" diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index 0d126bd449..d8a74f459a 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -93,6 +93,7 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di monkeypatch.setattr("devtools.run_tests._clear_pytest_report", lambda _cmd: None) monkeypatch.setattr("devtools.run_tests._run", _fake_run) monkeypatch.setattr("devtools.run_tests.git_head", lambda _root: "abc123") + monkeypatch.setattr("devtools.run_tests.append_verify_history", lambda payload: captured.update(history=payload)) assert run_tests.main(["tests/unit/pipeline", "--json"]) == 0 assert "--json" not in captured["cmd"] assert "tests/unit/pipeline" in captured["cmd"] @@ -102,6 +103,8 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di assert isinstance(captured["run"]._payload["git_dirty"], bool) assert captured["run"]._payload["verification_scope"] == "affected" assert captured["run"]._payload["release_baseline_allowed"] is False + assert captured["history"]["run_id"] == captured["run"].run_id + assert captured["history"]["status"] == "success" def test_main_returns_pytest_exit_code(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit/devtools/test_task_history.py b/tests/unit/devtools/test_task_history.py index 87fc917881..69d16a2c53 100644 --- a/tests/unit/devtools/test_task_history.py +++ b/tests/unit/devtools/test_task_history.py @@ -83,20 +83,6 @@ def test_log_and_recent_round_trip(isolated_task_history_file: Path, capsys: pyt assert entry["exit_code"] == 0 -def test_default_history_path_uses_user_state_across_worktrees( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - monkeypatch.delenv("POLYLOGUE_TASK_HISTORY_FILE", raising=False) - monkeypatch.setattr(task_history, "DEVTOOLS_STATE_DIR", tmp_path / "state" / "polylogue" / "devtools") - - path = task_history.task_history_file_path() - task_history.record_invocation(command="verify", args=[], duration_ms=1.0, exit_code=0) - - assert path == tmp_path / "state" / "polylogue" / "devtools" / "task-history.jsonl" - assert json.loads(path.read_text(encoding="utf-8"))["command"] == "verify" - - def test_stats_by_class_and_slowest(isolated_task_history_file: Path, capsys: pytest.CaptureFixture[str]) -> None: samples = [ ("verify", 1000.0, 0), diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 6e14ce1023..357db3a02a 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1013,6 +1013,41 @@ def test_aggregate_pytest_statistics_reduces_phases_fixtures_and_resources(tmp_p assert result["cleanup"]["complete"] is True +def test_print_history_accepts_verify_and_focused_run_records( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + verify, + "_load_history", + lambda: [ + { + "timestamp": "2026-08-12T20:00:00+00:00", + "tier": "quick", + "git_head": "a" * 40, + "total_duration_s": 2.0, + "exit_code": 0, + "steps": [{"name": "ruff", "duration_s": 1.0, "exit": 0}], + }, + { + "finished_at": "2026-08-12T20:01:00+00:00", + "tier": "focused-test", + "git_head": "b" * 40, + "duration_s": 3.0, + "exit_code": 1, + "steps": [{"name": "pytest focused", "duration_s": None, "exit": 1}], + }, + ], + ) + + verify._print_history() + + output = capsys.readouterr().out + assert "quick" in output + assert "focused-" in output + assert "pytest focused(0s FAIL)" in output + + def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) From ad6704f9a476f9e24cc24473229921de6e155886 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 22:46:57 +0200 Subject: [PATCH 07/53] fix(test): serialize durable history appends --- devtools/verify_runs.py | 9 ++++++++- tests/unit/devtools/test_verify.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index b0f5ac35c4..dd00c146de 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -8,6 +8,7 @@ from __future__ import annotations import contextlib +import fcntl import hashlib import json import os @@ -82,7 +83,13 @@ def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTO payload = (json.dumps(dict(entry), ensure_ascii=False) + "\n").encode() descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) try: - os.write(descriptor, payload) + fcntl.flock(descriptor, fcntl.LOCK_EX) + remaining = memoryview(payload) + while remaining: + written = os.write(descriptor, remaining) + if written <= 0: + raise OSError("verification history append made no progress") + remaining = remaining[written:] finally: os.close(descriptor) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 357db3a02a..5ab3b36cfc 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -7,6 +7,7 @@ import sqlite3 import subprocess import sys +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from unittest.mock import patch @@ -80,6 +81,7 @@ adaptive_pytest_runtime_policy, adaptive_pytest_worker_count, aggregate_pytest_statistics, + append_verify_history, apply_managed_pytest_runtime_policy, classify_pytest_result, cleanup_managed_pytest_basetemp, @@ -1048,6 +1050,16 @@ def test_print_history_accepts_verify_and_focused_run_records( assert "pytest focused(0s FAIL)" in output +def test_verify_history_appends_concurrent_records_without_interleaving(tmp_path: Path) -> None: + history = tmp_path / "state" / "verify-history.jsonl" + + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda sequence: append_verify_history({"sequence": sequence}, path=history), range(64))) + + rows = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] + assert sorted(row["sequence"] for row in rows) == list(range(64)) + + def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) From 23e4b2f8c4605b6e9cbe8f7ebc2b93a30bf25c69 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 23:03:52 +0200 Subject: [PATCH 08/53] fix(test): retain compact statistics across cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Embed each pytest step’s reduced timing, storage, and resource statistics in the shared run-history row. Detailed artifacts remain checkout-local, but deleting a merged worktree no longer discards the aggregate evidence used for later performance comparisons. --- devtools/verify_runs.py | 11 ++++++++--- tests/unit/devtools/test_verify.py | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index dd00c146de..62ec092209 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -75,9 +75,10 @@ class PytestResourceError(RuntimeError): def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTORY_PATH) -> None: """Append one complete invocation to the cross-worktree run history. - A single ``O_APPEND`` write keeps concurrent worktrees from overwriting or - interleaving their records. Detailed artifacts remain checkout-local; this - history is the compact durable index used to find and compare them. + ``O_APPEND`` plus an advisory lock keeps concurrent worktrees from + overwriting or interleaving their records, including short writes. + Detailed artifacts remain checkout-local; this history is the compact + durable index used to find and compare them. """ path.parent.mkdir(parents=True, exist_ok=True) payload = (json.dumps(dict(entry), ensure_ascii=False) + "\n").encode() @@ -471,6 +472,10 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> None: with contextlib.suppress(OSError): shutil.copyfile(statistics_path, self.root / CURRENT_STATISTICS_PATH) step["statistics_path"] = str(self.relative_run_dir / "steps" / step_id / "statistics.json") + # Keep the compact aggregate in the cross-worktree history + # itself. The detailed artifact path is checkout-local and + # may disappear when a merged lane is cleaned up. + step["statistics"] = statistics break self.write() diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 5ab3b36cfc..b3105b0979 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1015,6 +1015,31 @@ def test_aggregate_pytest_statistics_reduces_phases_fixtures_and_resources(tmp_p assert result["cleanup"]["complete"] is True +def test_verify_run_embeds_compact_statistics_before_worktree_cleanup(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="pytest focused", cmd=["pytest", "tests/unit/example.py"]) + artifacts.events_merged_path.write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/unit/example.py::test_one", + "when": "call", + "duration_s": 0.25, + "outcome": "passed", + "worker_id": "controller", + } + ) + + "\n" + ) + + run.finish_step(step_id=artifacts.step_id, result={"exit": 0, "duration_s": 0.25}) + payload = run.finish(exit_code=0, duration_s=0.25) + + statistics = payload["steps"][0]["statistics"] + assert statistics["node_count"] == 1 + assert statistics["phases"]["call"]["p50_s"] == 0.25 + + def test_print_history_accepts_verify_and_focused_run_records( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], From e3dd927893e43fe25f19a87a5e6dc697024b4b5a Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 23:24:58 +0200 Subject: [PATCH 09/53] fix(test): repair verify run evidence review Problem: current-head review found stale shared-history consumers, duplicated xdist phase accounting, incomplete interrupted-run evidence, and basetemp authority gaps.\n\nWhat changed: use shared XDG history, aggregate only real pytest steps from canonical worker reports, merge shards before finalization, retain explicit diagnostics, and carry explicit basetemp and checkout authority through verification evidence.\n\nVerification: focused verify-evidence harness (213 passed) and devtools verify --quick (25 steps passed). --- devtools/evidence_dashboard.py | 8 +- devtools/pytest_progress_plugin.py | 5 + devtools/verify.py | 92 ++++++++--- devtools/verify_runs.py | 147 +++++++++++------ tests/conftest.py | 9 +- .../unit/devtools/test_evidence_dashboard.py | 30 ++++ .../devtools/test_pytest_progress_plugin.py | 17 ++ tests/unit/devtools/test_verify.py | 156 +++++++++++++++++- tests/unit/test_pytest_temp_policy.py | 22 ++- 9 files changed, 403 insertions(+), 83 deletions(-) create mode 100644 tests/unit/devtools/test_evidence_dashboard.py diff --git a/devtools/evidence_dashboard.py b/devtools/evidence_dashboard.py index 6959909d41..de15f60d11 100644 --- a/devtools/evidence_dashboard.py +++ b/devtools/evidence_dashboard.py @@ -9,7 +9,7 @@ - pytest health from ``.cache/verify/last-pytest.json``; - coverage from ``.coverage`` / ``coverage.xml`` when present; - benchmark/SLO catalog rows and their required-artifact coverage; -- static gate status from ``.cache/verify-history.jsonl``; +- static gate status from the shared XDG verify history; - witness lifecycle counts; - mutation/benchmark campaign freshness. @@ -28,12 +28,12 @@ from typing import Any from devtools import repo_root as _get_root +from devtools.verify_runs import VERIFY_HISTORY_PATH ROOT = _get_root() # Artifact paths (relative to repo root). PYTEST_REPORT_REL = Path(".cache/verify/last-pytest.json") -VERIFY_HISTORY_REL = Path(".cache/verify-history.jsonl") LAST_VERIFY_RESULT_REL = Path(".cache/last-verify-result.json") COVERAGE_DATA_REL = Path(".coverage") COVERAGE_XML_REL = Path("coverage.xml") @@ -225,7 +225,7 @@ def _benchmark_slo(root: Path, *, now: datetime) -> dict[str, Any]: def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: - history_path = root / VERIFY_HISTORY_REL + history_path = VERIFY_HISTORY_PATH last_result_path = root / LAST_VERIFY_RESULT_REL # Prefer last-verify-result.json (the most recent run) then walk back through @@ -291,7 +291,7 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: failing = [g for g in gates if g.get("status") == "fail"] return { "available": last_result_path.exists() or history_path.exists(), - "history_path": str(VERIFY_HISTORY_REL), + "history_path": str(history_path), "last_result_path": str(LAST_VERIFY_RESULT_REL), "total_gates_tracked": len(_STATIC_GATE_NAMES), "gates_with_status": sum(1 for g in gates if g.get("available")), diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index a792a6499a..5014722d3f 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -251,6 +251,11 @@ def pytest_runtest_makereport(item: Any, call: Any) -> Any: @pytest.hookimpl def pytest_runtest_logreport(report: Any) -> None: """Retain the direct/log-hook fallback used by older pytest plugins/tests.""" + # xdist forwards each worker's report to the controller. The worker has + # already written the authoritative shard event through makereport; avoid + # recording that deserialized controller copy a second time. + if not os.environ.get("PYTEST_XDIST_WORKER") and getattr(report, "worker_id", None): + return _record_phase_report(report) diff --git a/devtools/verify.py b/devtools/verify.py index b55420bb64..0522f06629 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -81,6 +81,7 @@ CURRENT_EVENTS_DIR, CURRENT_POSTMORTEM_PATH, CURRENT_RESOURCES_PATH, + PYTEST_EXPLICIT_BASETEMP_ENV, VERIFY_HISTORY_PATH, PytestResourceError, PytestStepArtifacts, @@ -95,7 +96,6 @@ env_for_pytest_step, force_managed_pytest_scratch, latest_event_from_paths, - merge_worker_events, normalize_pytest_basetemp_env, pytest_basetemp_path, pytest_tmpfs_budget_kb, @@ -1520,6 +1520,9 @@ def _run( _clear_pytest_report(cmd) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None env = _subprocess_env() + explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd) + if explicit_basetemp is not None: + env[PYTEST_EXPLICIT_BASETEMP_ENV] = str(explicit_basetemp) pytest_tmpfs = False pytest_tmpfs_budget_mb: float | None = None runtime_policy = None @@ -1567,21 +1570,26 @@ def _run( env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" if run is not None and artifacts is not None: env = env_for_pytest_step(env, run=run, artifacts=artifacts) + interrupted = False if is_pytest: try: - result = _run_pytest_with_heartbeat(cmd, cwd=cwd, env=env, t0=t0, run=run, artifacts=artifacts) + try: + result = _run_pytest_with_heartbeat(cmd, cwd=cwd, env=env, t0=t0, run=run, artifacts=artifacts) + except KeyboardInterrupt: + interrupted = True + result = subprocess.CompletedProcess(args=cmd, returncode=130, stdout="", stderr="") finally: basetemp_cleanup = cleanup_managed_pytest_basetemp( root=ROOT, run_id=env.get("POLYLOGUE_PYTEST_RUN_ID", ""), env=env, ) - if artifacts is not None: - merge_worker_events(artifacts.events_dir, artifacts.events_merged_path) - with contextlib.suppress(FileNotFoundError): - shutil.copyfile(PYTEST_PROGRESS_PATH, artifacts.progress_path) else: - result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + try: + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + except KeyboardInterrupt: + interrupted = True + result = subprocess.CompletedProcess(args=cmd, returncode=130, stdout="", stderr="") elapsed = time.monotonic() - t0 metadata: dict[str, Any] = {} if artifacts is not None: @@ -1761,6 +1769,9 @@ def _run( summary=summary if isinstance(summary, dict) else None, progress_event=metadata.get("progress_event") if isinstance(metadata.get("progress_event"), str) else None, ) + if interrupted: + diagnosis = "pytest_interrupted" + metadata["termination_reason"] = "operator_interrupt" metadata["diagnosis"] = diagnosis termination_reason = ( metadata.get("termination_reason") if isinstance(metadata.get("termination_reason"), str) else None @@ -1797,17 +1808,8 @@ def _run( **resource_summary, } artifacts.postmortem_path.write_text(json.dumps(postmortem, indent=2, ensure_ascii=False) + "\n") - copy_current_pytest_artifacts( - Path.cwd(), - artifacts, - legacy_paths={ - "progress_path": PYTEST_PROGRESS_PATH, - "events_merged_path": PYTEST_EVENTS_PATH, - "selection_path": PYTEST_SELECTION_PATH, - "summary_path": PYTEST_SUMMARY_PATH, - "output_path": PYTEST_OUTPUT_PATH, - }, - ) + elif interrupted: + metadata = {"diagnosis": "verification_interrupted", "termination_reason": "operator_interrupt"} if result.returncode == 0: sys.stderr.write(f"ok ({elapsed:.1f}s)\n") else: @@ -1820,9 +1822,37 @@ def _run( run.finish_step( step_id=artifacts.step_id, result={"duration_s": round(elapsed, 2), "exit": result.returncode, **metadata} ) + if is_pytest and artifacts is not None: + copy_current_pytest_artifacts( + Path.cwd(), + artifacts, + legacy_paths={ + "progress_path": PYTEST_PROGRESS_PATH, + "events_merged_path": PYTEST_EVENTS_PATH, + "selection_path": PYTEST_SELECTION_PATH, + "summary_path": PYTEST_SUMMARY_PATH, + "output_path": PYTEST_OUTPUT_PATH, + }, + ) return result.returncode, elapsed, metadata +def _pytest_command_basetemp(cmd: Sequence[str], *, cwd: str | None) -> Path | None: + """Return the effective explicit pytest basetemp, if the command has one.""" + raw_path: str | None = None + for index, argument in enumerate(cmd): + if argument.startswith("--basetemp="): + raw_path = argument.partition("=")[2] + elif argument == "--basetemp" and index + 1 < len(cmd): + raw_path = cmd[index + 1] + if not raw_path: + return None + path = Path(raw_path) + if path.is_absolute(): + return path + return (Path(cwd) if cwd is not None else Path.cwd()) / path + + def _subprocess_env() -> dict[str, str]: env = normalize_pytest_basetemp_env(os.environ) env["POLYLOGUE_ROOT"] = str(ROOT) @@ -2115,7 +2145,18 @@ def _compare_against_last(step_results: list[dict[str, Any]]) -> list[str]: entries = _load_history() if len(entries) < 1: return [] - last = entries[-1] + current_names = {str(step.get("name")) for step in step_results if isinstance(step.get("name"), str)} + last = next( + ( + entry + for entry in reversed(entries) + if entry.get("tier") != "focused-test" + and any(isinstance(step, dict) and step.get("name") in current_names for step in entry.get("steps", [])) + ), + None, + ) + if last is None: + return [] last_steps = {s["name"]: s["duration_s"] for s in last.get("steps", [])} flags: list[str] = [] for s in step_results: @@ -3405,11 +3446,12 @@ def main(argv: list[str] | None = None) -> int: ) except RuntimeError as exc: sys.stderr.write(f"verify: {exc}\n") - verify_run.finish( + early_payload = verify_run.finish( exit_code=125, duration_s=time.monotonic() - t0, diagnosis="testmon_environment_identity_unavailable", ) + _save_history(early_payload) return 125 resume_testmon_seed = _testmon_seed_can_resume(seed_identity) prepared_seed_attempt = _prepare_testmon_seed_attempt( @@ -3441,7 +3483,12 @@ def main(argv: list[str] | None = None) -> int: ) except PytestResourceError as exc: sys.stderr.write(f"verify: {exc}\n") - verify_run.finish(exit_code=125, duration_s=time.monotonic() - t0, diagnosis="pytest_resource_preflight_failed") + early_payload = verify_run.finish( + exit_code=125, + duration_s=time.monotonic() - t0, + diagnosis="pytest_resource_preflight_failed", + ) + _save_history(early_payload) return 125 step_results: list[dict[str, Any]] = [] @@ -3580,7 +3627,7 @@ def main(argv: list[str] | None = None) -> int: _refresh_testmon_selection_attempt(step=step_result, run=verify_run, exit_code=rc) if rc != 0: exit_code = rc - if _stop_after_failed_step(label): + if rc == 130 or _stop_after_failed_step(label): break seed_receipt: dict[str, Any] | None = None @@ -3605,6 +3652,7 @@ def main(argv: list[str] | None = None) -> int: "git_head": head, "tier": tier, "run_id": verify_run.run_id, + "checkout_root": str(ROOT.resolve()), "artifact_dir": str(verify_run.relative_run_dir), "steps": step_results, "total_duration_s": total_duration, diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 62ec092209..829e35a57e 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -49,6 +49,7 @@ PYTEST_BASETEMP_MIN_FREE_MB_ENV = "POLYLOGUE_PYTEST_BASETEMP_MIN_FREE_MB" DEFAULT_PYTEST_BASETEMP_MIN_FREE_MB = 1024 PYTEST_BASETEMP_REQUIRED_MB_ENV = "POLYLOGUE_PYTEST_BASETEMP_REQUIRED_MB" +PYTEST_EXPLICIT_BASETEMP_ENV = "POLYLOGUE_PYTEST_EXPLICIT_BASETEMP" PYTEST_MEMORY_ENVELOPE_WORKERS = 4 PYTEST_MEMORY_ENVELOPE_PSS_KB = 4_353_168 PYTEST_MEMORY_ENVELOPE_TMPFS_KB = 1_472_636 @@ -219,6 +220,7 @@ def aggregate_pytest_statistics( phase_outcomes: dict[str, dict[str, int]] = {"setup": {}, "call": {}, "teardown": {}} nodes: set[str] = set() workers: set[str] = set() + reports: dict[tuple[str, str], dict[str, Any]] = {} for row in events: worker = row.get("worker_id") if isinstance(worker, str): @@ -227,17 +229,43 @@ def aggregate_pytest_statistics( if isinstance(nodeid, str) and nodeid: nodes.add(nodeid) event = row.get("event") - if event == "test_report": - when = row.get("when") - duration = row.get("duration_s") - if when in phases and isinstance(duration, (int, float)): - phases[when].append(float(duration)) - outcome = row.get("outcome") - if isinstance(outcome, str) and when in phase_outcomes: - bucket = phase_outcomes[when] - bucket[outcome] = bucket.get(outcome, 0) + 1 - if when == "call": - outcomes[outcome] = outcomes.get(outcome, 0) + 1 + if event != "test_report" or not isinstance(nodeid, str) or not nodeid: + continue + when = row.get("when") + if when not in phases: + continue + key = (nodeid, when) + prior = reports.get(key) + # xdist sends the worker's original report to the controller. Prefer + # the worker event when both arrive, while accepting old/controller-only + # artifacts produced before that forwarding copy was suppressed. + if prior is None or (prior.get("worker_id") == "controller" and row.get("worker_id") != "controller"): + reports[key] = row + + reports_by_node: dict[str, dict[str, dict[str, Any]]] = {} + for (nodeid, when), row in reports.items(): + reports_by_node.setdefault(nodeid, {})[when] = row + duration = row.get("duration_s") + if isinstance(duration, (int, float)): + phases[when].append(float(duration)) + outcome = row.get("outcome") + if isinstance(outcome, str): + bucket = phase_outcomes[when] + bucket[outcome] = bucket.get(outcome, 0) + 1 + + for node_reports in reports_by_node.values(): + setup = node_reports.get("setup", {}).get("outcome") + call = node_reports.get("call", {}).get("outcome") + teardown = node_reports.get("teardown", {}).get("outcome") + if setup == "failed" or teardown == "failed": + terminal = "error" + elif isinstance(call, str): + terminal = call + elif setup == "skipped" or teardown == "skipped": + terminal = "skipped" + else: + continue + outcomes[terminal] = outcomes.get(terminal, 0) + 1 resources: list[dict[str, Any]] = [] resources_path = step_dir / "resources.jsonl" @@ -270,6 +298,7 @@ def aggregate_pytest_statistics( if isinstance(raw, dict): containment = raw + parent_cleanup = (step_result or {}).get("basetemp_cleanup") return { "schema_version": 1, "command": [str(value) for value in command], @@ -312,7 +341,9 @@ def aggregate_pytest_statistics( ), }, "cleanup": { - "complete": containment.get("tmpfs_cleanup_complete"), + "complete": True + if isinstance(parent_cleanup, str) and parent_cleanup + else containment.get("tmpfs_cleanup_complete"), "termination_reason": containment.get("termination_reason"), "escalated_to_sigkill": containment.get("escalated_to_sigkill"), "exit_code": containment.get("exit_code", (step_result or {}).get("exit")), @@ -461,10 +492,17 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> None: step.update(result) step["finished_at"] = utc_now() step["status"] = "success" if result.get("exit") == 0 else "failed" - statistics_path = self.run_dir / "steps" / step_id / "statistics.json" + step_dir = self.run_dir / "steps" / step_id + if not str(step.get("name", "")).startswith("pytest"): + break + # An interrupted runner never returns through the normal + # post-subprocess merge. Fold shards here, before every + # aggregation path, so completed worker evidence survives. + merge_worker_events(step_dir / "events", step_dir / "events.jsonl") + statistics_path = step_dir / "statistics.json" with contextlib.suppress(OSError, ValueError): statistics = aggregate_pytest_statistics( - self.run_dir / "steps" / step_id, + step_dir, command=step.get("cmd", []), step_result=result, ) @@ -759,40 +797,22 @@ def _fs_usage(path: Path) -> dict[str, int] | None: return None -def _dir_size_kb(path: Path) -> int | None: +def _dir_usage_kb(path: Path) -> tuple[int | None, int | None]: + """Measure apparent and allocated file bytes in one filesystem walk.""" if not path.exists(): - return None - total = 0 + return None, None + logical_total = 0 + allocated_total = 0 try: for item in path.rglob("*"): with contextlib.suppress(OSError): if item.is_file(): - total += item.stat().st_size + item_stat = item.stat() + logical_total += item_stat.st_size + allocated_total += item_stat.st_blocks * 512 except OSError: - return None - return int(total / 1024) - - -def _dir_allocated_kb(path: Path) -> int | None: - """Return filesystem blocks charged to files beneath *path*. - - This is deliberately reported alongside apparent bytes. On btrfs it is - the filesystem's per-file block charge (and may differ from compressed or - shared physical allocation); on tmpfs it is the RAM-backed block charge. - It is still the useful apples-to-apples signal available without requiring - filesystem-specific ioctl tooling in the test harness. - """ - if not path.exists(): - return None - total = 0 - try: - for item in path.rglob("*"): - with contextlib.suppress(OSError): - if item.is_file(): - total += item.stat().st_blocks * 512 - except OSError: - return None - return int(total / 1024) + return None, None + return int(logical_total / 1024), int(allocated_total / 1024) def checkout_hash(root: Path) -> str: @@ -995,14 +1015,21 @@ def apply_managed_pytest_runtime_policy( unrelated command minutes or hours later. """ normalized = normalize_pytest_basetemp_env(env) + explicit_basetemp = normalized.get(PYTEST_EXPLICIT_BASETEMP_ENV) default_full_suite_scratch = ( full_suite + and explicit_basetemp is None and not normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") and "POLYLOGUE_PYTEST_TMPFS" not in normalized ) configured_root = normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") configured_tmpfs = configured_root is not None and _is_beneath(Path(configured_root), PYTEST_TMPFS_ROOT) - manages_tmpfs = configured_tmpfs or (configured_root is None and normalized.get("POLYLOGUE_PYTEST_TMPFS") != "0") + explicit_tmpfs = explicit_basetemp is not None and _is_beneath(Path(explicit_basetemp), PYTEST_TMPFS_ROOT) + manages_tmpfs = ( + explicit_tmpfs + or configured_tmpfs + or (explicit_basetemp is None and configured_root is None and normalized.get("POLYLOGUE_PYTEST_TMPFS") != "0") + ) policy = adaptive_pytest_runtime_policy( worker_count=worker_count, shm_free_kb=None if manages_tmpfs else 0, @@ -1036,11 +1063,28 @@ def apply_managed_pytest_runtime_policy( # Keep that ceiling for explicit tmpfs runs; use NVMe for the default # broad route instead of guessing the next aggregate peak. normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" - selected_root, selected_label = resolve_pytest_basetemp_root(normalized) - if not normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") and selected_root != PYTEST_TMPFS_ROOT: + if explicit_basetemp is not None: + selected_root = Path(explicit_basetemp) + selected_label = "explicit" + free_kb = _headroom_kb(selected_root) + min_free_kb = pytest_basetemp_min_free_kb(normalized) + if free_kb is None or free_kb < min_free_kb: + raise PytestResourceError( + "explicit pytest basetemp does not have enough free space " + f"({selected_root}: {free_kb / 1024:.0f} MiB free, need >= {min_free_kb / 1024:.0f} MiB)" + if free_kb is not None + else f"explicit pytest basetemp is unreachable: {selected_root}" + ) + else: + selected_root, selected_label = resolve_pytest_basetemp_root(normalized) + free_kb = _headroom_kb(selected_root) + if ( + explicit_basetemp is None + and not normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") + and selected_root != PYTEST_TMPFS_ROOT + ): normalized["POLYLOGUE_PYTEST_BASETEMP_ROOT"] = str(selected_root) normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" - free_kb = _headroom_kb(selected_root) required_kb = pytest_basetemp_required_kb(normalized) policy = replace( policy, @@ -1228,6 +1272,9 @@ def pytest_basetemp_path(*, root: Path, run_id: str, env: dict[str, str]) -> Pat refusal here would just be noise for a monitoring/cleanup path. Fall back to the top placement candidate, ignoring headroom, rather than raising. """ + explicit = env.get(PYTEST_EXPLICIT_BASETEMP_ENV) + if explicit: + return Path(explicit) try: scratch_root, _label = resolve_pytest_basetemp_root(env) except PytestResourceError: @@ -1245,7 +1292,8 @@ def pytest_basetemp_path(*, root: Path, run_id: str, env: dict[str, str]) -> Pat def pytest_tmpfs_budget_kb(env: Mapping[str, str]) -> int | None: """Return the bounded per-run tmpfs budget shared by all pytest workers.""" - configured_root = env.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") + explicit = env.get(PYTEST_EXPLICIT_BASETEMP_ENV) + configured_root = explicit or env.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") configured_tmpfs = configured_root is not None and _is_beneath(Path(configured_root), PYTEST_TMPFS_ROOT) if env.get("POLYLOGUE_PYTEST_TMPFS") != "1" or (configured_root is not None and not configured_tmpfs): return None @@ -1266,6 +1314,8 @@ def cleanup_managed_pytest_basetemp(*, root: Path, run_id: str, env: dict[str, s basetemps immediately instead of waiting for the next pytest startup sweep. """ + if env.get(PYTEST_EXPLICIT_BASETEMP_ENV): + return None basetemp = pytest_basetemp_path(root=root, run_id=run_id, env=env) if not basetemp.name.startswith("pytest-polylogue-") or "-seeded-" in basetemp.name: return None @@ -1338,8 +1388,7 @@ def _sample_basetemp_sizes(self, *, event: str) -> tuple[int | None, int | None] or now - self._last_basetemp_size_sample_at >= self._basetemp_size_interval_s ) if should_sample: - self._last_basetemp_size_kb = _dir_size_kb(self._basetemp) - self._last_basetemp_allocated_kb = _dir_allocated_kb(self._basetemp) + self._last_basetemp_size_kb, self._last_basetemp_allocated_kb = _dir_usage_kb(self._basetemp) self._last_basetemp_size_sample_at = now return self._last_basetemp_size_kb, self._last_basetemp_allocated_kb diff --git a/tests/conftest.py b/tests/conftest.py index 6c27fa58e7..8abd97ae15 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -321,6 +321,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: @pytest.fixture(autouse=True) def _reclaim_test_tmp_path( tmp_path: Path, + request: pytest.FixtureRequest, ) -> Iterator[None]: """Release each test's private tree as soon as its teardown finishes. @@ -328,8 +329,14 @@ def _reclaim_test_tmp_path( not in an unbounded filesystem witness. Retaining every failed tree made full-suite tmpfs usage proportional to the number of failures and caused a calm 8-worker run to exceed 2 GiB before completing. A failing node can - still be rerun with an explicit basetemp when its files matter. + still be rerun with an explicit basetemp when its files matter. That + explicit diagnostic path retains its per-test trees for inspection. """ + configured = getattr(request.config.option, "basetemp", None) + managed = os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") + if configured and str(configured) != managed: + yield + return try: yield finally: diff --git a/tests/unit/devtools/test_evidence_dashboard.py b/tests/unit/devtools/test_evidence_dashboard.py new file mode 100644 index 0000000000..403cf59e54 --- /dev/null +++ b/tests/unit/devtools/test_evidence_dashboard.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from devtools import evidence_dashboard + + +def test_static_gates_read_shared_verify_history(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + history = tmp_path / "xdg-state" / "polylogue" / "devtools" / "verify-history.jsonl" + history.parent.mkdir(parents=True) + history.write_text( + json.dumps( + { + "timestamp": "2026-08-12T00:00:00+00:00", + "steps": [{"name": "ruff check", "duration_s": 1.0, "exit": 0}], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + ruff = next(gate for gate in gates["gates"] if gate["name"] == "ruff check") + assert gates["history_path"] == str(history) + assert ruff["status"] == "ok" diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index 0f9c3d0dde..cd7f0f4844 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -50,6 +50,7 @@ class _Report: outcome: str duration: float = 0.0 longrepr: str = "" + worker_id: str | None = None def test_progress_plugin_records_call_and_setup_failures( @@ -75,6 +76,22 @@ def test_progress_plugin_records_call_and_setup_failures( assert events[2]["longrepr"] == "fixture exploded" +def test_progress_plugin_skips_xdist_controller_forwarding_copy( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + events_path = tmp_path / "events.jsonl" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_PATH", str(events_path)) + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw0") + pytest_progress_plugin.pytest_runtest_logreport(_Report("test_one", "call", "passed", worker_id="gw0")) + + monkeypatch.delenv("PYTEST_XDIST_WORKER") + pytest_progress_plugin.pytest_runtest_logreport(_Report("test_one", "call", "passed", worker_id="gw0")) + + events = [json.loads(line) for line in events_path.read_text().splitlines()] + assert [(event["nodeid"], event["when"], event["worker_id"]) for event in events] == [("test_one", "call", "gw0")] + + def test_managed_event_ledger_survives_test_host_environment_scrub(tmp_path: Path) -> None: events_dir = tmp_path / "events" checkout_root = Path(__file__).resolve().parents[3] diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index b3105b0979..9cfdd84819 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1003,9 +1003,13 @@ def test_aggregate_pytest_statistics_reduces_phases_fixtures_and_resources(tmp_p ) + "\n" ) - (step / "containment.json").write_text(json.dumps({"tmpfs_cleanup_complete": True, "exit_code": 0})) + (step / "containment.json").write_text(json.dumps({"tmpfs_cleanup_complete": False, "exit_code": 0})) - result = aggregate_pytest_statistics(step, command=["pytest"], step_result={"exit": 0}) + result = aggregate_pytest_statistics( + step, + command=["pytest"], + step_result={"exit": 0, "basetemp_cleanup": "/realm/tmp/polylogue-pytest/pytest-polylogue-run"}, + ) assert result["node_count"] == 1 assert result["phases"]["call"]["p50_s"] == 2.0 @@ -1015,6 +1019,63 @@ def test_aggregate_pytest_statistics_reduces_phases_fixtures_and_resources(tmp_p assert result["cleanup"]["complete"] is True +def test_aggregate_pytest_statistics_deduplicates_xdist_reports_and_terminal_failures(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + rows = [ + { + "event": "test_report", + "nodeid": "test_setup", + "when": "setup", + "outcome": "failed", + "duration_s": 1.0, + "worker_id": "gw0", + }, + { + "event": "test_report", + "nodeid": "test_setup", + "when": "setup", + "outcome": "failed", + "duration_s": 1.0, + "worker_id": "controller", + }, + { + "event": "test_report", + "nodeid": "test_teardown", + "when": "call", + "outcome": "passed", + "duration_s": 0.2, + "worker_id": "gw1", + }, + { + "event": "test_report", + "nodeid": "test_teardown", + "when": "teardown", + "outcome": "failed", + "duration_s": 0.3, + "worker_id": "gw1", + }, + ] + (step / "events.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) + + result = aggregate_pytest_statistics(step, command=["pytest", "-n", "2"]) + + assert result["phases"]["setup"]["count"] == 1 + assert result["xdist"]["worker_count"] == 2 + assert result["outcomes"] == {"error": 2} + + +def test_verify_run_statistics_only_cover_pytest_steps(tmp_path: Path) -> None: + run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="ruff check", cmd=["ruff", "check"]) + + run.finish_step(step_id=artifacts.step_id, result={"exit": 0, "duration_s": 0.1}) + + step = run._payload["steps"][0] + assert "statistics" not in step + assert not artifacts.statistics_path.exists() + + def test_verify_run_embeds_compact_statistics_before_worktree_cleanup(tmp_path: Path) -> None: run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) artifacts = run.start_step(label="pytest focused", cmd=["pytest", "tests/unit/example.py"]) @@ -1034,12 +1095,37 @@ def test_verify_run_embeds_compact_statistics_before_worktree_cleanup(tmp_path: run.finish_step(step_id=artifacts.step_id, result={"exit": 0, "duration_s": 0.25}) payload = run.finish(exit_code=0, duration_s=0.25) + shutil.rmtree(run.run_dir) statistics = payload["steps"][0]["statistics"] assert statistics["node_count"] == 1 assert statistics["phases"]["call"]["p50_s"] == 0.25 +def test_interrupted_run_merges_worker_events_before_statistics(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="pytest focused", cmd=["pytest", "tests/unit/example.py"]) + artifacts.events_dir.mkdir() + (artifacts.events_dir / "gw0-1.jsonl").write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/unit/example.py::test_one", + "when": "call", + "outcome": "passed", + "duration_s": 0.2, + "worker_id": "gw0", + } + ) + + "\n" + ) + + run.finish_interrupted_steps(exit_code=130, diagnosis="pytest_interrupted") + + assert artifacts.events_merged_path.exists() + assert run._payload["steps"][0]["statistics"]["node_count"] == 1 + + def test_print_history_accepts_verify_and_focused_run_records( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], @@ -1085,6 +1171,21 @@ def test_verify_history_appends_concurrent_records_without_interleaving(tmp_path assert sorted(row["sequence"] for row in rows) == list(range(64)) +def test_compare_against_last_skips_intervening_focused_history(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + verify, + "_load_history", + lambda: [ + {"tier": "quick", "steps": [{"name": "ruff check", "duration_s": 1.0}]}, + {"tier": "focused-test", "steps": [{"name": "pytest focused", "duration_s": 999.0}]}, + ], + ) + + flags = verify._compare_against_last([{"name": "ruff check", "duration_s": 7.0}]) + + assert flags and "ruff check" in flags[0] + + def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) @@ -2020,12 +2121,12 @@ def test_resource_sampler_throttles_basetemp_size_walk(tmp_path: Path, monkeypat (basetemp / "artifact.txt").write_text("payload") calls = 0 - def counted_size(_path: Path) -> int: + def counted_usage(_path: Path) -> tuple[int, int]: nonlocal calls calls += 1 - return calls + return calls, calls + 1 - monkeypatch.setattr("devtools.verify_runs._dir_size_kb", counted_size) + monkeypatch.setattr("devtools.verify_runs._dir_usage_kb", counted_usage) sampler = ResourceSampler( root_pid=os.getpid(), run_id="test-run", @@ -2038,6 +2139,7 @@ def counted_size(_path: Path) -> int: second = sampler.sample(event="sample") assert first["basetemp_size_kb"] == 1 + assert first["basetemp_allocated_kb"] == 2 assert second["basetemp_size_kb"] == 1 assert calls == 1 @@ -2864,6 +2966,45 @@ def test_explicit_basetemp_root_retains_managed_resource_monitoring( assert metadata["resource_sample_count"] >= 1 +def test_run_propagates_explicit_basetemp_to_resource_policy(tmp_path: Path) -> None: + explicit = tmp_path / "diagnostic-basetemp" + captured: dict[str, str] = {} + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + + def apply_policy(env: dict[str, str], **_kwargs: object) -> tuple[dict[str, str], None]: + captured.update(env) + return env, None + + with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", side_effect=apply_policy), + patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), + patch("devtools.verify._read_pytest_report", return_value=None), + ): + rc, _elapsed, _metadata = _run("pytest focused", ["pytest", "--basetemp", str(explicit)]) + + assert rc == 0 + assert captured["POLYLOGUE_PYTEST_EXPLICIT_BASETEMP"] == str(explicit) + + +def test_explicit_basetemp_policy_uses_actual_path_for_admission( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=15_190) + explicit = tmp_path / "diagnostic-basetemp" + explicit.mkdir() + monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 32 * 1024 * 1024) + + _env, policy = apply_managed_pytest_runtime_policy( + {"POLYLOGUE_PYTEST_EXPLICIT_BASETEMP": str(explicit)}, worker_count=0, full_suite=False + ) + + assert policy is not None + assert policy.basetemp_root == str(explicit) + assert policy.basetemp_label == "explicit" + + def test_run_receipt_uses_capped_pytest_command_concurrency() -> None: completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") @@ -3536,7 +3677,7 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo patch("devtools.verify._run", side_effect=fake_run), patch("devtools.verify.build_verify_steps", return_value=[("pytest full", ["pytest"])]), patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._save_history"), + patch("devtools.verify._save_history") as save_history, patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), ): @@ -3546,6 +3687,7 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo assert payload["verification_scope"] == expected_scope assert payload["release_baseline_allowed"] is expected_permission assert payload["terminal_authorization"] == ("narrow-terminal" if expected_permission else None) + assert save_history.call_args.args[0]["checkout_root"] == str(ROOT.resolve()) def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.CaptureFixture[str]) -> None: @@ -3554,11 +3696,13 @@ def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.Ca patch("devtools.verify._git_head", return_value="head"), patch("devtools.verify._testmon_preflight", return_value=None), patch("devtools.verify._run") as run, + patch("devtools.verify._save_history") as save_history, ): rc = main(["--json"]) assert rc == 125 run.assert_not_called() + assert save_history.call_args.args[0]["diagnosis"] == "pytest_resource_preflight_failed" assert "only 0.50 GiB available" in capsys.readouterr().err diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index ccc1c9389a..7c1ea41bf5 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -58,7 +58,8 @@ def test_test_tmp_path_reclamation_runs_after_failure_or_interrupt( tree = tmp_path / "test-private" tree.mkdir() fixture_generator = cast(Any, conftest._reclaim_test_tmp_path).__wrapped__ - cleanup = cast("Generator[None, BaseException, None]", fixture_generator(tree)) + request = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=None))) + cleanup = cast("Generator[None, BaseException, None]", fixture_generator(tree, request)) assert next(cleanup) is None with pytest.raises(type(exception)): @@ -67,6 +68,25 @@ def test_test_tmp_path_reclamation_runs_after_failure_or_interrupt( assert not tree.exists() +def test_test_tmp_path_reclamation_keeps_explicit_diagnostic_basetemp( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + explicit = tmp_path / "diagnostic" + tree = explicit / "test-private" + tree.mkdir(parents=True) + fixture_generator = cast(Any, conftest._reclaim_test_tmp_path).__wrapped__ + request = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(explicit)))) + monkeypatch.delenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", raising=False) + cleanup = cast("Generator[None, BaseException, None]", fixture_generator(tree, request)) + + assert next(cleanup) is None + with pytest.raises(RuntimeError): + cleanup.throw(RuntimeError("failed diagnostic rerun")) + + assert tree.exists() + + def test_managed_pytest_temp_root_defaults_to_scratch( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From cf67f448b5914cff305ef4ed10c800ad1a8f4181 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 23:35:53 +0200 Subject: [PATCH 10/53] fix(test): retain terminal verify evidence Problem: exact-head review found that ordinary verify history omitted embedded pytest aggregates and that interruption could finalize before the supervisor stopped.\n\nWhat changed: propagate finalized statistics into ordinary verify step results and wait for containment, with bounded forced cleanup, before interrupt cleanup or finalization.\n\nVerification: focused verify-evidence harness (215 passed) and devtools verify --quick (25 steps passed). --- devtools/verify.py | 39 ++++++++++++++++++++++++++-- tests/unit/devtools/test_verify.py | 41 +++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 0522f06629..514324be56 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -783,6 +783,29 @@ def _request_supervisor_termination( process.send_signal(signal.SIGTERM) +def _await_interrupted_pytest_containment( + process: subprocess.Popen[bytes], + launch: SupervisorLaunch, + *, + term_grace_s: float, + preserved_runner_descendants: Sequence[tuple[int, int]], +) -> None: + """Wait for an interrupted pytest supervisor before its caller cleans up.""" + if process.poll() is None: + _request_supervisor_termination(process, launch, reason="pytest runner interrupted") + try: + process.wait(timeout=max(1.0, term_grace_s + 1.0)) + except subprocess.TimeoutExpired: + _force_kill_owned_run( + process, + launch, + preserved_runner_descendants=preserved_runner_descendants, + ) + with contextlib.suppress(subprocess.TimeoutExpired): + process.wait(timeout=1.0) + reap_exited_children() + + def _force_kill_owned_run( process: subprocess.Popen[bytes], launch: SupervisorLaunch, @@ -1424,8 +1447,12 @@ def _refresh_progress_marker(at: float, latest: dict[str, Any] | None = None) -> if process.poll() is not None and not selector.get_map(): break except BaseException: - if process.poll() is None: - _request_supervisor_termination(process, launch, reason="pytest runner interrupted") + _await_interrupted_pytest_containment( + process, + launch, + term_grace_s=term_grace_s, + preserved_runner_descendants=preserved_runner_descendants, + ) raise finally: selector.close() @@ -1822,6 +1849,14 @@ def _run( run.finish_step( step_id=artifacts.step_id, result={"duration_s": round(elapsed, 2), "exit": result.returncode, **metadata} ) + finalized_step = next( + (step for step in run._payload["steps"] if step.get("step_id") == artifacts.step_id), + None, + ) + if isinstance(finalized_step, dict): + for key in ("statistics", "statistics_path"): + if key in finalized_step: + metadata[key] = finalized_step[key] if is_pytest and artifacts is not None: copy_current_pytest_artifacts( Path.cwd(), diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 9cfdd84819..085d2ed7da 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -9,7 +9,7 @@ import sys from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest @@ -1126,6 +1126,45 @@ def test_interrupted_run_merges_worker_events_before_statistics(tmp_path: Path) assert run._payload["steps"][0]["statistics"]["node_count"] == 1 +def test_run_returns_finalized_statistics_for_verify_history(tmp_path: Path) -> None: + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) + + with ( + patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), + patch("devtools.verify._read_pytest_report", return_value=None), + ): + rc, _elapsed, metadata = _run("pytest testmon", ["pytest", "-n", "0"], run=run) + + assert rc == 0 + assert metadata["statistics"]["node_count"] == 0 + assert metadata["statistics_path"].endswith("statistics.json") + + +def test_interrupted_pytest_waits_for_containment_before_cleanup() -> None: + process = MagicMock() + process.poll.return_value = None + process.wait.side_effect = [subprocess.TimeoutExpired(cmd="pytest", timeout=2.0), None] + launch = MagicMock() + + with ( + patch("devtools.verify._request_supervisor_termination") as request_termination, + patch("devtools.verify._force_kill_owned_run") as force_kill, + patch("devtools.verify.reap_exited_children") as reap, + ): + verify._await_interrupted_pytest_containment( + process, + launch, + term_grace_s=1.0, + preserved_runner_descendants=(), + ) + + request_termination.assert_called_once() + force_kill.assert_called_once_with(process, launch, preserved_runner_descendants=()) + assert process.wait.call_count == 2 + reap.assert_called_once() + + def test_print_history_accepts_verify_and_focused_run_records( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], From c95cd8bdedabb9c809c1da05208d143426b1d27c Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 12 Aug 2026 23:44:19 +0200 Subject: [PATCH 11/53] fix(test): finalize verify run evidence Problem: review found that ordinary verify history could omit finalized pytest aggregates, and interruption cleanup could continue without confirmed process containment.\n\nWhat changed: expose finalized step data through VerifyRun for ordinary history rows, and fail closed when forced containment cannot quiesce before cleanup or finalization.\n\nVerification: focused verify-evidence harness (218 passed, 25.17s) and devtools verify --quick --json (25 steps passed, 142.94s). --- devtools/verify.py | 31 ++++++--- devtools/verify_runs.py | 4 +- tests/unit/devtools/test_verify.py | 107 ++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 15 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 514324be56..0265fe307c 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -783,6 +783,10 @@ def _request_supervisor_termination( process.send_signal(signal.SIGTERM) +class PytestContainmentError(RuntimeError): + """Raised when an interrupted pytest supervisor cannot be confirmed stopped.""" + + def _await_interrupted_pytest_containment( process: subprocess.Popen[bytes], launch: SupervisorLaunch, @@ -801,8 +805,12 @@ def _await_interrupted_pytest_containment( launch, preserved_runner_descendants=preserved_runner_descendants, ) - with contextlib.suppress(subprocess.TimeoutExpired): + try: process.wait(timeout=1.0) + except subprocess.TimeoutExpired as exc: + raise PytestContainmentError( + "pytest containment did not quiesce after forced termination; leaving its basetemp intact" + ) from exc reap_exited_children() @@ -1598,19 +1606,24 @@ def _run( if run is not None and artifacts is not None: env = env_for_pytest_step(env, run=run, artifacts=artifacts) interrupted = False + pytest_containment_quiescent = True if is_pytest: try: try: result = _run_pytest_with_heartbeat(cmd, cwd=cwd, env=env, t0=t0, run=run, artifacts=artifacts) + except PytestContainmentError: + pytest_containment_quiescent = False + raise except KeyboardInterrupt: interrupted = True result = subprocess.CompletedProcess(args=cmd, returncode=130, stdout="", stderr="") finally: - basetemp_cleanup = cleanup_managed_pytest_basetemp( - root=ROOT, - run_id=env.get("POLYLOGUE_PYTEST_RUN_ID", ""), - env=env, - ) + if pytest_containment_quiescent: + basetemp_cleanup = cleanup_managed_pytest_basetemp( + root=ROOT, + run_id=env.get("POLYLOGUE_PYTEST_RUN_ID", ""), + env=env, + ) else: try: result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) @@ -1846,13 +1859,9 @@ def _run( if result.stderr.strip(): sys.stderr.write(result.stderr + "\n") if run is not None and artifacts is not None: - run.finish_step( + finalized_step = run.finish_step( step_id=artifacts.step_id, result={"duration_s": round(elapsed, 2), "exit": result.returncode, **metadata} ) - finalized_step = next( - (step for step in run._payload["steps"] if step.get("step_id") == artifacts.step_id), - None, - ) if isinstance(finalized_step, dict): for key in ("statistics", "statistics_path"): if key in finalized_step: diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 829e35a57e..83b749bc43 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -486,7 +486,8 @@ def start_step(self, *, label: str, cmd: list[str]) -> PytestStepArtifacts: self.write() return artifacts - def finish_step(self, *, step_id: str, result: dict[str, Any]) -> None: + def finish_step(self, *, step_id: str, result: dict[str, Any]) -> dict[str, Any] | None: + """Finalize one step and return its durable compact representation.""" for step in self._payload["steps"]: if step.get("step_id") == step_id: step.update(result) @@ -516,6 +517,7 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> None: step["statistics"] = statistics break self.write() + return next((dict(step) for step in self._payload["steps"] if step.get("step_id") == step_id), None) def finish_interrupted_steps(self, *, exit_code: int, diagnosis: str) -> None: """Close any open step when the outer runner receives Ctrl-C.""" diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 085d2ed7da..474268481c 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -9,6 +9,7 @@ import sys from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from typing import Any from unittest.mock import MagicMock, patch import pytest @@ -1129,19 +1130,48 @@ def test_interrupted_run_merges_worker_events_before_statistics(tmp_path: Path) def test_run_returns_finalized_statistics_for_verify_history(tmp_path: Path) -> None: completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) + history_path = tmp_path / "state" / "verify-history.jsonl" + + def _complete_with_evidence( + *_args: object, artifacts: object, **_kwargs: object + ) -> subprocess.CompletedProcess[str]: + assert isinstance(artifacts, verify_runs.PytestStepArtifacts) + artifacts.events_merged_path.write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/unit/example.py::test_one", + "when": "call", + "duration_s": 0.25, + "outcome": "passed", + "worker_id": "controller", + } + ) + + "\n" + ) + artifacts.resources_path.write_text(json.dumps({"tree_rss_kb": 512}) + "\n") + return completed with ( - patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), + patch("devtools.verify._run_pytest_with_heartbeat", side_effect=_complete_with_evidence), patch("devtools.verify._read_pytest_report", return_value=None), + patch("devtools.verify.copy_current_pytest_artifacts"), ): rc, _elapsed, metadata = _run("pytest testmon", ["pytest", "-n", "0"], run=run) assert rc == 0 - assert metadata["statistics"]["node_count"] == 0 + append_verify_history( + {"tier": "quick", "steps": [{"name": "pytest testmon", "exit": rc, **metadata}]}, path=history_path + ) + shutil.rmtree(run.run_dir) + + durable_row = json.loads(history_path.read_text(encoding="utf-8")) + assert durable_row["steps"][0]["statistics"]["node_count"] == 1 + assert durable_row["steps"][0]["statistics"]["resources"]["peak_tree_rss_kb"] == 512 assert metadata["statistics_path"].endswith("statistics.json") -def test_interrupted_pytest_waits_for_containment_before_cleanup() -> None: +def test_interrupted_pytest_waits_for_forced_containment_quiescence() -> None: process = MagicMock() process.poll.return_value = None process.wait.side_effect = [subprocess.TimeoutExpired(cmd="pytest", timeout=2.0), None] @@ -1165,6 +1195,77 @@ def test_interrupted_pytest_waits_for_containment_before_cleanup() -> None: reap.assert_called_once() +def test_interrupted_pytest_refuses_cleanup_without_containment_quiescence() -> None: + process = MagicMock() + process.poll.return_value = None + process.wait.side_effect = [ + subprocess.TimeoutExpired(cmd="pytest", timeout=2.0), + subprocess.TimeoutExpired(cmd="pytest", timeout=1.0), + ] + launch = MagicMock() + + with ( + patch("devtools.verify._request_supervisor_termination"), + patch("devtools.verify._force_kill_owned_run") as force_kill, + patch("devtools.verify.reap_exited_children") as reap, + pytest.raises(verify.PytestContainmentError, match="did not quiesce"), + ): + verify._await_interrupted_pytest_containment( + process, + launch, + term_grace_s=1.0, + preserved_runner_descendants=(), + ) + + force_kill.assert_called_once_with(process, launch, preserved_runner_descendants=()) + reap.assert_not_called() + + +def test_run_cleans_and_finalizes_only_after_contained_interrupt(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + order: list[str] = [] + original_finish_step = run.finish_step + + def _contained_interrupt(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + order.append("contained") + raise KeyboardInterrupt + + def _cleanup(**_kwargs: object) -> None: + order.append("cleanup") + return None + + def _finish_step(*, step_id: str, result: dict[str, Any]) -> dict[str, Any] | None: + order.append("finalize") + return original_finish_step(step_id=step_id, result=result) + + with ( + patch("devtools.verify._run_pytest_with_heartbeat", side_effect=_contained_interrupt), + patch("devtools.verify.cleanup_managed_pytest_basetemp", side_effect=_cleanup), + patch.object(run, "finish_step", side_effect=_finish_step), + ): + rc, _elapsed, _metadata = _run("pytest focused", ["pytest", "-n", "0"], run=run) + + assert rc == 130 + assert order == ["contained", "cleanup", "finalize"] + + +def test_run_leaves_basetemp_and_step_open_when_containment_fails(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + + with ( + patch( + "devtools.verify._run_pytest_with_heartbeat", + side_effect=verify.PytestContainmentError("still running"), + ), + patch("devtools.verify.cleanup_managed_pytest_basetemp") as cleanup, + pytest.raises(verify.PytestContainmentError, match="still running"), + ): + _run("pytest focused", ["pytest", "-n", "0"], run=run) + + cleanup.assert_not_called() + assert run._payload["steps"][0]["status"] == "running" + + def test_print_history_accepts_verify_and_focused_run_records( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], From 782b0ef7ad7d16de86bb797d0d120b9b6901b4e8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 00:20:15 +0200 Subject: [PATCH 12/53] fix(test): preserve complete verify-run evidence Repair history framing and per-step comparisons, preserve interrupted and\nresource-refused pytest statistics, and require full containment quiescence\nbefore cleanup.\n\nKeep explicit diagnostic basetemps operator-owned while retaining xdist\ncontroller timing summaries without duplicating event evidence.\n\nRef #3962 --- devtools/pytest_progress_plugin.py | 15 +- devtools/verify.py | 69 ++++++--- devtools/verify_runs.py | 47 +++++- .../devtools/test_pytest_progress_plugin.py | 31 ++++ tests/unit/devtools/test_verify.py | 145 ++++++++++++++++++ 5 files changed, 274 insertions(+), 33 deletions(-) diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index 5014722d3f..283e6d5bc0 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -215,7 +215,7 @@ def pytest_runtest_logfinish(nodeid: str, location: tuple[str, int | None, str]) @pytest.hookimpl -def _record_phase_report(report: Any) -> None: +def _record_phase_report(report: Any, *, write_event: bool = True) -> None: """Append one phase report so slow setup/call/teardown remains visible.""" when = str(getattr(report, "when", "")) nodeid = str(getattr(report, "nodeid", "")) @@ -237,7 +237,8 @@ def _record_phase_report(report: Any) -> None: if payload["outcome"] == "failed": payload["longrepr"] = str(getattr(report, "longrepr", "")) _remember_report(payload) - _write_event(payload) + if write_event: + _write_event(payload) @pytest.hookimpl(hookwrapper=True) @@ -252,9 +253,10 @@ def pytest_runtest_makereport(item: Any, call: Any) -> Any: def pytest_runtest_logreport(report: Any) -> None: """Retain the direct/log-hook fallback used by older pytest plugins/tests.""" # xdist forwards each worker's report to the controller. The worker has - # already written the authoritative shard event through makereport; avoid - # recording that deserialized controller copy a second time. + # already written the authoritative shard event through makereport. Keep + # its timing in the controller's summary, but do not duplicate the ledger. if not os.environ.get("PYTEST_XDIST_WORKER") and getattr(report, "worker_id", None): + _record_phase_report(report, write_event=False) return _record_phase_report(report) @@ -263,6 +265,11 @@ def pytest_runtest_logreport(report: Any) -> None: def pytest_sessionfinish(session: Any, exitstatus: int) -> None: """Write a compact post-run diagnosis artifact independent of pytest-json-report.""" del session + # Worker processes have their own in-memory slowest lists. The controller + # receives the forwarded timings and is the only writer for the shared + # summary path, so an empty worker summary cannot overwrite it. + if os.environ.get("PYTEST_XDIST_WORKER"): + return payload: dict[str, Any] = { "exitstatus": int(exitstatus), "selected_count": _SELECTED_COUNT, diff --git a/devtools/verify.py b/devtools/verify.py index 0265fe307c..f5b001a821 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -812,6 +812,28 @@ def _await_interrupted_pytest_containment( "pytest containment did not quiesce after forced termination; leaving its basetemp intact" ) from exc reap_exited_children() + receipt = read_receipt(launch.receipt_path) + remaining_descendants = tuple( + identity + for identity in descendant_process_identities(os.getpid()) + if identity not in preserved_runner_descendants + ) + if ( + receipt is None + or receipt.get("status") not in {"finished", "terminated"} + or receipt.get("controller_group_alive") is not False + or remaining_descendants + ): + raise PytestContainmentError( + "pytest containment did not quiesce its owned process tree; leaving its basetemp intact" + ) + + +def _supervised_tmpfs_cleanup_path(*, root: Path, run_id: str, env: dict[str, str]) -> Path | None: + """Return only a supervisor-owned tmpfs path eligible for cleanup.""" + if env.get(PYTEST_EXPLICIT_BASETEMP_ENV) or pytest_tmpfs_budget_kb(env) is None: + return None + return pytest_basetemp_path(root=root, run_id=run_id, env=env) def _force_kill_owned_run( @@ -971,14 +993,10 @@ def _run_pytest_with_heartbeat( else Path(env.get("POLYLOGUE_PYTEST_CONTAINMENT_PATH", str(Path.cwd() / PYTEST_CONTAINMENT_PATH))) ) pytest_run_id = run.run_id if run is not None else env.get("POLYLOGUE_PYTEST_RUN_ID", str(os.getpid())) - tmpfs_cleanup_path = ( - pytest_basetemp_path( - root=Path(cwd) if cwd is not None else Path.cwd(), - run_id=pytest_run_id, - env=env, - ) - if tmpfs_budget_kb is not None - else None + tmpfs_cleanup_path = _supervised_tmpfs_cleanup_path( + root=Path(cwd) if cwd is not None else Path.cwd(), + run_id=pytest_run_id, + env=env, ) launch = build_supervisor_launch( cmd, @@ -1590,10 +1608,14 @@ def _run( "release_baseline_allowed": False, } if run is not None and artifacts is not None: - run.finish_step( + finalized_step = run.finish_step( step_id=artifacts.step_id, result={"duration_s": round(elapsed, 2), "exit": 125, **refusal_metadata}, ) + if isinstance(finalized_step, dict): + for key in ("statistics", "statistics_path"): + if key in finalized_step: + refusal_metadata[key] = finalized_step[key] return 125, elapsed, refusal_metadata pytest_tmpfs = env.get("POLYLOGUE_PYTEST_TMPFS") == "1" budget_kb = pytest_tmpfs_budget_kb(env) @@ -2189,22 +2211,23 @@ def _compare_against_last(step_results: list[dict[str, Any]]) -> list[str]: entries = _load_history() if len(entries) < 1: return [] - current_names = {str(step.get("name")) for step in step_results if isinstance(step.get("name"), str)} - last = next( - ( - entry - for entry in reversed(entries) - if entry.get("tier") != "focused-test" - and any(isinstance(step, dict) and step.get("name") in current_names for step in entry.get("steps", [])) - ), - None, - ) - if last is None: - return [] - last_steps = {s["name"]: s["duration_s"] for s in last.get("steps", [])} flags: list[str] = [] for s in step_results: - prev = last_steps.get(s["name"]) + name = s.get("name") + if not isinstance(name, str): + continue + prev = next( + ( + prior.get("duration_s") + for entry in reversed(entries) + if entry.get("tier") != "focused-test" + for prior in entry.get("steps", []) + if isinstance(prior, dict) + and prior.get("name") == name + and isinstance(prior.get("duration_s"), (int, float)) + ), + None, + ) if prev is not None and prev > 0: delta = s["duration_s"] - prev pct = (delta / prev) * 100 diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 83b749bc43..969e396a8d 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -83,9 +83,28 @@ def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTO """ path.parent.mkdir(parents=True, exist_ok=True) payload = (json.dumps(dict(entry), ensure_ascii=False) + "\n").encode() - descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_APPEND, 0o600) try: fcntl.flock(descriptor, fcntl.LOCK_EX) + end = os.lseek(descriptor, 0, os.SEEK_END) + if end: + os.lseek(descriptor, 0, os.SEEK_SET) + existing = bytearray() + while chunk := os.read(descriptor, 64 * 1024): + existing.extend(chunk) + if not existing.endswith(b"\n"): + last_newline = existing.rfind(b"\n") + trailing = bytes(existing[last_newline + 1 :]) + try: + json.loads(trailing) + except (UnicodeDecodeError, json.JSONDecodeError): + os.ftruncate(descriptor, last_newline + 1) + else: + # A complete JSON record can lose only its framing newline + # during an interrupted append. Preserve it before adding + # the next durable record. + os.lseek(descriptor, 0, os.SEEK_END) + os.write(descriptor, b"\n") remaining = memoryview(payload) while remaining: written = os.write(descriptor, remaining) @@ -253,7 +272,8 @@ def aggregate_pytest_statistics( bucket = phase_outcomes[when] bucket[outcome] = bucket.get(outcome, 0) + 1 - for node_reports in reports_by_node.values(): + for nodeid in nodes: + node_reports = reports_by_node.get(nodeid, {}) setup = node_reports.get("setup", {}).get("outcome") call = node_reports.get("call", {}).get("outcome") teardown = node_reports.get("teardown", {}).get("outcome") @@ -264,7 +284,10 @@ def aggregate_pytest_statistics( elif setup == "skipped" or teardown == "skipped": terminal = "skipped" else: - continue + # A test may have emitted its start event just before an interrupt + # or forced containment cleanup. Keep that missing terminal phase + # visible so outcome totals still account for every started node. + terminal = "interrupted" outcomes[terminal] = outcomes.get(terminal, 0) + 1 resources: list[dict[str, Any]] = [] @@ -1069,11 +1092,18 @@ def apply_managed_pytest_runtime_policy( selected_root = Path(explicit_basetemp) selected_label = "explicit" free_kb = _headroom_kb(selected_root) - min_free_kb = pytest_basetemp_min_free_kb(normalized) - if free_kb is None or free_kb < min_free_kb: + required_kb = pytest_basetemp_required_kb(normalized) + min_free_kb = max(pytest_basetemp_min_free_kb(normalized), required_kb or 0) + explicit_required_kb = min_free_kb + if _is_beneath(selected_root, PYTEST_TMPFS_ROOT) and normalized.get("POLYLOGUE_PYTEST_TMPFS") == "1": + explicit_required_kb = pytest_basetemp_min_free_kb(normalized) + max( + required_kb or 0, + pytest_tmpfs_budget_kb(normalized) or 0, + ) + if free_kb is None or free_kb < explicit_required_kb: raise PytestResourceError( "explicit pytest basetemp does not have enough free space " - f"({selected_root}: {free_kb / 1024:.0f} MiB free, need >= {min_free_kb / 1024:.0f} MiB)" + f"({selected_root}: {free_kb / 1024:.0f} MiB free, need >= {explicit_required_kb / 1024:.0f} MiB)" if free_kb is not None else f"explicit pytest basetemp is unreachable: {selected_root}" ) @@ -1321,6 +1351,11 @@ def cleanup_managed_pytest_basetemp(*, root: Path, run_id: str, env: dict[str, s basetemp = pytest_basetemp_path(root=root, run_id=run_id, env=env) if not basetemp.name.startswith("pytest-polylogue-") or "-seeded-" in basetemp.name: return None + # A serial pytest child may already have reclaimed this exact run-owned + # directory in sessionfinish. That is a completed cleanup, not an absent + # receipt for the durable summary to misclassify. + if not basetemp.exists(): + return basetemp with contextlib.suppress(OSError): if basetemp.exists(): shutil.rmtree(basetemp) diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index cd7f0f4844..962b22681d 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -92,6 +92,37 @@ def test_progress_plugin_skips_xdist_controller_forwarding_copy( assert [(event["nodeid"], event["when"], event["worker_id"]) for event in events] == [("test_one", "call", "gw0")] +def test_progress_plugin_keeps_xdist_worker_timings_in_controller_summary( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + events_path = tmp_path / "events.jsonl" + summary_path = tmp_path / "summary.json" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_PATH", str(events_path)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SUMMARY_PATH", str(summary_path)) + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw0") + pytest_progress_plugin.pytest_runtest_logreport( + _Report("test_slow", "call", "passed", duration=1.5, worker_id="gw0") + ) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + monkeypatch.delenv("PYTEST_XDIST_WORKER") + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_runtest_logreport( + _Report("test_slow", "call", "passed", duration=1.5, worker_id="gw0") + ) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + events = [ + json.loads(line) + for line in events_path.read_text().splitlines() + if json.loads(line).get("event") == "test_report" + ] + summary = json.loads(summary_path.read_text()) + assert [(event["nodeid"], event["worker_id"]) for event in events] == [("test_slow", "gw0")] + assert [report["nodeid"] for report in summary["slowest_reports"]] == ["test_slow"] + + def test_managed_event_ledger_survives_test_host_environment_scrub(tmp_path: Path) -> None: events_dir = tmp_path / "events" checkout_root = Path(__file__).resolve().parents[3] diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 474268481c..31eb09264c 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1066,6 +1066,30 @@ def test_aggregate_pytest_statistics_deduplicates_xdist_reports_and_terminal_fai assert result["outcomes"] == {"error": 2} +def test_aggregate_pytest_statistics_accounts_for_started_node_without_a_phase(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + rows = [ + {"event": "test_started", "nodeid": "tests/a.py::test_completed", "worker_id": "gw0"}, + { + "event": "test_report", + "nodeid": "tests/a.py::test_completed", + "when": "call", + "outcome": "passed", + "duration_s": 0.1, + "worker_id": "gw0", + }, + {"event": "test_started", "nodeid": "tests/a.py::test_interrupted", "worker_id": "gw1"}, + ] + (step / "events.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) + + result = aggregate_pytest_statistics(step) + + assert result["node_count"] == 2 + assert result["outcomes"] == {"passed": 1, "interrupted": 1} + assert sum(result["outcomes"].values()) == result["node_count"] + + def test_verify_run_statistics_only_cover_pytest_steps(tmp_path: Path) -> None: run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) artifacts = run.start_step(label="ruff check", cmd=["ruff", "check"]) @@ -1181,6 +1205,11 @@ def test_interrupted_pytest_waits_for_forced_containment_quiescence() -> None: patch("devtools.verify._request_supervisor_termination") as request_termination, patch("devtools.verify._force_kill_owned_run") as force_kill, patch("devtools.verify.reap_exited_children") as reap, + patch( + "devtools.verify.read_receipt", + return_value={"status": "terminated", "controller_group_alive": False}, + ), + patch("devtools.verify.descendant_process_identities", return_value=()), ): verify._await_interrupted_pytest_containment( process, @@ -1221,6 +1250,30 @@ def test_interrupted_pytest_refuses_cleanup_without_containment_quiescence() -> reap.assert_not_called() +def test_interrupted_pytest_refuses_cleanup_when_controller_group_survives() -> None: + process = MagicMock() + process.poll.return_value = None + process.wait.return_value = None + launch = MagicMock() + + with ( + patch("devtools.verify._request_supervisor_termination"), + patch("devtools.verify.reap_exited_children"), + patch( + "devtools.verify.read_receipt", + return_value={"status": "terminated", "controller_group_alive": True}, + ), + patch("devtools.verify.descendant_process_identities", return_value=()), + pytest.raises(verify.PytestContainmentError, match="owned process tree"), + ): + verify._await_interrupted_pytest_containment( + process, + launch, + term_grace_s=1.0, + preserved_runner_descendants=(), + ) + + def test_run_cleans_and_finalizes_only_after_contained_interrupt(tmp_path: Path) -> None: run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) order: list[str] = [] @@ -1311,6 +1364,20 @@ def test_verify_history_appends_concurrent_records_without_interleaving(tmp_path assert sorted(row["sequence"] for row in rows) == list(range(64)) +def test_verify_history_repairs_or_frames_an_incomplete_trailing_record(tmp_path: Path) -> None: + history = tmp_path / "state" / "verify-history.jsonl" + history.parent.mkdir(parents=True) + history.write_text('{"sequence": 0}', encoding="utf-8") + + append_verify_history({"sequence": 1}, path=history) + + history.write_text(history.read_text(encoding="utf-8") + '{"interrupted":', encoding="utf-8") + append_verify_history({"sequence": 2}, path=history) + + rows = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] + assert rows == [{"sequence": 0}, {"sequence": 1}, {"sequence": 2}] + + def test_compare_against_last_skips_intervening_focused_history(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( verify, @@ -1326,6 +1393,33 @@ def test_compare_against_last_skips_intervening_focused_history(monkeypatch: pyt assert flags and "ruff check" in flags[0] +def test_compare_against_last_selects_prior_run_independently_per_step(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + verify, + "_load_history", + lambda: [ + { + "tier": "default", + "steps": [ + {"name": "ruff check", "duration_s": 1.0}, + {"name": "pytest testmon", "duration_s": 2.0}, + ], + }, + {"tier": "quick", "steps": [{"name": "ruff check", "duration_s": 1.0}]}, + ], + ) + + flags = verify._compare_against_last( + [ + {"name": "ruff check", "duration_s": 1.1}, + {"name": "pytest testmon", "duration_s": 8.0}, + ] + ) + + assert len(flags) == 1 + assert "pytest testmon" in flags[0] + + def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) TESTMON_DATA.parent.mkdir(parents=True) @@ -2990,6 +3084,15 @@ def test_cleanup_managed_pytest_basetemp_removes_run_root(tmp_path: Path) -> Non assert not basetemp.exists() +def test_cleanup_managed_pytest_basetemp_recognizes_child_cleanup(tmp_path: Path) -> None: + env = {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)} + basetemp = pytest_basetemp_path(root=tmp_path, run_id="run-cleaned-by-child", env=env) + + cleaned = cleanup_managed_pytest_basetemp(root=tmp_path, run_id="run-cleaned-by-child", env=env) + + assert cleaned == basetemp + + def test_cleanup_managed_pytest_basetemp_does_not_receipt_residual_tree( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -3145,6 +3248,48 @@ def test_explicit_basetemp_policy_uses_actual_path_for_admission( assert policy.basetemp_label == "explicit" +def test_explicit_tmpfs_basetemp_requires_declared_demand_and_headroom( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=15_190) + explicit = shm / "pytest-polylogue-diagnostic" + monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 2500 * 1024) + + with pytest.raises(PytestResourceError, match="need >= 3072 MiB"): + apply_managed_pytest_runtime_policy( + {verify_runs.PYTEST_EXPLICIT_BASETEMP_ENV: str(explicit)}, worker_count=4, full_suite=True + ) + + +def test_supervisor_never_cleans_an_explicit_tmpfs_basetemp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(verify_runs, "PYTEST_TMPFS_ROOT", tmp_path / "dev-shm") + explicit = verify_runs.PYTEST_TMPFS_ROOT / "pytest-polylogue-diagnostic" + + cleanup_path = verify._supervised_tmpfs_cleanup_path( + root=tmp_path, + run_id="run-1", + env={verify_runs.PYTEST_EXPLICIT_BASETEMP_ENV: str(explicit), "POLYLOGUE_PYTEST_TMPFS": "1"}, + ) + + assert cleanup_path is None + + +def test_run_resource_refusal_returns_finalized_compact_statistics(tmp_path: Path) -> None: + run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) + + with patch( + "devtools.verify.apply_managed_pytest_runtime_policy", + side_effect=PytestResourceError("starved basetemp"), + ): + rc, _elapsed, metadata = _run("pytest testmon", ["pytest", "-n", "0"], run=run) + + assert rc == 125 + assert metadata["statistics"]["node_count"] == 0 + assert metadata["statistics_path"].endswith("statistics.json") + + def test_run_receipt_uses_capped_pytest_command_concurrency() -> None: completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") From 107ce5408b6180faa88be7bf3248d7b42cf731c5 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 00:46:00 +0200 Subject: [PATCH 13/53] fix(test): refuse unsafe explicit tmpfs basetemps Problem: an explicit /dev/shm basetemp could clear its bounded mode when adaptive memory admission fell below declared demand, then pass a free-space-only preflight. What changed: refuse that explicit tmpfs admission and cover the 3 GiB capacity path with adequate filesystem headroom. --- devtools/verify_runs.py | 6 ++++++ tests/unit/devtools/test_verify.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 969e396a8d..286b55b89c 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -1076,6 +1076,12 @@ def apply_managed_pytest_runtime_policy( and effective_tmpfs_budget_kb is not None and effective_tmpfs_budget_kb < required_basetemp_kb ): + if explicit_tmpfs: + raise PytestResourceError( + "explicit pytest basetemp declared demand exceeds its safe adaptive tmpfs budget " + f"({explicit_basetemp}: declared demand={required_basetemp_kb / 1024:.0f} MiB, " + f"safe tmpfs budget={effective_tmpfs_budget_kb / 1024:.0f} MiB)" + ) normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" if configured_tmpfs: # The configured tmpfs root has become unsafe for this run. diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 31eb09264c..bd3a41a503 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -3263,6 +3263,24 @@ def test_explicit_tmpfs_basetemp_requires_declared_demand_and_headroom( ) +def test_explicit_tmpfs_basetemp_refuses_declared_demand_above_adaptive_memory_cap( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=3072) + explicit = shm / "pytest-polylogue-diagnostic" + monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 4 * 1024 * 1024) + + with pytest.raises( + PytestResourceError, + match=r"declared demand=1522 MiB, safe tmpfs budget=1082 MiB", + ): + apply_managed_pytest_runtime_policy( + {verify_runs.PYTEST_EXPLICIT_BASETEMP_ENV: str(explicit)}, worker_count=0, full_suite=True + ) + + def test_supervisor_never_cleans_an_explicit_tmpfs_basetemp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(verify_runs, "PYTEST_TMPFS_ROOT", tmp_path / "dev-shm") explicit = verify_runs.PYTEST_TMPFS_ROOT / "pytest-polylogue-diagnostic" From 2c835eddb7702f9b86a9f8f8a61506e3cdf81004 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 01:09:32 +0200 Subject: [PATCH 14/53] fix(test): close tmpfs basetemp ownership gaps Problem: unsafe tmpfs admission could omit filesystem evidence, configured roots could clear their cap, and explicit caller paths could later enter stale cleanup. What changed: combine admission diagnostics, refuse configured tmpfs roots at the adaptive cap, and mark explicit caller-owned basetemps so stale cleanup retains them. --- devtools/verify_runs.py | 34 +++++++++++++++++++---- tests/conftest.py | 19 ++++++++++++- tests/unit/devtools/test_verify.py | 39 ++++++++++++++++++++++----- tests/unit/test_pytest_temp_policy.py | 21 +++++++++++++++ 4 files changed, 101 insertions(+), 12 deletions(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 286b55b89c..00a9100dcc 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -1028,6 +1028,27 @@ def adaptive_pytest_runtime_policy( ) +def _tmpfs_admission_refusal( + *, + kind: str, + path: Path, + declared_demand_kb: int, + safe_budget_kb: int, + headroom_kb: int, +) -> PytestResourceError: + """Describe all failed tmpfs admission constraints in one refusal.""" + free_kb = _headroom_kb(path) + required_headroom_kb = headroom_kb + max(declared_demand_kb, safe_budget_kb) + available = f"{free_kb / 1024:.0f} MiB" if free_kb is not None else "unknown" + return PytestResourceError( + f"{kind} pytest basetemp declared demand exceeds its safe adaptive tmpfs budget " + f"({path}: declared demand={declared_demand_kb / 1024:.0f} MiB, " + f"safe tmpfs budget={safe_budget_kb / 1024:.0f} MiB, " + f"available filesystem space={available}, " + f"required filesystem headroom={required_headroom_kb / 1024:.0f} MiB)" + ) + + def apply_managed_pytest_runtime_policy( env: Mapping[str, str], *, worker_count: int | None = None, full_suite: bool = True ) -> tuple[dict[str, str], PytestRuntimePolicy | None]: @@ -1076,11 +1097,14 @@ def apply_managed_pytest_runtime_policy( and effective_tmpfs_budget_kb is not None and effective_tmpfs_budget_kb < required_basetemp_kb ): - if explicit_tmpfs: - raise PytestResourceError( - "explicit pytest basetemp declared demand exceeds its safe adaptive tmpfs budget " - f"({explicit_basetemp}: declared demand={required_basetemp_kb / 1024:.0f} MiB, " - f"safe tmpfs budget={effective_tmpfs_budget_kb / 1024:.0f} MiB)" + if explicit_tmpfs or configured_tmpfs: + path = Path(explicit_basetemp or configured_root or PYTEST_TMPFS_ROOT) + raise _tmpfs_admission_refusal( + kind="explicit" if explicit_tmpfs else "configured", + path=path, + declared_demand_kb=required_basetemp_kb, + safe_budget_kb=effective_tmpfs_budget_kb, + headroom_kb=pytest_basetemp_min_free_kb(normalized), ) normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" if configured_tmpfs: diff --git a/tests/conftest.py b/tests/conftest.py index 8abd97ae15..c4764eca49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -103,6 +103,11 @@ def pytest_configure(config: pytest.Config) -> None: "scale_large: large-tier scale fixture (~10k convs / ~100k msgs); nightly CI / campaigns only (#1183)", ) + if config.option.basetemp is not None: + if not hasattr(config, "workerinput"): + _mark_caller_owned_basetemp(Path(str(config.option.basetemp))) + return + if config.option.basetemp is None: normalized_basetemp_env = normalize_pytest_basetemp_env(os.environ) if ( @@ -148,6 +153,7 @@ def pytest_configure(config: pytest.Config) -> None: _STALE_BASETEMP_MAX_AGE_S = 30 * 60 _STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S = 6 * 60 * 60 _OWNER_PID_MARKER = ".owner-pid" +_CALLER_OWNED_BASETEMP_MARKER = ".polylogue-caller-owned-basetemp" def _managed_pytest_temp_root() -> tuple[Path, str]: @@ -175,6 +181,13 @@ def _mark_basetemp_owner(basetemp: Path) -> None: (basetemp / _OWNER_PID_MARKER).write_text(identity, encoding="utf-8") +def _mark_caller_owned_basetemp(basetemp: Path) -> None: + """Record that an explicit ``--basetemp`` remains owned by its caller.""" + with contextlib.suppress(OSError): + basetemp.mkdir(parents=True, exist_ok=True) + (basetemp / _CALLER_OWNED_BASETEMP_MARKER).write_text("explicit\n", encoding="utf-8") + + def _basetemp_owner_alive(entry: Path) -> bool | None: """True/False when the owner marker resolves a live/dead process, else None.""" marker = entry / _OWNER_PID_MARKER @@ -265,7 +278,9 @@ def _sweep_stale_polylogue_basetemps( (written in ``pytest_configure``); a confirmed-dead owner uses the normal threshold, an unconfirmable owner (no marker — e.g. a directory from before this mechanism existed, or a startup race) uses a much longer - threshold before being reclaimed at all. Seeded corpora + threshold before being reclaimed at all. Explicit caller-owned paths + carry a separate marker and are excluded regardless of name or age. + Seeded corpora (``pytest-polylogue-*-seeded-*``) are never touched here — they are shared, reusable, and built once behind their own ``.build.done`` guard. """ @@ -279,6 +294,8 @@ def _sweep_stale_polylogue_basetemps( try: if not entry.is_dir(): continue + if (entry / _CALLER_OWNED_BASETEMP_MARKER).is_file(): + continue owner_alive = _basetemp_owner_alive(entry) if owner_alive: continue diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index bd3a41a503..20c9240e3d 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -2646,6 +2646,10 @@ def test_managed_pytest_policy_preserves_headroom_for_explicit_tmpfs_root( def test_full_suite_explicit_root_requires_measured_basetemp_space( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: + # The managed test harness itself uses /dev/shm, so keep this custom-root + # regression on a distinct disk route rather than accidentally admitting + # it as a configured tmpfs path. + monkeypatch.setattr(verify_runs, "PYTEST_TMPFS_ROOT", tmp_path / "other-shm") monkeypatch.setattr(verify_runs, "_meminfo", lambda: {"MemAvailable": 8 * 1024 * 1024}) monkeypatch.setattr(verify_runs, "read_cgroup_memory_headroom_bytes", lambda: None) monkeypatch.setattr(verify_runs, "_pressure", lambda _kind: {"full_avg10": 0.0}) @@ -3263,22 +3267,45 @@ def test_explicit_tmpfs_basetemp_requires_declared_demand_and_headroom( ) -def test_explicit_tmpfs_basetemp_refuses_declared_demand_above_adaptive_memory_cap( +def test_explicit_tmpfs_basetemp_reports_adaptive_and_filesystem_refusals_together( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=3072) explicit = shm / "pytest-polylogue-diagnostic" - monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 4 * 1024 * 1024) + monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 2 * 1024 * 1024) - with pytest.raises( - PytestResourceError, - match=r"declared demand=1522 MiB, safe tmpfs budget=1082 MiB", - ): + with pytest.raises(PytestResourceError) as excinfo: apply_managed_pytest_runtime_policy( {verify_runs.PYTEST_EXPLICIT_BASETEMP_ENV: str(explicit)}, worker_count=0, full_suite=True ) + message = str(excinfo.value) + + assert "declared demand=1522 MiB" in message + assert "safe tmpfs budget=1082 MiB" in message + assert "available filesystem space=2048 MiB" in message + assert "required filesystem headroom=2546 MiB" in message + + +def test_configured_tmpfs_basetemp_refuses_declared_demand_above_adaptive_memory_cap( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=3072) + configured = shm / "pytest-polylogue-configured" + monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 4 * 1024 * 1024) + + with pytest.raises(PytestResourceError) as excinfo: + apply_managed_pytest_runtime_policy( + {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(configured)}, worker_count=0, full_suite=True + ) + + message = str(excinfo.value) + assert "configured pytest basetemp" in message + assert "declared demand=1522 MiB" in message + assert "safe tmpfs budget=1082 MiB" in message def test_supervisor_never_cleans_an_explicit_tmpfs_basetemp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index 7c1ea41bf5..6b62b0d970 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -280,6 +280,27 @@ def test_sweep_stale_polylogue_basetemps_preserves_seeded_and_recent( assert unrelated.exists() +def test_explicit_basetemp_remains_outside_a_later_startup_stale_sweep( + tmp_path: Path, + frozen_clock: FrozenClock, +) -> None: + explicit = tmp_path / "pytest-polylogue-debug" + config = SimpleNamespace( + option=SimpleNamespace(basetemp=str(explicit)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + conftest.pytest_configure(cast("pytest.Config", config)) + assert (explicit / conftest._CALLER_OWNED_BASETEMP_MARKER).is_file() + old = frozen_clock.time() - conftest._STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S - 1 + os.utime(explicit, (old, old)) + + conftest._sweep_stale_polylogue_basetemps(roots=(tmp_path,)) + + assert explicit.exists() + + def test_sweep_stale_polylogue_basetemps_never_deletes_a_live_owner( tmp_path: Path, frozen_clock: FrozenClock, From d2d601db3dc4d864caf9c3c73ea01d5576b8b577 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 01:53:27 +0200 Subject: [PATCH 15/53] fix: preserve pytest basetemp ownership evidence Problem: pytest can replace explicit basetemps after configuration, while stale cleanup and xdist summaries lost ownership and collection facts. What changed: keep durable adjacent claims with per-basetemp locking, admit unmanaged configured tmpfs routes to scratch, and merge worker collection facts deterministically in the controller. Co-Authored-By: Codex --- devtools/pytest_progress_plugin.py | 98 +++++++++--- devtools/verify_runs.py | 23 +++ tests/conftest.py | 133 ++++++++++++----- .../devtools/test_pytest_progress_plugin.py | 37 +++++ tests/unit/test_pytest_temp_policy.py | 139 +++++++++++++++--- 5 files changed, 359 insertions(+), 71 deletions(-) diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index 283e6d5bc0..d700b4057a 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -32,6 +32,7 @@ _COLLECTION_DURATION_S: float | None = None _SLOW_REPORT_LIMIT = 20 _DEFAULT_SELECTION_NODEID_LIMIT = 500 +_COLLECTION_FACT_SUFFIX = ".collection.json" def _selection_nodeid_limit() -> int: @@ -85,6 +86,68 @@ def _write_selection(payload: dict[str, Any]) -> None: tmp.replace(path) +def _write_worker_collection_fact(payload: dict[str, Any]) -> None: + """Publish one worker-local collection fact for controller aggregation.""" + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + raw_dir = os.environ.get(_EVENTS_DIR_ENV) + if not worker_id or not raw_dir: + return + path = Path(raw_dir) / f"{worker_id.replace('/', '-')}-{os.getpid()}{_COLLECTION_FACT_SUFFIX}" + payload = {"worker_id": worker_id, "pid": os.getpid(), **payload} + with contextlib.suppress(OSError): + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + tmp.replace(path) + + +def _worker_collection_payloads() -> list[dict[str, Any]]: + """Read worker collection facts in a stable order for the controller.""" + raw_dir = os.environ.get(_EVENTS_DIR_ENV) + if not raw_dir: + return [] + payloads: list[tuple[str, int, str, dict[str, Any]]] = [] + for path in Path(raw_dir).glob(f"*{_COLLECTION_FACT_SUFFIX}"): + with contextlib.suppress(OSError, json.JSONDecodeError): + payload = json.loads(path.read_text(encoding="utf-8")) + worker_id = payload.get("worker_id") + pid = payload.get("pid") + if isinstance(worker_id, str) and isinstance(pid, int): + payloads.append((worker_id, pid, path.name, payload)) + return [payload for _worker_id, _pid, _name, payload in sorted(payloads)] + + +def _collection_payload() -> dict[str, Any]: + """Return this process's complete collection fact.""" + limit = _selection_nodeid_limit() + payload: dict[str, Any] = { + "selected_count": _SELECTED_COUNT, + "deselected_count": _DESELECTED_COUNT, + "selected_nodeids": [], + "selected_node_markers": {}, + "selected_nodeids_omitted": _SELECTED_COUNT, + "deselected_nodeids": list(_DESELECTED_NODEIDS_SAMPLE), + "deselected_nodeids_omitted": max(0, _DESELECTED_COUNT - len(_DESELECTED_NODEIDS_SAMPLE)), + "nodeid_sample_limit": limit, + } + if _COLLECTION_DURATION_S is not None: + payload["collection_duration_s"] = _COLLECTION_DURATION_S + return payload + + +def _merge_worker_collection_payloads() -> dict[str, Any] | None: + """Choose one canonical xdist collection set and the slowest wall time.""" + payloads = _worker_collection_payloads() + if not payloads: + return None + merged = dict(payloads[0]) + durations = [payload.get("collection_duration_s") for payload in payloads] + numeric_durations = [duration for duration in durations if isinstance(duration, (int, float))] + if numeric_durations: + merged["collection_duration_s"] = max(numeric_durations) + return merged + + def _write_summary(payload: dict[str, Any]) -> None: raw_path = os.environ.get(_SUMMARY_ENV) if not raw_path: @@ -167,19 +230,18 @@ def pytest_collection_modifyitems(session: Any, config: Any, items: list[Any]) - ) for item in items } - payload: dict[str, Any] = { - "selected_count": _SELECTED_COUNT, - "deselected_count": _DESELECTED_COUNT, - "selected_nodeids": selected_nodeids, - "selected_node_markers": selected_node_markers, - "selected_nodeids_omitted": max(0, _SELECTED_COUNT - len(selected_nodeids)), - "deselected_nodeids": list(_DESELECTED_NODEIDS_SAMPLE), - "deselected_nodeids_omitted": max(0, _DESELECTED_COUNT - len(_DESELECTED_NODEIDS_SAMPLE)), - "nodeid_sample_limit": limit, - } - if _COLLECTION_DURATION_S is not None: - payload["collection_duration_s"] = _COLLECTION_DURATION_S - _write_selection(payload) + payload = _collection_payload() + payload.update( + { + "selected_nodeids": selected_nodeids, + "selected_node_markers": selected_node_markers, + "selected_nodeids_omitted": max(0, _SELECTED_COUNT - len(selected_nodeids)), + } + ) + if os.environ.get("PYTEST_XDIST_WORKER"): + _write_worker_collection_fact(payload) + else: + _write_selection(payload) _write_event( { "event": "collection_finished", @@ -270,12 +332,14 @@ def pytest_sessionfinish(session: Any, exitstatus: int) -> None: # summary path, so an empty worker summary cannot overwrite it. if os.environ.get("PYTEST_XDIST_WORKER"): return + collection_payload = _merge_worker_collection_payloads() or _collection_payload() + _write_selection(collection_payload) payload: dict[str, Any] = { "exitstatus": int(exitstatus), - "selected_count": _SELECTED_COUNT, - "deselected_count": _DESELECTED_COUNT, + "selected_count": collection_payload["selected_count"], + "deselected_count": collection_payload["deselected_count"], "slowest_reports": list(_SLOWEST_REPORTS), } - if _COLLECTION_DURATION_S is not None: - payload["collection_duration_s"] = _COLLECTION_DURATION_S + if "collection_duration_s" in collection_payload: + payload["collection_duration_s"] = collection_payload["collection_duration_s"] _write_summary(payload) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 00a9100dcc..f6f67a1ce1 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -847,6 +847,7 @@ def checkout_hash(root: Path) -> str: DEFAULT_PYTEST_BASETEMP_ROOT = Path("/realm/tmp/polylogue-pytest") _CLOUD_PYTEST_BASETEMP_ROOT = Path("/tmp/polylogue-pytest") PYTEST_TMPFS_ROOT = Path("/dev/shm") +_PYTEST_BASETEMP_CLAIM_PREFIX = ".polylogue-pytest-claim-" def _is_beneath(path: Path, root: Path) -> bool: @@ -858,6 +859,26 @@ def _is_beneath(path: Path, root: Path) -> bool: return True +def pytest_basetemp_claim_path(basetemp: Path, *, kind: str) -> Path: + """Return the durable, adjacent claim path for one pytest basetemp. + + Pytest lazily clears an explicit ``--basetemp`` before first use, so an + ownership record inside that tree cannot survive normal initialization. + Claims live beside the tree and are keyed by its absolute path, not by a + reusable basename. + """ + digest = hashlib.sha256(str(basetemp.absolute()).encode("utf-8")).hexdigest()[:20] + return basetemp.parent / f"{_PYTEST_BASETEMP_CLAIM_PREFIX}{kind}-{digest}" + + +def clear_managed_pytest_basetemp_claim(basetemp: Path) -> None: + """Remove the durable claim after a managed run's tree is reclaimed.""" + with contextlib.suppress(OSError): + pytest_basetemp_claim_path(basetemp, kind="managed").unlink() + with contextlib.suppress(OSError): + pytest_basetemp_claim_path(basetemp, kind="lock").unlink() + + def normalize_pytest_basetemp_env(env: Mapping[str, str]) -> dict[str, str]: """Keep cloud pytest defaults from escaping a workstation scratch volume. @@ -1385,11 +1406,13 @@ def cleanup_managed_pytest_basetemp(*, root: Path, run_id: str, env: dict[str, s # directory in sessionfinish. That is a completed cleanup, not an absent # receipt for the durable summary to misclassify. if not basetemp.exists(): + clear_managed_pytest_basetemp_claim(basetemp) return basetemp with contextlib.suppress(OSError): if basetemp.exists(): shutil.rmtree(basetemp) if not basetemp.exists(): + clear_managed_pytest_basetemp_claim(basetemp) return basetemp return None diff --git a/tests/conftest.py b/tests/conftest.py index c4764eca49..1c264f48f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import fcntl import hashlib import json import os @@ -15,7 +16,7 @@ from collections.abc import AsyncIterator, Callable, Iterator, Mapping from pathlib import Path from types import FrameType, ModuleType -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TextIO import pytest from hypothesis import HealthCheck, settings @@ -37,7 +38,13 @@ resolved_polylogue_path, ) from devtools.pytest_supervisor import _process_start_ticks -from devtools.verify_runs import PytestResourceError, normalize_pytest_basetemp_env, resolve_pytest_basetemp_root +from devtools.verify_runs import ( + PytestResourceError, + clear_managed_pytest_basetemp_claim, + normalize_pytest_basetemp_env, + resolve_pytest_basetemp_root, +) +from devtools.verify_runs import pytest_basetemp_claim_path as _basetemp_claim_path # Resolve (but don't yet raise on) the polylogue-vs-checkout mismatch check # before the first `from polylogue...` import below: a shared/editable venv's @@ -110,9 +117,12 @@ def pytest_configure(config: pytest.Config) -> None: if config.option.basetemp is None: normalized_basetemp_env = normalize_pytest_basetemp_env(os.environ) - if ( - "POLYLOGUE_VERIFY_RUN_ID" not in os.environ - and "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in normalized_basetemp_env + configured_root = normalized_basetemp_env.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") + unmanaged_tmpfs_root = configured_root is not None and verify_runs._is_beneath( + Path(configured_root), verify_runs.PYTEST_TMPFS_ROOT + ) + if "POLYLOGUE_VERIFY_RUN_ID" not in os.environ and ( + "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in normalized_basetemp_env or unmanaged_tmpfs_root ): # Bare pytest has no devtools supervisor to enforce a tmpfs cap. # Keep its basetemp on scratch; managed devtools runs carry the @@ -151,9 +161,8 @@ def pytest_configure(config: pytest.Config) -> None: # ``devtools.verify_runs.resolve_pytest_basetemp_root`` — this module only # adds the mkdir/no-CoW-marking side effects and the stale-directory sweep. _STALE_BASETEMP_MAX_AGE_S = 30 * 60 -_STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S = 6 * 60 * 60 -_OWNER_PID_MARKER = ".owner-pid" -_CALLER_OWNED_BASETEMP_MARKER = ".polylogue-caller-owned-basetemp" +_BASE_TEMP_CLAIM_LOCKS: dict[Path, TextIO] = {} +_BASE_TEMP_CLAIM_THREAD_LOCKS: dict[Path, threading.Lock] = {} def _managed_pytest_temp_root() -> tuple[Path, str]: @@ -172,25 +181,66 @@ def _managed_pytest_temp_root() -> tuple[Path, str]: def _mark_basetemp_owner(basetemp: Path) -> None: - """Record the owning process identity so a sweep never races a live run.""" + """Claim a managed tree outside pytest's replaceable basetemp directory.""" + handle = _acquire_basetemp_claim_lock(basetemp, blocking=False) + if handle is None: + # A nested raw pytest can inherit the controller's run id and its + # exact basetemp. That controller still owns the durable claim; do + # not block the nested process on a lock it cannot usefully replace. + return + pid = os.getpid() + start_ticks = _process_start_ticks(pid) + identity = f"{pid}:{start_ticks}" if start_ticks is not None else str(pid) with contextlib.suppress(OSError): - basetemp.mkdir(parents=True, exist_ok=True) - pid = os.getpid() - start_ticks = _process_start_ticks(pid) - identity = f"{pid}:{start_ticks}" if start_ticks is not None else str(pid) - (basetemp / _OWNER_PID_MARKER).write_text(identity, encoding="utf-8") + _basetemp_claim_path(basetemp, kind="managed").write_text(identity, encoding="utf-8") def _mark_caller_owned_basetemp(basetemp: Path) -> None: - """Record that an explicit ``--basetemp`` remains owned by its caller.""" + """Claim an explicit ``--basetemp`` before pytest may replace its tree.""" + handle = _acquire_basetemp_claim_lock(basetemp, blocking=True) + assert handle is not None with contextlib.suppress(OSError): basetemp.mkdir(parents=True, exist_ok=True) - (basetemp / _CALLER_OWNED_BASETEMP_MARKER).write_text("explicit\n", encoding="utf-8") + clear_managed_pytest_basetemp_claim(basetemp) + _basetemp_claim_path(basetemp, kind="caller-owned").write_text("explicit\n", encoding="utf-8") + + +def _acquire_basetemp_claim_lock(basetemp: Path, *, blocking: bool) -> TextIO | None: + """Serialize a claim against a stale sweep for the same basetemp path.""" + lock_path = _basetemp_claim_path(basetemp, kind="lock") + thread_lock = _BASE_TEMP_CLAIM_THREAD_LOCKS.setdefault(lock_path, threading.Lock()) + if not thread_lock.acquire(blocking=blocking): + return None + with contextlib.suppress(OSError): + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+", encoding="utf-8") + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)) + except BlockingIOError: + handle.close() + thread_lock.release() + return None + _BASE_TEMP_CLAIM_LOCKS[lock_path] = handle + return handle + thread_lock.release() + return None + + +def _release_basetemp_claim_lock(basetemp: Path) -> None: + """Release this pytest process's claim lock after its session ends.""" + lock_path = _basetemp_claim_path(basetemp, kind="lock") + handle = _BASE_TEMP_CLAIM_LOCKS.pop(lock_path, None) + if handle is not None: + with contextlib.suppress(OSError): + handle.close() + thread_lock = _BASE_TEMP_CLAIM_THREAD_LOCKS.get(lock_path) + if thread_lock is not None and thread_lock.locked(): + thread_lock.release() def _basetemp_owner_alive(entry: Path) -> bool | None: """True/False when the owner marker resolves a live/dead process, else None.""" - marker = entry / _OWNER_PID_MARKER + marker = _basetemp_claim_path(entry, kind="managed") try: raw_identity = marker.read_text(encoding="utf-8").strip() raw_pid, separator, raw_start_ticks = raw_identity.partition(":") @@ -271,22 +321,18 @@ def _sweep_stale_polylogue_basetemps( ) -> None: """Best-effort reclaim of per-run basetemps left by crashed runs. - Safety invariant: never delete a basetemp whose owning process is still - alive, regardless of age. Age alone is not a liveness proxy — a - long-running scale/lab test can legitimately outlive the stale-age - threshold. Each managed basetemp carries a ``.owner-pid`` marker - (written in ``pytest_configure``); a confirmed-dead owner uses the normal - threshold, an unconfirmable owner (no marker — e.g. a directory from - before this mechanism existed, or a startup race) uses a much longer - threshold before being reclaimed at all. Explicit caller-owned paths - carry a separate marker and are excluded regardless of name or age. + Safety invariant: reclamation requires a durable, positive managed claim + plus a confirmed-dead owner. Unknown paths are never deleted: they may be + an explicit caller path racing a startup sweep. The claim lock makes the + decision and a caller's claim mutually exclusive, while the claim itself + survives pytest's lazy replacement of the basetemp directory. Explicit + caller-owned paths carry their own durable claim and are excluded. Seeded corpora (``pytest-polylogue-*-seeded-*``) are never touched here — they are shared, reusable, and built once behind their own ``.build.done`` guard. """ cutoff = time.time() - max_age_s - unknown_owner_cutoff = time.time() - _STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S for root in roots or _polylogue_basetemp_roots(): for entry in root.glob("pytest-polylogue-*"): if "-seeded-" in entry.name: @@ -294,17 +340,26 @@ def _sweep_stale_polylogue_basetemps( try: if not entry.is_dir(): continue - if (entry / _CALLER_OWNED_BASETEMP_MARKER).is_file(): + if _basetemp_claim_path(entry, kind="caller-owned").is_file(): continue - owner_alive = _basetemp_owner_alive(entry) - if owner_alive: + if not _basetemp_claim_path(entry, kind="managed").is_file(): continue - mtime = entry.stat().st_mtime - if owner_alive is False: - if mtime < cutoff: + handle = _acquire_basetemp_claim_lock(entry, blocking=False) + if handle is None: + continue + try: + if _basetemp_claim_path(entry, kind="caller-owned").is_file(): + continue + if not _basetemp_claim_path(entry, kind="managed").is_file(): + continue + owner_alive = _basetemp_owner_alive(entry) + if owner_alive is not False: + continue + if entry.stat().st_mtime < cutoff: _remove_stale_basetemp(entry) - elif mtime < unknown_owner_cutoff: - _remove_stale_basetemp(entry) + clear_managed_pytest_basetemp_claim(entry) + finally: + _release_basetemp_claim_lock(entry) except OSError: pass @@ -331,8 +386,12 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if not basetemp: return basetemp_path = Path(str(basetemp)) - if str(basetemp_path) == os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP"): - shutil.rmtree(basetemp_path, ignore_errors=True) + try: + if str(basetemp_path) == os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP"): + shutil.rmtree(basetemp_path, ignore_errors=True) + clear_managed_pytest_basetemp_claim(basetemp_path) + finally: + _release_basetemp_claim_lock(basetemp_path) @pytest.fixture(autouse=True) diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index 962b22681d..32dfc337e8 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -322,3 +322,40 @@ def test_progress_plugin_records_collection_duration_and_summary( "collection_finished", ] assert events[2]["duration_s"] == 2.5 + + +def test_progress_plugin_merges_xdist_collection_facts_without_double_counting( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + events_dir = tmp_path / "events" + selection_path = tmp_path / "selection.json" + summary_path = tmp_path / "summary.json" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_DIR", str(events_dir)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SELECTION_PATH", str(selection_path)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SUMMARY_PATH", str(summary_path)) + + for worker_id, duration in (("gw1", 1.5), ("gw0", 2.5)): + ticks = iter([10.0, 10.0 + duration]) + monkeypatch.setattr("devtools.pytest_progress_plugin.time.monotonic", lambda ticks=ticks: next(ticks)) + monkeypatch.setenv("PYTEST_XDIST_WORKER", worker_id) + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_collection(object()) + pytest_progress_plugin.pytest_deselected([_Item("tests/a.py::test_skip")]) + pytest_progress_plugin.pytest_collection_modifyitems( + _Session(["tests/a.py::test_keep"]), object(), [_Item("tests/a.py::test_keep")] + ) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + monkeypatch.delenv("PYTEST_XDIST_WORKER") + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + selection = json.loads(selection_path.read_text()) + summary = json.loads(summary_path.read_text()) + assert selection["selected_count"] == 1 + assert selection["deselected_count"] == 1 + assert selection["selected_nodeids"] == ["tests/a.py::test_keep"] + assert summary["selected_count"] == 1 + assert summary["deselected_count"] == 1 + assert summary["collection_duration_s"] == 2.5 diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index 6b62b0d970..a25be9000b 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -1,6 +1,9 @@ from __future__ import annotations import os +import subprocess +import sys +import threading from collections.abc import Generator from pathlib import Path from types import SimpleNamespace @@ -257,7 +260,31 @@ def test_bare_pytest_ignores_leaked_cloud_basetemp_on_workstation( assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" -def test_sweep_stale_polylogue_basetemps_preserves_seeded_and_recent( +def test_bare_pytest_routes_an_environment_configured_tmpfs_root_to_scratch( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _make_real_candidates(monkeypatch, tmp_path) + monkeypatch.setenv("POLYLOGUE_PYTEST_BASETEMP_ROOT", str(shm / "configured")) + monkeypatch.delenv("POLYLOGUE_VERIFY_RUN_ID", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_TMPFS", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_RUN_ID", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_CHECKOUT", raising=False) + monkeypatch.delenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", raising=False) + config = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + conftest.pytest_configure(cast("pytest.Config", config)) + + assert Path(str(config.option.basetemp)).parent == scratch + assert "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in os.environ + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" + + +def test_sweep_stale_polylogue_basetemps_preserves_unknown_seeded_and_recent( tmp_path: Path, frozen_clock: FrozenClock, ) -> None: @@ -268,13 +295,13 @@ def test_sweep_stale_polylogue_basetemps_preserves_seeded_and_recent( for path in (stale, seeded, recent, unrelated): path.mkdir() - old = frozen_clock.time() - conftest._STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S - 1 + old = frozen_clock.time() - 24 * 60 * 60 os.utime(stale, (old, old)) os.utime(seeded, (old, old)) conftest._sweep_stale_polylogue_basetemps(max_age_s=60, roots=(tmp_path,)) - assert not stale.exists() + assert stale.exists() assert seeded.exists() assert recent.exists() assert unrelated.exists() @@ -292,8 +319,8 @@ def test_explicit_basetemp_remains_outside_a_later_startup_stale_sweep( ) conftest.pytest_configure(cast("pytest.Config", config)) - assert (explicit / conftest._CALLER_OWNED_BASETEMP_MARKER).is_file() - old = frozen_clock.time() - conftest._STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S - 1 + assert verify_runs.pytest_basetemp_claim_path(explicit, kind="caller-owned").is_file() + old = frozen_clock.time() - 24 * 60 * 60 os.utime(explicit, (old, old)) conftest._sweep_stale_polylogue_basetemps(roots=(tmp_path,)) @@ -301,6 +328,86 @@ def test_explicit_basetemp_remains_outside_a_later_startup_stale_sweep( assert explicit.exists() +def test_explicit_basetemp_claim_survives_real_pytest_basetemp_replacement(tmp_path: Path) -> None: + """Exercise pytest's lazy TempPathFactory clearing against our real conftest.""" + explicit = tmp_path / "pytest-polylogue-diagnostic" + explicit.mkdir() + cleared_by_pytest = explicit / "cleared-by-temp-path-factory" + cleared_by_pytest.write_text("old", encoding="utf-8") + repo_root = Path(__file__).resolve().parents[2] + env = {key: value for key, value in os.environ.items() if not key.startswith("POLYLOGUE_PYTEST_")} + env.pop("POLYLOGUE_VERIFY_RUN_ID", None) + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--basetemp", + str(explicit), + "tests/unit/test_pytest_temp_policy.py::test_archive_template_clone_is_private", + ], + cwd=repo_root, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert not cleared_by_pytest.exists() + assert verify_runs.pytest_basetemp_claim_path(explicit, kind="caller-owned").is_file() + + +def test_stale_sweep_and_explicit_claim_are_atomic_for_one_path( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + frozen_clock: FrozenClock, +) -> None: + basetemp = tmp_path / "pytest-polylogue-race-123" + basetemp.mkdir() + conftest._mark_basetemp_owner(basetemp) + verify_runs.pytest_basetemp_claim_path(basetemp, kind="managed").write_text("999999999", encoding="utf-8") + conftest._release_basetemp_claim_lock(basetemp) + old = frozen_clock.time() - 120 + os.utime(basetemp, (old, old)) + sweep_checked = threading.Event() + allow_sweep = threading.Event() + caller_claimed = threading.Event() + original_owner_alive = conftest._basetemp_owner_alive + + def pause_after_admission(entry: Path) -> bool | None: + sweep_checked.set() + assert allow_sweep.wait(timeout=2) + return original_owner_alive(entry) + + monkeypatch.setattr(conftest, "_basetemp_owner_alive", pause_after_admission) + sweeper = threading.Thread( + target=conftest._sweep_stale_polylogue_basetemps, + kwargs={"max_age_s": 60, "roots": (tmp_path,)}, + ) + sweeper.start() + assert sweep_checked.wait(timeout=2) + + def claim_and_use() -> None: + conftest._mark_caller_owned_basetemp(basetemp) + basetemp.mkdir(exist_ok=True) + caller_claimed.set() + + caller = threading.Thread(target=claim_and_use) + caller.start() + assert not caller_claimed.wait(timeout=0.1) + allow_sweep.set() + sweeper.join(timeout=2) + caller.join(timeout=2) + + assert not sweeper.is_alive() + assert not caller.is_alive() + assert caller_claimed.is_set() + assert basetemp.exists() + assert verify_runs.pytest_basetemp_claim_path(basetemp, kind="caller-owned").is_file() + + def test_sweep_stale_polylogue_basetemps_never_deletes_a_live_owner( tmp_path: Path, frozen_clock: FrozenClock, @@ -328,7 +435,7 @@ def test_sweep_stale_polylogue_basetemps_reclaims_a_confirmed_dead_owner( dead.mkdir() # A pid that is guaranteed not to be alive right now (max pid + 1 territory # would flake on hosts near pid rollover; /proc simply never has this one). - (dead / conftest._OWNER_PID_MARKER).write_text("999999999", encoding="utf-8") + verify_runs.pytest_basetemp_claim_path(dead, kind="managed").write_text("999999999", encoding="utf-8") old = frozen_clock.time() - 120 os.utime(dead, (old, old)) @@ -344,7 +451,7 @@ def test_sweep_stale_polylogue_basetemps_reclaims_reused_pid_identity( ) -> None: stale = tmp_path / "pytest-polylogue-reused-pid-123" stale.mkdir() - (stale / conftest._OWNER_PID_MARKER).write_text(f"{os.getpid()}:100", encoding="utf-8") + verify_runs.pytest_basetemp_claim_path(stale, kind="managed").write_text(f"{os.getpid()}:100", encoding="utf-8") monkeypatch.setattr(conftest, "_process_start_ticks", lambda _pid: 200) old = frozen_clock.time() - 120 os.utime(stale, (old, old)) @@ -366,7 +473,8 @@ def test_sweep_stale_polylogue_basetemps_reclaims_read_only_fixture_tree( payload.chmod(0o400) nested.chmod(0o500) (stale / "published").chmod(0o500) - old = frozen_clock.time() - conftest._STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S - 1 + verify_runs.pytest_basetemp_claim_path(stale, kind="managed").write_text("999999999", encoding="utf-8") + old = frozen_clock.time() - 120 os.utime(stale, (old, old)) conftest._sweep_stale_polylogue_basetemps(max_age_s=60, roots=(tmp_path,)) @@ -386,7 +494,7 @@ def test_sweep_stale_polylogue_basetemps_does_not_follow_top_level_symlink( payload.chmod(0o400) nested.chmod(0o500) target.chmod(0o500) - old = frozen_clock.time() - conftest._STALE_BASETEMP_UNKNOWN_OWNER_MAX_AGE_S - 1 + old = frozen_clock.time() - 24 * 60 * 60 os.utime(target, (old, old)) link = tmp_path / "pytest-polylogue-stale-symlink-123" @@ -401,18 +509,15 @@ def test_sweep_stale_polylogue_basetemps_does_not_follow_top_level_symlink( assert (target.stat().st_mode, nested.stat().st_mode, payload.stat().st_mode) == before_modes -def test_sweep_stale_polylogue_basetemps_gives_unknown_owner_a_long_grace_period( +def test_sweep_stale_polylogue_basetemps_never_deletes_an_unknown_owner( tmp_path: Path, ) -> None: - """A directory with no owner marker (pre-fix leftover, or a startup - race) cannot be confirmed dead, so it gets a much longer grace period - rather than the normal stale-age cutoff.""" + """Unknown paths may be an explicit caller racing a sweep, so retain them.""" unmarked = tmp_path / "pytest-polylogue-unmarked-123" unmarked.mkdir() - # Past the normal (60s, for this test) stale-age cutoff, but nowhere near - # the multi-hour unknown-owner grace period. Derive "now" from the - # directory's own just-created mtime (filesystem metadata) rather than a - # direct host-clock read, which test code may not perform (clock_guard). + # Derive "now" from the directory's own just-created mtime (filesystem + # metadata) rather than a direct host-clock read, which test code may not + # perform (clock_guard). now = unmarked.stat().st_mtime past_normal_cutoff = now - 120 os.utime(unmarked, (past_normal_cutoff, past_normal_cutoff)) From 5e7efa3a0c20c669b24f7befd649967530c5b088 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 02:23:36 +0200 Subject: [PATCH 16/53] fix(test): preserve pytest ownership evidence Keep the basetemp lock inode stable, reject managed ownership collisions, retain failed-cleanup claims, meter tmpfs by allocated blocks, and terminalize unproven containment without reclaiming evidence. --- devtools/verify.py | 29 +++++++-- devtools/verify_runs.py | 13 +++- tests/conftest.py | 19 +++--- tests/unit/devtools/test_verify.py | 74 +++++++++++++++++++++-- tests/unit/test_pytest_temp_policy.py | 86 +++++++++++++++++++++++++++ 5 files changed, 200 insertions(+), 21 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index f5b001a821..6e1057ffdd 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -98,6 +98,7 @@ latest_event_from_paths, normalize_pytest_basetemp_env, pytest_basetemp_path, + pytest_tmpfs_budget_exceeded, pytest_tmpfs_budget_kb, utc_now, xdist_uninterruptible_stall_reason, @@ -1446,15 +1447,14 @@ def _refresh_progress_marker(at: float, latest: dict[str, Any] | None = None) -> and sample_now - last_resource_sample >= resource_interval_s ): resource_sample = sampler.sample(event="sample") - basetemp_size_kb = resource_sample.get("basetemp_size_kb") if ( termination_reason is None and tmpfs_budget_kb is not None - and isinstance(basetemp_size_kb, int) - and basetemp_size_kb > tmpfs_budget_kb + and pytest_tmpfs_budget_exceeded(resource_sample, budget_kb=tmpfs_budget_kb) ): + basetemp_allocated_kb = int(resource_sample["basetemp_allocated_kb"]) termination_reason = ( - f"pytest tmpfs budget exceeded: {basetemp_size_kb / 1024:.1f} MiB " + f"pytest tmpfs budget exceeded: {basetemp_allocated_kb / 1024:.1f} MiB allocated " f"> {tmpfs_budget_kb / 1024:.0f} MiB" ) if resource_sample.get("all_xdist_workers_uninterruptible") is True: @@ -1629,13 +1629,15 @@ def _run( env = env_for_pytest_step(env, run=run, artifacts=artifacts) interrupted = False pytest_containment_quiescent = True + containment_error: str | None = None if is_pytest: try: try: result = _run_pytest_with_heartbeat(cmd, cwd=cwd, env=env, t0=t0, run=run, artifacts=artifacts) - except PytestContainmentError: + except PytestContainmentError as exc: pytest_containment_quiescent = False - raise + containment_error = str(exc) + result = subprocess.CompletedProcess(args=cmd, returncode=125, stdout="", stderr=str(exc)) except KeyboardInterrupt: interrupted = True result = subprocess.CompletedProcess(args=cmd, returncode=130, stdout="", stderr="") @@ -1658,6 +1660,9 @@ def _run( metadata["run_id"] = run.run_id if run is not None else None metadata["artifact_dir"] = str(artifacts.step_dir.relative_to(Path.cwd())) if is_pytest: + if containment_error is not None: + metadata["diagnosis"] = "pytest_containment_unproven" + metadata["termination_reason"] = f"pytest containment did not quiesce: {containment_error}" metadata.update(_pytest_command_metadata(cmd)) metadata["heartbeat_s"] = _pytest_heartbeat_interval() metadata["timeout_s"] = _pytest_timeout_s() @@ -1753,6 +1758,7 @@ def _run( peak_swap_pss: int | None = None peak_process_count = 0 peak_basetemp_size_kb: int | None = None + peak_basetemp_allocated_kb: int | None = None with artifacts.resources_path.open(encoding="utf-8") as resource_handle: for line in resource_handle: if not line.strip(): @@ -1782,6 +1788,11 @@ def _run( peak_basetemp_size_kb or 0, int(row["basetemp_size_kb"]), ) + if row.get("basetemp_allocated_kb") is not None: + peak_basetemp_allocated_kb = max( + peak_basetemp_allocated_kb or 0, + int(row["basetemp_allocated_kb"]), + ) if sample_count: resource_summary = { "resource_sample_count": sample_count, @@ -1820,6 +1831,10 @@ def _run( "peak_basetemp_size_mb": ( round(peak_basetemp_size_kb / 1024, 1) if peak_basetemp_size_kb is not None else None ), + "peak_basetemp_allocated_kb": peak_basetemp_allocated_kb, + "peak_basetemp_allocated_mb": ( + round(peak_basetemp_allocated_kb / 1024, 1) if peak_basetemp_allocated_kb is not None else None + ), } metadata.update(resource_summary) diagnosis = classify_pytest_result( @@ -1831,6 +1846,8 @@ def _run( summary=summary if isinstance(summary, dict) else None, progress_event=metadata.get("progress_event") if isinstance(metadata.get("progress_event"), str) else None, ) + if containment_error is not None: + diagnosis = "pytest_containment_unproven" if interrupted: diagnosis = "pytest_interrupted" metadata["termination_reason"] = "operator_interrupt" diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index f6f67a1ce1..f50b9ff31d 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -875,8 +875,17 @@ def clear_managed_pytest_basetemp_claim(basetemp: Path) -> None: """Remove the durable claim after a managed run's tree is reclaimed.""" with contextlib.suppress(OSError): pytest_basetemp_claim_path(basetemp, kind="managed").unlink() - with contextlib.suppress(OSError): - pytest_basetemp_claim_path(basetemp, kind="lock").unlink() + + +def pytest_tmpfs_budget_exceeded(sample: Mapping[str, Any], *, budget_kb: int) -> bool: + """Return whether a sampled basetemp exceeds its tmpfs allocation cap. + + ``st_size`` remains forensic evidence because sparse files can expose a + large logical extent. Tmpfs capacity is consumed by allocated blocks, so + the admission limit must use that physical measure. + """ + allocated_kb = sample.get("basetemp_allocated_kb") + return isinstance(allocated_kb, int) and allocated_kb > budget_kb def normalize_pytest_basetemp_env(env: Mapping[str, str]) -> dict[str, str]: diff --git a/tests/conftest.py b/tests/conftest.py index 1c264f48f5..bc034eb223 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -139,16 +139,16 @@ def pytest_configure(config: pytest.Config) -> None: os.environ["POLYLOGUE_PYTEST_RUN_ID"] = run_id try: root, label = _managed_pytest_temp_root() + basetemp = root / f"pytest-polylogue-{checkout}-{run_id}" + if not hasattr(config, "workerinput"): + _mark_basetemp_owner(basetemp) except PytestResourceError as exc: # Fail loudly and early: refuse before pytest starts collecting, # rather than crashing an unrelated command later with a bare # OSError once the chosen basetemp fills up. raise pytest.UsageError(f"pytest: {exc}") from exc - basetemp = root / f"pytest-polylogue-{checkout}-{run_id}" config.option.basetemp = str(basetemp) os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = str(basetemp) - if not hasattr(config, "workerinput"): - _mark_basetemp_owner(basetemp) sys.stderr.write(f"pytest: basetemp → {config.option.basetemp} ({label})\n") @@ -184,15 +184,15 @@ def _mark_basetemp_owner(basetemp: Path) -> None: """Claim a managed tree outside pytest's replaceable basetemp directory.""" handle = _acquire_basetemp_claim_lock(basetemp, blocking=False) if handle is None: - # A nested raw pytest can inherit the controller's run id and its - # exact basetemp. That controller still owns the durable claim; do - # not block the nested process on a lock it cannot usefully replace. - return + raise PytestResourceError(f"managed pytest basetemp is already claimed: {basetemp}") pid = os.getpid() start_ticks = _process_start_ticks(pid) identity = f"{pid}:{start_ticks}" if start_ticks is not None else str(pid) - with contextlib.suppress(OSError): + try: _basetemp_claim_path(basetemp, kind="managed").write_text(identity, encoding="utf-8") + except OSError as exc: + _release_basetemp_claim_lock(basetemp) + raise PytestResourceError(f"cannot record managed pytest basetemp claim: {basetemp}") from exc def _mark_caller_owned_basetemp(basetemp: Path) -> None: @@ -389,7 +389,8 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: try: if str(basetemp_path) == os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP"): shutil.rmtree(basetemp_path, ignore_errors=True) - clear_managed_pytest_basetemp_claim(basetemp_path) + if not basetemp_path.exists(): + clear_managed_pytest_basetemp_claim(basetemp_path) finally: _release_basetemp_claim_lock(basetemp_path) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 20c9240e3d..ad88b2e29f 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -88,6 +88,7 @@ cleanup_managed_pytest_basetemp, pytest_basetemp_known_roots, pytest_basetemp_path, + pytest_tmpfs_budget_exceeded, pytest_tmpfs_budget_kb, resolve_pytest_basetemp_root, xdist_uninterruptible_stall_reason, @@ -1302,7 +1303,7 @@ def _finish_step(*, step_id: str, result: dict[str, Any]) -> dict[str, Any] | No assert order == ["contained", "cleanup", "finalize"] -def test_run_leaves_basetemp_and_step_open_when_containment_fails(tmp_path: Path) -> None: +def test_run_terminalizes_containment_failure_without_cleaning_basetemp(tmp_path: Path) -> None: run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) with ( @@ -1311,12 +1312,50 @@ def test_run_leaves_basetemp_and_step_open_when_containment_fails(tmp_path: Path side_effect=verify.PytestContainmentError("still running"), ), patch("devtools.verify.cleanup_managed_pytest_basetemp") as cleanup, - pytest.raises(verify.PytestContainmentError, match="still running"), ): - _run("pytest focused", ["pytest", "-n", "0"], run=run) + rc, _elapsed, metadata = _run("pytest focused", ["pytest", "-n", "0"], run=run) cleanup.assert_not_called() - assert run._payload["steps"][0]["status"] == "running" + assert rc == 125 + assert metadata["diagnosis"] == "pytest_containment_unproven" + assert run._payload["steps"][0]["status"] == "failed" + assert run._payload["steps"][0]["termination_reason"].startswith("pytest containment did not quiesce") + + +def test_verify_main_records_containment_failure_as_terminal_history( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + history_path = tmp_path / "verify-history.jsonl" + monkeypatch.setattr(verify, "HISTORY_PATH", history_path) + + with ( + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._testmon_preflight", return_value=None), + patch("devtools.verify.build_verify_steps", return_value=[("pytest containment", ["pytest", "-n", "0"])]), + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), + patch( + "devtools.verify._run_pytest_with_heartbeat", + side_effect=verify.PytestContainmentError("owned child still running"), + ), + patch("devtools.verify.cleanup_managed_pytest_basetemp") as cleanup, + patch("devtools.verify._notify"), + ): + rc = main(["--json"]) + + history = json.loads(history_path.read_text(encoding="utf-8")) + run_json = next((tmp_path / ".cache" / "verify" / "runs").glob("*/run.json")) + run_payload = json.loads(run_json.read_text(encoding="utf-8")) + payload = json.loads(capsys.readouterr().out) + + assert rc == 125 + cleanup.assert_not_called() + assert payload["diagnosis"] == "pytest_containment_unproven" + assert history["exit_code"] == 125 + assert history["diagnosis"] == "pytest_containment_unproven" + assert run_payload["status"] == "failed" + assert run_payload["steps"][0]["status"] == "failed" def test_print_history_accepts_verify_and_focused_run_records( @@ -2378,6 +2417,33 @@ def counted_usage(_path: Path) -> tuple[int, int]: assert calls == 1 +def test_sparse_basetemp_enforces_allocated_tmpfs_bytes_and_retains_logical_evidence(tmp_path: Path) -> None: + env = { + "POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path), + "POLYLOGUE_PYTEST_TMPFS": "1", + "POLYLOGUE_VERIFY_BASETEMP_SIZE_INTERVAL_S": "1", + } + run_id = "sparse-physical-accounting" + basetemp = pytest_basetemp_path(root=tmp_path, run_id=run_id, env=env) + basetemp.mkdir(parents=True) + with (basetemp / "sparse.bin").open("wb") as handle: + handle.seek(64 * 1024 * 1024) + handle.write(b"x") + sampler = ResourceSampler( + root_pid=os.getpid(), run_id=run_id, root=tmp_path, env=env, output_path=tmp_path / "resources.jsonl" + ) + + sample = sampler.sample(event="sample") + logical_kb = sample["basetemp_size_kb"] + allocated_kb = sample["basetemp_allocated_kb"] + + assert isinstance(logical_kb, int) + assert isinstance(allocated_kb, int) + assert logical_kb > allocated_kb + assert not pytest_tmpfs_budget_exceeded(sample, budget_kb=allocated_kb + 1) + assert pytest_tmpfs_budget_exceeded(sample, budget_kb=allocated_kb - 1) + + def test_pytest_basetemp_path_tracks_tmpfs_opt_in(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: shm, _scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) path = pytest_basetemp_path(root=tmp_path, run_id="run-1", env={"POLYLOGUE_PYTEST_TMPFS": "1"}) diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index a25be9000b..fe710ccf9d 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import shutil import subprocess import sys import threading @@ -359,6 +360,71 @@ def test_explicit_basetemp_claim_survives_real_pytest_basetemp_replacement(tmp_p assert verify_runs.pytest_basetemp_claim_path(explicit, kind="caller-owned").is_file() +def test_claim_lock_inode_stays_contended_after_managed_claim_clear(tmp_path: Path) -> None: + """A second process cannot lock a replacement inode for this basetemp.""" + basetemp = tmp_path / "pytest-polylogue-lock-inode" + conftest._mark_caller_owned_basetemp(basetemp) + lock_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="lock") + try: + verify_runs.clear_managed_pytest_basetemp_claim(basetemp) + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import fcntl, sys\n" + "with open(sys.argv[1], 'a+', encoding='utf-8') as handle:\n" + " try:\n" + " fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)\n" + " except BlockingIOError:\n" + " raise SystemExit(0)\n" + "raise SystemExit(1)\n" + ), + str(lock_path), + ], + capture_output=True, + text=True, + timeout=10, + ) + finally: + conftest._release_basetemp_claim_lock(basetemp) + + assert result.returncode == 0, result.stdout + result.stderr + assert lock_path.is_file() + + +def test_managed_basetemp_claim_collision_is_rejected_across_processes(tmp_path: Path) -> None: + basetemp = tmp_path / "pytest-polylogue-managed-collision" + conftest._mark_basetemp_owner(basetemp) + try: + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "from pathlib import Path\n" + "import sys\n" + "import tests.conftest as conftest\n" + "from devtools.verify_runs import PytestResourceError\n" + "try:\n" + " conftest._mark_basetemp_owner(Path(sys.argv[1]))\n" + "except PytestResourceError:\n" + " raise SystemExit(0)\n" + "raise SystemExit(1)\n" + ), + str(basetemp), + ], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + timeout=10, + ) + finally: + conftest._release_basetemp_claim_lock(basetemp) + + assert result.returncode == 0, result.stdout + result.stderr + + def test_stale_sweep_and_explicit_claim_are_atomic_for_one_path( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -558,6 +624,26 @@ def test_sessionfinish_reclaims_only_its_managed_basetemp( assert not basetemp.exists() +def test_sessionfinish_retains_managed_claim_after_failed_rmtree( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + basetemp = tmp_path / "pytest-polylogue-rmtree-failure" + basetemp.mkdir() + managed_claim = verify_runs.pytest_basetemp_claim_path(basetemp, kind="managed") + managed_claim.write_text("999999999", encoding="utf-8") + session = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(basetemp), numprocesses=0))) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "run-123") + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(basetemp)) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + monkeypatch.setattr(shutil, "rmtree", lambda _path, **_kwargs: None) + + conftest.pytest_sessionfinish(cast("pytest.Session", session), 1) + + assert basetemp.exists() + assert managed_claim.is_file() + + def test_sessionfinish_retains_explicit_diagnostic_basetemp( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From 044acb1433d18d1d2065152d10f1fa55c61210db Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 02:49:47 +0200 Subject: [PATCH 17/53] fix(test): auto-reroute configured tmpfs roots --- devtools/verify_runs.py | 6 +++--- tests/unit/devtools/test_verify.py | 22 +--------------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index f50b9ff31d..448330a6dd 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -1127,10 +1127,10 @@ def apply_managed_pytest_runtime_policy( and effective_tmpfs_budget_kb is not None and effective_tmpfs_budget_kb < required_basetemp_kb ): - if explicit_tmpfs or configured_tmpfs: - path = Path(explicit_basetemp or configured_root or PYTEST_TMPFS_ROOT) + if explicit_tmpfs: + path = Path(explicit_basetemp or PYTEST_TMPFS_ROOT) raise _tmpfs_admission_refusal( - kind="explicit" if explicit_tmpfs else "configured", + kind="explicit", path=path, declared_demand_kb=required_basetemp_kb, safe_budget_kb=effective_tmpfs_budget_kb, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index ad88b2e29f..e7ed1917a1 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -2858,7 +2858,7 @@ def test_inherited_512_mib_tmpfs_cap_reroutes_measured_demand_to_scratch( assert env["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(scratch) -def test_explicit_tmpfs_root_reroutes_to_scratch_when_its_cap_is_too_small( +def test_configured_tmpfs_root_reroutes_to_scratch_when_its_cap_is_too_small( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) @@ -3354,26 +3354,6 @@ def test_explicit_tmpfs_basetemp_reports_adaptive_and_filesystem_refusals_togeth assert "required filesystem headroom=2546 MiB" in message -def test_configured_tmpfs_basetemp_refuses_declared_demand_above_adaptive_memory_cap( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) - _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=3072) - configured = shm / "pytest-polylogue-configured" - monkeypatch.setattr("devtools.verify_runs._headroom_kb", lambda _path: 4 * 1024 * 1024) - - with pytest.raises(PytestResourceError) as excinfo: - apply_managed_pytest_runtime_policy( - {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(configured)}, worker_count=0, full_suite=True - ) - - message = str(excinfo.value) - assert "configured pytest basetemp" in message - assert "declared demand=1522 MiB" in message - assert "safe tmpfs budget=1082 MiB" in message - - def test_supervisor_never_cleans_an_explicit_tmpfs_basetemp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(verify_runs, "PYTEST_TMPFS_ROOT", tmp_path / "dev-shm") explicit = verify_runs.PYTEST_TMPFS_ROOT / "pytest-polylogue-diagnostic" From 17f8b20c0b9391b7e7f232f5243cd8143f4d6f3b Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 03:10:42 +0200 Subject: [PATCH 18/53] fix(test): preserve exact pytest run evidence Scope shared history by checkout and HEAD. Retain allocated tmpfs accounting, captured output, serialized cleanup, canonical claim IDs, and interrupted xdist collection facts. --- devtools/evidence_dashboard.py | 6 +- devtools/verify.py | 95 ++++++++-- devtools/verify_runs.py | 87 +++++++-- .../unit/devtools/test_evidence_dashboard.py | 14 ++ tests/unit/devtools/test_verify.py | 165 ++++++++++++++++++ 5 files changed, 337 insertions(+), 30 deletions(-) diff --git a/devtools/evidence_dashboard.py b/devtools/evidence_dashboard.py index de15f60d11..66fe545012 100644 --- a/devtools/evidence_dashboard.py +++ b/devtools/evidence_dashboard.py @@ -28,7 +28,7 @@ from typing import Any from devtools import repo_root as _get_root -from devtools.verify_runs import VERIFY_HISTORY_PATH +from devtools.verify_runs import VERIFY_HISTORY_PATH, git_head ROOT = _get_root() @@ -227,6 +227,8 @@ def _benchmark_slo(root: Path, *, now: datetime) -> dict[str, Any]: def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: history_path = VERIFY_HISTORY_PATH last_result_path = root / LAST_VERIFY_RESULT_REL + checkout_root = str(root.resolve()) + checkout_head = git_head(root) # Prefer last-verify-result.json (the most recent run) then walk back through # history to find the last status for each gate. @@ -263,6 +265,8 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: # appearance in history. if history_entries: for entry in reversed(history_entries): + if entry.get("checkout_root") != checkout_root or entry.get("git_head") != checkout_head: + continue steps = entry.get("steps", []) for step in steps: if not isinstance(step, dict): diff --git a/devtools/verify.py b/devtools/verify.py index 6e1057ffdd..153b4568ab 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -466,7 +466,8 @@ def _pytest_workload_receipt( peak_swap_pss_kb = resource_summary.get("peak_tree_swap_pss_kb") read_bytes = resource_summary.get("tree_read_bytes_delta") write_bytes = resource_summary.get("tree_write_bytes_delta") - peak_basetemp_kb = resource_summary.get("peak_basetemp_size_kb") + peak_basetemp_kb = resource_summary.get("peak_basetemp_allocated_kb") + logical_basetemp_kb = resource_summary.get("peak_basetemp_size_kb") final_rss_kb = last_resource_sample.get("tree_rss_kb") if last_resource_sample is not None else None final_pss_kb = last_resource_sample.get("tree_pss_kb") if last_resource_sample is not None else None total_cpu_s = last_resource_sample.get("tree_cpu_s") if last_resource_sample is not None else None @@ -564,7 +565,14 @@ def _pytest_workload_receipt( ), cancellation_requested=termination_reason is not None, cleanup_complete=True if basetemp_cleanup is not None else None, - notes=("Managed pytest process-tree sampler adapter.",), + notes=( + "Managed pytest process-tree sampler adapter.", + ( + f"Logical basetemp peak retained as diagnostic evidence: {logical_basetemp_kb * 1024} bytes." + if isinstance(logical_basetemp_kb, int) + else "Logical basetemp peak unavailable." + ), + ), ) return dict(receipt.to_payload()) @@ -615,6 +623,15 @@ def _write_pytest_output(stdout: str, stderr: str) -> None: PYTEST_OUTPUT_PATH.write_text(stdout + ("\n" if stdout and stderr else "") + stderr, encoding="utf-8") +def _persist_pytest_output(stdout: str, stderr: str, *, artifacts: PytestStepArtifacts | None) -> None: + """Persist drained pytest output on both ordinary and exceptional exits.""" + _write_pytest_output(stdout, stderr) + if artifacts is not None: + artifacts.stdout_path.write_text(stdout, encoding="utf-8") + artifacts.stderr_path.write_text(stderr, encoding="utf-8") + artifacts.output_path.write_text(stdout + stderr, encoding="utf-8") + + def _write_pytest_progress( *, event: str, @@ -1473,12 +1490,19 @@ def _refresh_progress_marker(at: float, latest: dict[str, Any] | None = None) -> if process.poll() is not None and not selector.get_map(): break except BaseException: - _await_interrupted_pytest_containment( - process, - launch, - term_grace_s=term_grace_s, - preserved_runner_descendants=preserved_runner_descendants, - ) + try: + _await_interrupted_pytest_containment( + process, + launch, + term_grace_s=term_grace_s, + preserved_runner_descendants=preserved_runner_descendants, + ) + finally: + _persist_pytest_output( + b"".join(output["stdout"]).decode(errors="replace"), + b"".join(output["stderr"]).decode(errors="replace"), + artifacts=artifacts, + ) raise finally: selector.close() @@ -1547,11 +1571,7 @@ def _refresh_progress_marker(at: float, latest: dict[str, Any] | None = None) -> events_path=events_path, events_dir=events_dir, ) - _write_pytest_output(stdout, stderr) - if artifacts is not None: - artifacts.stdout_path.write_text(stdout, encoding="utf-8") - artifacts.stderr_path.write_text(stderr, encoding="utf-8") - artifacts.output_path.write_text(stdout + stderr, encoding="utf-8") + _persist_pytest_output(stdout, stderr, artifacts=artifacts) return subprocess.CompletedProcess(cmd, returncode, stdout, stderr) @@ -1715,6 +1735,11 @@ def _run( if isinstance(termination_reason, str): metadata["termination_reason"] = termination_reason selection_path = artifacts.selection_path if artifacts is not None else PYTEST_SELECTION_PATH + if interrupted or containment_error is not None: + _recover_worker_collection_facts( + events_dir=artifacts.events_dir if artifacts is not None else Path(env["POLYLOGUE_PYTEST_EVENTS_DIR"]), + selection_path=selection_path, + ) selection = _read_json_artifact(selection_path) if selection is not None: selected_count = selection.get("selected_count") @@ -2584,6 +2609,50 @@ def _read_testmon_seed_attempt() -> dict[str, Any] | None: return payload if isinstance(payload, dict) else None +def _recover_worker_collection_facts(*, events_dir: Path, selection_path: Path) -> bool: + """Publish xdist worker collection facts when its controller never finishes. + + The progress plugin normally merges these facts during controller + ``pytest_sessionfinish``. Interrupted containment bypasses that hook, so + the runner recovers the same canonical worker fact before it terminalizes + the durable step record. + """ + payloads: list[tuple[str, int, str, dict[str, Any]]] = [] + for path in events_dir.glob("*.collection.json"): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + worker_id = payload.get("worker_id") + pid = payload.get("pid") + if isinstance(worker_id, str) and isinstance(pid, int): + payloads.append((worker_id, pid, path.name, payload)) + if not payloads: + return False + payloads.sort() + selection = dict(payloads[0][3]) + durations = [payload.get("collection_duration_s") for *_ignored, payload in payloads] + numeric_durations = [duration for duration in durations if isinstance(duration, int | float)] + if numeric_durations: + selection["collection_duration_s"] = max(numeric_durations) + selection.update( + { + "updated_at": datetime.now(timezone.utc).isoformat(), + "worker_id": "runner", + "pid": os.getpid(), + "recovered_after_interruption": True, + } + ) + try: + selection_path.parent.mkdir(parents=True, exist_ok=True) + temporary = selection_path.with_name(f"{selection_path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(selection, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + temporary.replace(selection_path) + except OSError: + return False + return True + + def _flatten_seed_outcomes(attempt: Mapping[str, Any] | None) -> list[dict[str, Any]]: """Flatten outcomes from every interrupted attempt, newest result winning.""" if attempt is None: diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 448330a6dd..da889dfb58 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -22,7 +22,7 @@ from dataclasses import dataclass, replace from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, TextIO from polylogue.core.metrics import read_cgroup_memory_headroom_bytes @@ -864,11 +864,16 @@ def pytest_basetemp_claim_path(basetemp: Path, *, kind: str) -> Path: Pytest lazily clears an explicit ``--basetemp`` before first use, so an ownership record inside that tree cannot survive normal initialization. - Claims live beside the tree and are keyed by its absolute path, not by a - reusable basename. + Claims live beside the tree and are keyed by its canonical filesystem + path, not by a reusable basename. A configured symlink and an explicit + real-path spelling must therefore serialize through the same claim. """ - digest = hashlib.sha256(str(basetemp.absolute()).encode("utf-8")).hexdigest()[:20] - return basetemp.parent / f"{_PYTEST_BASETEMP_CLAIM_PREFIX}{kind}-{digest}" + try: + canonical = basetemp.resolve() + except OSError: + canonical = basetemp.absolute() + digest = hashlib.sha256(str(canonical).encode("utf-8")).hexdigest()[:20] + return canonical.parent / f"{_PYTEST_BASETEMP_CLAIM_PREFIX}{kind}-{digest}" def clear_managed_pytest_basetemp_claim(basetemp: Path) -> None: @@ -877,6 +882,39 @@ def clear_managed_pytest_basetemp_claim(basetemp: Path) -> None: pytest_basetemp_claim_path(basetemp, kind="managed").unlink() +def _try_acquire_pytest_basetemp_claim_lock(basetemp: Path) -> TextIO | None: + """Acquire the adjacent claim lock without waiting on another pytest run.""" + lock_path = pytest_basetemp_claim_path(basetemp, kind="lock") + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+", encoding="utf-8") + except OSError: + return None + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except (BlockingIOError, OSError): + handle.close() + return None + return handle + + +def _managed_pytest_basetemp_owner_alive(basetemp: Path) -> bool | None: + """Return whether a positive managed claim still names a live process.""" + try: + raw_identity = pytest_basetemp_claim_path(basetemp, kind="managed").read_text(encoding="utf-8").strip() + raw_pid, separator, raw_start_ticks = raw_identity.partition(":") + pid = int(raw_pid) + start_ticks = int(raw_start_ticks) if separator else None + except (OSError, ValueError): + return None + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1].split() + current_start_ticks = int(fields[19]) + except (OSError, ValueError, IndexError): + return False + return start_ticks is None or current_start_ticks == start_ticks + + def pytest_tmpfs_budget_exceeded(sample: Mapping[str, Any], *, budget_kb: int) -> bool: """Return whether a sampled basetemp exceeds its tmpfs allocation cap. @@ -1411,18 +1449,35 @@ def cleanup_managed_pytest_basetemp(*, root: Path, run_id: str, env: dict[str, s basetemp = pytest_basetemp_path(root=root, run_id=run_id, env=env) if not basetemp.name.startswith("pytest-polylogue-") or "-seeded-" in basetemp.name: return None - # A serial pytest child may already have reclaimed this exact run-owned - # directory in sessionfinish. That is a completed cleanup, not an absent - # receipt for the durable summary to misclassify. - if not basetemp.exists(): - clear_managed_pytest_basetemp_claim(basetemp) - return basetemp - with contextlib.suppress(OSError): - if basetemp.exists(): - shutil.rmtree(basetemp) - if not basetemp.exists(): + claim_lock = _try_acquire_pytest_basetemp_claim_lock(basetemp) + if claim_lock is None: + # A successor with the same inherited run id owns this path. Leave + # both its claim and its fixture tree for that invocation to finish. + return None + try: + owner_alive = _managed_pytest_basetemp_owner_alive(basetemp) + if owner_alive is True: + return None + # A serial pytest child may already have reclaimed this exact run-owned + # directory in sessionfinish. That is a completed cleanup, not an absent + # receipt for the durable summary to misclassify. + if not basetemp.exists(): + if owner_alive is False: clear_managed_pytest_basetemp_claim(basetemp) - return basetemp + return basetemp + # Reclaim only a positively claimed tree whose owner is confirmed dead. + # An unknown claim/tree may be caller-owned or belong to a newer runner. + if owner_alive is not False: + return None + with contextlib.suppress(OSError): + if basetemp.exists(): + shutil.rmtree(basetemp) + if not basetemp.exists(): + clear_managed_pytest_basetemp_claim(basetemp) + return basetemp + finally: + with contextlib.suppress(OSError): + claim_lock.close() return None diff --git a/tests/unit/devtools/test_evidence_dashboard.py b/tests/unit/devtools/test_evidence_dashboard.py index 403cf59e54..bdddecbe62 100644 --- a/tests/unit/devtools/test_evidence_dashboard.py +++ b/tests/unit/devtools/test_evidence_dashboard.py @@ -16,15 +16,29 @@ def test_static_gates_read_shared_verify_history(monkeypatch: pytest.MonkeyPatch json.dumps( { "timestamp": "2026-08-12T00:00:00+00:00", + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", "steps": [{"name": "ruff check", "duration_s": 1.0, "exit": 0}], } ) + "\n" + + json.dumps( + { + "timestamp": "2026-08-12T00:01:00+00:00", + "checkout_root": str((tmp_path / "other-worktree").resolve()), + "git_head": "other-head", + "steps": [{"name": "mypy", "duration_s": 1.0, "exit": 0}], + } + ) + + "\n" ) monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) ruff = next(gate for gate in gates["gates"] if gate["name"] == "ruff check") assert gates["history_path"] == str(history) assert ruff["status"] == "ok" + mypy = next(gate for gate in gates["gates"] if gate["name"] == "mypy") + assert mypy["available"] is False diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index e7ed1917a1..2e0f8f3ae2 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1,5 +1,6 @@ from __future__ import annotations +import fcntl import hashlib import json import os @@ -7,8 +8,10 @@ import sqlite3 import subprocess import sys +import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock, patch @@ -77,6 +80,7 @@ ) from devtools.verify_runs import ( PytestResourceError, + PytestStepArtifacts, ResourceSampler, VerifyRun, adaptive_pytest_runtime_policy, @@ -1322,6 +1326,39 @@ def test_run_terminalizes_containment_failure_without_cleaning_basetemp(tmp_path assert run._payload["steps"][0]["termination_reason"].startswith("pytest containment did not quiesce") +def test_run_recovers_xdist_collection_facts_after_containment_failure(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + + def _write_worker_facts(*_args: object, artifacts: PytestStepArtifacts, **_kwargs: object) -> None: + artifacts.events_dir.mkdir(parents=True, exist_ok=True) + for worker_id, pid, duration in (("gw1", 11, 1.5), ("gw0", 10, 2.5)): + (artifacts.events_dir / f"{worker_id}-{pid}.collection.json").write_text( + json.dumps( + { + "worker_id": worker_id, + "pid": pid, + "selected_count": 3, + "deselected_count": 2, + "selected_nodeids": ["tests/unit/example.py::test_selected"], + "collection_duration_s": duration, + } + ), + encoding="utf-8", + ) + raise verify.PytestContainmentError("controller interrupted") + + with patch("devtools.verify._run_pytest_with_heartbeat", side_effect=_write_worker_facts): + rc, _elapsed, metadata = _run("pytest focused", ["pytest", "-n", "2"], run=run) + + assert rc == 125 + assert metadata["selected_count"] == 3 + assert metadata["deselected_count"] == 2 + assert metadata["collection_duration_s"] == 2.5 + selection = json.loads((run.run_dir / "steps" / "01-pytest-focused" / "selection.json").read_text()) + assert selection["recovered_after_interruption"] is True + assert selection["worker_id"] == "runner" + + def test_verify_main_records_containment_failure_as_terminal_history( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -3147,6 +3184,7 @@ def test_cleanup_managed_pytest_basetemp_removes_run_root(tmp_path: Path) -> Non env = {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)} basetemp = pytest_basetemp_path(root=tmp_path, run_id="run-1", env=env) (basetemp / "worker-output").mkdir(parents=True) + verify_runs.pytest_basetemp_claim_path(basetemp, kind="managed").write_text("999999:1", encoding="utf-8") cleaned = cleanup_managed_pytest_basetemp(root=tmp_path, run_id="run-1", env=env) @@ -3154,6 +3192,56 @@ def test_cleanup_managed_pytest_basetemp_removes_run_root(tmp_path: Path) -> Non assert not basetemp.exists() +def test_pytest_basetemp_claim_path_canonicalizes_symlink_aliases(tmp_path: Path) -> None: + real_root = tmp_path / "real-root" + real_root.mkdir() + linked_root = tmp_path / "linked-root" + linked_root.symlink_to(real_root, target_is_directory=True) + + real_basetemp = real_root / "pytest-polylogue-run" + linked_basetemp = linked_root / "pytest-polylogue-run" + + assert verify_runs.pytest_basetemp_claim_path(real_basetemp, kind="lock") == verify_runs.pytest_basetemp_claim_path( + linked_basetemp, kind="lock" + ) + + +def test_cleanup_managed_pytest_basetemp_leaves_successor_claim_while_locked(tmp_path: Path) -> None: + env = {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)} + basetemp = pytest_basetemp_path(root=tmp_path, run_id="reused-run", env=env) + basetemp.mkdir(parents=True) + (basetemp / "successor-fixture").write_text("live", encoding="utf-8") + claim_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="managed") + claim_path.write_text("999999:1", encoding="utf-8") + lock_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="lock") + + with lock_path.open("a+", encoding="utf-8") as lock_handle: + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) + assert cleanup_managed_pytest_basetemp(root=tmp_path, run_id="reused-run", env=env) is None + + assert basetemp.exists() + assert claim_path.exists() + + +def test_pytest_workload_receipt_uses_allocated_basetemp_peak() -> None: + receipt = verify._pytest_workload_receipt( + label="pytest sparse", + cmd=["pytest", "tests/unit/example.py"], + elapsed_s=1.0, + returncode=0, + termination_reason=None, + resource_summary={"peak_basetemp_size_kb": 64 * 1024, "peak_basetemp_allocated_kb": 8}, + last_resource_sample={"tree_rss_kb": 0, "tree_pss_kb": 0, "tree_cpu_s": 0.0}, + tmpfs_budget_mb=1, + basetemp_cleanup=None, + concurrency=1, + ) + + execute = next(phase for phase in receipt["phases"] if phase["name"] == "execute") + assert execute["temp_storage_bytes"] == 8 * 1024 + assert "Logical basetemp peak retained as diagnostic evidence: 67108864 bytes." in receipt["notes"] + + def test_cleanup_managed_pytest_basetemp_recognizes_child_cleanup(tmp_path: Path) -> None: env = {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)} basetemp = pytest_basetemp_path(root=tmp_path, run_id="run-cleaned-by-child", env=env) @@ -3252,6 +3340,83 @@ def test_run_records_managed_basetemp_cleanup_metadata(tmp_path: Path) -> None: assert metadata["basetemp_cleanup"] == str(cleaned) +@pytest.mark.uses_real_clock("the heartbeat loop computes elapsed containment time") +def test_heartbeat_persists_drained_output_before_interrupting(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="pytest buffered", cmd=["pytest"]) + stdout_read, stdout_write = os.pipe() + stderr_read, stderr_write = os.pipe() + os.write(stdout_write, b"buffered stdout before interrupt\n") + os.close(stdout_write) + os.close(stderr_write) + stdout_pipe = os.fdopen(stdout_read, "rb", closefd=True) + stderr_pipe = os.fdopen(stderr_read, "rb", closefd=True) + + class _Process: + pid = os.getpid() + stdout = stdout_pipe + stderr = stderr_pipe + returncode = None + + def poll(self) -> None: + return None + + class _InterruptingSelector: + def __init__(self) -> None: + self.calls = 0 + + def register(self, _fileobj: object, _events: int, _data: str) -> None: + return None + + def get_map(self) -> dict[int, object]: + return {1: object()} + + def select(self, timeout: float | None = None) -> list[tuple[SimpleNamespace, int]]: + del timeout + if self.calls == 0: + self.calls += 1 + return [(SimpleNamespace(fd=stdout_pipe.fileno(), data="stdout", fileobj=stdout_pipe), 1)] + raise KeyboardInterrupt + + def close(self) -> None: + return None + + launch = SimpleNamespace( + argv=["pytest"], + receipt_path=tmp_path / "containment.json", + request_path=tmp_path / "request.json", + mode="process-group", + unit=None, + cgroup_path=None, + fallback_argv=None, + runtime_cap_s=0.0, + ) + try: + with ( + patch("devtools.verify.enable_child_subreaper", return_value=True), + patch("devtools.verify.descendant_process_identities", return_value=()), + patch("devtools.verify.build_supervisor_launch", return_value=launch), + patch("devtools.verify.subprocess.Popen", return_value=_Process()), + patch("devtools.verify._wait_for_supervisor_start", return_value={"status": "started"}), + patch("devtools.verify.selectors.DefaultSelector", _InterruptingSelector), + patch("devtools.verify._await_interrupted_pytest_containment"), + patch("devtools.verify._write_pytest_progress"), + patch("devtools.verify.ResourceSampler", return_value=MagicMock()), + ): + with pytest.raises(KeyboardInterrupt): + verify._run_pytest_with_heartbeat( + ["pytest"], cwd=str(tmp_path), env={}, t0=time.monotonic(), run=run, artifacts=artifacts + ) + finally: + stdout_pipe.close() + stderr_pipe.close() + + assert artifacts.stdout_path.read_text(encoding="utf-8") == "buffered stdout before interrupt\n" + assert artifacts.stderr_path.read_text(encoding="utf-8") == "" + assert artifacts.output_path.read_text(encoding="utf-8") == "buffered stdout before interrupt\n" + assert (tmp_path / PYTEST_OUTPUT_PATH).read_text(encoding="utf-8") == "buffered stdout before interrupt\n" + + def test_explicit_basetemp_root_retains_managed_resource_monitoring( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From ff3066ebbe861a9bb1b0810232d80d1f3598f971 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 03:24:35 +0200 Subject: [PATCH 19/53] fix(test): retain failed pytest admission evidence Preserve configured tmpfs admission diagnostics through scratch refusal and defer supervisor cleanup until the receipt proves the controller group is quiescent. --- devtools/pytest_supervisor.py | 4 +- devtools/verify_runs.py | 24 +++++++++-- tests/unit/devtools/test_pytest_supervisor.py | 40 +++++++++++++++++++ tests/unit/devtools/test_verify.py | 30 ++++++++++++++ 4 files changed, 93 insertions(+), 5 deletions(-) diff --git a/devtools/pytest_supervisor.py b/devtools/pytest_supervisor.py index 2cb1b5a02d..265f1a47de 100644 --- a/devtools/pytest_supervisor.py +++ b/devtools/pytest_supervisor.py @@ -931,7 +931,9 @@ def main(argv: Sequence[str] | None = None) -> int: runtime_cap_s=args.runtime_cap_s, ) finally: - cleanup_complete = cleanup_managed_tmpfs_path(args.cleanup_path) + receipt = read_receipt(args.receipt) + receipt_quiescent = receipt is not None and receipt.get("controller_group_alive") is False + cleanup_complete = cleanup_managed_tmpfs_path(args.cleanup_path) if receipt_quiescent else False if args.cleanup_path is not None: with contextlib.suppress(OSError): update_receipt( diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index da889dfb58..081caaa610 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -1149,6 +1149,7 @@ def apply_managed_pytest_runtime_policy( shm_free_kb=None if manages_tmpfs else 0, full_suite=full_suite, ) + rejected_candidates: tuple[str, ...] = () if full_suite and policy.tmpfs_predicted_mb is not None: normalized.setdefault(PYTEST_BASETEMP_REQUIRED_MB_ENV, str(policy.tmpfs_predicted_mb)) if manages_tmpfs: @@ -1174,12 +1175,23 @@ def apply_managed_pytest_runtime_policy( safe_budget_kb=effective_tmpfs_budget_kb, headroom_kb=pytest_basetemp_min_free_kb(normalized), ) - normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" if configured_tmpfs: # The configured tmpfs root has become unsafe for this run. # Leaving it in place would make the resolver select it even # though tmpfs has just been disabled, without its cap. + rejected_candidates = ( + str( + _tmpfs_admission_refusal( + kind="configured", + path=Path(configured_root or PYTEST_TMPFS_ROOT), + declared_demand_kb=required_basetemp_kb, + safe_budget_kb=effective_tmpfs_budget_kb, + headroom_kb=pytest_basetemp_min_free_kb(normalized), + ) + ), + ) normalized.pop("POLYLOGUE_PYTEST_BASETEMP_ROOT", None) + normalized["POLYLOGUE_PYTEST_TMPFS"] = "0" if default_full_suite_scratch: # Broad-suite demand grows with the fixture universe and has exceeded # the supervised 2 GiB ceiling while tests were still progressing. @@ -1206,7 +1218,9 @@ def apply_managed_pytest_runtime_policy( else f"explicit pytest basetemp is unreachable: {selected_root}" ) else: - selected_root, selected_label = resolve_pytest_basetemp_root(normalized) + selected_root, selected_label = resolve_pytest_basetemp_root( + normalized, rejected_candidates=rejected_candidates + ) free_kb = _headroom_kb(selected_root) if ( explicit_basetemp is None @@ -1316,7 +1330,9 @@ def _basetemp_refusal(checked: list[str], min_free_kb: int) -> PytestResourceErr ) -def resolve_pytest_basetemp_root(env: Mapping[str, str]) -> tuple[Path, str]: +def resolve_pytest_basetemp_root( + env: Mapping[str, str], *, rejected_candidates: tuple[str, ...] = () +) -> tuple[Path, str]: """Pick the ONE basetemp root pytest will use this run. Single resolution order, shared by ``tests/conftest.py`` (direct pytest @@ -1344,7 +1360,7 @@ def resolve_pytest_basetemp_root(env: Mapping[str, str]) -> tuple[Path, str]: required_kb = pytest_basetemp_required_kb(env) min_free_kb = max(pytest_basetemp_min_free_kb(env), required_kb or 0) normalized = normalize_pytest_basetemp_env(env) - checked: list[str] = [] + checked = list(rejected_candidates) configured = normalized.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") if configured: diff --git a/tests/unit/devtools/test_pytest_supervisor.py b/tests/unit/devtools/test_pytest_supervisor.py index 35c1bb13b1..c374651719 100644 --- a/tests/unit/devtools/test_pytest_supervisor.py +++ b/tests/unit/devtools/test_pytest_supervisor.py @@ -12,6 +12,7 @@ import time from collections.abc import Callable, Mapping, Sequence from pathlib import Path +from unittest.mock import patch import pytest import tomllib @@ -441,6 +442,45 @@ def test_surviving_owned_process_forces_nonzero_containment_result( assert receipt["termination_reason"] == "owned pytest processes survived cleanup" +def test_supervisor_main_retains_tmpfs_tree_when_receipt_is_not_quiescent( + tmp_path: Path, +) -> None: + receipt_path = tmp_path / "containment.json" + cleanup_path = Path("/dev/shm") / "pytest-polylogue-retained" + + def fake_supervise(*_args: object, **_kwargs: object) -> int: + receipt_path.write_text(json.dumps({"controller_group_alive": True}), encoding="utf-8") + return 125 + + with ( + patch("devtools.pytest_supervisor.supervise", side_effect=fake_supervise), + patch("devtools.pytest_supervisor.cleanup_managed_tmpfs_path") as cleanup, + ): + rc = pytest_supervisor.main( + [ + "--receipt", + str(receipt_path), + "--owner-pid", + str(os.getpid()), + "--timeout-s", + "1", + "--term-grace-s", + "1", + "--mode", + "process-group", + "--cleanup-path", + str(cleanup_path), + "--", + "pytest", + ] + ) + + assert rc == 125 + cleanup.assert_not_called() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["tmpfs_cleanup_complete"] is False + + def test_managed_runner_retains_responsible_node_for_per_test_timeout( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 2e0f8f3ae2..cc8e14e2f0 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -2917,6 +2917,36 @@ def test_configured_tmpfs_root_reroutes_to_scratch_when_its_cap_is_too_small( assert env["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(scratch) +def test_configured_tmpfs_reroute_keeps_admission_evidence_when_scratch_refuses( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) + _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=15_190) + configured = shm / "configured" + configured.mkdir() + + def constrained_headroom(path: Path) -> int | None: + if path == scratch: + return 1 * 1024 + return 8 * 1024 * 1024 + + monkeypatch.setattr(verify_runs, "_headroom_kb", constrained_headroom) + + with pytest.raises(PytestResourceError) as excinfo: + apply_managed_pytest_runtime_policy( + { + "POLYLOGUE_PYTEST_BASETEMP_ROOT": str(configured), + "POLYLOGUE_PYTEST_TMPFS_MAX_MB": "512", + }, + worker_count=4, + ) + + message = str(excinfo.value) + assert f"configured pytest basetemp declared demand exceeds its safe adaptive tmpfs budget ({configured}" in message + assert "safe tmpfs budget=512 MiB" in message + assert f"{scratch} (scratch): 1 MiB free" in message + + def test_focused_policy_keeps_full_suite_basetemp_demand_out_of_scratch_preflight( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 0697f57c4932fc3ea32eff751381bd8f58363e0e Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 03:58:15 +0200 Subject: [PATCH 20/53] fix(test): preserve exact pytest outcome evidence --- devtools/pytest_progress_plugin.py | 13 ++++- devtools/verify.py | 19 +++++-- devtools/verify_runs.py | 6 +-- .../devtools/test_pytest_progress_plugin.py | 35 ++++++++++++ tests/unit/devtools/test_verify.py | 54 +++++++++++++++++++ 5 files changed, 118 insertions(+), 9 deletions(-) diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index d700b4057a..4eefcd77e7 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -173,6 +173,17 @@ def _remember_report(payload: dict[str, Any]) -> None: del _SLOWEST_REPORTS[_SLOW_REPORT_LIMIT:] +def _durable_report_outcome(report: Any, outcome: str) -> str: + """Preserve pytest's xfail semantics in the append-only event ledger.""" + if not getattr(report, "wasxfail", None): + return outcome + if outcome == "skipped": + return "xfailed" + if outcome == "passed": + return "xpassed" + return outcome + + @pytest.hookimpl def pytest_sessionstart(session: Any) -> None: """Reset per-session ledgers when tests invoke pytest in-process.""" @@ -281,7 +292,7 @@ def _record_phase_report(report: Any, *, write_event: bool = True) -> None: """Append one phase report so slow setup/call/teardown remains visible.""" when = str(getattr(report, "when", "")) nodeid = str(getattr(report, "nodeid", "")) - outcome = str(getattr(report, "outcome", "")) + outcome = _durable_report_outcome(report, str(getattr(report, "outcome", ""))) duration = float(getattr(report, "duration", 0.0) or 0.0) report_key = (id(report), when, nodeid, outcome, duration) if report_key in _RECORDED_REPORT_KEYS: diff --git a/devtools/verify.py b/devtools/verify.py index 153b4568ab..58e257add5 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -81,6 +81,7 @@ CURRENT_EVENTS_DIR, CURRENT_POSTMORTEM_PATH, CURRENT_RESOURCES_PATH, + CURRENT_STATISTICS_PATH, PYTEST_EXPLICIT_BASETEMP_ENV, VERIFY_HISTORY_PATH, PytestResourceError, @@ -609,6 +610,7 @@ def _clear_pytest_report(cmd: Sequence[str] = ()) -> None: CURRENT_RESOURCES_PATH, CURRENT_POSTMORTEM_PATH, CURRENT_CONTAINMENT_PATH, + CURRENT_STATISTICS_PATH, ): with contextlib.suppress(FileNotFoundError): if path.is_dir(): @@ -1593,7 +1595,7 @@ def _run( _clear_pytest_report(cmd) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None env = _subprocess_env() - explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd) + explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd, env=env) if explicit_basetemp is not None: env[PYTEST_EXPLICIT_BASETEMP_ENV] = str(explicit_basetemp) pytest_tmpfs = False @@ -1945,14 +1947,21 @@ def _run( return result.returncode, elapsed, metadata -def _pytest_command_basetemp(cmd: Sequence[str], *, cwd: str | None) -> Path | None: +def _pytest_command_basetemp( + cmd: Sequence[str], *, cwd: str | None, env: Mapping[str, str] | None = None +) -> Path | None: """Return the effective explicit pytest basetemp, if the command has one.""" raw_path: str | None = None - for index, argument in enumerate(cmd): + addopts: list[str] = [] + if env is not None: + with contextlib.suppress(ValueError): + addopts = shlex.split(env.get("PYTEST_ADDOPTS", "")) + arguments = [*addopts, *cmd] + for index, argument in enumerate(arguments): if argument.startswith("--basetemp="): raw_path = argument.partition("=")[2] - elif argument == "--basetemp" and index + 1 < len(cmd): - raw_path = cmd[index + 1] + elif argument == "--basetemp" and index + 1 < len(arguments): + raw_path = arguments[index + 1] if not raw_path: return None path = Path(raw_path) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 081caaa610..f32a8c9ec0 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -823,7 +823,7 @@ def _fs_usage(path: Path) -> dict[str, int] | None: def _dir_usage_kb(path: Path) -> tuple[int | None, int | None]: - """Measure apparent and allocated file bytes in one filesystem walk.""" + """Measure apparent and allocated bytes owned by one basetemp tree.""" if not path.exists(): return None, None logical_total = 0 @@ -831,8 +831,8 @@ def _dir_usage_kb(path: Path) -> tuple[int | None, int | None]: try: for item in path.rglob("*"): with contextlib.suppress(OSError): - if item.is_file(): - item_stat = item.stat() + item_stat = item.lstat() + if not stat.S_ISDIR(item_stat.st_mode): logical_total += item_stat.st_size allocated_total += item_stat.st_blocks * 512 except OSError: diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index 32dfc337e8..52b8067dbb 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -12,6 +12,7 @@ import pytest from devtools import pytest_progress_plugin +from devtools.verify_runs import aggregate_pytest_statistics @pytest.fixture(autouse=True) @@ -51,6 +52,7 @@ class _Report: duration: float = 0.0 longrepr: str = "" worker_id: str | None = None + wasxfail: str | None = None def test_progress_plugin_records_call_and_setup_failures( @@ -76,6 +78,28 @@ def test_progress_plugin_records_call_and_setup_failures( assert events[2]["longrepr"] == "fixture exploded" +def test_progress_plugin_preserves_xfail_and_xpass_in_durable_statistics( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + step = tmp_path / "step" + step.mkdir() + events_path = step / "events.jsonl" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_PATH", str(events_path)) + pytest_progress_plugin.pytest_sessionstart(object()) + + pytest_progress_plugin.pytest_runtest_logstart("test_xfailed", ("tests/a.py", 1, "test_xfailed")) + pytest_progress_plugin.pytest_runtest_logreport( + _Report("test_xfailed", "call", "skipped", wasxfail="known failure") + ) + pytest_progress_plugin.pytest_runtest_logstart("test_xpassed", ("tests/a.py", 2, "test_xpassed")) + pytest_progress_plugin.pytest_runtest_logreport(_Report("test_xpassed", "call", "passed", wasxfail="known failure")) + + statistics = aggregate_pytest_statistics(step) + + assert statistics["outcomes"] == {"xfailed": 1, "xpassed": 1} + + def test_progress_plugin_skips_xdist_controller_forwarding_copy( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -132,8 +156,19 @@ def test_managed_event_ledger_survives_test_host_environment_scrub(tmp_path: Pat # resulting cache to leak into later tests. (checkout_root / ".cache" / "testmon").mkdir(parents=True, exist_ok=True) env = os.environ.copy() + for name in ( + "POLYLOGUE_PYTEST_BASETEMP_ROOT", + "POLYLOGUE_PYTEST_TMPFS", + "POLYLOGUE_PYTEST_RUN_ID", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", + ): + env.pop(name, None) env.update( { + # Keep the nested real pytest away from the host-only scratch + # fallback while preserving the scrubbed event/testmon scenario. + "POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path / "pytest-basetemp"), + "POLYLOGUE_PYTEST_TMPFS": "0", "POLYLOGUE_PYTEST_EVENTS_DIR": str(events_dir), "POLYLOGUE_PYTEST_SELECTION_PATH": str(tmp_path / "selection.json"), "POLYLOGUE_PYTEST_SUMMARY_PATH": str(tmp_path / "summary.json"), diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index cc8e14e2f0..fae943b50a 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -2481,6 +2481,24 @@ def test_sparse_basetemp_enforces_allocated_tmpfs_bytes_and_retains_logical_evid assert pytest_tmpfs_budget_exceeded(sample, budget_kb=allocated_kb - 1) +def test_resource_sampler_does_not_charge_symlink_targets_to_managed_basetemp(tmp_path: Path) -> None: + env = {"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)} + run_id = "symlink-accounting" + basetemp = pytest_basetemp_path(root=tmp_path, run_id=run_id, env=env) + basetemp.mkdir(parents=True) + outside_target = tmp_path / "outside-target.bin" + outside_target.write_bytes(b"x" * (4 * 1024 * 1024)) + (basetemp / "external-link").symlink_to(outside_target) + sampler = ResourceSampler( + root_pid=os.getpid(), run_id=run_id, root=tmp_path, env=env, output_path=tmp_path / "resources.jsonl" + ) + + sample = sampler.sample(event="sample") + + assert sample["basetemp_size_kb"] < 512 + assert sample["basetemp_allocated_kb"] < 512 + + def test_pytest_basetemp_path_tracks_tmpfs_opt_in(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: shm, _scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) path = pytest_basetemp_path(root=tmp_path, run_id="run-1", env={"POLYLOGUE_PYTEST_TMPFS": "1"}) @@ -3494,6 +3512,42 @@ def apply_policy(env: dict[str, str], **_kwargs: object) -> tuple[dict[str, str] assert captured["POLYLOGUE_PYTEST_EXPLICIT_BASETEMP"] == str(explicit) +def test_run_propagates_pytest_addopts_basetemp_to_resource_policy( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + explicit = tmp_path / "diagnostic-basetemp" + captured: dict[str, str] = {} + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + monkeypatch.setenv("PYTEST_ADDOPTS", f"--basetemp {explicit}") + + def apply_policy(env: dict[str, str], **_kwargs: object) -> tuple[dict[str, str], None]: + captured.update(env) + return env, None + + with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", side_effect=apply_policy), + patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), + patch("devtools.verify._read_pytest_report", return_value=None), + ): + rc, _elapsed, _metadata = _run("pytest focused", ["pytest"]) + + assert rc == 0 + assert captured["POLYLOGUE_PYTEST_EXPLICIT_BASETEMP"] == str(explicit) + + +def test_run_clears_stale_current_statistics_before_an_interrupted_pytest_step(tmp_path: Path) -> None: + stale_statistics = tmp_path / verify_runs.CURRENT_STATISTICS_PATH + stale_statistics.parent.mkdir(parents=True) + stale_statistics.write_text('{"node_count": 99}\n', encoding="utf-8") + + with patch("devtools.verify._run_pytest_with_heartbeat", side_effect=KeyboardInterrupt): + rc, _elapsed, metadata = _run("pytest focused", ["pytest"]) + + assert rc == 130 + assert metadata["diagnosis"] == "pytest_interrupted" + assert not stale_statistics.exists() + + def test_explicit_basetemp_policy_uses_actual_path_for_admission( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From 066769ccf2389db60db81ba8880699dbb860d572 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 04:10:33 +0200 Subject: [PATCH 21/53] test(devtools): isolate verification anchor regression Keep the terminal containment-history regression independent of the caller's pytest basetemp location. This preserves the production checkout anchoring behavior while making the test own its temporary artifact paths. --- tests/unit/devtools/test_verify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index fae943b50a..39a970c0bd 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1368,6 +1368,7 @@ def test_verify_main_records_containment_failure_as_terminal_history( monkeypatch.setattr(verify, "HISTORY_PATH", history_path) with ( + patch("devtools.verify._anchor_verification_paths"), patch("devtools.verify._git_head", return_value="head"), patch("devtools.verify._testmon_preflight", return_value=None), patch("devtools.verify.build_verify_steps", return_value=[("pytest containment", ["pytest", "-n", "0"])]), From b0eaa5bc24da9a6af5ed1ef1543c8db1bbcdf6c5 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 04:33:42 +0200 Subject: [PATCH 22/53] fix(test): bind terminal evidence to exact checkout state Preserve xfail and xpass through pytest aggregation and resumable testmon seeds, bind dashboard gates to the exact clean worktree fingerprint, and anchor focused-test artifacts to the checkout root. This prevents expected failures from making complete seeds look interrupted and prevents stale evidence from authorizing a different source state. --- devtools/evidence_dashboard.py | 43 ++++++++- devtools/run_tests.py | 13 +++ devtools/testmon_state.py | 17 +++- devtools/verify.py | 31 ++++-- devtools/verify_runs.py | 12 ++- .../unit/devtools/test_evidence_dashboard.py | 95 +++++++++++++++++++ .../devtools/test_pytest_progress_plugin.py | 8 +- tests/unit/devtools/test_run_tests.py | 41 ++++++++ tests/unit/devtools/test_testmon_state.py | 43 +++++++++ tests/unit/devtools/test_verify.py | 50 +++++++++- 10 files changed, 322 insertions(+), 31 deletions(-) diff --git a/devtools/evidence_dashboard.py b/devtools/evidence_dashboard.py index 66fe545012..9c0cf9acfe 100644 --- a/devtools/evidence_dashboard.py +++ b/devtools/evidence_dashboard.py @@ -28,7 +28,8 @@ from typing import Any from devtools import repo_root as _get_root -from devtools.verify_runs import VERIFY_HISTORY_PATH, git_head +from devtools.verify import _worktree_fingerprint +from devtools.verify_runs import VERIFY_HISTORY_PATH, git_dirty, git_head ROOT = _get_root() @@ -224,11 +225,28 @@ def _benchmark_slo(root: Path, *, now: datetime) -> dict[str, Any]: ) +def _static_evidence_is_bound( + entry: dict[str, Any], + *, + checkout_root: str, + checkout_head: str | None, + worktree_fingerprint: str, +) -> bool: + """Accept only evidence tied to the exact checkout contents being viewed.""" + return ( + entry.get("checkout_root") == checkout_root + and entry.get("git_head") == checkout_head + and entry.get("worktree_fingerprint") == worktree_fingerprint + ) + + def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: history_path = VERIFY_HISTORY_PATH last_result_path = root / LAST_VERIFY_RESULT_REL checkout_root = str(root.resolve()) checkout_head = git_head(root) + checkout_dirty = git_dirty(root) + worktree_fingerprint = None if checkout_dirty else _worktree_fingerprint(root) # Prefer last-verify-result.json (the most recent run) then walk back through # history to find the last status for each gate. @@ -238,7 +256,16 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: try: data = json.loads(last_result_path.read_text()) result = data.get("result") if isinstance(data, dict) else None - if isinstance(result, dict): + if ( + isinstance(result, dict) + and worktree_fingerprint is not None + and _static_evidence_is_bound( + result, + checkout_root=checkout_root, + checkout_head=checkout_head, + worktree_fingerprint=worktree_fingerprint, + ) + ): for step in result.get("steps", []): if isinstance(step, dict) and isinstance(step.get("name"), str): last_steps[step["name"]] = step @@ -265,7 +292,12 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: # appearance in history. if history_entries: for entry in reversed(history_entries): - if entry.get("checkout_root") != checkout_root or entry.get("git_head") != checkout_head: + if worktree_fingerprint is None or not _static_evidence_is_bound( + entry, + checkout_root=checkout_root, + checkout_head=checkout_head, + worktree_fingerprint=worktree_fingerprint, + ): continue steps = entry.get("steps", []) for step in steps: @@ -279,7 +311,8 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: for gate_name in _STATIC_GATE_NAMES: step = last_steps.get(gate_name) if step is None: - gates.append({"name": gate_name, "available": False, "reason": "no run observed in cached history"}) + reason = "checkout has uncommitted changes" if checkout_dirty else "no bound run observed in cached history" + gates.append({"name": gate_name, "available": False, "reason": reason}) continue exit_code = step.get("exit", -1) gates.append( @@ -294,7 +327,7 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: ) failing = [g for g in gates if g.get("status") == "fail"] return { - "available": last_result_path.exists() or history_path.exists(), + "available": bool(last_steps) and not checkout_dirty, "history_path": str(history_path), "last_result_path": str(LAST_VERIFY_RESULT_REL), "total_gates_tracked": len(_STATIC_GATE_NAMES), diff --git a/devtools/run_tests.py b/devtools/run_tests.py index da4886abcb..fc375f2bee 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -46,6 +46,7 @@ PYTEST_SUMMARY_PATH, _clear_pytest_report, _run, + _worktree_fingerprint, ) from devtools.verify_runs import VerifyRun, append_verify_history, git_head @@ -53,6 +54,16 @@ _LOCK_PATH = ROOT / ".cache" / "test-run.lock" +def _anchor_test_paths() -> None: + """Anchor focused-test artifacts to this checkout when called below it.""" + current = Path.cwd().resolve() + try: + current.relative_to(ROOT.resolve()) + except ValueError: + return + os.chdir(ROOT) + + def _has_worker_flag(selection: list[str]) -> bool: """True when the caller already chose an xdist worker count.""" return any(arg.startswith(("-n", "--numprocesses")) for arg in selection) @@ -112,6 +123,7 @@ def _run_lock(*, enabled: bool) -> Iterator[None]: def main(argv: list[str] | None = None) -> int: + _anchor_test_paths() try: fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools test") except CheckoutImportMismatchError as exc: @@ -147,6 +159,7 @@ def main(argv: list[str] | None = None) -> int: root=ROOT, polylogue_import_path=str(polylogue_import_path), environment_fingerprint=environment_fingerprint, + worktree_fingerprint=_worktree_fingerprint(ROOT), ) started = time.monotonic() try: diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py index fd3dc16e65..8ac4bdaa14 100644 --- a/devtools/testmon_state.py +++ b/devtools/testmon_state.py @@ -78,7 +78,12 @@ class TerminalAuthorization(StrEnum): NARROW_TERMINAL = "narrow-terminal" -_TERMINAL_NODE_OUTCOMES = frozenset({"passed", "failed", "error", "skipped"}) +# Pytest reports expected failures separately from ordinary skips/passes. They +# still finish the selected node and therefore make its dependency capture +# reusable. ``xpassed`` remains subject to pytest's configured strictness via +# the process exit code; it is not independently recast as a failure here. +TERMINAL_NODE_OUTCOMES = frozenset({"passed", "failed", "error", "skipped", "xfailed", "xpassed"}) +SUCCESSFUL_NODE_OUTCOMES = frozenset({"passed", "skipped", "xfailed", "xpassed"}) def seed_shard_plan( @@ -179,7 +184,7 @@ def validate_seed_shard_ledger( return None if status is SeedShardStatus.COMPLETE and ( set(outcome_by_node) != set(nodeids) - or any(item.get("outcome") not in _TERMINAL_NODE_OUTCOMES for item in outcome_by_node.values()) + or any(item.get("outcome") not in TERMINAL_NODE_OUTCOMES for item in outcome_by_node.values()) ): return None if ( @@ -771,7 +776,7 @@ def attempt_is_checkout_bound( nodeids = [item.get("nodeid") for item in outcomes if isinstance(item, Mapping)] if len(nodeids) != len(outcomes) or set(nodeids) != set(expected) or len(set(nodeids)) != len(nodeids): return False - if any(item.get("outcome") not in {"passed", "failed", "error", "skipped"} for item in outcomes): + if any(item.get("outcome") not in TERMINAL_NODE_OUTCOMES for item in outcomes): return False return True @@ -1048,7 +1053,7 @@ def stamp_from_attempt( not isinstance(nodeid, str) or not nodeid for nodeid in outcome_by_node ): return None - if any(outcome not in {"passed", "failed", "error", "skipped"} for outcome in outcome_by_node.values()): + if any(outcome not in TERMINAL_NODE_OUTCOMES for outcome in outcome_by_node.values()): return None exit_code = attempt.get("exit_code") if not isinstance(exit_code, int) or isinstance(exit_code, bool): @@ -1066,7 +1071,7 @@ def stamp_from_attempt( BaselineStatus.GREEN if attempt.get("status") == "complete" and exit_code == 0 - and all(outcome in {"passed", "skipped"} for outcome in outcome_by_node.values()) + and all(outcome in SUCCESSFUL_NODE_OUTCOMES for outcome in outcome_by_node.values()) and not graph.failed_nodeids else BaselineStatus.RED ) @@ -1133,6 +1138,8 @@ def stamp_from_attempt( "SeedAttemptOutcome", "TestmonBinding", "TestmonIdentity", + "SUCCESSFUL_NODE_OUTCOMES", + "TERMINAL_NODE_OUTCOMES", "TestmonSeedStamp", "TerminalAuthorization", "VerificationScope", diff --git a/devtools/verify.py b/devtools/verify.py index 58e257add5..7b1fd23e44 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -59,6 +59,8 @@ ) from devtools.testmon_bootstrap import maybe_bootstrap_testmon_seed from devtools.testmon_state import ( + SUCCESSFUL_NODE_OUTCOMES, + TERMINAL_NODE_OUTCOMES, BindingMode, GraphStatus, SeedAttemptOutcome, @@ -2540,15 +2542,16 @@ def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: temporary.replace(path) -def _worktree_fingerprint() -> str: +def _worktree_fingerprint(root: Path | None = None) -> str: """Fingerprint tracked changes plus exact non-ignored untracked content.""" + checkout_root = (root or Path.cwd()).resolve() digest = hashlib.sha256() for command in ( ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"], ["git", "diff", "--binary", "HEAD", "--"], ): try: - result = subprocess.run(command, capture_output=True, timeout=30) + result = subprocess.run(command, capture_output=True, timeout=30, cwd=checkout_root) except (OSError, subprocess.TimeoutExpired): return "unavailable" if result.returncode != 0: @@ -2560,6 +2563,7 @@ def _worktree_fingerprint() -> str: ["git", "ls-files", "--others", "--exclude-standard", "-z"], capture_output=True, timeout=30, + cwd=checkout_root, ) except (OSError, subprocess.TimeoutExpired): return "unavailable" @@ -2568,7 +2572,7 @@ def _worktree_fingerprint() -> str: for raw_path in sorted(path for path in untracked.stdout.split(b"\0") if path): try: path_text = os.fsdecode(raw_path) - path = Path(path_text) + path = checkout_root / path_text mode = path.lstat().st_mode digest.update(raw_path) digest.update(b"\0") @@ -2991,7 +2995,7 @@ def _checkpoint_testmon_seed_shard( prior_node_outcomes=prior, use_database_fallback=False, ) - terminal = all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in outcomes) + terminal = all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in outcomes) selection_matches = selected == nodeids shard.update( { @@ -3098,6 +3102,10 @@ def _seed_node_outcomes_from_events( outcome, reason = "timeout", "pytest-timeout report" elif any(report.get("when") in {"setup", "teardown"} for report in failed_reports): outcome, reason = "error", "fixture setup/teardown failed" + elif any(report.get("outcome") == "xfailed" for report in node_reports): + outcome, reason = "xfailed", "pytest expected failure" + elif any(report.get("outcome") == "xpassed" for report in node_reports): + outcome, reason = "xpassed", "pytest unexpected pass" elif any(report.get("outcome") == "failed" for report in call_reports): outcome, reason = "failed", "test call failed" elif any(report.get("outcome") == "passed" for report in call_reports): @@ -3136,7 +3144,7 @@ def _seed_node_outcomes_from_events( elif prior_node_outcomes is not None and nodeid in prior_node_outcomes: prior = prior_node_outcomes[nodeid] prior_outcome = prior.get("outcome") - if prior_outcome in {"passed", "failed", "error", "skipped"}: + if prior_outcome in TERMINAL_NODE_OUTCOMES: outcome, reason = str(prior_outcome), "terminal outcome carried from the prior seed attempt" else: outcome, reason = "missing", "prior seed attempt has no terminal outcome" @@ -3271,7 +3279,7 @@ def _finalize_testmon_seed_attempt( }, ) unsuccessful_nodeids = [ - str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} + str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in SUCCESSFUL_NODE_OUTCOMES ] green_complete = ( exit_code == 0 @@ -3296,7 +3304,7 @@ def _finalize_testmon_seed_attempt( and not database["missing_nodeids"] and database["orphan_execution_edges"] == 0 and database["orphan_fingerprint_edges"] == 0 - and all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in node_outcomes) + and all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) ) outcome = _seed_attempt_outcome( release_eligible=release_eligible, @@ -3312,7 +3320,7 @@ def _finalize_testmon_seed_attempt( { "status": ( SeedShardStatus.COMPLETE.value - if all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in node_outcomes) + if all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) else SeedShardStatus.INCOMPLETE.value ), "node_outcomes": node_outcomes, @@ -3451,7 +3459,7 @@ def _refresh_testmon_selection_attempt( and database.get("orphan_execution_edges") == 0 and database.get("orphan_fingerprint_edges") == 0 ) - terminal = all(item.get("outcome") in {"passed", "failed", "error", "skipped"} for item in node_outcomes) + terminal = all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) prior_selection = attempt.get("selection") payload = { **attempt, @@ -3482,7 +3490,7 @@ def _refresh_testmon_selection_attempt( ) ), "unsuccessful_nodeids": [ - str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in {"passed", "skipped"} + str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in SUCCESSFUL_NODE_OUTCOMES ], "testmon_data": _file_fingerprint(TESTMON_DATA), "run_id": run.run_id, @@ -3587,12 +3595,14 @@ def main(argv: list[str] | None = None) -> int: head = _git_head() t0 = time.monotonic() + worktree_fingerprint = _worktree_fingerprint() verify_run = VerifyRun( tier=tier, argv=list(sys.argv[1:] if argv is None else argv), git_head=head, polylogue_import_path=str(polylogue_import_path), environment_fingerprint=environment_fingerprint, + worktree_fingerprint=worktree_fingerprint, ) seed_identity: dict[str, Any] | None = None resume_testmon_seed = False @@ -3815,6 +3825,7 @@ def main(argv: list[str] | None = None) -> int: "tier": tier, "run_id": verify_run.run_id, "checkout_root": str(ROOT.resolve()), + "worktree_fingerprint": worktree_fingerprint, "artifact_dir": str(verify_run.relative_run_dir), "steps": step_results, "total_duration_s": total_duration, diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index f32a8c9ec0..c17d7ca920 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -281,8 +281,8 @@ def aggregate_pytest_statistics( terminal = "error" elif isinstance(call, str): terminal = call - elif setup == "skipped" or teardown == "skipped": - terminal = "skipped" + elif setup in {"skipped", "xfailed", "xpassed"} or teardown in {"skipped", "xfailed", "xpassed"}: + terminal = setup if setup in {"skipped", "xfailed", "xpassed"} else teardown else: # A test may have emitted its start event just before an interrupt # or forced containment cleanup. Keep that missing terminal phase @@ -374,9 +374,9 @@ def aggregate_pytest_statistics( } -def git_dirty() -> bool: +def git_dirty(cwd: Path | None = None) -> bool: try: - result = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, timeout=5) + result = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, timeout=5, cwd=cwd) except (OSError, subprocess.TimeoutExpired): return True return bool(result.stdout.strip()) @@ -435,6 +435,7 @@ def __init__( root: Path | None = None, polylogue_import_path: str | None = None, environment_fingerprint: Mapping[str, Any] | None = None, + worktree_fingerprint: str | None = None, ) -> None: self.root = root or Path.cwd() self.run_id = make_run_id(tier=tier) @@ -444,7 +445,7 @@ def __init__( "tier": tier, "argv": list(argv), "git_head": git_head, - "git_dirty": git_dirty(), + "git_dirty": git_dirty(self.root), # Receipt for the worktree-import hazard (devtools/checkout_guard.py): # the resolved `polylogue` package path this run actually used, so a # wrong-tree run is visible after the fact from the run artifact @@ -452,6 +453,7 @@ def __init__( # caller and this fired for a different process boundary. "polylogue_import_path": polylogue_import_path, "environment_fingerprint": dict(environment_fingerprint) if environment_fingerprint is not None else None, + "worktree_fingerprint": worktree_fingerprint, # A VerifyRun can be constructed by maintenance/test helpers that # do not have a checkout fingerprint. Keep its current-run marker # attributable to this checkout either way. diff --git a/tests/unit/devtools/test_evidence_dashboard.py b/tests/unit/devtools/test_evidence_dashboard.py index bdddecbe62..1cae6600ca 100644 --- a/tests/unit/devtools/test_evidence_dashboard.py +++ b/tests/unit/devtools/test_evidence_dashboard.py @@ -18,6 +18,7 @@ def test_static_gates_read_shared_verify_history(monkeypatch: pytest.MonkeyPatch "timestamp": "2026-08-12T00:00:00+00:00", "checkout_root": str(tmp_path.resolve()), "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", "steps": [{"name": "ruff check", "duration_s": 1.0, "exit": 0}], } ) @@ -34,6 +35,8 @@ def test_static_gates_read_shared_verify_history(monkeypatch: pytest.MonkeyPatch ) monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) @@ -42,3 +45,95 @@ def test_static_gates_read_shared_verify_history(monkeypatch: pytest.MonkeyPatch assert ruff["status"] == "ok" mypy = next(gate for gate in gates["gates"] if gate["name"] == "mypy") assert mypy["available"] is False + + +def test_static_gates_accept_exactly_bound_last_verify_result(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + result_path = tmp_path / evidence_dashboard.LAST_VERIFY_RESULT_REL + result_path.parent.mkdir(parents=True) + result_path.write_text( + json.dumps( + { + "result": { + "timestamp": "2026-08-12T00:00:00+00:00", + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "steps": [{"name": "ruff check", "duration_s": 1.0, "exit": 0}], + } + } + ) + ) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is True + assert next(gate for gate in gates["gates"] if gate["name"] == "ruff check")["status"] == "ok" + + +def test_static_gates_withhold_evidence_for_dirty_checkout(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + history = tmp_path / "history.jsonl" + history.write_text( + json.dumps( + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "steps": [{"name": "ruff check", "exit": 0}], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: True) + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["reason"] == "checkout has uncommitted changes" for gate in gates["gates"]) + + +def test_static_gates_reject_wrong_checkout_fingerprint_and_legacy_evidence( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + history = tmp_path / "history.jsonl" + legacy_result = tmp_path / evidence_dashboard.LAST_VERIFY_RESULT_REL + legacy_result.parent.mkdir(parents=True) + legacy_result.write_text(json.dumps({"result": {"steps": [{"name": "ruff check", "exit": 0}]}})) + history.write_text( + "\n".join( + json.dumps(entry) + for entry in ( + { + "checkout_root": str(tmp_path / "other"), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "steps": [{"name": "ruff check", "exit": 0}], + }, + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "other-fingerprint", + "steps": [{"name": "mypy", "exit": 0}], + }, + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "steps": [{"name": "render all", "exit": 0}], + }, + ) + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["available"] is False for gate in gates["gates"]) diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index 52b8067dbb..a0ab00fdcd 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -92,12 +92,16 @@ def test_progress_plugin_preserves_xfail_and_xpass_in_durable_statistics( pytest_progress_plugin.pytest_runtest_logreport( _Report("test_xfailed", "call", "skipped", wasxfail="known failure") ) - pytest_progress_plugin.pytest_runtest_logstart("test_xpassed", ("tests/a.py", 2, "test_xpassed")) + pytest_progress_plugin.pytest_runtest_logstart("test_setup_xfailed", ("tests/a.py", 2, "test_setup_xfailed")) + pytest_progress_plugin.pytest_runtest_logreport( + _Report("test_setup_xfailed", "setup", "skipped", wasxfail="fixture calls pytest.xfail()") + ) + pytest_progress_plugin.pytest_runtest_logstart("test_xpassed", ("tests/a.py", 3, "test_xpassed")) pytest_progress_plugin.pytest_runtest_logreport(_Report("test_xpassed", "call", "passed", wasxfail="known failure")) statistics = aggregate_pytest_statistics(step) - assert statistics["outcomes"] == {"xfailed": 1, "xpassed": 1} + assert statistics["outcomes"] == {"xfailed": 2, "xpassed": 1} def test_progress_plugin_skips_xdist_controller_forwarding_copy( diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index d8a74f459a..f0b6cd436a 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -5,6 +5,7 @@ import subprocess import sys from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest @@ -117,6 +118,46 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di assert run_tests.main(["tests/unit/does_not_exist"]) == 5 +def test_main_anchors_and_refreshes_root_artifacts_from_a_subdirectory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + root = tmp_path / "checkout" + subdirectory = root / "devtools" + subdirectory.mkdir(parents=True) + stale_report = root / verify.PYTEST_REPORT_PATH + stale_statistics = root / verify.CURRENT_STATISTICS_PATH + stale_report.parent.mkdir(parents=True) + stale_report.write_text('{"stale": true}') + stale_statistics.write_text('{"stale": true}') + captured: dict[str, object] = {} + + def fake_run(_label: str, _cmd: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: + captured["cwd"] = kwargs["cwd"] + Path(verify.PYTEST_REPORT_PATH).write_text('{"fresh": true}') + return 0, 0.01, {"diagnosis": "pytest_passed"} + + monkeypatch.setattr(run_tests, "ROOT", root) + monkeypatch.setattr(run_tests, "_LOCK_PATH", root / ".cache" / "test-run.lock") + monkeypatch.setattr( + run_tests, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=root / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setattr(run_tests, "git_head", lambda _root: "head") + monkeypatch.setattr(run_tests, "_worktree_fingerprint", lambda _root: "fingerprint") + monkeypatch.setattr(run_tests, "_run", fake_run) + monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.chdir(subdirectory) + + assert run_tests.main(["tests/unit/example.py"]) == 0 + + assert captured["cwd"] == str(root) + assert stale_report.read_text() == '{"fresh": true}' + assert not stale_statistics.exists() + assert not (subdirectory / ".cache").exists() + + def test_git_head_records_checkout_head(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: def _fake_run(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: assert cmd == ["git", "rev-parse", "HEAD"] diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py index 0a01f3c9a2..3df6ce4440 100644 --- a/tests/unit/devtools/test_testmon_state.py +++ b/tests/unit/devtools/test_testmon_state.py @@ -122,6 +122,49 @@ def test_seed_shard_ledger_rejects_duplicate_nodes_across_shards() -> None: assert testmon_state.validate_seed_shard_ledger([shard, duplicate], expected_nodeids=[NODEIDS[0]]) is None +def test_seed_shard_ledger_accepts_expected_and_unexpected_xfail_outcomes() -> None: + nodes = sorted(NODEIDS) + shard = { + "index": 1, + "nodeids": nodes, + "nodeid_count": len(nodes), + "nodeid_digest": hashlib.sha256("\n".join(nodes).encode()).hexdigest(), + "status": "complete", + "node_outcomes": [ + {"nodeid": nodes[0], "outcome": "xfailed"}, + {"nodeid": nodes[1], "outcome": "xpassed"}, + ], + } + + assert testmon_state.validate_seed_shard_ledger([shard], expected_nodeids=nodes) == [shard] + + +def test_expected_failure_and_non_strict_xpass_preserve_green_baseline(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + _write_graph(data) + attempt = _attempt(data, outcomes=("xfailed", "xpassed")) + attempt.update(status="complete", exit_code=0, release_baseline_allowed=True, verification_scope="release-baseline") + identity = attempt["identity"] + assert isinstance(identity, dict) + identity.update(skip_slow=False, terminal_authorization=None) + + green = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert green is not None + assert green.baseline_status is BaselineStatus.GREEN + serialized = green.as_dict() + assert _TestmonSeedStamp.from_mapping(serialized, protocol_version=PROTOCOL).baseline_status is BaselineStatus.GREEN + + # pytest controls strict-xpass behavior through its process exit code. A + # strict xpass therefore remains reusable graph evidence but is red. + attempt.update(status="reusable", exit_code=1, release_baseline_allowed=False) + strict = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) + + assert strict is not None + assert strict.baseline_status is BaselineStatus.RED + assert strict.affected_selection_allowed + + def test_testmon_database_canonicalizes_xdist_group_names(tmp_path: Path) -> None: data = tmp_path / "testmondata" _write_graph(data) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 39a970c0bd..34441a810e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -445,7 +445,7 @@ def test_seed_shard_checkpoint_preserves_completed_shards_for_resume( "event": "test_report", "nodeid": f"{ordered[0]}@web-reader", "when": "call", - "outcome": "passed", + "outcome": "xfailed", } ) + "\n" @@ -459,7 +459,7 @@ def test_seed_shard_checkpoint_preserves_completed_shards_for_resume( assert checkpointed["shards"][0]["status"] == "complete" assert checkpointed["shards"][1]["status"] == "pending" - assert json.loads(TESTMON_SEED_ATTEMPT.read_text())["shards"][0]["node_outcomes"][0]["outcome"] == "passed" + assert json.loads(TESTMON_SEED_ATTEMPT.read_text())["shards"][0]["node_outcomes"][0]["outcome"] == "xfailed" resumed = _prepare_testmon_seed_attempt( identity={ "git_head": "head", @@ -1968,6 +1968,39 @@ def test_seed_node_outcomes_accept_setup_skip_as_terminal_skip(tmp_path: Path) - ] +def test_seed_node_outcomes_preserve_call_and_fixture_xfail_xpass(tmp_path: Path) -> None: + """Durable pytest reports, including fixture ``pytest.xfail()``, finish seed nodes.""" + events = tmp_path / "events.jsonl" + nodes = [ + "tests/test_a.py::test_call_xfailed", + "tests/test_a.py::test_call_xpassed", + "tests/test_a.py::test_setup_xfailed", + ] + events.write_text( + "\n".join( + json.dumps(event) + for event in ( + {"event": "test_report", "nodeid": nodes[0], "when": "call", "outcome": "xfailed"}, + {"event": "test_report", "nodeid": nodes[1], "when": "call", "outcome": "xpassed"}, + {"event": "test_report", "nodeid": nodes[2], "when": "setup", "outcome": "xfailed"}, + ) + ) + + "\n" + ) + + outcomes = _seed_node_outcomes_from_events( + events, + expected_nodeids=nodes, + database={"node_outcomes": {}}, + pytest_step={}, + use_database_fallback=False, + ) + + assert {item["nodeid"]: item["outcome"] for item in outcomes} == dict( + zip(nodes, ("xfailed", "xpassed", "xfailed"), strict=True) + ) + + def test_resumed_seed_carries_forward_prior_terminal_outcome(tmp_path: Path) -> None: events = tmp_path / "events.jsonl" events.write_text( @@ -1978,18 +2011,27 @@ def test_resumed_seed_carries_forward_prior_terminal_outcome(tmp_path: Path) -> ) outcomes = _seed_node_outcomes_from_events( events, - expected_nodeids=["tests/test_a.py::test_repaired", "tests/test_b.py::test_prior"], + expected_nodeids=[ + "tests/test_a.py::test_repaired", + "tests/test_b.py::test_prior", + "tests/test_c.py::test_expected_failure", + ], database={"node_outcomes": {"tests/test_b.py::test_prior": "passed"}}, pytest_step={}, use_database_fallback=False, prior_node_outcomes={ - "tests/test_b.py::test_prior": {"nodeid": "tests/test_b.py::test_prior", "outcome": "passed"} + "tests/test_b.py::test_prior": {"nodeid": "tests/test_b.py::test_prior", "outcome": "passed"}, + "tests/test_c.py::test_expected_failure": { + "nodeid": "tests/test_c.py::test_expected_failure", + "outcome": "xfailed", + }, }, ) assert {item["nodeid"]: item["outcome"] for item in outcomes} == { "tests/test_a.py::test_repaired": "passed", "tests/test_b.py::test_prior": "passed", + "tests/test_c.py::test_expected_failure": "xfailed", } From f0fcd1cc2a76a3a6786bab5a02ea30ce2f29ced1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 04:49:06 +0200 Subject: [PATCH 23/53] fix(test): make focused xdist runs isolation-safe --- devtools/run_tests.py | 23 ++++++++++----- .../devtools/test_pytest_progress_plugin.py | 3 ++ tests/unit/devtools/test_run_tests.py | 29 +++++++++++++++---- tests/unit/devtools/test_verify.py | 2 ++ 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/devtools/run_tests.py b/devtools/run_tests.py index fc375f2bee..a650a84de5 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -45,6 +45,7 @@ PYTEST_SELECTION_PATH, PYTEST_SUMMARY_PATH, _clear_pytest_report, + _pytest_command_worker_request, _run, _worktree_fingerprint, ) @@ -55,12 +56,7 @@ def _anchor_test_paths() -> None: - """Anchor focused-test artifacts to this checkout when called below it.""" - current = Path.cwd().resolve() - try: - current.relative_to(ROOT.resolve()) - except ValueError: - return + """Anchor focused-test execution and artifacts to this checkout.""" os.chdir(ROOT) @@ -77,8 +73,20 @@ def _worker_args(selection: list[str]) -> list[str]: return ["-n", workers] +def _xdist_distribution_args(selection: list[str], worker_args: list[str]) -> list[str]: + """Keep declared shared-state groups together whenever xdist is active.""" + if any(arg == "--dist" or arg.startswith("--dist=") for arg in selection): + return [] + command = [*selection, *worker_args] + request = _pytest_command_worker_request(command) + if request in {None, "0"}: + return [] + return ["--dist=loadgroup"] + + def build_pytest_cmd(selection: list[str]) -> list[str]: """Compose the pytest command for a focused selection.""" + worker_args = _worker_args(selection) return [ sys.executable, "-m", @@ -89,7 +97,8 @@ def build_pytest_cmd(selection: list[str]) -> list[str]: "--json-report-omit=collectors,log,streams,warnings", f"--json-report-file={PYTEST_REPORT_PATH}", *selection, - *_worker_args(selection), + *worker_args, + *_xdist_distribution_args(selection, worker_args), ] diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index a0ab00fdcd..e4ab65dcc6 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -14,6 +14,8 @@ from devtools import pytest_progress_plugin from devtools.verify_runs import aggregate_pytest_statistics +pytestmark = pytest.mark.xdist_group("checkout-testmon") + @pytest.fixture(autouse=True) def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[None]: @@ -24,6 +26,7 @@ def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> It "POLYLOGUE_PYTEST_EVENTS_PATH", "POLYLOGUE_PYTEST_SELECTION_PATH", "POLYLOGUE_PYTEST_SUMMARY_PATH", + "PYTEST_XDIST_WORKER", ): monkeypatch.delenv(name, raising=False) selected_count = pytest_progress_plugin._SELECTED_COUNT diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index f0b6cd436a..b2fe877c68 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -31,7 +31,7 @@ def test_build_pytest_cmd_respects_explicit_worker_flag() -> None: cmd = run_tests.build_pytest_cmd(["tests/unit", "-n", "4"]) # No injected -n when the caller already chose one. assert cmd.count("-n") == 1 - assert cmd[-2:] == ["-n", "4"] + assert cmd[-3:] == ["-n", "4", "--dist=loadgroup"] @pytest.mark.parametrize( @@ -60,7 +60,20 @@ def test_build_pytest_cmd_forwards_exactly_one_xdist_worker_request( def test_build_pytest_cmd_honors_workers_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("POLYLOGUE_PYTEST_WORKERS", "8") cmd = run_tests.build_pytest_cmd(["tests/unit"]) - assert cmd[-2:] == ["-n", "8"] + assert cmd[-3:] == ["-n", "8", "--dist=loadgroup"] + + +def test_build_pytest_cmd_preserves_explicit_xdist_distribution() -> None: + cmd = run_tests.build_pytest_cmd(["tests/unit", "-n", "4", "--dist=worksteal"]) + + assert cmd.count("--dist=worksteal") == 1 + assert "--dist=loadgroup" not in cmd + + +def test_build_pytest_cmd_does_not_add_distribution_for_serial_run() -> None: + cmd = run_tests.build_pytest_cmd(["tests/unit", "-n", "0"]) + + assert not any(arg.startswith("--dist") for arg in cmd) def test_subprocess_env_anchors_pytest_artifacts_to_checkout(monkeypatch: pytest.MonkeyPatch) -> None: @@ -118,12 +131,15 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di assert run_tests.main(["tests/unit/does_not_exist"]) == 5 -def test_main_anchors_and_refreshes_root_artifacts_from_a_subdirectory( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path +@pytest.mark.parametrize("invocation_location", ["inside", "external"]) +def test_main_anchors_and_refreshes_root_artifacts_from_any_invocation_directory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, invocation_location: str ) -> None: root = tmp_path / "checkout" subdirectory = root / "devtools" + external_directory = tmp_path / "unrelated" subdirectory.mkdir(parents=True) + external_directory.mkdir() stale_report = root / verify.PYTEST_REPORT_PATH stale_statistics = root / verify.CURRENT_STATISTICS_PATH stale_report.parent.mkdir(parents=True) @@ -148,14 +164,15 @@ def fake_run(_label: str, _cmd: list[str], **kwargs: Any) -> tuple[int, float, d monkeypatch.setattr(run_tests, "_run", fake_run) monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") - monkeypatch.chdir(subdirectory) + invocation_directory = subdirectory if invocation_location == "inside" else external_directory + monkeypatch.chdir(invocation_directory) assert run_tests.main(["tests/unit/example.py"]) == 0 assert captured["cwd"] == str(root) assert stale_report.read_text() == '{"fresh": true}' assert not stale_statistics.exists() - assert not (subdirectory / ".cache").exists() + assert not (invocation_directory / ".cache").exists() def test_git_head_records_checkout_head(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 34441a810e..f28f476a97 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -98,6 +98,8 @@ xdist_uninterruptible_stall_reason, ) +pytestmark = pytest.mark.xdist_group("checkout-testmon") + @pytest.fixture(autouse=True) def _isolate_verify_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: From 8640677d6ee0875fc9dfca1db3b75008188e3cc6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 04:58:12 +0200 Subject: [PATCH 24/53] fix(test): preserve canonical completed-run evidence --- devtools/verify.py | 8 +++-- devtools/verify_runs.py | 43 +++++++++++++++++++++-- tests/unit/devtools/test_run_tests.py | 4 +-- tests/unit/devtools/test_verify.py | 49 +++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 7 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 7b1fd23e44..ccca18ae5f 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -84,6 +84,7 @@ CURRENT_POSTMORTEM_PATH, CURRENT_RESOURCES_PATH, CURRENT_STATISTICS_PATH, + PYTEST_CANONICAL_REPORT_NAME, PYTEST_EXPLICIT_BASETEMP_ENV, VERIFY_HISTORY_PATH, PytestResourceError, @@ -1716,6 +1717,10 @@ def _run( if report is not None: metadata.update(_pytest_metadata_from_report(report, report_path=report_path)) metadata["report_status"] = "present" + if artifacts is not None: + durable_report_path = artifacts.step_dir / PYTEST_CANONICAL_REPORT_NAME + shutil.copyfile(report_path, durable_report_path) + metadata["report_path"] = str(durable_report_path.relative_to(ROOT)) else: # Fallback: terminal scraping when the structured report is # missing (pytest crashed before writing it, or the plugin is @@ -2352,7 +2357,7 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: workers = adaptive_pytest_worker_count(os.environ) if maximum is not None: workers = min(workers, maximum) - return ["-n", str(workers)] + return ["--dist=loadgroup", "-n", str(workers)] BROAD_PYTEST_STEP_LABELS = { @@ -2931,7 +2936,6 @@ def _seed_shard_command( else: command.extend( [ - "--dist=loadgroup", *_pytest_worker_args(maximum=10), "--testmon", "--testmon-noselect", diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index c17d7ca920..e694e236f9 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -35,6 +35,7 @@ CURRENT_POSTMORTEM_PATH = VERIFY_CACHE / "current-pytest-postmortem.json" CURRENT_CONTAINMENT_PATH = VERIFY_CACHE / "current-pytest-containment.json" CURRENT_STATISTICS_PATH = VERIFY_CACHE / "current-pytest-statistics.json" +PYTEST_CANONICAL_REPORT_NAME = "pytest-report.json" CURRENT_EVENTS_DIR = VERIFY_CACHE / "current-pytest-events" DEFAULT_BASETEMP_SIZE_SAMPLE_INTERVAL_S = 15.0 DEFAULT_TMPFS_SIZE_SAMPLE_INTERVAL_S = 2.0 @@ -261,6 +262,36 @@ def aggregate_pytest_statistics( if prior is None or (prior.get("worker_id") == "controller" and row.get("worker_id") != "controller"): reports[key] = row + canonical_outcomes: dict[str, str] = {} + canonical_report_path = step_dir / PYTEST_CANONICAL_REPORT_NAME + if canonical_report_path.exists(): + with contextlib.suppress(OSError, json.JSONDecodeError): + canonical_report = json.loads(canonical_report_path.read_text(encoding="utf-8")) + canonical_tests = canonical_report.get("tests") if isinstance(canonical_report, dict) else None + if isinstance(canonical_tests, list): + for test in canonical_tests: + if not isinstance(test, dict): + continue + nodeid = test.get("nodeid") + outcome = test.get("outcome") + if not isinstance(nodeid, str) or not nodeid or not isinstance(outcome, str): + continue + nodes.add(nodeid) + canonical_outcomes[nodeid] = outcome + for when in phases: + phase = test.get(when) + if not isinstance(phase, dict) or (nodeid, when) in reports: + continue + phase_outcome = phase.get("outcome") + duration = phase.get("duration") + reports[(nodeid, when)] = { + "nodeid": nodeid, + "when": when, + "outcome": phase_outcome, + "duration_s": duration, + "worker_id": "canonical-report", + } + reports_by_node: dict[str, dict[str, dict[str, Any]]] = {} for (nodeid, when), row in reports.items(): reports_by_node.setdefault(nodeid, {})[when] = row @@ -277,12 +308,17 @@ def aggregate_pytest_statistics( setup = node_reports.get("setup", {}).get("outcome") call = node_reports.get("call", {}).get("outcome") teardown = node_reports.get("teardown", {}).get("outcome") - if setup == "failed" or teardown == "failed": + canonical_outcome = canonical_outcomes.get(nodeid) + if canonical_outcome is not None: + terminal = canonical_outcome + elif setup == "failed" or teardown == "failed": terminal = "error" elif isinstance(call, str): terminal = call - elif setup in {"skipped", "xfailed", "xpassed"} or teardown in {"skipped", "xfailed", "xpassed"}: - terminal = setup if setup in {"skipped", "xfailed", "xpassed"} else teardown + elif setup in {"skipped", "xfailed", "xpassed"}: + terminal = str(setup) + elif teardown in {"skipped", "xfailed", "xpassed"}: + terminal = str(teardown) else: # A test may have emitted its start event just before an interrupt # or forced containment cleanup. Keep that missing terminal phase @@ -324,6 +360,7 @@ def aggregate_pytest_statistics( parent_cleanup = (step_result or {}).get("basetemp_cleanup") return { "schema_version": 1, + "canonical_report_status": "present" if canonical_outcomes else "missing", "command": [str(value) for value in command], "node_count": len(nodes), "outcomes": outcomes, diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index b2fe877c68..e3d27a3df0 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -11,7 +11,7 @@ import pytest from devtools import run_tests, verify -from devtools.verify_runs import git_head +from devtools.verify_runs import CURRENT_STATISTICS_PATH, git_head def test_build_pytest_cmd_defaults_to_single_process() -> None: @@ -141,7 +141,7 @@ def test_main_anchors_and_refreshes_root_artifacts_from_any_invocation_directory subdirectory.mkdir(parents=True) external_directory.mkdir() stale_report = root / verify.PYTEST_REPORT_PATH - stale_statistics = root / verify.CURRENT_STATISTICS_PATH + stale_statistics = root / CURRENT_STATISTICS_PATH stale_report.parent.mkdir(parents=True) stale_report.write_text('{"stale": true}') stale_statistics.write_text('{"stale": true}') diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index f28f476a97..444442d5e7 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -272,8 +272,10 @@ def test_default_verify_uses_adaptive_pytest_testmon(monkeypatch: pytest.MonkeyP assert "--testmon" in command assert "--testmon-noselect" not in command assert "--testmon-forceselect" in command + assert "--dist=loadgroup" in command assert "-n" in command assert command[command.index("-n") + 1] == "8" + assert "--dist=loadgroup" in command def test_broad_default_verify_uses_parallel_testmon(monkeypatch: pytest.MonkeyPatch) -> None: @@ -639,6 +641,7 @@ def test_full_verify_includes_full_pytest_without_testmon(monkeypatch: pytest.Mo assert "--testmon" not in bulk_command assert "-n" in bulk_command assert bulk_command[bulk_command.index("-n") + 1] == "8" + assert "--dist=loadgroup" in bulk_command isolated_label, isolated_command = steps[-1] assert isolated_label == "pytest load-sensitive (isolated)" @@ -1097,6 +1100,52 @@ def test_aggregate_pytest_statistics_accounts_for_started_node_without_a_phase(t assert sum(result["outcomes"].values()) == result["node_count"] +def test_aggregate_pytest_statistics_uses_completed_report_to_fill_event_gaps(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + (step / "events.jsonl").write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/a.py::test_event", + "when": "call", + "outcome": "passed", + "duration_s": 0.1, + "worker_id": "gw0", + } + ) + + "\n" + ) + (step / "pytest-report.json").write_text( + json.dumps( + { + "tests": [ + { + "nodeid": "tests/a.py::test_event", + "outcome": "passed", + "call": {"outcome": "passed", "duration": 0.1}, + }, + { + "nodeid": "tests/a.py::test_redirected", + "outcome": "xfailed", + "setup": {"outcome": "passed", "duration": 0.2}, + "call": {"outcome": "skipped", "duration": 0.3}, + "teardown": {"outcome": "passed", "duration": 0.1}, + }, + ] + } + ) + ) + + result = aggregate_pytest_statistics(step) + + assert result["canonical_report_status"] == "present" + assert result["node_count"] == 2 + assert result["outcomes"] == {"passed": 1, "xfailed": 1} + assert result["phases"]["setup"]["count"] == 1 + assert result["phases"]["call"]["count"] == 2 + + def test_verify_run_statistics_only_cover_pytest_steps(tmp_path: Path) -> None: run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) artifacts = run.start_step(label="ruff check", cmd=["ruff", "check"]) From dcee6b78248182ee4c83edb4e6363201a8bac39d Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 04:59:53 +0200 Subject: [PATCH 25/53] fix(test): reject unverifiable cached gate evidence --- devtools/evidence_dashboard.py | 14 ++++++-- .../unit/devtools/test_evidence_dashboard.py | 33 +++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/devtools/evidence_dashboard.py b/devtools/evidence_dashboard.py index 9c0cf9acfe..7f3c28ad57 100644 --- a/devtools/evidence_dashboard.py +++ b/devtools/evidence_dashboard.py @@ -246,7 +246,9 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: checkout_root = str(root.resolve()) checkout_head = git_head(root) checkout_dirty = git_dirty(root) - worktree_fingerprint = None if checkout_dirty else _worktree_fingerprint(root) + fingerprint = None if checkout_dirty or checkout_head is None else _worktree_fingerprint(root) + worktree_fingerprint = None if fingerprint == "unavailable" else fingerprint + identity_available = checkout_head is not None and worktree_fingerprint is not None # Prefer last-verify-result.json (the most recent run) then walk back through # history to find the last status for each gate. @@ -311,7 +313,13 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: for gate_name in _STATIC_GATE_NAMES: step = last_steps.get(gate_name) if step is None: - reason = "checkout has uncommitted changes" if checkout_dirty else "no bound run observed in cached history" + reason = ( + "checkout has uncommitted changes" + if checkout_dirty + else "checkout Git identity is unavailable" + if not identity_available + else "no bound run observed in cached history" + ) gates.append({"name": gate_name, "available": False, "reason": reason}) continue exit_code = step.get("exit", -1) @@ -327,7 +335,7 @@ def _static_gates(root: Path, *, now: datetime) -> dict[str, Any]: ) failing = [g for g in gates if g.get("status") == "fail"] return { - "available": bool(last_steps) and not checkout_dirty, + "available": bool(last_steps) and not checkout_dirty and identity_available, "history_path": str(history_path), "last_result_path": str(LAST_VERIFY_RESULT_REL), "total_gates_tracked": len(_STATIC_GATE_NAMES), diff --git a/tests/unit/devtools/test_evidence_dashboard.py b/tests/unit/devtools/test_evidence_dashboard.py index 1cae6600ca..2d1035ffaf 100644 --- a/tests/unit/devtools/test_evidence_dashboard.py +++ b/tests/unit/devtools/test_evidence_dashboard.py @@ -96,6 +96,39 @@ def test_static_gates_withhold_evidence_for_dirty_checkout(monkeypatch: pytest.M assert all(gate["reason"] == "checkout has uncommitted changes" for gate in gates["gates"]) +@pytest.mark.parametrize( + ("checkout_head", "fingerprint"), + [(None, "unavailable"), ("current-head", "unavailable")], +) +def test_static_gates_withhold_evidence_when_git_identity_is_unavailable( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + checkout_head: str | None, + fingerprint: str, +) -> None: + history = tmp_path / "history.jsonl" + history.write_text( + json.dumps( + { + "checkout_root": str(tmp_path.resolve()), + "git_head": checkout_head, + "worktree_fingerprint": fingerprint, + "steps": [{"name": "ruff check", "exit": 0}], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: checkout_head) + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: fingerprint) + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["reason"] == "checkout Git identity is unavailable" for gate in gates["gates"]) + + def test_static_gates_reject_wrong_checkout_fingerprint_and_legacy_evidence( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 258306ce348d5b60b2dad609b46250062cf2a3df Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 05:06:31 +0200 Subject: [PATCH 26/53] fix(devtools): recognize empty pytest reports Treat a successfully parsed canonical pytest report as present even when it contains zero tests. This preserves truthful completed-run evidence for zero-selection executions. --- devtools/verify_runs.py | 4 +++- tests/unit/devtools/test_verify.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index e694e236f9..5de910c5b8 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -263,12 +263,14 @@ def aggregate_pytest_statistics( reports[key] = row canonical_outcomes: dict[str, str] = {} + canonical_report_present = False canonical_report_path = step_dir / PYTEST_CANONICAL_REPORT_NAME if canonical_report_path.exists(): with contextlib.suppress(OSError, json.JSONDecodeError): canonical_report = json.loads(canonical_report_path.read_text(encoding="utf-8")) canonical_tests = canonical_report.get("tests") if isinstance(canonical_report, dict) else None if isinstance(canonical_tests, list): + canonical_report_present = True for test in canonical_tests: if not isinstance(test, dict): continue @@ -360,7 +362,7 @@ def aggregate_pytest_statistics( parent_cleanup = (step_result or {}).get("basetemp_cleanup") return { "schema_version": 1, - "canonical_report_status": "present" if canonical_outcomes else "missing", + "canonical_report_status": "present" if canonical_report_present else "missing", "command": [str(value) for value in command], "node_count": len(nodes), "outcomes": outcomes, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 444442d5e7..8389e543e2 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1146,6 +1146,18 @@ def test_aggregate_pytest_statistics_uses_completed_report_to_fill_event_gaps(tm assert result["phases"]["call"]["count"] == 2 +def test_aggregate_pytest_statistics_recognizes_completed_empty_report(tmp_path: Path) -> None: + step = tmp_path / "step" + step.mkdir() + (step / "pytest-report.json").write_text(json.dumps({"tests": []})) + + result = aggregate_pytest_statistics(step) + + assert result["canonical_report_status"] == "present" + assert result["node_count"] == 0 + assert result["outcomes"] == {} + + def test_verify_run_statistics_only_cover_pytest_steps(tmp_path: Path) -> None: run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) artifacts = run.start_step(label="ruff check", cmd=["ruff", "check"]) From 2f89c21ceabac6a094d3b5ee91f141659745bd1e Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 05:33:50 +0200 Subject: [PATCH 27/53] fix(test): isolate nested testmon state --- devtools/verify.py | 2 +- .../devtools/test_pytest_progress_plugin.py | 15 ++----- tests/unit/devtools/test_verify.py | 39 +------------------ 3 files changed, 5 insertions(+), 51 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index ccca18ae5f..7254909c4f 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -1720,7 +1720,7 @@ def _run( if artifacts is not None: durable_report_path = artifacts.step_dir / PYTEST_CANONICAL_REPORT_NAME shutil.copyfile(report_path, durable_report_path) - metadata["report_path"] = str(durable_report_path.relative_to(ROOT)) + metadata["report_path"] = str(durable_report_path.relative_to(run.root)) else: # Fallback: terminal scraping when the structured report is # missing (pytest crashed before writing it, or the plugin is diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index e4ab65dcc6..b51393484b 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -2,7 +2,6 @@ import json import os -import shutil import subprocess import sys from collections.abc import Iterator @@ -14,11 +13,9 @@ from devtools import pytest_progress_plugin from devtools.verify_runs import aggregate_pytest_statistics -pytestmark = pytest.mark.xdist_group("checkout-testmon") - @pytest.fixture(autouse=True) -def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[None]: +def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: # Unit tests own their event destinations; do not let a surrounding # managed verify invocation redirect them into its step artifacts. for name in ( @@ -36,9 +33,6 @@ def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> It collection_started_at = pytest_progress_plugin._COLLECTION_STARTED_AT collection_duration_s = pytest_progress_plugin._COLLECTION_DURATION_S yield - checkout_cache = Path(__file__).resolve().parents[3] / ".cache" / "testmon" - if checkout_cache.exists(): - shutil.move(str(checkout_cache), str(tmp_path / "checkout-testmon-generated")) pytest_progress_plugin._SELECTED_COUNT = selected_count pytest_progress_plugin._DESELECTED_COUNT = deselected_count pytest_progress_plugin._DESELECTED_NODEIDS_SAMPLE[:] = deselected_nodeids @@ -157,11 +151,6 @@ def test_progress_plugin_keeps_xdist_worker_timings_in_controller_summary( def test_managed_event_ledger_survives_test_host_environment_scrub(tmp_path: Path) -> None: events_dir = tmp_path / "events" checkout_root = Path(__file__).resolve().parents[3] - # The real testmon plugin receives no TESTMON_DATAFILE here by design: - # this regression test models a child process after the host scrub. Give - # its default relative path a parent directory without permitting the - # resulting cache to leak into later tests. - (checkout_root / ".cache" / "testmon").mkdir(parents=True, exist_ok=True) env = os.environ.copy() for name in ( "POLYLOGUE_PYTEST_BASETEMP_ROOT", @@ -180,8 +169,10 @@ def test_managed_event_ledger_survives_test_host_environment_scrub(tmp_path: Pat "POLYLOGUE_PYTEST_SELECTION_PATH": str(tmp_path / "selection.json"), "POLYLOGUE_PYTEST_SUMMARY_PATH": str(tmp_path / "summary.json"), "POLYLOGUE_VERIFY_RUN_ID": "subprocess-regression", + "TESTMON_DATAFILE": str(tmp_path / "testmon" / "testmon.sqlite"), } ) + Path(env["TESTMON_DATAFILE"]).parent.mkdir(parents=True) result = subprocess.run( [ sys.executable, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 8389e543e2..c1dd19483a 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -98,21 +98,11 @@ xdist_uninterruptible_stall_reason, ) -pytestmark = pytest.mark.xdist_group("checkout-testmon") - @pytest.fixture(autouse=True) def _isolate_verify_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Keep supervisor and testmon receipts private to each test. - - These tests exercise the real checkout guard, so leaving a synthetic - ``.cache/testmon`` behind makes a later guard test observe a fixture - artifact as if it were a developer's checkout state. - """ + """Keep supervisor and testmon receipts private to each test.""" monkeypatch.chdir(tmp_path) - checkout_cache = ROOT / ".cache" / "testmon" - if checkout_cache.exists(): - shutil.move(str(checkout_cache), str(tmp_path / "checkout-testmon-generated")) for name in ( "TESTMON_DATA", "TESTMON_SEED_STAMP", @@ -124,33 +114,6 @@ def _isolate_verify_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) - monkeypatch.setattr(sys.modules[__name__], name, isolated) -@pytest.fixture(scope="session", autouse=True) -def _quarantine_checkout_testmon(tmp_path_factory: pytest.TempPathFactory) -> object: - """Prevent subprocess-backed verify tests from contaminating the checkout. - - A few tests intentionally re-anchor verification to ``ROOT``. Their child - pytest process therefore uses the real checkout's relative testmon path, - even though the parent test has a private working directory. Keep any - pre-existing state safe for restoration and quarantine only state created - during this test module. - """ - checkout_cache = ROOT / ".cache" / "testmon" - quarantine = tmp_path_factory.mktemp("checkout-testmon") - original: Path | None = None - if checkout_cache.exists(): - original = quarantine / "original" - original.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(checkout_cache), str(original)) - try: - yield - finally: - if checkout_cache.exists(): - shutil.move(str(checkout_cache), str(quarantine / "generated")) - if original is not None and not checkout_cache.exists(): - checkout_cache.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(original), str(checkout_cache)) - - def _pytest_marker_expr(command: list[str]) -> str: marker_indexes = [idx for idx, item in enumerate(command) if item == "-m"] assert marker_indexes From 3b87b213ec478df09c51d27c7d19bd6f7c5d9e30 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 05:44:53 +0200 Subject: [PATCH 28/53] fix(verify): anchor nested report paths --- devtools/verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/verify.py b/devtools/verify.py index 7254909c4f..4e6e190d7b 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -1720,7 +1720,7 @@ def _run( if artifacts is not None: durable_report_path = artifacts.step_dir / PYTEST_CANONICAL_REPORT_NAME shutil.copyfile(report_path, durable_report_path) - metadata["report_path"] = str(durable_report_path.relative_to(run.root)) + metadata["report_path"] = str(durable_report_path.relative_to(run.root if run is not None else ROOT)) else: # Fallback: terminal scraping when the structured report is # missing (pytest crashed before writing it, or the plugin is From 867d57b9aee7709168bfe3b2d0c23df0f591e8b3 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 05:57:47 +0200 Subject: [PATCH 29/53] fix(devtools): parse streamed merge receipts --- devtools/merge_gate.py | 39 ++++++++++++++++---------- tests/unit/devtools/test_merge_gate.py | 27 ++++++++++++++++++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index 198cf027b2..459e363dac 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -219,13 +219,28 @@ def _command_skips_tests(command: str) -> bool: return not any(marker in lowered for marker in _LOOKS_LIKE_TESTS_MARKERS) +def _structured_verification_receipt(stdout: str) -> dict[str, Any] | None: + """Read a final JSON receipt even when the verifier streamed progress first.""" + + candidate_start = len(stdout) + while candidate_start: + candidate_start = stdout.rfind("\n{", 0, candidate_start) + start = candidate_start + 1 if candidate_start >= 0 else 0 + candidate = stdout[start:].strip() + try: + payload = json.loads(candidate) + except (TypeError, json.JSONDecodeError): + if candidate_start < 0: + return None + continue + return payload if isinstance(payload, dict) else None + return None + + def _release_baseline_permission(stdout: str) -> bool | None: """Read the structured verify decision when the command emitted one.""" - try: - payload = json.loads(stdout) - except (TypeError, json.JSONDecodeError): - return None - if not isinstance(payload, dict): + payload = _structured_verification_receipt(stdout) + if payload is None: return None value = payload.get("release_baseline_allowed") return value if isinstance(value, bool) else None @@ -233,11 +248,8 @@ def _release_baseline_permission(stdout: str) -> bool | None: def _verification_scope(stdout: str) -> str | None: """Read the typed verification scope from a structured verify receipt.""" - try: - payload = json.loads(stdout) - except (TypeError, json.JSONDecodeError): - return None - if not isinstance(payload, dict): + payload = _structured_verification_receipt(stdout) + if payload is None: return None value = payload.get("verification_scope") return value if value in {scope.value for scope in VerificationScope} else None @@ -245,11 +257,8 @@ def _verification_scope(stdout: str) -> str | None: def _terminal_authorization(stdout: str) -> str | None: """Read the typed terminal authorization from a structured receipt.""" - try: - payload = json.loads(stdout) - except (TypeError, json.JSONDecodeError): - return None - if not isinstance(payload, dict): + payload = _structured_verification_receipt(stdout) + if payload is None: return None value = payload.get("terminal_authorization") return value if value in {authorization.value for authorization in TerminalAuthorization} else None diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index ca1a98adf7..003033cfbc 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -165,6 +165,33 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: assert receipt["release_baseline_allowed"] is False +def test_record_consumes_receipt_after_streamed_verifier_progress( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + payload = { + "verification_scope": "narrow-terminal", + "release_baseline_allowed": False, + "terminal_authorization": "narrow-terminal", + } + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + return MagicMock(returncode=0, stdout=f"pytest progress\n{json.dumps(payload, indent=2)}\n", stderr="") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] == "narrow-terminal" + assert receipt["release_baseline_allowed"] is False + assert receipt["terminal_authorization"] == "narrow-terminal" + + def test_record_accepts_authoritative_dependabot_dependency_only_pr_without_carrier( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From e1345262b6df28503703493ed486bbdaa3f8e71b Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 06:06:00 +0200 Subject: [PATCH 30/53] fix(devtools): bind merge gate to run artifact --- devtools/merge_gate.py | 48 ++++++++++++++++-- tests/unit/devtools/test_merge_gate.py | 69 ++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index 459e363dac..deabfd3379 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -80,6 +80,7 @@ from devtools import pr_scope from devtools.testmon_state import TerminalAuthorization, VerificationScope +from devtools.verify_runs import CURRENT_RUN_PATH _RECEIPT_DIR = Path(".cache/verify/merge-gate") _DEFAULT_MAX_AGE_S = 3600 @@ -237,6 +238,40 @@ def _structured_verification_receipt(stdout: str) -> dict[str, Any] | None: return None +def _current_run_bytes() -> bytes | None: + try: + return CURRENT_RUN_PATH.read_bytes() + except OSError: + return None + + +def _current_run_receipt( + *, + previous: bytes | None, + head_sha: str, + command_exit: int, +) -> dict[str, Any] | None: + """Load the exact run artifact produced when a verifier writes no stdout receipt.""" + + current = _current_run_bytes() + if current is None or current == previous: + return None + try: + payload = json.loads(current) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + if payload.get("git_head") != head_sha or payload.get("exit_code") != command_exit: + return None + checkout_root = payload.get("checkout_root") + if not isinstance(checkout_root, str) or Path(checkout_root).resolve(strict=False) != Path.cwd().resolve( + strict=False + ): + return None + return payload + + def _release_baseline_permission(stdout: str) -> bool | None: """Read the structured verify decision when the command emitted one.""" payload = _structured_verification_receipt(stdout) @@ -330,6 +365,7 @@ def cmd_record(pr: int, command: str) -> int: if not argv: print("REFUSING to record: --command is empty after shell splitting.", file=sys.stderr) return 2 + previous_current_run = _current_run_bytes() started = time.time() try: result = subprocess.run(argv, capture_output=True, text=True) @@ -337,6 +373,12 @@ def cmd_record(pr: int, command: str) -> int: print(f"REFUSING to record: could not run {command!r}: {exc}", file=sys.stderr) return 2 duration_s = round(time.time() - started, 2) + verification_receipt = _structured_verification_receipt(result.stdout) or _current_run_receipt( + previous=previous_current_run, + head_sha=head_sha, + command_exit=result.returncode, + ) + verification_payload = json.dumps(verification_receipt) if verification_receipt is not None else "" receipt = { "pr": pr, @@ -353,9 +395,9 @@ def cmd_record(pr: int, command: str) -> int: "branch": info["headRefName"], "command": command, "skips_tests": _command_skips_tests(command), - "verification_scope": _verification_scope(result.stdout), - "release_baseline_allowed": _release_baseline_permission(result.stdout), - "terminal_authorization": _terminal_authorization(result.stdout), + "verification_scope": _verification_scope(verification_payload), + "release_baseline_allowed": _release_baseline_permission(verification_payload), + "terminal_authorization": _terminal_authorization(verification_payload), "exit_code": result.returncode, "duration_s": duration_s, "recorded_at": time.time(), diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 003033cfbc..80597d3951 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -192,6 +192,75 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: assert receipt["terminal_authorization"] == "narrow-terminal" +def test_record_consumes_exact_new_current_run_when_verifier_writes_only_stderr( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + current_run = tmp_path / merge_gate.CURRENT_RUN_PATH + current_run.parent.mkdir(parents=True) + current_run.write_text(json.dumps({"run_id": "old"})) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + current_run.write_text( + json.dumps( + { + "run_id": "new", + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "affected", + "release_baseline_allowed": False, + "terminal_authorization": None, + } + ) + ) + return MagicMock(returncode=0, stdout="", stderr="pytest progress") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] == "affected" + assert receipt["release_baseline_allowed"] is False + + +def test_record_rejects_current_run_from_another_head(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + current_run = tmp_path / merge_gate.CURRENT_RUN_PATH + current_run.parent.mkdir(parents=True) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + current_run.write_text( + json.dumps( + { + "git_head": "different", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "affected", + "release_baseline_allowed": False, + } + ) + ) + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] is None + assert receipt["release_baseline_allowed"] is None + + def test_record_accepts_authoritative_dependabot_dependency_only_pr_without_carrier( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From d38940fd52f71e6d16d90959321ecd14ece6b308 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 06:29:56 +0200 Subject: [PATCH 31/53] fix(devtools): expose current run evidence path --- devtools/merge_gate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index deabfd3379..884992cc0c 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -80,7 +80,7 @@ from devtools import pr_scope from devtools.testmon_state import TerminalAuthorization, VerificationScope -from devtools.verify_runs import CURRENT_RUN_PATH +from devtools.verify_runs import CURRENT_RUN_PATH as CURRENT_RUN_PATH _RECEIPT_DIR = Path(".cache/verify/merge-gate") _DEFAULT_MAX_AGE_S = 3600 From c5a3e0e56b132c3acd2691f86fd7e5f47b4b3994 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 06:56:23 +0200 Subject: [PATCH 32/53] fix(devtools): bind merge evidence to invocation --- devtools/merge_gate.py | 104 ++++++++++++++++--------- devtools/verify.py | 9 ++- devtools/verify_runs.py | 13 ++++ tests/unit/devtools/test_merge_gate.py | 92 +++++++++++++++++++--- tests/unit/devtools/test_verify.py | 24 ++++++ 5 files changed, 194 insertions(+), 48 deletions(-) diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index 884992cc0c..4255846694 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -70,17 +70,21 @@ import argparse import json +import os import shlex import subprocess import sys +import tempfile import time +import uuid from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any from devtools import pr_scope from devtools.testmon_state import TerminalAuthorization, VerificationScope -from devtools.verify_runs import CURRENT_RUN_PATH as CURRENT_RUN_PATH +from devtools.verify_runs import VERIFICATION_INVOCATION_ID_ENV as VERIFICATION_INVOCATION_ID_ENV +from devtools.verify_runs import VERIFICATION_RECEIPT_PATH_ENV as VERIFICATION_RECEIPT_PATH_ENV _RECEIPT_DIR = Path(".cache/verify/merge-gate") _DEFAULT_MAX_AGE_S = 3600 @@ -193,6 +197,18 @@ def _git_is_clean() -> bool: return result.returncode == 0 and not result.stdout.strip() +def _repository_root() -> Path: + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + timeout=15, + ) + if result.returncode == 0 and result.stdout.strip(): + return Path(result.stdout.strip()).resolve(strict=False) + return Path.cwd().resolve(strict=False) + + @dataclass class GateVerdict: pr: int @@ -206,11 +222,11 @@ class GateVerdict: def _receipt_path(pr: int) -> Path: - return _RECEIPT_DIR / f"pr-{pr}.json" + return _repository_root() / _RECEIPT_DIR / f"pr-{pr}.json" def _ack_path(pr: int) -> Path: - return _RECEIPT_DIR / f"pr-{pr}-acks.json" + return _repository_root() / _RECEIPT_DIR / f"pr-{pr}-acks.json" def _command_skips_tests(command: str) -> bool: @@ -238,34 +254,30 @@ def _structured_verification_receipt(stdout: str) -> dict[str, Any] | None: return None -def _current_run_bytes() -> bytes | None: - try: - return CURRENT_RUN_PATH.read_bytes() - except OSError: - return None - - -def _current_run_receipt( +def _invocation_receipt( *, - previous: bytes | None, + path: Path, + invocation_id: str, head_sha: str, command_exit: int, + checkout_root: Path, ) -> dict[str, Any] | None: - """Load the exact run artifact produced when a verifier writes no stdout receipt.""" + """Load the exact run artifact bound to the launched verifier process.""" - current = _current_run_bytes() - if current is None or current == previous: - return None try: - payload = json.loads(current) - except (UnicodeDecodeError, json.JSONDecodeError): + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): return None if not isinstance(payload, dict): return None - if payload.get("git_head") != head_sha or payload.get("exit_code") != command_exit: + if ( + payload.get("invocation_id") != invocation_id + or payload.get("git_head") != head_sha + or payload.get("exit_code") != command_exit + ): return None - checkout_root = payload.get("checkout_root") - if not isinstance(checkout_root, str) or Path(checkout_root).resolve(strict=False) != Path.cwd().resolve( + recorded_root = payload.get("checkout_root") + if not isinstance(recorded_root, str) or Path(recorded_root).resolve(strict=False) != checkout_root.resolve( strict=False ): return None @@ -305,7 +317,13 @@ def _base_sha(info: dict[str, Any]) -> str | None: return value if isinstance(value, str) else None -def _scope_verdict(pr: int, info: dict[str, Any], *, head_sha: str) -> pr_scope.ScopeVerdict: +def _scope_verdict( + pr: int, + info: dict[str, Any], + *, + head_sha: str, + checkout_root: Path, +) -> pr_scope.ScopeVerdict: """Use the same carrier or typed bot exception for record and check.""" author = info.get("author") files = info.get("files") @@ -328,6 +346,7 @@ def _scope_verdict(pr: int, info: dict[str, Any], *, head_sha: str) -> pr_scope. info.get("body") or "", head_sha=head_sha, is_draft=bool(info.get("isDraft")), + beads_path=checkout_root / ".beads" / "issues.jsonl", base_sha=_base_sha(info), ) @@ -335,6 +354,7 @@ def _scope_verdict(pr: int, info: dict[str, Any], *, head_sha: str) -> pr_scope. def cmd_record(pr: int, command: str) -> int: info = _gh_json(["pr", "view", str(pr), "--json", "headRefOid,headRefName,baseRefOid,body,isDraft,author,files"]) head_sha = info["headRefOid"] + checkout_root = _repository_root() local_head = _git_head_sha() if local_head != head_sha: @@ -354,7 +374,7 @@ def cmd_record(pr: int, command: str) -> int: ) return 2 - scope = _scope_verdict(pr, info, head_sha=head_sha) + scope = _scope_verdict(pr, info, head_sha=head_sha, checkout_root=checkout_root) if not scope.ok: print(f"REFUSING to record: PR #{pr} has an invalid structured pr-scope carrier:", file=sys.stderr) for reason in scope.reasons: @@ -365,19 +385,26 @@ def cmd_record(pr: int, command: str) -> int: if not argv: print("REFUSING to record: --command is empty after shell splitting.", file=sys.stderr) return 2 - previous_current_run = _current_run_bytes() + invocation_id = uuid.uuid4().hex started = time.time() - try: - result = subprocess.run(argv, capture_output=True, text=True) - except OSError as exc: - print(f"REFUSING to record: could not run {command!r}: {exc}", file=sys.stderr) - return 2 + with tempfile.TemporaryDirectory(prefix="polylogue-merge-gate-") as temp_dir: + receipt_path = Path(temp_dir) / "run.json" + env = dict(os.environ) + env[VERIFICATION_INVOCATION_ID_ENV] = invocation_id + env[VERIFICATION_RECEIPT_PATH_ENV] = str(receipt_path) + try: + result = subprocess.run(argv, capture_output=True, text=True, cwd=checkout_root, env=env) + except OSError as exc: + print(f"REFUSING to record: could not run {command!r}: {exc}", file=sys.stderr) + return 2 + verification_receipt = _structured_verification_receipt(result.stdout) or _invocation_receipt( + path=receipt_path, + invocation_id=invocation_id, + head_sha=head_sha, + command_exit=result.returncode, + checkout_root=checkout_root, + ) duration_s = round(time.time() - started, 2) - verification_receipt = _structured_verification_receipt(result.stdout) or _current_run_receipt( - previous=previous_current_run, - head_sha=head_sha, - command_exit=result.returncode, - ) verification_payload = json.dumps(verification_receipt) if verification_receipt is not None else "" receipt = { @@ -404,8 +431,9 @@ def cmd_record(pr: int, command: str) -> int: "stdout_tail": result.stdout[-4000:], "stderr_tail": result.stderr[-4000:], } - _RECEIPT_DIR.mkdir(parents=True, exist_ok=True) - _receipt_path(pr).write_text(json.dumps(receipt, indent=2)) + receipt_path = _receipt_path(pr) + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt, indent=2)) print(f"recorded receipt for PR #{pr} @ {head_sha[:8]}: exit={result.returncode} ({duration_s}s)") if receipt["skips_tests"]: @@ -436,7 +464,7 @@ def cmd_ack(pr: int, comment_id: int, *, reason: str) -> int: else: acks = {} acks[str(comment_id)] = {"head_sha": head_sha, "reason": reason, "acked_at": time.time()} - _RECEIPT_DIR.mkdir(parents=True, exist_ok=True) + ack_path.parent.mkdir(parents=True, exist_ok=True) ack_path.write_text(json.dumps(acks, indent=2)) print(f"acknowledged comment {comment_id} on PR #{pr} @ {head_sha[:8]}: {reason}") return 0 @@ -549,7 +577,7 @@ def cmd_check( "current checkout has uncommitted changes; merge-gate check requires committed PR content" ) - scope = _scope_verdict(pr, info, head_sha=head_sha) + scope = _scope_verdict(pr, info, head_sha=head_sha, checkout_root=_repository_root()) verdict.pr_scope = asdict(scope) if not scope.ok: verdict.ok = False diff --git a/devtools/verify.py b/devtools/verify.py index 4e6e190d7b..871ab576be 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -3914,7 +3914,14 @@ def main(argv: list[str] | None = None) -> int: # Persist history and stamp. _save_history(history_entry) - verify_run.finish(exit_code=exit_code, duration_s=total_duration, diagnosis=pytest_diagnosis) + verify_run.finish( + exit_code=exit_code, + duration_s=total_duration, + diagnosis=pytest_diagnosis, + verification_scope=verification_scope.value, + release_baseline_allowed=release_baseline_allowed, + terminal_authorization=args.terminal_authorization, + ) if exit_code == 0: _stamp_head() diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 5de910c5b8..62a9dd2a50 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -31,6 +31,8 @@ DEVTOOLS_STATE_DIR = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state")) / "polylogue" / "devtools" VERIFY_HISTORY_PATH = DEVTOOLS_STATE_DIR / "verify-history.jsonl" CURRENT_RUN_PATH = VERIFY_CACHE / "current-run.json" +VERIFICATION_INVOCATION_ID_ENV = "POLYLOGUE_VERIFICATION_INVOCATION_ID" +VERIFICATION_RECEIPT_PATH_ENV = "POLYLOGUE_VERIFICATION_RECEIPT_PATH" CURRENT_RESOURCES_PATH = VERIFY_CACHE / "current-pytest-resources.jsonl" CURRENT_POSTMORTEM_PATH = VERIFY_CACHE / "current-pytest-postmortem.json" CURRENT_CONTAINMENT_PATH = VERIFY_CACHE / "current-pytest-containment.json" @@ -503,6 +505,9 @@ def __init__( "steps": [], "artifact_dir": str(VERIFY_RUNS_DIR / self.run_id), } + invocation_id = os.environ.get(VERIFICATION_INVOCATION_ID_ENV) + if invocation_id: + self._payload["invocation_id"] = invocation_id self.run_dir.mkdir(parents=True, exist_ok=True) self.write() @@ -512,6 +517,9 @@ def relative_run_dir(self) -> Path: def write(self) -> None: _write_json(self.run_dir / "run.json", self._payload) + invocation_receipt = os.environ.get(VERIFICATION_RECEIPT_PATH_ENV) + if invocation_receipt: + _write_json(Path(invocation_receipt), self._payload) current_path = self.root / CURRENT_RUN_PATH if not _current_owner_is_other_live_run(current_path): _write_json(current_path, self._payload) @@ -623,6 +631,11 @@ def finish( def env_for_pytest_step(env: dict[str, str], *, run: VerifyRun, artifacts: PytestStepArtifacts) -> dict[str, str]: updated = dict(env) + # The merge-gate invocation receipt belongs to the top-level devtools + # process. Pytest and any nested harness commands must not inherit the + # token and overwrite that receipt with a child run. + updated.pop(VERIFICATION_INVOCATION_ID_ENV, None) + updated.pop(VERIFICATION_RECEIPT_PATH_ENV, None) updated["POLYLOGUE_VERIFY_RUN_ID"] = run.run_id updated["POLYLOGUE_PYTEST_RUN_ID"] = run.run_id updated["POLYLOGUE_PYTEST_EVENTS_DIR"] = str(artifacts.events_dir) diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 80597d3951..339589f678 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -78,6 +78,8 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: call_count["round"] += 1 return MagicMock(returncode=0, stdout=json.dumps([comment_rounds[round_index]]), stderr="") if cmd[:2] == ["git", "rev-parse"]: + if "--show-toplevel" in cmd: + return MagicMock(returncode=0, stdout=str(Path.cwd()) + "\n", stderr="") return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: return MagicMock(returncode=0, stdout=" M dirty.py\n" if dirty else "", stderr="") @@ -192,25 +194,25 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: assert receipt["terminal_authorization"] == "narrow-terminal" -def test_record_consumes_exact_new_current_run_when_verifier_writes_only_stderr( +def test_record_consumes_exact_invocation_receipt_when_verifier_writes_only_stderr( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.chdir(tmp_path) pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) - current_run = tmp_path / merge_gate.CURRENT_RUN_PATH - current_run.parent.mkdir(parents=True) - current_run.write_text(json.dumps({"run_id": "old"})) def _run(cmd: list[str], **kwargs: object) -> MagicMock: if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): return base(cmd, **kwargs) if cmd[:3] == ["gh", "pr", "view"]: return base(cmd, **kwargs) - current_run.write_text( + env = cast(dict[str, str], kwargs["env"]) + receipt_path = Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]) + receipt_path.write_text( json.dumps( { "run_id": "new", + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], "git_head": "abc123", "checkout_root": str(tmp_path), "exit_code": 0, @@ -229,21 +231,21 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: assert receipt["release_baseline_allowed"] is False -def test_record_rejects_current_run_from_another_head(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +def test_record_rejects_invocation_receipt_from_another_head(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) - current_run = tmp_path / merge_gate.CURRENT_RUN_PATH - current_run.parent.mkdir(parents=True) def _run(cmd: list[str], **kwargs: object) -> MagicMock: if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): return base(cmd, **kwargs) if cmd[:3] == ["gh", "pr", "view"]: return base(cmd, **kwargs) - current_run.write_text( + env = cast(dict[str, str], kwargs["env"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( json.dumps( { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], "git_head": "different", "checkout_root": str(tmp_path), "exit_code": 0, @@ -261,6 +263,78 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: assert receipt["release_baseline_allowed"] is None +def test_record_rejects_unrelated_invocation_receipt_with_same_head_and_exit( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + env = cast(dict[str, str], kwargs["env"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": "another-command", + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "affected", + "release_baseline_allowed": False, + } + ) + ) + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] is None + assert receipt["release_baseline_allowed"] is None + + +def test_record_anchors_invocation_to_repository_root_from_subdirectory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + subdirectory = tmp_path / "devtools" + subdirectory.mkdir() + monkeypatch.chdir(subdirectory) + pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} + base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) + + def _run(cmd: list[str], **kwargs: object) -> MagicMock: + if cmd == ["git", "rev-parse", "--show-toplevel"]: + return MagicMock(returncode=0, stdout=str(tmp_path) + "\n", stderr="") + if cmd[:2] in (["git", "rev-parse"], ["git", "status"]): + return base(cmd, **kwargs) + if cmd[:3] == ["gh", "pr", "view"]: + return base(cmd, **kwargs) + assert kwargs["cwd"] == tmp_path + env = cast(dict[str, str], kwargs["env"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "affected", + "release_baseline_allowed": False, + } + ) + ) + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", _run) + assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 + receipt = json.loads(merge_gate._receipt_path(42).read_text()) + assert receipt["verification_scope"] == "affected" + + def test_record_accepts_authoritative_dependabot_dependency_only_pr_without_carrier( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index c1dd19483a..7acb839685 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -929,6 +929,30 @@ def test_focused_run_can_record_typed_affected_scope(tmp_path: Path) -> None: assert json.loads((tmp_path / ".cache" / "verify" / "current-run.json").read_text()) == payload +def test_verify_run_writes_invocation_receipt_without_leaking_token_to_pytest( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + receipt = tmp_path / "invocation" / "run.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "invocation-1") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="pytest focused", cmd=["pytest", "tests/unit/example.py"]) + + payload = run.finish( + exit_code=0, + duration_s=0.1, + verification_scope="affected", + release_baseline_allowed=False, + ) + child_env = verify_runs.env_for_pytest_step(dict(os.environ), run=run, artifacts=artifacts) + + assert json.loads(receipt.read_text()) == payload + assert payload["invocation_id"] == "invocation-1" + assert verify_runs.VERIFICATION_INVOCATION_ID_ENV not in child_env + assert verify_runs.VERIFICATION_RECEIPT_PATH_ENV not in child_env + + def test_aggregate_pytest_statistics_reduces_phases_fixtures_and_resources(tmp_path: Path) -> None: step = tmp_path / "step" step.mkdir() From d536ab349623538e4f798097705f5876524e2492 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 07:26:47 +0200 Subject: [PATCH 33/53] fix(devtools): preserve exact pytest evidence --- TESTING.md | 5 +- devtools/evidence_dashboard.py | 2 +- devtools/pytest_progress_plugin.py | 26 ++- devtools/run_tests.py | 39 +++- devtools/verify.py | 198 ++++++++---------- devtools/verify_runs.py | 116 ++++++++-- tests/conftest.py | 68 +++--- .../unit/devtools/test_evidence_dashboard.py | 1 + .../devtools/test_pytest_progress_plugin.py | 64 ++++++ tests/unit/devtools/test_run_tests.py | 27 ++- tests/unit/devtools/test_verify.py | 97 ++++++++- tests/unit/test_pytest_temp_policy.py | 58 ++++- 12 files changed, 500 insertions(+), 201 deletions(-) diff --git a/TESTING.md b/TESTING.md index 86a00aff9b..77eeee105a 100644 --- a/TESTING.md +++ b/TESTING.md @@ -125,8 +125,9 @@ stale per-run dirs from every known root (`/dev/shm`, `/realm/tmp/polylogue-pyte `/tmp/polylogue-pytest`, plus any explicit configured root) — never based on age alone: each managed basetemp carries a PID plus process-start identity, and a directory whose exact owner process is still alive is never removed -regardless of age. An owner that cannot be confirmed dead (no marker) gets a -multi-hour grace period rather than the normal ~30-minute one. The sweeper +regardless of age. A tree without a valid managed claim or whose owner cannot +be confirmed dead is never removed. The thirty-minute age threshold applies +only after a positive managed claim identifies a dead owner. The sweeper restores owner-write permission only after a tree is adjudicated stale, so published read-only fixture copies cannot leak tmpfs indefinitely. Shared `pytest-polylogue-*-seeded-*` caches are never touched by the sweep — they diff --git a/devtools/evidence_dashboard.py b/devtools/evidence_dashboard.py index 7f3c28ad57..f7e02c1aaa 100644 --- a/devtools/evidence_dashboard.py +++ b/devtools/evidence_dashboard.py @@ -28,8 +28,8 @@ from typing import Any from devtools import repo_root as _get_root -from devtools.verify import _worktree_fingerprint from devtools.verify_runs import VERIFY_HISTORY_PATH, git_dirty, git_head +from devtools.verify_runs import worktree_fingerprint as _worktree_fingerprint ROOT = _get_root() diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index 4eefcd77e7..99dbec5c62 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -30,6 +30,7 @@ _RECORDED_REPORT_KEYS: set[tuple[int, str, str, str, float]] = set() _COLLECTION_STARTED_AT: float | None = None _COLLECTION_DURATION_S: float | None = None +_CONTROLLER_COLLECTION_PAYLOAD: dict[str, Any] | None = None _SLOW_REPORT_LIMIT = 20 _DEFAULT_SELECTION_NODEID_LIMIT = 500 _COLLECTION_FACT_SUFFIX = ".collection.json" @@ -101,13 +102,15 @@ def _write_worker_collection_fact(payload: dict[str, Any]) -> None: tmp.replace(path) -def _worker_collection_payloads() -> list[dict[str, Any]]: +def _worker_collection_payloads(events_dir: Path | None = None) -> list[dict[str, Any]]: """Read worker collection facts in a stable order for the controller.""" - raw_dir = os.environ.get(_EVENTS_DIR_ENV) - if not raw_dir: - return [] + if events_dir is None: + raw_dir = os.environ.get(_EVENTS_DIR_ENV) + if not raw_dir: + return [] + events_dir = Path(raw_dir) payloads: list[tuple[str, int, str, dict[str, Any]]] = [] - for path in Path(raw_dir).glob(f"*{_COLLECTION_FACT_SUFFIX}"): + for path in events_dir.glob(f"*{_COLLECTION_FACT_SUFFIX}"): with contextlib.suppress(OSError, json.JSONDecodeError): payload = json.loads(path.read_text(encoding="utf-8")) worker_id = payload.get("worker_id") @@ -135,9 +138,9 @@ def _collection_payload() -> dict[str, Any]: return payload -def _merge_worker_collection_payloads() -> dict[str, Any] | None: +def merge_worker_collection_payloads(events_dir: Path | None = None) -> dict[str, Any] | None: """Choose one canonical xdist collection set and the slowest wall time.""" - payloads = _worker_collection_payloads() + payloads = _worker_collection_payloads(events_dir) if not payloads: return None merged = dict(payloads[0]) @@ -188,7 +191,8 @@ def _durable_report_outcome(report: Any, outcome: str) -> str: def pytest_sessionstart(session: Any) -> None: """Reset per-session ledgers when tests invoke pytest in-process.""" del session - global _COLLECTION_STARTED_AT, _COLLECTION_DURATION_S, _DESELECTED_COUNT, _SELECTED_COUNT + global _COLLECTION_STARTED_AT, _COLLECTION_DURATION_S, _CONTROLLER_COLLECTION_PAYLOAD + global _DESELECTED_COUNT, _SELECTED_COUNT _DESELECTED_NODEIDS_SAMPLE.clear() _DESELECTED_COUNT = 0 _SELECTED_COUNT = 0 @@ -196,6 +200,7 @@ def pytest_sessionstart(session: Any) -> None: _RECORDED_REPORT_KEYS.clear() _COLLECTION_STARTED_AT = None _COLLECTION_DURATION_S = None + _CONTROLLER_COLLECTION_PAYLOAD = None # The worker environment is assigned after process exec, so it is not # reliably visible through /proc//environ. Emit the identity from # inside the worker for the supervisor's process-state sampler. @@ -226,7 +231,7 @@ def pytest_deselected(items: list[Any]) -> None: def pytest_collection_modifyitems(session: Any, config: Any, items: list[Any]) -> None: """Write the final selected test set after pytest/testmon deselection.""" del config - global _COLLECTION_DURATION_S, _SELECTED_COUNT + global _COLLECTION_DURATION_S, _CONTROLLER_COLLECTION_PAYLOAD, _SELECTED_COUNT if _COLLECTION_STARTED_AT is not None: _COLLECTION_DURATION_S = round(time.monotonic() - _COLLECTION_STARTED_AT, 4) _SELECTED_COUNT = len(items) @@ -252,6 +257,7 @@ def pytest_collection_modifyitems(session: Any, config: Any, items: list[Any]) - if os.environ.get("PYTEST_XDIST_WORKER"): _write_worker_collection_fact(payload) else: + _CONTROLLER_COLLECTION_PAYLOAD = dict(payload) _write_selection(payload) _write_event( { @@ -343,7 +349,7 @@ def pytest_sessionfinish(session: Any, exitstatus: int) -> None: # summary path, so an empty worker summary cannot overwrite it. if os.environ.get("PYTEST_XDIST_WORKER"): return - collection_payload = _merge_worker_collection_payloads() or _collection_payload() + collection_payload = merge_worker_collection_payloads() or _CONTROLLER_COLLECTION_PAYLOAD or _collection_payload() _write_selection(collection_payload) payload: dict[str, Any] = { "exitstatus": int(exitstatus), diff --git a/devtools/run_tests.py b/devtools/run_tests.py index a650a84de5..09cbf9adc8 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -45,16 +45,41 @@ PYTEST_SELECTION_PATH, PYTEST_SUMMARY_PATH, _clear_pytest_report, - _pytest_command_worker_request, _run, - _worktree_fingerprint, ) -from devtools.verify_runs import VerifyRun, append_verify_history, git_head +from devtools.verify_runs import ( + VerifyRun, + append_verify_history, + git_head, + pytest_command_worker_request, + worktree_fingerprint, +) ROOT = Path(__file__).resolve().parent.parent _LOCK_PATH = ROOT / ".cache" / "test-run.lock" +def _normalize_selection_paths(selection: list[str], *, invocation_directory: Path) -> list[str]: + """Preserve path selections relative to the directory that invoked devtools.""" + normalized: list[str] = [] + for argument in selection: + if argument.startswith("-"): + normalized.append(argument) + continue + path_text, separator, node_suffix = argument.partition("::") + candidate = Path(path_text) + if candidate.is_absolute() or not (invocation_directory / candidate).exists(): + normalized.append(argument) + continue + resolved = (invocation_directory / candidate).resolve() + try: + anchored = resolved.relative_to(ROOT).as_posix() + except ValueError: + anchored = str(resolved) + normalized.append(f"{anchored}{separator}{node_suffix}") + return normalized + + def _anchor_test_paths() -> None: """Anchor focused-test execution and artifacts to this checkout.""" os.chdir(ROOT) @@ -78,7 +103,7 @@ def _xdist_distribution_args(selection: list[str], worker_args: list[str]) -> li if any(arg == "--dist" or arg.startswith("--dist=") for arg in selection): return [] command = [*selection, *worker_args] - request = _pytest_command_worker_request(command) + request = pytest_command_worker_request(command) if request in {None, "0"}: return [] return ["--dist=loadgroup"] @@ -132,6 +157,9 @@ def _run_lock(*, enabled: bool) -> Iterator[None]: def main(argv: list[str] | None = None) -> int: + invocation_directory = Path.cwd() + selection = list(sys.argv[1:] if argv is None else argv) + selection = _normalize_selection_paths(selection, invocation_directory=invocation_directory) _anchor_test_paths() try: fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools test") @@ -142,7 +170,6 @@ def main(argv: list[str] | None = None) -> int: environment_fingerprint = fingerprint.as_dict() sys.stderr.write(f"devtools test: polylogue package → {polylogue_import_path}\n") - selection = list(sys.argv[1:] if argv is None else argv) use_json = "--json" in selection # The control-plane dispatch may append a bare ``--json`` machine-readable # flag; it is meaningless for a streamed test run, so drop it before pytest. @@ -168,7 +195,7 @@ def main(argv: list[str] | None = None) -> int: root=ROOT, polylogue_import_path=str(polylogue_import_path), environment_fingerprint=environment_fingerprint, - worktree_fingerprint=_worktree_fingerprint(ROOT), + worktree_fingerprint=worktree_fingerprint(ROOT), ) started = time.monotonic() try: diff --git a/devtools/verify.py b/devtools/verify.py index 871ab576be..c97af8ab98 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -31,10 +31,10 @@ import shlex import shutil import signal -import stat import subprocess import sys import time +import uuid from collections.abc import Mapping, Sequence from datetime import datetime, timezone from pathlib import Path @@ -44,6 +44,7 @@ CheckoutImportMismatchError, assert_polylogue_matches_checkout, ) +from devtools.pytest_progress_plugin import merge_worker_collection_payloads from devtools.pytest_supervisor import ( SupervisorLaunch, build_supervisor_launch, @@ -102,9 +103,11 @@ latest_event_from_paths, normalize_pytest_basetemp_env, pytest_basetemp_path, + pytest_command_worker_request, pytest_tmpfs_budget_exceeded, pytest_tmpfs_budget_kb, utc_now, + worktree_fingerprint, xdist_uninterruptible_stall_reason, ) from polylogue.scenarios.workload import ( @@ -287,14 +290,28 @@ def _print_history(file: Path | None = None) -> None: tier = str(entry.get("tier") or "unknown")[:8] head = str(entry.get("git_head") or "unknown")[:8] duration = entry.get("total_duration_s", entry.get("duration_s", 0.0)) - dur = f"{float(duration or 0.0):.0f}s" - ec = int(entry.get("exit_code", 1)) + try: + dur = f"{float(duration or 0.0):.0f}s" + except (TypeError, ValueError): + dur = "0s" + raw_exit = entry.get("exit_code", 1) + try: + ec = int(raw_exit if raw_exit is not None else 1) + except (TypeError, ValueError): + ec = 1 rendered_steps: list[str] = [] for step in entry.get("steps", []): if not isinstance(step, dict): continue - step_duration = float(step.get("duration_s") or 0.0) - step_exit = int(step.get("exit", 1)) + try: + step_duration = float(step.get("duration_s") or 0.0) + except (TypeError, ValueError): + step_duration = 0.0 + raw_step_exit = step.get("exit", 1) + try: + step_exit = int(raw_step_exit if raw_step_exit is not None else 1) + except (TypeError, ValueError): + step_exit = 1 rendered_steps.append(f"{step.get('name', 'unknown')}({step_duration:.0f}s{' FAIL' if step_exit else ''})") steps = ", ".join(rendered_steps) print(f"{ts:<20} {tier:<8} {head:<10} {dur:>7} {ec:>4} {steps}") @@ -394,7 +411,7 @@ def _pytest_metadata_from_report(report: dict[str, Any], *, report_path: Path) - def _pytest_command_metadata(cmd: list[str]) -> dict[str, Any]: """Return verify metadata that explains the pytest worker policy.""" metadata: dict[str, Any] = {} - metadata["pytest_workers"] = _pytest_command_worker_request(cmd) or "unset" + metadata["pytest_workers"] = pytest_command_worker_request(cmd) or "unset" if "--testmon" in cmd: metadata["pytest_selection"] = "testmon-noselect" if "--testmon-noselect" in cmd else "testmon" else: @@ -630,11 +647,16 @@ def _write_pytest_output(stdout: str, stderr: str) -> None: def _persist_pytest_output(stdout: str, stderr: str, *, artifacts: PytestStepArtifacts | None) -> None: """Persist drained pytest output on both ordinary and exceptional exits.""" - _write_pytest_output(stdout, stderr) + with contextlib.suppress(OSError): + _write_pytest_output(stdout, stderr) if artifacts is not None: - artifacts.stdout_path.write_text(stdout, encoding="utf-8") - artifacts.stderr_path.write_text(stderr, encoding="utf-8") - artifacts.output_path.write_text(stdout + stderr, encoding="utf-8") + for path, content in ( + (artifacts.stdout_path, stdout), + (artifacts.stderr_path, stderr), + (artifacts.output_path, stdout + stderr), + ): + with contextlib.suppress(OSError): + path.write_text(content, encoding="utf-8") def _write_pytest_progress( @@ -698,10 +720,12 @@ def _write_pytest_progress( } if latest_event.get("event") == "test_started" and isinstance(latest_event.get("nodeid"), str): payload["current_test_nodeid"] = latest_event["nodeid"] - PYTEST_PROGRESS_PATH.parent.mkdir(parents=True, exist_ok=True) - tmp = PYTEST_PROGRESS_PATH.with_name(f"{PYTEST_PROGRESS_PATH.name}.{os.getpid()}.{time.monotonic_ns()}.tmp") - tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n") - tmp.replace(PYTEST_PROGRESS_PATH) + targets = [PYTEST_PROGRESS_PATH] + if artifact_dir is not None: + targets.insert(0, Path(artifact_dir) / "progress.json") + for target in dict.fromkeys(targets): + with contextlib.suppress(OSError): + _atomic_write_json(target, payload) def _process_cpu_seconds(pid: int) -> float | None: @@ -1594,6 +1618,9 @@ def _run( # ``bench slo`` starts pytest-benchmark itself, so it needs the same # bounded temp policy and run marker as a direct pytest step. has_managed_pytest_child = label == "bench slo" + if is_pytest and run is not None: + isolated_report = run.run_dir / f"pytest-report-{uuid.uuid4().hex}.json" + cmd = [f"--json-report-file={isolated_report}" if arg.startswith("--json-report-file=") else arg for arg in cmd] if is_pytest: _clear_pytest_report(cmd) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None @@ -1719,8 +1746,17 @@ def _run( metadata["report_status"] = "present" if artifacts is not None: durable_report_path = artifacts.step_dir / PYTEST_CANONICAL_REPORT_NAME - shutil.copyfile(report_path, durable_report_path) - metadata["report_path"] = str(durable_report_path.relative_to(run.root if run is not None else ROOT)) + try: + shutil.copyfile(report_path, durable_report_path) + except OSError: + metadata["report_path"] = None + else: + metadata["report_path"] = str( + durable_report_path.relative_to(run.root if run is not None else ROOT) + ) + if report_path != durable_report_path: + with contextlib.suppress(OSError): + report_path.unlink() else: # Fallback: terminal scraping when the structured report is # missing (pytest crashed before writing it, or the plugin is @@ -1746,7 +1782,11 @@ def _run( selection_path = artifacts.selection_path if artifacts is not None else PYTEST_SELECTION_PATH if interrupted or containment_error is not None: _recover_worker_collection_facts( - events_dir=artifacts.events_dir if artifacts is not None else Path(env["POLYLOGUE_PYTEST_EVENTS_DIR"]), + events_dir=( + artifacts.events_dir + if artifacts is not None + else Path(env.get("POLYLOGUE_PYTEST_EVENTS_DIR", str(PYTEST_EVENTS_DIR))) + ), selection_path=selection_path, ) selection = _read_json_artifact(selection_path) @@ -1922,7 +1962,7 @@ def _run( } artifacts.postmortem_path.write_text(json.dumps(postmortem, indent=2, ensure_ascii=False) + "\n") elif interrupted: - metadata = {"diagnosis": "verification_interrupted", "termination_reason": "operator_interrupt"} + metadata.update({"diagnosis": "verification_interrupted", "termination_reason": "operator_interrupt"}) if result.returncode == 0: sys.stderr.write(f"ok ({elapsed:.1f}s)\n") else: @@ -2369,25 +2409,6 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: } -def _pytest_command_worker_request(cmd: Sequence[str]) -> str | None: - """Return the last xdist worker request from a final pytest command. - - ``devtools test`` forwards pytest arguments unchanged, so this accepts - both xdist spellings and their compact forms. The final occurrence wins, - matching pytest's normal option precedence. - """ - request: str | None = None - for index, arg in enumerate(cmd): - if arg in {"-n", "--numprocesses"}: - if index + 1 < len(cmd): - request = cmd[index + 1] - elif arg.startswith("--numprocesses="): - request = arg.removeprefix("--numprocesses=") - elif arg.startswith("-n") and len(arg) > 2: - request = arg[2:].removeprefix("=") - return request - - def _pytest_command_concurrency(cmd: Sequence[str], *, env: Mapping[str, str] | None = None) -> int: """Return a fail-closed reservation for the final pytest command. @@ -2395,7 +2416,7 @@ def _pytest_command_concurrency(cmd: Sequence[str], *, env: Mapping[str, str] | instead of guessing one worker; an unrecognised xdist value is treated the same way so malformed or future values cannot weaken admission. """ - request = _pytest_command_worker_request(cmd) + request = pytest_command_worker_request(cmd) if request is None: return 0 if request == "auto": @@ -2455,7 +2476,7 @@ def _changed_executable_paths() -> tuple[str, ...]: def _testmon_coverage_identity(executable_paths: Sequence[str]) -> dict[str, Any]: """Identify the exact worktree contents covered by an affected/full run.""" return { - "worktree_fingerprint": _worktree_fingerprint(), + "worktree_fingerprint": worktree_fingerprint(), "executable_paths": list(executable_paths), } @@ -2547,56 +2568,6 @@ def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: temporary.replace(path) -def _worktree_fingerprint(root: Path | None = None) -> str: - """Fingerprint tracked changes plus exact non-ignored untracked content.""" - checkout_root = (root or Path.cwd()).resolve() - digest = hashlib.sha256() - for command in ( - ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"], - ["git", "diff", "--binary", "HEAD", "--"], - ): - try: - result = subprocess.run(command, capture_output=True, timeout=30, cwd=checkout_root) - except (OSError, subprocess.TimeoutExpired): - return "unavailable" - if result.returncode != 0: - return "unavailable" - digest.update(result.stdout) - digest.update(b"\0") - try: - untracked = subprocess.run( - ["git", "ls-files", "--others", "--exclude-standard", "-z"], - capture_output=True, - timeout=30, - cwd=checkout_root, - ) - except (OSError, subprocess.TimeoutExpired): - return "unavailable" - if untracked.returncode != 0: - return "unavailable" - for raw_path in sorted(path for path in untracked.stdout.split(b"\0") if path): - try: - path_text = os.fsdecode(raw_path) - path = checkout_root / path_text - mode = path.lstat().st_mode - digest.update(raw_path) - digest.update(b"\0") - if stat.S_ISLNK(mode): - digest.update(b"symlink\0") - digest.update(os.fsencode(os.readlink(path))) - elif stat.S_ISREG(mode): - digest.update(b"file\0") - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - else: - digest.update(f"mode:{stat.S_IFMT(mode):o}".encode()) - digest.update(b"\0") - except OSError: - return "unavailable" - return digest.hexdigest() - - def _testmon_seed_identity( *, git_head: str | None, @@ -2612,7 +2583,7 @@ def _testmon_seed_identity( return { "git_head": git_head, "git_tree": git_tree, - "worktree_fingerprint": _worktree_fingerprint(), + "worktree_fingerprint": worktree_fingerprint(), "python": sys.version, "skip_slow": skip_slow, "lab": lab, @@ -2635,24 +2606,10 @@ def _recover_worker_collection_facts(*, events_dir: Path, selection_path: Path) the runner recovers the same canonical worker fact before it terminalizes the durable step record. """ - payloads: list[tuple[str, int, str, dict[str, Any]]] = [] - for path in events_dir.glob("*.collection.json"): - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - continue - worker_id = payload.get("worker_id") - pid = payload.get("pid") - if isinstance(worker_id, str) and isinstance(pid, int): - payloads.append((worker_id, pid, path.name, payload)) - if not payloads: + merged = merge_worker_collection_payloads(events_dir) + if merged is None: return False - payloads.sort() - selection = dict(payloads[0][3]) - durations = [payload.get("collection_duration_s") for *_ignored, payload in payloads] - numeric_durations = [duration for duration in durations if isinstance(duration, int | float)] - if numeric_durations: - selection["collection_duration_s"] = max(numeric_durations) + selection = dict(merged) selection.update( { "updated_at": datetime.now(timezone.utc).isoformat(), @@ -3599,14 +3556,14 @@ def main(argv: list[str] | None = None) -> int: head = _git_head() t0 = time.monotonic() - worktree_fingerprint = _worktree_fingerprint() + checkout_fingerprint = worktree_fingerprint() verify_run = VerifyRun( tier=tier, argv=list(sys.argv[1:] if argv is None else argv), git_head=head, polylogue_import_path=str(polylogue_import_path), environment_fingerprint=environment_fingerprint, - worktree_fingerprint=worktree_fingerprint, + worktree_fingerprint=checkout_fingerprint, ) seed_identity: dict[str, Any] | None = None resume_testmon_seed = False @@ -3820,6 +3777,26 @@ def main(argv: list[str] | None = None) -> int: f"inspect {TESTMON_SEED_ATTEMPT}.\n" ) + final_checkout_fingerprint = worktree_fingerprint() + if ( + checkout_fingerprint != "unavailable" + and final_checkout_fingerprint != "unavailable" + and final_checkout_fingerprint != checkout_fingerprint + ): + step_results.append( + { + "name": "checkout stability", + "duration_s": 0.0, + "exit": 125, + "diagnosis": "checkout_changed_during_verification", + "initial_worktree_fingerprint": checkout_fingerprint, + "final_worktree_fingerprint": final_checkout_fingerprint, + } + ) + if exit_code == 0: + exit_code = 125 + sys.stderr.write("verify: checkout contents changed during verification; evidence is not exact-head.\n") + total_duration = round(time.monotonic() - t0, 2) # Build history entry. @@ -3829,7 +3806,8 @@ def main(argv: list[str] | None = None) -> int: "tier": tier, "run_id": verify_run.run_id, "checkout_root": str(ROOT.resolve()), - "worktree_fingerprint": worktree_fingerprint, + "worktree_fingerprint": checkout_fingerprint, + "final_worktree_fingerprint": final_checkout_fingerprint, "artifact_dir": str(verify_run.relative_run_dir), "steps": step_results, "total_duration_s": total_duration, diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 62a9dd2a50..133a411751 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -18,7 +18,7 @@ import subprocess import time import uuid -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace from datetime import UTC, datetime from pathlib import Path @@ -28,7 +28,10 @@ VERIFY_CACHE = Path(".cache/verify") VERIFY_RUNS_DIR = VERIFY_CACHE / "runs" -DEVTOOLS_STATE_DIR = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state")) / "polylogue" / "devtools" +_XDG_STATE_HOME = os.environ.get("XDG_STATE_HOME", "").strip() +DEVTOOLS_STATE_DIR = ( + (Path(_XDG_STATE_HOME) if _XDG_STATE_HOME else Path.home() / ".local" / "state") / "polylogue" / "devtools" +) VERIFY_HISTORY_PATH = DEVTOOLS_STATE_DIR / "verify-history.jsonl" CURRENT_RUN_PATH = VERIFY_CACHE / "current-run.json" VERIFICATION_INVOCATION_ID_ENV = "POLYLOGUE_VERIFICATION_INVOCATION_ID" @@ -76,6 +79,22 @@ class PytestResourceError(RuntimeError): """Raised when the host cannot safely start a managed pytest run.""" +def _trailing_history_record(descriptor: int, *, end: int) -> tuple[int, bytes]: + """Read only the final unterminated JSONL record and its start offset.""" + cursor = end + suffix: list[bytes] = [] + while cursor > 0: + start = max(0, cursor - 64 * 1024) + os.lseek(descriptor, start, os.SEEK_SET) + chunk = os.read(descriptor, cursor - start) + delimiter = chunk.rfind(b"\n") + if delimiter >= 0: + return start + delimiter + 1, chunk[delimiter + 1 :] + b"".join(reversed(suffix)) + suffix.append(chunk) + cursor = start + return 0, b"".join(reversed(suffix)) + + def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTORY_PATH) -> None: """Append one complete invocation to the cross-worktree run history. @@ -91,17 +110,13 @@ def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTO fcntl.flock(descriptor, fcntl.LOCK_EX) end = os.lseek(descriptor, 0, os.SEEK_END) if end: - os.lseek(descriptor, 0, os.SEEK_SET) - existing = bytearray() - while chunk := os.read(descriptor, 64 * 1024): - existing.extend(chunk) - if not existing.endswith(b"\n"): - last_newline = existing.rfind(b"\n") - trailing = bytes(existing[last_newline + 1 :]) + os.lseek(descriptor, end - 1, os.SEEK_SET) + if os.read(descriptor, 1) != b"\n": + trailing_start, trailing = _trailing_history_record(descriptor, end=end) try: json.loads(trailing) except (UnicodeDecodeError, json.JSONDecodeError): - os.ftruncate(descriptor, last_newline + 1) + os.ftruncate(descriptor, trailing_start) else: # A complete JSON record can lose only its framing newline # during an interrupted append. Preserve it before adding @@ -118,6 +133,70 @@ def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTO os.close(descriptor) +def pytest_command_worker_request(cmd: Sequence[str]) -> str | None: + """Return the last xdist worker request from a final pytest command.""" + request: str | None = None + for index, argument in enumerate(cmd): + if argument in {"-n", "--numprocesses"}: + if index + 1 < len(cmd): + request = cmd[index + 1] + elif argument.startswith("--numprocesses="): + request = argument.removeprefix("--numprocesses=") + elif argument.startswith("-n") and len(argument) > 2: + request = argument[2:].removeprefix("=") + return request + + +def worktree_fingerprint(root: Path | None = None) -> str: + """Fingerprint tracked changes plus exact non-ignored untracked content.""" + checkout_root = (root or Path.cwd()).resolve() + digest = hashlib.sha256() + for command in ( + ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"], + ["git", "diff", "--binary", "HEAD", "--"], + ): + try: + result = subprocess.run(command, capture_output=True, timeout=30, cwd=checkout_root) + except (OSError, subprocess.TimeoutExpired): + return "unavailable" + if result.returncode != 0: + return "unavailable" + digest.update(result.stdout) + digest.update(b"\0") + try: + untracked = subprocess.run( + ["git", "ls-files", "--others", "--exclude-standard", "-z"], + capture_output=True, + timeout=30, + cwd=checkout_root, + ) + except (OSError, subprocess.TimeoutExpired): + return "unavailable" + if untracked.returncode != 0: + return "unavailable" + for raw_path in sorted(path for path in untracked.stdout.split(b"\0") if path): + try: + path_text = os.fsdecode(raw_path) + path = checkout_root / path_text + mode = path.lstat().st_mode + digest.update(raw_path) + digest.update(b"\0") + if stat.S_ISLNK(mode): + digest.update(b"symlink\0") + digest.update(os.fsencode(os.readlink(path))) + elif stat.S_ISREG(mode): + digest.update(b"file\0") + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + else: + digest.update(f"mode:{stat.S_IFMT(mode):o}".encode()) + digest.update(b"\0") + except OSError: + return "unavailable" + return digest.hexdigest() + + @dataclass(frozen=True) class PytestRuntimePolicy: """One start-time resource decision for a managed pytest run.""" @@ -339,12 +418,10 @@ def aggregate_pytest_statistics( if isinstance(row, dict): resources.append(row) explicit_worker_count: int | None = None - command_values = [str(value) for value in command] - for index, value in enumerate(command_values[:-1]): - if value in {"-n", "--numprocesses"}: - with contextlib.suppress(ValueError): - explicit_worker_count = int(command_values[index + 1]) - break + worker_request = pytest_command_worker_request([str(value) for value in command]) + if worker_request is not None: + with contextlib.suppress(ValueError): + explicit_worker_count = int(worker_request) basetemp_sizes = [ int(size_value) * 1024 for row in resources if isinstance((size_value := row.get("basetemp_size_kb")), int) ] @@ -571,7 +648,8 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> dict[str, Any] # An interrupted runner never returns through the normal # post-subprocess merge. Fold shards here, before every # aggregation path, so completed worker evidence survives. - merge_worker_events(step_dir / "events", step_dir / "events.jsonl") + with contextlib.suppress(OSError): + merge_worker_events(step_dir / "events", step_dir / "events.jsonl") statistics_path = step_dir / "statistics.json" with contextlib.suppress(OSError, ValueError): statistics = aggregate_pytest_statistics( @@ -952,7 +1030,7 @@ def _try_acquire_pytest_basetemp_claim_lock(basetemp: Path) -> TextIO | None: return handle -def _managed_pytest_basetemp_owner_alive(basetemp: Path) -> bool | None: +def managed_pytest_basetemp_owner_alive(basetemp: Path) -> bool | None: """Return whether a positive managed claim still names a live process.""" try: raw_identity = pytest_basetemp_claim_path(basetemp, kind="managed").read_text(encoding="utf-8").strip() @@ -1525,7 +1603,7 @@ def cleanup_managed_pytest_basetemp(*, root: Path, run_id: str, env: dict[str, s # both its claim and its fixture tree for that invocation to finish. return None try: - owner_alive = _managed_pytest_basetemp_owner_alive(basetemp) + owner_alive = managed_pytest_basetemp_owner_alive(basetemp) if owner_alive is True: return None # A serial pytest child may already have reclaimed this exact run-owned diff --git a/tests/conftest.py b/tests/conftest.py index bc034eb223..c908af8bc2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,14 +23,9 @@ from hypothesis.configuration import set_hypothesis_home_dir from hypothesis.database import DirectoryBasedExampleDatabase -# --------------------------------------------------------------------------- -# Place pytest temp directories on the NVMe scratch area by default. -# Test SQLite databases are write-heavy; /tmp lives on the root SSD on the -# operator workstation, while /realm/tmp is the high-write-budget scratch -# volume. /dev/shm remains available for explicit performance lanes, but it -# must not be the default: interrupted full/xdist runs can otherwise leave -# multi-GiB RAM-backed basetemps resident until reboot. -# --------------------------------------------------------------------------- +# Basetemp placement is selected once by ``resolve_pytest_basetemp_root``. +# This conftest owns only pytest-side claims, stale-tree reclamation, and the +# optional btrfs no-CoW mark for a disk-backed selection. from devtools import verify_runs from devtools.checkout_guard import ( CheckoutImportMismatchError, @@ -198,7 +193,8 @@ def _mark_basetemp_owner(basetemp: Path) -> None: def _mark_caller_owned_basetemp(basetemp: Path) -> None: """Claim an explicit ``--basetemp`` before pytest may replace its tree.""" handle = _acquire_basetemp_claim_lock(basetemp, blocking=True) - assert handle is not None + if handle is None: + raise pytest.UsageError(f"pytest: cannot claim the explicit basetemp: {basetemp}") with contextlib.suppress(OSError): basetemp.mkdir(parents=True, exist_ok=True) clear_managed_pytest_basetemp_claim(basetemp) @@ -211,19 +207,20 @@ def _acquire_basetemp_claim_lock(basetemp: Path, *, blocking: bool) -> TextIO | thread_lock = _BASE_TEMP_CLAIM_THREAD_LOCKS.setdefault(lock_path, threading.Lock()) if not thread_lock.acquire(blocking=blocking): return None - with contextlib.suppress(OSError): + try: lock_path.parent.mkdir(parents=True, exist_ok=True) handle = lock_path.open("a+", encoding="utf-8") - try: - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)) - except BlockingIOError: - handle.close() - thread_lock.release() - return None - _BASE_TEMP_CLAIM_LOCKS[lock_path] = handle - return handle - thread_lock.release() - return None + except OSError: + thread_lock.release() + return None + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)) + except OSError: + handle.close() + thread_lock.release() + return None + _BASE_TEMP_CLAIM_LOCKS[lock_path] = handle + return handle def _release_basetemp_claim_lock(basetemp: Path) -> None: @@ -238,21 +235,6 @@ def _release_basetemp_claim_lock(basetemp: Path) -> None: thread_lock.release() -def _basetemp_owner_alive(entry: Path) -> bool | None: - """True/False when the owner marker resolves a live/dead process, else None.""" - marker = _basetemp_claim_path(entry, kind="managed") - try: - raw_identity = marker.read_text(encoding="utf-8").strip() - raw_pid, separator, raw_start_ticks = raw_identity.partition(":") - pid = int(raw_pid) - start_ticks = int(raw_start_ticks) if separator else None - except (OSError, ValueError): - return None - if not Path(f"/proc/{pid}").exists(): - return False - return start_ticks is None or _process_start_ticks(pid) == start_ticks - - def _remove_stale_basetemp(entry: Path) -> None: """Remove an already-adjudicated stale tree, including read-only fixtures.""" if entry.is_symlink(): @@ -352,7 +334,7 @@ def _sweep_stale_polylogue_basetemps( continue if not _basetemp_claim_path(entry, kind="managed").is_file(): continue - owner_alive = _basetemp_owner_alive(entry) + owner_alive = verify_runs.managed_pytest_basetemp_owner_alive(entry) if owner_alive is not False: continue if entry.stat().st_mtime < cutoff: @@ -375,18 +357,18 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: teardown, where xdist/json-report may still be flushing controller/worker artifacts. """ - if os.environ.get("PYTEST_XDIST_WORKER"): - return - if not os.environ.get("POLYLOGUE_PYTEST_RUN_ID"): - return - numprocesses = getattr(session.config.option, "numprocesses", None) - if numprocesses not in (None, 0, "0"): - return basetemp = session.config.option.basetemp if not basetemp: return basetemp_path = Path(str(basetemp)) try: + if os.environ.get("PYTEST_XDIST_WORKER"): + return + if not os.environ.get("POLYLOGUE_PYTEST_RUN_ID"): + return + numprocesses = getattr(session.config.option, "numprocesses", None) + if numprocesses not in (None, 0, "0"): + return if str(basetemp_path) == os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP"): shutil.rmtree(basetemp_path, ignore_errors=True) if not basetemp_path.exists(): diff --git a/tests/unit/devtools/test_evidence_dashboard.py b/tests/unit/devtools/test_evidence_dashboard.py index 2d1035ffaf..39daf2e677 100644 --- a/tests/unit/devtools/test_evidence_dashboard.py +++ b/tests/unit/devtools/test_evidence_dashboard.py @@ -63,6 +63,7 @@ def test_static_gates_accept_exactly_bound_last_verify_result(monkeypatch: pytes } ) ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", tmp_path / "history.jsonl") monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index b51393484b..6aabf59160 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -32,6 +32,9 @@ def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: slowest_reports = list(pytest_progress_plugin._SLOWEST_REPORTS) collection_started_at = pytest_progress_plugin._COLLECTION_STARTED_AT collection_duration_s = pytest_progress_plugin._COLLECTION_DURATION_S + controller_collection_payload = pytest_progress_plugin._CONTROLLER_COLLECTION_PAYLOAD + recorded_report_keys = set(pytest_progress_plugin._RECORDED_REPORT_KEYS) + pytest_progress_plugin._RECORDED_REPORT_KEYS.clear() yield pytest_progress_plugin._SELECTED_COUNT = selected_count pytest_progress_plugin._DESELECTED_COUNT = deselected_count @@ -39,6 +42,9 @@ def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: pytest_progress_plugin._SLOWEST_REPORTS[:] = slowest_reports pytest_progress_plugin._COLLECTION_STARTED_AT = collection_started_at pytest_progress_plugin._COLLECTION_DURATION_S = collection_duration_s + pytest_progress_plugin._CONTROLLER_COLLECTION_PAYLOAD = controller_collection_payload + pytest_progress_plugin._RECORDED_REPORT_KEYS.clear() + pytest_progress_plugin._RECORDED_REPORT_KEYS.update(recorded_report_keys) @dataclass(frozen=True) @@ -101,6 +107,44 @@ def test_progress_plugin_preserves_xfail_and_xpass_in_durable_statistics( assert statistics["outcomes"] == {"xfailed": 2, "xpassed": 1} +def test_progress_plugin_observes_real_pytest_xfail_outcome(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + test_path = tmp_path / "test_xfail.py" + test_path.write_text( + "import pytest\n\n@pytest.mark.xfail(reason='known failure')\ndef test_expected_failure():\n assert False\n" + ) + env = os.environ.copy() + env["POLYLOGUE_PYTEST_EVENTS_PATH"] = str(events_path) + checkout_root = Path(__file__).resolve().parents[3] + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "devtools.pytest_progress_plugin", + "-p", + "no:testmon", + str(test_path), + ], + cwd=checkout_root, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + reports = [ + json.loads(line) + for line in events_path.read_text().splitlines() + if json.loads(line).get("event") == "test_report" + ] + assert any(report["when"] == "call" and report["outcome"] == "xfailed" for report in reports) + + def test_progress_plugin_skips_xdist_controller_forwarding_copy( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -357,6 +401,26 @@ def test_progress_plugin_records_collection_duration_and_summary( assert events[2]["duration_s"] == 2.5 +def test_progress_plugin_retains_controller_selection_through_session_finish( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + selection_path = tmp_path / "selection.json" + monkeypatch.setenv("POLYLOGUE_PYTEST_SELECTION_PATH", str(selection_path)) + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_collection_modifyitems( + _Session(["tests/a.py::test_keep"]), + object(), + [_Item("tests/a.py::test_keep")], + ) + + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + selection = json.loads(selection_path.read_text()) + assert selection["selected_nodeids"] == ["tests/a.py::test_keep"] + assert selection["selected_nodeids_omitted"] == 0 + + def test_progress_plugin_merges_xdist_collection_facts_without_double_counting( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index e3d27a3df0..4fbfce624f 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -11,7 +11,7 @@ import pytest from devtools import run_tests, verify -from devtools.verify_runs import CURRENT_STATISTICS_PATH, git_head +from devtools.verify_runs import CURRENT_STATISTICS_PATH, git_head, pytest_command_worker_request def test_build_pytest_cmd_defaults_to_single_process() -> None: @@ -54,7 +54,7 @@ def test_build_pytest_cmd_forwards_exactly_one_xdist_worker_request( arg for arg in command if arg in {"-n", "--numprocesses"} or arg.startswith(("-n", "--numprocesses=")) ] assert len(worker_flags) == 1 - assert verify._pytest_command_worker_request(command) == expected_request + assert pytest_command_worker_request(command) == expected_request def test_build_pytest_cmd_honors_workers_env(monkeypatch: pytest.MonkeyPatch) -> None: @@ -121,6 +121,26 @@ def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, di assert captured["history"]["status"] == "success" +def test_main_preserves_relative_selection_from_subdirectory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, Any] = {} + + def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + captured["cmd"] = cmd + return 0, 0.01, {"diagnosis": "pytest_passed"} + + monkeypatch.chdir(run_tests.ROOT / "tests" / "unit") + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr("devtools.run_tests._clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr("devtools.run_tests._run", _fake_run) + monkeypatch.setattr("devtools.run_tests.append_verify_history", lambda _payload: None) + + assert run_tests.main(["core/test_identity_law.py::test_session_id_is_origin_native_id"]) == 0 + + assert "tests/unit/core/test_identity_law.py::test_session_id_is_origin_native_id" in captured["cmd"] + + def test_main_returns_pytest_exit_code(monkeypatch: pytest.MonkeyPatch) -> None: def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: return 5, 0.01, {"diagnosis": "pytest_failed"} @@ -143,6 +163,7 @@ def test_main_anchors_and_refreshes_root_artifacts_from_any_invocation_directory stale_report = root / verify.PYTEST_REPORT_PATH stale_statistics = root / CURRENT_STATISTICS_PATH stale_report.parent.mkdir(parents=True) + stale_statistics.parent.mkdir(parents=True, exist_ok=True) stale_report.write_text('{"stale": true}') stale_statistics.write_text('{"stale": true}') captured: dict[str, object] = {} @@ -160,7 +181,7 @@ def fake_run(_label: str, _cmd: list[str], **kwargs: Any) -> tuple[int, float, d lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=root / "polylogue", as_dict=lambda: {}), ) monkeypatch.setattr(run_tests, "git_head", lambda _root: "head") - monkeypatch.setattr(run_tests, "_worktree_fingerprint", lambda _root: "fingerprint") + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "fingerprint") monkeypatch.setattr(run_tests, "_run", fake_run) monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 7acb839685..1826474792 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -74,7 +74,6 @@ _testmon_database_state, _testmon_preflight, _testmon_seed_can_resume, - _worktree_fingerprint, build_verify_steps, main, ) @@ -97,6 +96,9 @@ resolve_pytest_basetemp_root, xdist_uninterruptible_stall_reason, ) +from devtools.verify_runs import ( + worktree_fingerprint as _worktree_fingerprint, +) @pytest.fixture(autouse=True) @@ -235,7 +237,6 @@ def test_default_verify_uses_adaptive_pytest_testmon(monkeypatch: pytest.MonkeyP assert "--testmon" in command assert "--testmon-noselect" not in command assert "--testmon-forceselect" in command - assert "--dist=loadgroup" in command assert "-n" in command assert command[command.index("-n") + 1] == "8" assert "--dist=loadgroup" in command @@ -1062,6 +1063,9 @@ def test_aggregate_pytest_statistics_deduplicates_xdist_reports_and_terminal_fai assert result["xdist"]["worker_count"] == 2 assert result["outcomes"] == {"error": 2} + compact_result = aggregate_pytest_statistics(step, command=["pytest", "--numprocesses=3"]) + assert compact_result["xdist"]["worker_count"] == 3 + def test_aggregate_pytest_statistics_accounts_for_started_node_without_a_phase(tmp_path: Path) -> None: step = tmp_path / "step" @@ -1470,6 +1474,14 @@ def test_print_history_accepts_verify_and_focused_run_records( "exit_code": 1, "steps": [{"name": "pytest focused", "duration_s": None, "exit": 1}], }, + { + "finished_at": "2026-08-12T20:02:00+00:00", + "tier": "focused-test", + "git_head": "c" * 40, + "duration_s": "invalid", + "exit_code": None, + "steps": [{"name": "pytest interrupted", "duration_s": "invalid", "exit": None}], + }, ], ) @@ -1479,6 +1491,7 @@ def test_print_history_accepts_verify_and_focused_run_records( assert "quick" in output assert "focused-" in output assert "pytest focused(0s FAIL)" in output + assert "pytest interrupted(0s FAIL)" in output def test_verify_history_appends_concurrent_records_without_interleaving(tmp_path: Path) -> None: @@ -1505,6 +1518,30 @@ def test_verify_history_repairs_or_frames_an_incomplete_trailing_record(tmp_path assert rows == [{"sequence": 0}, {"sequence": 1}, {"sequence": 2}] +def test_verify_history_append_reads_only_the_trailing_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + history = tmp_path / "state" / "verify-history.jsonl" + history.parent.mkdir(parents=True) + history.write_bytes(b'{"padding":"' + (b"x" * (5 * 1024 * 1024)) + b'"}\n{"interrupted":') + bytes_read = 0 + real_read = os.read + + def measured_read(descriptor: int, count: int) -> bytes: + nonlocal bytes_read + payload = real_read(descriptor, count) + bytes_read += len(payload) + return payload + + monkeypatch.setattr(os, "read", measured_read) + + append_verify_history({"sequence": 1}, path=history) + + assert bytes_read < 128 * 1024 + assert json.loads(history.read_text(encoding="utf-8").splitlines()[-1]) == {"sequence": 1} + + def test_compare_against_last_skips_intervening_focused_history(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( verify, @@ -3864,6 +3901,7 @@ def test_run_reads_structured_pytest_report() -> None: with ( patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), patch("devtools.verify._read_pytest_report", return_value=report), + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), ): rc, _elapsed, metadata = _run("pytest testmon", ["pytest", "--testmon", "-n", "8"]) @@ -4006,6 +4044,55 @@ def test_pytest_run_preserves_other_lane_reports( assert metadata["junitxml_path"] == str(isolated_junit) +def test_managed_pytest_run_reads_only_its_invocation_report( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + run = VerifyRun(tier="focused-test", argv=[], git_head="head", root=tmp_path) + seen_report: Path | None = None + + def fake_pytest(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + nonlocal seen_report + seen_report = verify._pytest_json_report_path(cmd) + assert seen_report is not None + assert seen_report.parent == run.run_dir + seen_report.write_text('{"summary":{"passed":1,"total":1}}', encoding="utf-8") + return subprocess.CompletedProcess(cmd, 0, "1 passed in 0.01s\n", "") + + with patch("devtools.verify._run_pytest_with_heartbeat", side_effect=fake_pytest): + rc, _elapsed, metadata = _run( + "pytest focused", + [sys.executable, "-m", "pytest", f"--json-report-file={PYTEST_REPORT_PATH}"], + run=run, + ) + + assert rc == 0 + assert seen_report is not None and not seen_report.exists() + assert metadata["report_path"].endswith("/pytest-report.json") + + +def test_pytest_progress_is_durable_per_step_and_mirrored_current( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + artifact_dir = tmp_path / ".cache" / "verify" / "runs" / "run" / "steps" / "01-pytest" + + verify._write_pytest_progress( + event="running", + cmd=["pytest"], + started_at=0.0, + elapsed_s=0.0, + artifact_dir=str(artifact_dir), + ) + + durable = json.loads((artifact_dir / "progress.json").read_text(encoding="utf-8")) + current = json.loads((tmp_path / PYTEST_PROGRESS_PATH).read_text(encoding="utf-8")) + assert durable == current + assert durable["event"] == "running" + + def test_pytest_run_terminates_after_runtime_budget( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -4491,7 +4578,7 @@ def test_testmon_coverage_receipts_are_content_exact() -> None: assert _matching_testmon_coverage(paths) is None TESTMON_SEED_STAMP.unlink() - with patch("devtools.verify._worktree_fingerprint", return_value="affected"): + with patch("devtools.verify.worktree_fingerprint", return_value="affected"): _record_testmon_affected_coverage( executable_paths=paths, selected_count=3, @@ -4501,11 +4588,11 @@ def test_testmon_coverage_receipts_are_content_exact() -> None: assert _matching_testmon_coverage(paths) == "successful_affected_run" assert _matching_testmon_coverage(("polylogue/other.py",)) is None - with patch("devtools.verify._worktree_fingerprint", return_value="changed"): + with patch("devtools.verify.worktree_fingerprint", return_value="changed"): assert _matching_testmon_coverage(paths) is None TESTMON_AFFECTED_STAMP.write_text(json.dumps({"identity": {"worktree_fingerprint": "affected"}})) - with patch("devtools.verify._worktree_fingerprint", return_value="affected"): + with patch("devtools.verify.worktree_fingerprint", return_value="affected"): assert _matching_testmon_coverage(paths) is None diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index fe710ccf9d..0983fc98f1 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -1,5 +1,6 @@ from __future__ import annotations +import fcntl import os import shutil import subprocess @@ -393,6 +394,38 @@ def test_claim_lock_inode_stays_contended_after_managed_claim_clear(tmp_path: Pa assert lock_path.is_file() +def test_claim_lock_failure_closes_handle_and_releases_thread_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + basetemp = tmp_path / "pytest-polylogue-lock-failure" + lock_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="lock") + + with monkeypatch.context() as scoped: + + def fail_lock(*_args: object) -> None: + raise OSError("lock failed") + + scoped.setattr(fcntl, "flock", fail_lock) + assert conftest._acquire_basetemp_claim_lock(basetemp, blocking=True) is None + + assert not conftest._BASE_TEMP_CLAIM_THREAD_LOCKS[lock_path].locked() + handle = conftest._acquire_basetemp_claim_lock(basetemp, blocking=True) + assert handle is not None + conftest._release_basetemp_claim_lock(basetemp) + + +def test_explicit_basetemp_claim_failure_is_a_usage_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + basetemp = tmp_path / "unclaimable" + monkeypatch.setattr(conftest, "_acquire_basetemp_claim_lock", lambda *_args, **_kwargs: None) + + with pytest.raises(pytest.UsageError, match="cannot claim the explicit basetemp"): + conftest._mark_caller_owned_basetemp(basetemp) + + def test_managed_basetemp_claim_collision_is_rejected_across_processes(tmp_path: Path) -> None: basetemp = tmp_path / "pytest-polylogue-managed-collision" conftest._mark_basetemp_owner(basetemp) @@ -440,14 +473,14 @@ def test_stale_sweep_and_explicit_claim_are_atomic_for_one_path( sweep_checked = threading.Event() allow_sweep = threading.Event() caller_claimed = threading.Event() - original_owner_alive = conftest._basetemp_owner_alive + original_owner_alive = verify_runs.managed_pytest_basetemp_owner_alive def pause_after_admission(entry: Path) -> bool | None: sweep_checked.set() assert allow_sweep.wait(timeout=2) return original_owner_alive(entry) - monkeypatch.setattr(conftest, "_basetemp_owner_alive", pause_after_admission) + monkeypatch.setattr(verify_runs, "managed_pytest_basetemp_owner_alive", pause_after_admission) sweeper = threading.Thread( target=conftest._sweep_stale_polylogue_basetemps, kwargs={"max_age_s": 60, "roots": (tmp_path,)}, @@ -608,6 +641,27 @@ def test_sessionfinish_leaves_xdist_basetemp_for_supervisor_cleanup( assert basetemp.exists() +@pytest.mark.parametrize("worker_id", [None, "gw0"]) +def test_sessionfinish_releases_explicit_claim_without_managed_run( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + worker_id: str | None, +) -> None: + basetemp = tmp_path / "pytest-polylogue-explicit" + conftest._mark_caller_owned_basetemp(basetemp) + lock_path = verify_runs.pytest_basetemp_claim_path(basetemp, kind="lock") + session = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(basetemp), numprocesses=0))) + monkeypatch.delenv("POLYLOGUE_PYTEST_RUN_ID", raising=False) + if worker_id is None: + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + else: + monkeypatch.setenv("PYTEST_XDIST_WORKER", worker_id) + + conftest.pytest_sessionfinish(cast("pytest.Session", session), 0) + + assert not conftest._BASE_TEMP_CLAIM_THREAD_LOCKS[lock_path].locked() + + def test_sessionfinish_reclaims_only_its_managed_basetemp( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From ceca1781fd231b0abf1d0e51a66f4122debe4a09 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 07:33:09 +0200 Subject: [PATCH 34/53] fix(devtools): bind merge scope to checkout root --- devtools/merge_boundary.py | 15 +++++++++++++-- tests/unit/devtools/test_merge_boundary.py | 2 ++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index aefce1e8e7..361bc0837d 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -608,7 +608,13 @@ def cmd_merge( return 1 head_sha = info["headRefOid"] - scope = merge_gate._scope_verdict(pr, info, head_sha=head_sha) + checkout_root = merge_gate._repository_root() + scope = merge_gate._scope_verdict( + pr, + info, + head_sha=head_sha, + checkout_root=checkout_root, + ) if not scope.ok: print(f"REFUSING to merge PR #{pr}: invalid structured pr-scope carrier:", file=sys.stderr) @@ -666,7 +672,12 @@ def cmd_merge( file=sys.stderr, ) return 1 - final_scope = merge_gate._scope_verdict(pr, final_info, head_sha=head_sha) + final_scope = merge_gate._scope_verdict( + pr, + final_info, + head_sha=head_sha, + checkout_root=checkout_root, + ) initial_attestation = pr_scope.attestation_payload( scope, head_sha=head_sha, base_sha=merge_gate._base_sha(info) ).get("attestation_digest") diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index 9ad800ff38..517e131d80 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -88,6 +88,8 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return MagicMock(returncode=0, stdout=json.dumps([[]]), stderr="") if "/pulls/" in joined and "/comments" in joined: return MagicMock(returncode=0, stdout=json.dumps([comments]), stderr="") + if cmd[:3] == ["git", "rev-parse", "--show-toplevel"]: + return MagicMock(returncode=0, stdout=str(Path.cwd()) + "\n", stderr="") if cmd[:2] == ["git", "rev-parse"]: return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: From b1f1eb4d3cb05dcb492c9b4508371e141ea5b657 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 07:42:56 +0200 Subject: [PATCH 35/53] test(devtools): release direct basetemp claims --- tests/unit/test_pytest_temp_policy.py | 49 ++++++++++++++++----------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index 0983fc98f1..68df0c5c5d 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -7,6 +7,7 @@ import sys import threading from collections.abc import Generator +from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -18,6 +19,17 @@ from tests.infra.frozen_clock import FrozenClock +@contextmanager +def _configured_pytest(config: Any) -> Generator[None, None, None]: + """Run the configure hook directly without leaking its claim lock.""" + conftest.pytest_configure(cast("pytest.Config", config)) + basetemp = Path(str(config.option.basetemp)) + try: + yield + finally: + conftest._release_basetemp_claim_lock(basetemp) + + def _make_real_candidates( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, realm_mounted: bool = True ) -> tuple[Path, Path]: @@ -232,10 +244,9 @@ def test_bare_pytest_configure_defaults_to_scratch_without_a_supervisor( rootpath=tmp_path, ) - conftest.pytest_configure(cast("pytest.Config", config)) - - assert Path(str(config.option.basetemp)).parent == scratch - assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" + with _configured_pytest(config): + assert Path(str(config.option.basetemp)).parent == scratch + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" def test_bare_pytest_ignores_leaked_cloud_basetemp_on_workstation( @@ -255,11 +266,10 @@ def test_bare_pytest_ignores_leaked_cloud_basetemp_on_workstation( rootpath=tmp_path, ) - conftest.pytest_configure(cast("pytest.Config", config)) - - assert Path(str(config.option.basetemp)).parent == scratch - assert "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in os.environ - assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" + with _configured_pytest(config): + assert Path(str(config.option.basetemp)).parent == scratch + assert "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in os.environ + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" def test_bare_pytest_routes_an_environment_configured_tmpfs_root_to_scratch( @@ -279,11 +289,10 @@ def test_bare_pytest_routes_an_environment_configured_tmpfs_root_to_scratch( rootpath=tmp_path, ) - conftest.pytest_configure(cast("pytest.Config", config)) - - assert Path(str(config.option.basetemp)).parent == scratch - assert "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in os.environ - assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" + with _configured_pytest(config): + assert Path(str(config.option.basetemp)).parent == scratch + assert "POLYLOGUE_PYTEST_BASETEMP_ROOT" not in os.environ + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" def test_sweep_stale_polylogue_basetemps_preserves_unknown_seeded_and_recent( @@ -320,14 +329,14 @@ def test_explicit_basetemp_remains_outside_a_later_startup_stale_sweep( rootpath=tmp_path, ) - conftest.pytest_configure(cast("pytest.Config", config)) - assert verify_runs.pytest_basetemp_claim_path(explicit, kind="caller-owned").is_file() - old = frozen_clock.time() - 24 * 60 * 60 - os.utime(explicit, (old, old)) + with _configured_pytest(config): + assert verify_runs.pytest_basetemp_claim_path(explicit, kind="caller-owned").is_file() + old = frozen_clock.time() - 24 * 60 * 60 + os.utime(explicit, (old, old)) - conftest._sweep_stale_polylogue_basetemps(roots=(tmp_path,)) + conftest._sweep_stale_polylogue_basetemps(roots=(tmp_path,)) - assert explicit.exists() + assert explicit.exists() def test_explicit_basetemp_claim_survives_real_pytest_basetemp_replacement(tmp_path: Path) -> None: From 7dac899d56372deac92593b7731f0a1242570854 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 08:07:52 +0200 Subject: [PATCH 36/53] fix(devtools): reject mixed-checkout test evidence --- devtools/evidence_dashboard.py | 1 + devtools/run_tests.py | 43 ++++++++++++++++- devtools/verify_runs.py | 3 ++ tests/conftest.py | 6 +++ .../unit/devtools/test_evidence_dashboard.py | 30 ++++++++++++ tests/unit/devtools/test_run_tests.py | 48 ++++++++++++++++++- tests/unit/test_pytest_temp_policy.py | 21 ++++++++ 7 files changed, 150 insertions(+), 2 deletions(-) diff --git a/devtools/evidence_dashboard.py b/devtools/evidence_dashboard.py index f7e02c1aaa..348524b728 100644 --- a/devtools/evidence_dashboard.py +++ b/devtools/evidence_dashboard.py @@ -237,6 +237,7 @@ def _static_evidence_is_bound( entry.get("checkout_root") == checkout_root and entry.get("git_head") == checkout_head and entry.get("worktree_fingerprint") == worktree_fingerprint + and entry.get("final_worktree_fingerprint") == worktree_fingerprint ) diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 09cbf9adc8..c789568e29 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -57,12 +57,41 @@ ROOT = Path(__file__).resolve().parent.parent _LOCK_PATH = ROOT / ".cache" / "test-run.lock" +_PATH_VALUE_OPTIONS = frozenset( + { + "--basetemp", + "--confcutdir", + "--ignore", + "--junitxml", + "--rootdir", + } +) + + +def _absolute_option_path(value: str, *, invocation_directory: Path) -> str: + path = Path(value) + return str(path if path.is_absolute() else (invocation_directory / path).resolve()) def _normalize_selection_paths(selection: list[str], *, invocation_directory: Path) -> list[str]: """Preserve path selections relative to the directory that invoked devtools.""" normalized: list[str] = [] + option_value_pending = False for argument in selection: + if option_value_pending: + normalized.append(_absolute_option_path(argument, invocation_directory=invocation_directory)) + option_value_pending = False + continue + option_name, equals, option_value = argument.partition("=") + if option_name in _PATH_VALUE_OPTIONS: + if equals: + normalized.append( + f"{option_name}={_absolute_option_path(option_value, invocation_directory=invocation_directory)}" + ) + else: + normalized.append(argument) + option_value_pending = True + continue if argument.startswith("-"): normalized.append(argument) continue @@ -188,6 +217,7 @@ def main(argv: list[str] | None = None) -> int: no_lock = os.environ.get("POLYLOGUE_TEST_NO_LOCK") == "1" with _run_lock(enabled=not no_lock): _clear_pytest_report(cmd) + initial_worktree_fingerprint = worktree_fingerprint(ROOT) run = VerifyRun( tier="focused-test", argv=selection, @@ -195,7 +225,7 @@ def main(argv: list[str] | None = None) -> int: root=ROOT, polylogue_import_path=str(polylogue_import_path), environment_fingerprint=environment_fingerprint, - worktree_fingerprint=worktree_fingerprint(ROOT), + worktree_fingerprint=initial_worktree_fingerprint, ) started = time.monotonic() try: @@ -204,12 +234,23 @@ def main(argv: list[str] | None = None) -> int: rc = 130 metadata = {"diagnosis": "pytest_interrupted", "termination_reason": "operator_interrupt"} run.finish_interrupted_steps(exit_code=rc, diagnosis=str(metadata["diagnosis"])) + final_worktree_fingerprint = worktree_fingerprint(ROOT) + if ( + initial_worktree_fingerprint != "unavailable" + and final_worktree_fingerprint != "unavailable" + and final_worktree_fingerprint != initial_worktree_fingerprint + ): + metadata["diagnosis"] = "checkout_changed_during_focused_test" + if rc == 0: + rc = 125 + sys.stderr.write("devtools test: checkout contents changed during pytest; evidence is not exact-head.\n") payload = run.finish( exit_code=rc, duration_s=time.monotonic() - started, diagnosis=metadata.get("diagnosis"), verification_scope="affected", release_baseline_allowed=False, + final_worktree_fingerprint=final_worktree_fingerprint, ) append_verify_history(payload) if use_json: diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 133a411751..846d067604 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -692,6 +692,7 @@ def finish( verification_scope: str | None = None, release_baseline_allowed: bool | None = None, terminal_authorization: str | None = None, + final_worktree_fingerprint: str | None = None, ) -> dict[str, Any]: self._payload["finished_at"] = utc_now() self._payload["duration_s"] = round(duration_s, 2) @@ -699,6 +700,8 @@ def finish( self._payload["status"] = "success" if exit_code == 0 else "failed" if diagnosis: self._payload["diagnosis"] = diagnosis + if final_worktree_fingerprint is not None: + self._payload["final_worktree_fingerprint"] = final_worktree_fingerprint if verification_scope is not None: self._payload["verification_scope"] = verification_scope self._payload["release_baseline_allowed"] = release_baseline_allowed diff --git a/tests/conftest.py b/tests/conftest.py index c908af8bc2..d466dd4e73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -106,7 +106,13 @@ def pytest_configure(config: pytest.Config) -> None: ) if config.option.basetemp is not None: + # A second in-process pytest.main() inherits os.environ from the first + # run. Explicit basetemp ownership is per invocation, so stale managed + # markers must not turn the caller-owned diagnostic tree into cleanup + # fodder at session finish. if not hasattr(config, "workerinput"): + os.environ.pop("POLYLOGUE_PYTEST_RUN_ID", None) + os.environ.pop("POLYLOGUE_PYTEST_MANAGED_BASETEMP", None) _mark_caller_owned_basetemp(Path(str(config.option.basetemp))) return diff --git a/tests/unit/devtools/test_evidence_dashboard.py b/tests/unit/devtools/test_evidence_dashboard.py index 39daf2e677..b03a6ae7f5 100644 --- a/tests/unit/devtools/test_evidence_dashboard.py +++ b/tests/unit/devtools/test_evidence_dashboard.py @@ -19,6 +19,7 @@ def test_static_gates_read_shared_verify_history(monkeypatch: pytest.MonkeyPatch "checkout_root": str(tmp_path.resolve()), "git_head": "current-head", "worktree_fingerprint": "current-fingerprint", + "final_worktree_fingerprint": "current-fingerprint", "steps": [{"name": "ruff check", "duration_s": 1.0, "exit": 0}], } ) @@ -58,6 +59,7 @@ def test_static_gates_accept_exactly_bound_last_verify_result(monkeypatch: pytes "checkout_root": str(tmp_path.resolve()), "git_head": "current-head", "worktree_fingerprint": "current-fingerprint", + "final_worktree_fingerprint": "current-fingerprint", "steps": [{"name": "ruff check", "duration_s": 1.0, "exit": 0}], } } @@ -171,3 +173,31 @@ def test_static_gates_reject_wrong_checkout_fingerprint_and_legacy_evidence( assert gates["available"] is False assert all(gate["available"] is False for gate in gates["gates"]) + + +def test_static_gates_reject_a_run_whose_checkout_changed_mid_verification( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + history = tmp_path / "history.jsonl" + history.write_text( + json.dumps( + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "final_worktree_fingerprint": "changed-during-run", + "steps": [{"name": "ruff check", "exit": 0}], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["available"] is False for gate in gates["gates"]) diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index 4fbfce624f..4c5cce634a 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -6,7 +6,7 @@ import sys from pathlib import Path from types import SimpleNamespace -from typing import Any +from typing import Any, cast import pytest @@ -141,6 +141,52 @@ def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, assert "tests/unit/core/test_identity_law.py::test_session_id_is_origin_native_id" in captured["cmd"] +def test_main_preserves_path_valued_options_from_subdirectory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + captured: dict[str, Any] = {} + + def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + captured["cmd"] = cmd + return 0, 0.01, {"diagnosis": "pytest_passed"} + + invocation = tmp_path / "nested" + invocation.mkdir() + (invocation / "fixtures").mkdir() + monkeypatch.chdir(invocation) + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", _fake_run) + monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) + + assert run_tests.main(["-k", "proof", "--basetemp=diagnostic", "--rootdir", ".", "--ignore", "fixtures"]) == 0 + + command = cast(list[str], captured["cmd"]) + assert f"--basetemp={invocation / 'diagnostic'}" in command + assert command[command.index("--rootdir") + 1] == str(invocation) + assert command[command.index("--ignore") + 1] == str(invocation / "fixtures") + + +def test_main_withholds_success_when_checkout_changes_during_pytest( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, Any] = {} + fingerprints = iter(("initial", "changed")) + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", lambda *_args, **_kwargs: (0, 0.01, {"diagnosis": "pytest_passed"})) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprints)) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: captured.update(payload)) + + assert run_tests.main(["tests/unit/example.py"]) == 125 + assert captured["status"] == "failed" + assert captured["diagnosis"] == "checkout_changed_during_focused_test" + assert captured["worktree_fingerprint"] == "initial" + assert captured["final_worktree_fingerprint"] == "changed" + + def test_main_returns_pytest_exit_code(monkeypatch: pytest.MonkeyPatch) -> None: def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: return 5, 0.01, {"diagnosis": "pytest_failed"} diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index 68df0c5c5d..6d7adbf616 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -339,6 +339,27 @@ def test_explicit_basetemp_remains_outside_a_later_startup_stale_sweep( assert explicit.exists() +def test_explicit_basetemp_clears_stale_managed_identity_from_prior_in_process_run( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + explicit = tmp_path / "pytest-polylogue-debug" + config = SimpleNamespace( + option=SimpleNamespace(basetemp=str(explicit)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "prior-run") + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(explicit)) + + with _configured_pytest(config): + assert "POLYLOGUE_PYTEST_RUN_ID" not in os.environ + assert "POLYLOGUE_PYTEST_MANAGED_BASETEMP" not in os.environ + explicit.mkdir(exist_ok=True) + + assert explicit.exists() + + def test_explicit_basetemp_claim_survives_real_pytest_basetemp_replacement(tmp_path: Path) -> None: """Exercise pytest's lazy TempPathFactory clearing against our real conftest.""" explicit = tmp_path / "pytest-polylogue-diagnostic" From cc8b29a0d67fe705b1b6e36c496584db8b8d08a7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 08:38:59 +0200 Subject: [PATCH 37/53] fix(test): preserve nested pytest authority --- devtools/run_tests.py | 13 +++--- devtools/verify.py | 20 ++++++--- tests/conftest.py | 58 ++++++++++++++++++++++++++- tests/unit/devtools/test_run_tests.py | 41 ++++++++++++++++++- tests/unit/devtools/test_verify.py | 24 +++++++++++ tests/unit/test_pytest_temp_policy.py | 48 ++++++++++++++++++++++ 6 files changed, 192 insertions(+), 12 deletions(-) diff --git a/devtools/run_tests.py b/devtools/run_tests.py index c789568e29..33ea62bbd7 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -62,6 +62,8 @@ "--basetemp", "--confcutdir", "--ignore", + "--ignore-glob", + "--junit-xml", "--junitxml", "--rootdir", } @@ -235,11 +237,12 @@ def main(argv: list[str] | None = None) -> int: metadata = {"diagnosis": "pytest_interrupted", "termination_reason": "operator_interrupt"} run.finish_interrupted_steps(exit_code=rc, diagnosis=str(metadata["diagnosis"])) final_worktree_fingerprint = worktree_fingerprint(ROOT) - if ( - initial_worktree_fingerprint != "unavailable" - and final_worktree_fingerprint != "unavailable" - and final_worktree_fingerprint != initial_worktree_fingerprint - ): + if "unavailable" in {initial_worktree_fingerprint, final_worktree_fingerprint}: + metadata["diagnosis"] = "checkout_fingerprint_unavailable" + if rc == 0: + rc = 125 + sys.stderr.write("devtools test: checkout fingerprint unavailable; evidence is not exact-head.\n") + elif final_worktree_fingerprint != initial_worktree_fingerprint: metadata["diagnosis"] = "checkout_changed_during_focused_test" if rc == 0: rc = 125 diff --git a/devtools/verify.py b/devtools/verify.py index c97af8ab98..295bb455a1 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -3778,11 +3778,21 @@ def main(argv: list[str] | None = None) -> int: ) final_checkout_fingerprint = worktree_fingerprint() - if ( - checkout_fingerprint != "unavailable" - and final_checkout_fingerprint != "unavailable" - and final_checkout_fingerprint != checkout_fingerprint - ): + if "unavailable" in {checkout_fingerprint, final_checkout_fingerprint}: + step_results.append( + { + "name": "checkout stability", + "duration_s": 0.0, + "exit": 125, + "diagnosis": "checkout_fingerprint_unavailable", + "initial_worktree_fingerprint": checkout_fingerprint, + "final_worktree_fingerprint": final_checkout_fingerprint, + } + ) + if exit_code == 0: + exit_code = 125 + sys.stderr.write("verify: checkout fingerprint unavailable; evidence is not exact-head.\n") + elif final_checkout_fingerprint != checkout_fingerprint: step_results.append( { "name": "checkout stability", diff --git a/tests/conftest.py b/tests/conftest.py index d466dd4e73..e9d0923d96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,6 +69,15 @@ "tests.infra.clock_guard", ) +# Pytest supports nested in-process ``pytest.main()`` calls. Keep the active +# controller identity process-local so an explicit nested basetemp can suspend +# and later restore the outer run's managed ownership markers. A completed +# earlier invocation is not active and therefore cannot authorize restoration +# of stale environment values. +_ACTIVE_MANAGED_PYTEST_IDENTITIES: list[tuple[str, str]] = [] +_MANAGED_IDENTITY_ATTR = "_polylogue_managed_pytest_identity" +_SUSPENDED_IDENTITY_ATTR = "_polylogue_suspended_managed_pytest_identity" + if TYPE_CHECKING: from click.testing import CliRunner @@ -106,17 +115,37 @@ def pytest_configure(config: pytest.Config) -> None: ) if config.option.basetemp is not None: + configured_basetemp = str(config.option.basetemp) + run_id = os.environ.get("POLYLOGUE_PYTEST_RUN_ID") + managed_basetemp = os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") + supervised_managed = ( + run_id is not None + and os.environ.get("POLYLOGUE_VERIFY_RUN_ID") == run_id + and managed_basetemp == configured_basetemp + ) + if supervised_managed and not hasattr(config, "workerinput"): + assert run_id is not None + identity = (run_id, configured_basetemp) + _ACTIVE_MANAGED_PYTEST_IDENTITIES.append(identity) + setattr(config, _MANAGED_IDENTITY_ATTR, identity) + return # A second in-process pytest.main() inherits os.environ from the first # run. Explicit basetemp ownership is per invocation, so stale managed # markers must not turn the caller-owned diagnostic tree into cleanup # fodder at session finish. if not hasattr(config, "workerinput"): + if _ACTIVE_MANAGED_PYTEST_IDENTITIES: + setattr(config, _SUSPENDED_IDENTITY_ATTR, _ACTIVE_MANAGED_PYTEST_IDENTITIES[-1]) os.environ.pop("POLYLOGUE_PYTEST_RUN_ID", None) os.environ.pop("POLYLOGUE_PYTEST_MANAGED_BASETEMP", None) - _mark_caller_owned_basetemp(Path(str(config.option.basetemp))) + _mark_caller_owned_basetemp(Path(configured_basetemp)) return if config.option.basetemp is None: + if not hasattr(config, "workerinput") and _ACTIVE_MANAGED_PYTEST_IDENTITIES: + setattr(config, _SUSPENDED_IDENTITY_ATTR, _ACTIVE_MANAGED_PYTEST_IDENTITIES[-1]) + os.environ.pop("POLYLOGUE_PYTEST_RUN_ID", None) + os.environ.pop("POLYLOGUE_PYTEST_MANAGED_BASETEMP", None) normalized_basetemp_env = normalize_pytest_basetemp_env(os.environ) configured_root = normalized_basetemp_env.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") unmanaged_tmpfs_root = configured_root is not None and verify_runs._is_beneath( @@ -150,9 +179,35 @@ def pytest_configure(config: pytest.Config) -> None: raise pytest.UsageError(f"pytest: {exc}") from exc config.option.basetemp = str(basetemp) os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = str(basetemp) + if not hasattr(config, "workerinput"): + identity = (run_id, str(basetemp)) + _ACTIVE_MANAGED_PYTEST_IDENTITIES.append(identity) + setattr(config, _MANAGED_IDENTITY_ATTR, identity) sys.stderr.write(f"pytest: basetemp → {config.option.basetemp} ({label})\n") +def pytest_unconfigure(config: pytest.Config) -> None: + """Restore or retire process-local managed ownership after one invocation.""" + suspended = getattr(config, _SUSPENDED_IDENTITY_ATTR, None) + if isinstance(suspended, tuple) and len(suspended) == 2: + run_id, basetemp = suspended + os.environ["POLYLOGUE_PYTEST_RUN_ID"] = str(run_id) + os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = str(basetemp) + + managed = getattr(config, _MANAGED_IDENTITY_ATTR, None) + if not (isinstance(managed, tuple) and len(managed) == 2): + return + for index in range(len(_ACTIVE_MANAGED_PYTEST_IDENTITIES) - 1, -1, -1): + if _ACTIVE_MANAGED_PYTEST_IDENTITIES[index] == managed: + del _ACTIVE_MANAGED_PYTEST_IDENTITIES[index] + break + run_id, basetemp = managed + if os.environ.get("POLYLOGUE_PYTEST_RUN_ID") == run_id: + os.environ.pop("POLYLOGUE_PYTEST_RUN_ID", None) + if os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") == basetemp: + os.environ.pop("POLYLOGUE_PYTEST_MANAGED_BASETEMP", None) + + # Per-run basetemps are freed on sessionfinish. A run killed before # sessionfinish (SIGKILL, OOM) leaks its basetemp, so the controller reclaims # clearly-dead orphans on startup. Seeded corpora (``pytest-polylogue-seeded-*``) @@ -474,6 +529,7 @@ def _reclaim_test_tmp_path( "POLYLOGUE_VERIFY_RUN_ID", "POLYLOGUE_PYTEST_EVENTS_DIR", "POLYLOGUE_PYTEST_EVENTS_PATH", + "POLYLOGUE_PYTEST_RUN_ID", "POLYLOGUE_PYTEST_SELECTION_PATH", "POLYLOGUE_PYTEST_SUMMARY_PATH", "POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT", diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index 4c5cce634a..2bb81bc53b 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -160,12 +160,30 @@ def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, monkeypatch.setattr(run_tests, "_run", _fake_run) monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) - assert run_tests.main(["-k", "proof", "--basetemp=diagnostic", "--rootdir", ".", "--ignore", "fixtures"]) == 0 + assert ( + run_tests.main( + [ + "-k", + "proof", + "--basetemp=diagnostic", + "--rootdir", + ".", + "--ignore", + "fixtures", + "--ignore-glob=fixtures/*.json", + "--junit-xml", + "reports/results.xml", + ] + ) + == 0 + ) command = cast(list[str], captured["cmd"]) assert f"--basetemp={invocation / 'diagnostic'}" in command assert command[command.index("--rootdir") + 1] == str(invocation) assert command[command.index("--ignore") + 1] == str(invocation / "fixtures") + assert f"--ignore-glob={invocation / 'fixtures' / '*.json'}" in command + assert command[command.index("--junit-xml") + 1] == str(invocation / "reports" / "results.xml") def test_main_withholds_success_when_checkout_changes_during_pytest( @@ -187,6 +205,27 @@ def test_main_withholds_success_when_checkout_changes_during_pytest( assert captured["final_worktree_fingerprint"] == "changed" +@pytest.mark.parametrize("fingerprints", [("unavailable", "stable"), ("stable", "unavailable")]) +def test_main_withholds_success_when_checkout_fingerprint_is_unavailable( + monkeypatch: pytest.MonkeyPatch, + fingerprints: tuple[str, str], +) -> None: + captured: dict[str, Any] = {} + fingerprint_values = iter(fingerprints) + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", lambda *_args, **_kwargs: (0, 0.01, {"diagnosis": "pytest_passed"})) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprint_values)) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: captured.update(payload)) + + assert run_tests.main(["tests/unit/example.py"]) == 125 + assert captured["status"] == "failed" + assert captured["diagnosis"] == "checkout_fingerprint_unavailable" + assert captured["worktree_fingerprint"] == fingerprints[0] + assert captured["final_worktree_fingerprint"] == fingerprints[1] + + def test_main_returns_pytest_exit_code(monkeypatch: pytest.MonkeyPatch) -> None: def _fake_run(label: str, cmd: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: return 5, 0.01, {"diagnosis": "pytest_failed"} diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 1826474792..ad8b6d23f5 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -4310,6 +4310,30 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo assert payload["release_baseline_allowed"] is False +@pytest.mark.parametrize("fingerprints", [("unavailable", "stable"), ("stable", "unavailable")]) +def test_verify_withholds_success_when_checkout_fingerprint_is_unavailable( + capsys: pytest.CaptureFixture[str], + fingerprints: tuple[str, str], +) -> None: + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {})), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.worktree_fingerprint", side_effect=fingerprints), + ): + rc = main(["--quick", "--json"]) + + assert rc == 125 + payload = json.loads(capsys.readouterr().out) + assert payload["exit_code"] == 125 + checkout_step = next(step for step in payload["steps"] if step["name"] == "checkout stability") + assert checkout_step["diagnosis"] == "checkout_fingerprint_unavailable" + assert checkout_step["initial_worktree_fingerprint"] == fingerprints[0] + assert checkout_step["final_worktree_fingerprint"] == fingerprints[1] + + def test_verify_stops_after_failed_heavy_step(capsys: pytest.CaptureFixture[str]) -> None: calls: list[str] = [] diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index 6d7adbf616..ed3dcffd65 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -27,6 +27,7 @@ def _configured_pytest(config: Any) -> Generator[None, None, None]: try: yield finally: + conftest.pytest_unconfigure(cast("pytest.Config", config)) conftest._release_basetemp_claim_lock(basetemp) @@ -349,6 +350,7 @@ def test_explicit_basetemp_clears_stale_managed_identity_from_prior_in_process_r addinivalue_line=lambda *args, **kwargs: None, rootpath=tmp_path, ) + monkeypatch.setattr(conftest, "_ACTIVE_MANAGED_PYTEST_IDENTITIES", []) monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "prior-run") monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(explicit)) @@ -360,6 +362,52 @@ def test_explicit_basetemp_clears_stale_managed_identity_from_prior_in_process_r assert explicit.exists() +def test_nested_explicit_basetemp_restores_active_outer_managed_identity( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + ambient_identity = ( + os.environ.get("POLYLOGUE_PYTEST_RUN_ID"), + os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP"), + ) + assert all(ambient_identity) + _shm, _scratch = _make_real_candidates(monkeypatch, tmp_path) + for name in ( + "POLYLOGUE_VERIFY_RUN_ID", + "POLYLOGUE_PYTEST_BASETEMP_ROOT", + "POLYLOGUE_PYTEST_TMPFS", + "POLYLOGUE_PYTEST_RUN_ID", + "POLYLOGUE_PYTEST_CHECKOUT", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", + ): + monkeypatch.delenv(name, raising=False) + outer = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + explicit = tmp_path / "nested-diagnostic" + nested = SimpleNamespace( + option=SimpleNamespace(basetemp=str(explicit)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(outer): + outer_identity = ( + os.environ["POLYLOGUE_PYTEST_RUN_ID"], + os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"], + ) + with _configured_pytest(nested): + assert "POLYLOGUE_PYTEST_RUN_ID" not in os.environ + assert "POLYLOGUE_PYTEST_MANAGED_BASETEMP" not in os.environ + assert os.environ["POLYLOGUE_PYTEST_RUN_ID"] == outer_identity[0] + assert os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] == outer_identity[1] + + assert os.environ.get("POLYLOGUE_PYTEST_RUN_ID") == ambient_identity[0] + assert os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") == ambient_identity[1] + + def test_explicit_basetemp_claim_survives_real_pytest_basetemp_replacement(tmp_path: Path) -> None: """Exercise pytest's lazy TempPathFactory clearing against our real conftest.""" explicit = tmp_path / "pytest-polylogue-diagnostic" From 81a9b0e0d6dc85eb423de061ceab579ad9094b24 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 09:08:50 +0200 Subject: [PATCH 38/53] fix(test): preserve nested pytest evidence authority Normalize every supported focused-run pytest file option before changing directories, while matching pytest's rootdir-only environment expansion. Carry checkout-stability diagnoses and final fingerprints through every broad-run receipt, and represent nested explicit pytest invocations as ownership scopes so an unmanaged middle invocation cannot restore or reclaim a live outer basetemp. --- devtools/run_tests.py | 46 +++++++++++++--- devtools/verify.py | 16 ++++-- tests/conftest.py | 76 ++++++++++++++------------- tests/unit/devtools/test_run_tests.py | 57 ++++++++++++++++++++ tests/unit/devtools/test_verify.py | 45 ++++++++++++++++ tests/unit/test_pytest_temp_policy.py | 49 ++++++++++++++++- 6 files changed, 240 insertions(+), 49 deletions(-) diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 33ea62bbd7..c378a05e6e 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -59,18 +59,30 @@ _LOCK_PATH = ROOT / ".cache" / "test-run.lock" _PATH_VALUE_OPTIONS = frozenset( { + "-c", "--basetemp", + "--config-file", "--confcutdir", + "--debug", "--ignore", "--ignore-glob", "--junit-xml", "--junitxml", + "--log-file", "--rootdir", } ) +_ENV_EXPANDING_PATH_OPTIONS = frozenset({"--rootdir"}) -def _absolute_option_path(value: str, *, invocation_directory: Path) -> str: +def _absolute_option_path( + value: str, + *, + invocation_directory: Path, + expand_environment_variables: bool = False, +) -> str: + if expand_environment_variables: + value = os.path.expandvars(value) path = Path(value) return str(path if path.is_absolute() else (invocation_directory / path).resolve()) @@ -78,21 +90,39 @@ def _absolute_option_path(value: str, *, invocation_directory: Path) -> str: def _normalize_selection_paths(selection: list[str], *, invocation_directory: Path) -> list[str]: """Preserve path selections relative to the directory that invoked devtools.""" normalized: list[str] = [] - option_value_pending = False + pending_option: str | None = None for argument in selection: - if option_value_pending: - normalized.append(_absolute_option_path(argument, invocation_directory=invocation_directory)) - option_value_pending = False + if pending_option is not None: + normalized.append( + _absolute_option_path( + argument, + invocation_directory=invocation_directory, + expand_environment_variables=pending_option in _ENV_EXPANDING_PATH_OPTIONS, + ) + ) + pending_option = None continue option_name, equals, option_value = argument.partition("=") if option_name in _PATH_VALUE_OPTIONS: if equals: - normalized.append( - f"{option_name}={_absolute_option_path(option_value, invocation_directory=invocation_directory)}" + normalized_value = _absolute_option_path( + option_value, + invocation_directory=invocation_directory, + expand_environment_variables=option_name in _ENV_EXPANDING_PATH_OPTIONS, ) + normalized.append(f"{option_name}={normalized_value}") else: normalized.append(argument) - option_value_pending = True + pending_option = option_name + continue + if argument.startswith("-c") and len(argument) > len("-c"): + normalized.append( + "-c" + + _absolute_option_path( + argument[len("-c") :], + invocation_directory=invocation_directory, + ) + ) continue if argument.startswith("-"): normalized.append(argument) diff --git a/devtools/verify.py b/devtools/verify.py index 295bb455a1..ef8506558f 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -3823,6 +3823,14 @@ def main(argv: list[str] | None = None) -> int: "total_duration_s": total_duration, "exit_code": exit_code, } + checkout_stability_diagnosis = next( + ( + str(step["diagnosis"]) + for step in reversed(step_results) + if step.get("name") == "checkout stability" and "diagnosis" in step + ), + None, + ) fallback_pytest_diagnosis = next( ( str(step["diagnosis"]) @@ -3839,8 +3847,9 @@ def main(argv: list[str] | None = None) -> int: ), fallback_pytest_diagnosis, ) - if pytest_diagnosis is not None: - history_entry["diagnosis"] = pytest_diagnosis + run_diagnosis = checkout_stability_diagnosis or pytest_diagnosis + if run_diagnosis is not None: + history_entry["diagnosis"] = run_diagnosis if seed_receipt is not None: history_entry["testmon_seed"] = { "status": seed_receipt["status"], @@ -3905,10 +3914,11 @@ def main(argv: list[str] | None = None) -> int: verify_run.finish( exit_code=exit_code, duration_s=total_duration, - diagnosis=pytest_diagnosis, + diagnosis=run_diagnosis, verification_scope=verification_scope.value, release_baseline_allowed=release_baseline_allowed, terminal_authorization=args.terminal_authorization, + final_worktree_fingerprint=final_checkout_fingerprint, ) if exit_code == 0: _stamp_head() diff --git a/tests/conftest.py b/tests/conftest.py index e9d0923d96..ce28581624 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,14 +69,12 @@ "tests.infra.clock_guard", ) -# Pytest supports nested in-process ``pytest.main()`` calls. Keep the active -# controller identity process-local so an explicit nested basetemp can suspend -# and later restore the outer run's managed ownership markers. A completed -# earlier invocation is not active and therefore cannot authorize restoration -# of stale environment values. -_ACTIVE_MANAGED_PYTEST_IDENTITIES: list[tuple[str, str]] = [] -_MANAGED_IDENTITY_ATTR = "_polylogue_managed_pytest_identity" -_SUSPENDED_IDENTITY_ATTR = "_polylogue_suspended_managed_pytest_identity" +# Pytest supports nested in-process ``pytest.main()`` calls. Every controller +# invocation occupies this process-local stack, including caller-owned scopes. +# A completed invocation therefore cannot authorize stale environment values, +# and an unmanaged middle scope cannot resurrect a managed outer identity. +_ACTIVE_PYTEST_SCOPES: list[tuple[str, str] | None] = [] +_PYTEST_SCOPE_ATTR = "_polylogue_pytest_scope" if TYPE_CHECKING: from click.testing import CliRunner @@ -91,6 +89,23 @@ # --------------------------------------------------------------------------- +def _set_managed_pytest_identity(identity: tuple[str, str] | None) -> None: + """Expose only the managed identity owned by the active invocation.""" + if identity is None: + os.environ.pop("POLYLOGUE_PYTEST_RUN_ID", None) + os.environ.pop("POLYLOGUE_PYTEST_MANAGED_BASETEMP", None) + return + run_id, basetemp = identity + os.environ["POLYLOGUE_PYTEST_RUN_ID"] = run_id + os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = basetemp + + +def _push_pytest_scope(config: pytest.Config, identity: tuple[str, str] | None) -> None: + """Register one controller invocation, managed or caller-owned.""" + _ACTIVE_PYTEST_SCOPES.append(identity) + setattr(config, _PYTEST_SCOPE_ATTR, identity) + + def pytest_configure(config: pytest.Config) -> None: """Register custom markers and choose the managed test temp root.""" if _CHECKOUT_GUARD_ERROR is not None: @@ -126,26 +141,24 @@ def pytest_configure(config: pytest.Config) -> None: if supervised_managed and not hasattr(config, "workerinput"): assert run_id is not None identity = (run_id, configured_basetemp) - _ACTIVE_MANAGED_PYTEST_IDENTITIES.append(identity) - setattr(config, _MANAGED_IDENTITY_ATTR, identity) + _push_pytest_scope(config, identity) return # A second in-process pytest.main() inherits os.environ from the first # run. Explicit basetemp ownership is per invocation, so stale managed # markers must not turn the caller-owned diagnostic tree into cleanup # fodder at session finish. if not hasattr(config, "workerinput"): - if _ACTIVE_MANAGED_PYTEST_IDENTITIES: - setattr(config, _SUSPENDED_IDENTITY_ATTR, _ACTIVE_MANAGED_PYTEST_IDENTITIES[-1]) - os.environ.pop("POLYLOGUE_PYTEST_RUN_ID", None) - os.environ.pop("POLYLOGUE_PYTEST_MANAGED_BASETEMP", None) _mark_caller_owned_basetemp(Path(configured_basetemp)) + _push_pytest_scope(config, None) + _set_managed_pytest_identity(None) return if config.option.basetemp is None: - if not hasattr(config, "workerinput") and _ACTIVE_MANAGED_PYTEST_IDENTITIES: - setattr(config, _SUSPENDED_IDENTITY_ATTR, _ACTIVE_MANAGED_PYTEST_IDENTITIES[-1]) - os.environ.pop("POLYLOGUE_PYTEST_RUN_ID", None) - os.environ.pop("POLYLOGUE_PYTEST_MANAGED_BASETEMP", None) + prior_scope = ( + _ACTIVE_PYTEST_SCOPES[-1] if _ACTIVE_PYTEST_SCOPES and not hasattr(config, "workerinput") else None + ) + if _ACTIVE_PYTEST_SCOPES and not hasattr(config, "workerinput"): + _set_managed_pytest_identity(None) normalized_basetemp_env = normalize_pytest_basetemp_env(os.environ) configured_root = normalized_basetemp_env.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") unmanaged_tmpfs_root = configured_root is not None and verify_runs._is_beneath( @@ -173,6 +186,7 @@ def pytest_configure(config: pytest.Config) -> None: if not hasattr(config, "workerinput"): _mark_basetemp_owner(basetemp) except PytestResourceError as exc: + _set_managed_pytest_identity(prior_scope) # Fail loudly and early: refuse before pytest starts collecting, # rather than crashing an unrelated command later with a bare # OSError once the chosen basetemp fills up. @@ -181,31 +195,19 @@ def pytest_configure(config: pytest.Config) -> None: os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = str(basetemp) if not hasattr(config, "workerinput"): identity = (run_id, str(basetemp)) - _ACTIVE_MANAGED_PYTEST_IDENTITIES.append(identity) - setattr(config, _MANAGED_IDENTITY_ATTR, identity) + _push_pytest_scope(config, identity) sys.stderr.write(f"pytest: basetemp → {config.option.basetemp} ({label})\n") def pytest_unconfigure(config: pytest.Config) -> None: """Restore or retire process-local managed ownership after one invocation.""" - suspended = getattr(config, _SUSPENDED_IDENTITY_ATTR, None) - if isinstance(suspended, tuple) and len(suspended) == 2: - run_id, basetemp = suspended - os.environ["POLYLOGUE_PYTEST_RUN_ID"] = str(run_id) - os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = str(basetemp) - - managed = getattr(config, _MANAGED_IDENTITY_ATTR, None) - if not (isinstance(managed, tuple) and len(managed) == 2): + if not hasattr(config, _PYTEST_SCOPE_ATTR): return - for index in range(len(_ACTIVE_MANAGED_PYTEST_IDENTITIES) - 1, -1, -1): - if _ACTIVE_MANAGED_PYTEST_IDENTITIES[index] == managed: - del _ACTIVE_MANAGED_PYTEST_IDENTITIES[index] - break - run_id, basetemp = managed - if os.environ.get("POLYLOGUE_PYTEST_RUN_ID") == run_id: - os.environ.pop("POLYLOGUE_PYTEST_RUN_ID", None) - if os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") == basetemp: - os.environ.pop("POLYLOGUE_PYTEST_MANAGED_BASETEMP", None) + scope = getattr(config, _PYTEST_SCOPE_ATTR) + if not _ACTIVE_PYTEST_SCOPES or _ACTIVE_PYTEST_SCOPES[-1] != scope: + return + _ACTIVE_PYTEST_SCOPES.pop() + _set_managed_pytest_identity(_ACTIVE_PYTEST_SCOPES[-1] if _ACTIVE_PYTEST_SCOPES else None) # Per-run basetemps are freed on sessionfinish. A run killed before diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index 2bb81bc53b..1f6ebbc3a3 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -186,6 +186,63 @@ def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, assert command[command.index("--junit-xml") + 1] == str(invocation / "reports" / "results.xml") +def test_normalize_selection_paths_preserves_pytest_path_option_semantics( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + invocation = tmp_path / "invocation" + expanded_root = tmp_path / "expanded-root" + invocation.mkdir() + expanded_root.mkdir() + absolute_config = tmp_path / "absolute.ini" + monkeypatch.setenv("PYTEST_ROOT", str(expanded_root)) + + normalized = run_tests._normalize_selection_paths( + [ + "-cconfig/pytest.ini", + "-c", + "separate/pytest.ini", + "--config-file=other/pytest.ini", + "--config-file", + "separate-config/pytest.ini", + "--log-file", + "logs/test.log", + "--log-file=logs/joined.log", + "--debug", + "logs/debug-separated.log", + "--debug=logs/debug.log", + "--rootdir", + "$PYTEST_ROOT/relative", + "--rootdir=$PYTEST_ROOT/joined", + "--junitxml=reports/junit.xml", + "--junit-xml", + "reports/junit-alias.xml", + "--ignore-glob=fixtures/*.json", + "--basetemp", + str(absolute_config), + "--config-file", + "$PYTEST_ROOT/literal.ini", + ], + invocation_directory=invocation, + ) + + assert f"-c{invocation / 'config' / 'pytest.ini'}" in normalized + assert normalized[normalized.index("-c") + 1] == str(invocation / "separate" / "pytest.ini") + assert f"--config-file={invocation / 'other' / 'pytest.ini'}" in normalized + assert normalized[normalized.index("--config-file") + 1] == str(invocation / "separate-config" / "pytest.ini") + assert normalized[normalized.index("--log-file") + 1] == str(invocation / "logs" / "test.log") + assert f"--log-file={invocation / 'logs' / 'joined.log'}" in normalized + assert normalized[normalized.index("--debug") + 1] == str(invocation / "logs" / "debug-separated.log") + assert f"--debug={invocation / 'logs' / 'debug.log'}" in normalized + assert normalized[normalized.index("--rootdir") + 1] == str(expanded_root / "relative") + assert f"--rootdir={expanded_root / 'joined'}" in normalized + assert f"--junitxml={invocation / 'reports' / 'junit.xml'}" in normalized + assert normalized[normalized.index("--junit-xml") + 1] == str(invocation / "reports" / "junit-alias.xml") + assert f"--ignore-glob={invocation / 'fixtures' / '*.json'}" in normalized + assert normalized[normalized.index("--basetemp") + 1] == str(absolute_config) + assert normalized[-1] == str(invocation / "$PYTEST_ROOT" / "literal.ini") + + def test_main_withholds_success_when_checkout_changes_during_pytest( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index ad8b6d23f5..7ae7d1605a 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -4334,6 +4334,51 @@ def test_verify_withholds_success_when_checkout_fingerprint_is_unavailable( assert checkout_step["final_worktree_fingerprint"] == fingerprints[1] +@pytest.mark.parametrize( + ("fingerprints", "expected_diagnosis"), + [ + (("unavailable", "stable"), "checkout_fingerprint_unavailable"), + (("stable", "changed"), "checkout_changed_during_verification"), + ], +) +def test_checkout_stability_failure_controls_every_broad_run_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + fingerprints: tuple[str, str], + expected_diagnosis: str, +) -> None: + history: dict[str, Any] = {} + receipt = tmp_path / "invocation" / "run.json" + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.setattr( + verify, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "broad-invocation") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {"diagnosis": "pytest_passed"})), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.worktree_fingerprint", side_effect=fingerprints), + ): + assert main(["--quick", "--json"]) == 125 + + payload = json.loads(capsys.readouterr().out) + run_payload = json.loads(next((tmp_path / ".cache" / "verify" / "runs").glob("*/run.json")).read_text()) + current_payload = json.loads((tmp_path / ".cache" / "verify" / "current-run.json").read_text()) + receipt_payload = json.loads(receipt.read_text()) + + for durable_payload in (history, payload, run_payload, current_payload, receipt_payload): + assert durable_payload["diagnosis"] == expected_diagnosis + assert durable_payload["final_worktree_fingerprint"] == fingerprints[1] + + def test_verify_stops_after_failed_heavy_step(capsys: pytest.CaptureFixture[str]) -> None: calls: list[str] = [] diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index ed3dcffd65..76759f506a 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -350,7 +350,7 @@ def test_explicit_basetemp_clears_stale_managed_identity_from_prior_in_process_r addinivalue_line=lambda *args, **kwargs: None, rootpath=tmp_path, ) - monkeypatch.setattr(conftest, "_ACTIVE_MANAGED_PYTEST_IDENTITIES", []) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", "prior-run") monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(explicit)) @@ -408,6 +408,53 @@ def test_nested_explicit_basetemp_restores_active_outer_managed_identity( assert os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") == ambient_identity[1] +def test_nested_unmanaged_scopes_do_not_reclaim_the_live_outer_tree( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _shm, _scratch = _make_real_candidates(monkeypatch, tmp_path) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) + for name in ( + "POLYLOGUE_VERIFY_RUN_ID", + "POLYLOGUE_PYTEST_BASETEMP_ROOT", + "POLYLOGUE_PYTEST_TMPFS", + "POLYLOGUE_PYTEST_RUN_ID", + "POLYLOGUE_PYTEST_CHECKOUT", + "POLYLOGUE_PYTEST_MANAGED_BASETEMP", + ): + monkeypatch.delenv(name, raising=False) + outer = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + middle = SimpleNamespace( + option=SimpleNamespace(basetemp=str(tmp_path / "middle-diagnostic")), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + inner = SimpleNamespace( + option=SimpleNamespace(basetemp=str(tmp_path / "inner-diagnostic")), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(outer): + outer_basetemp = Path(str(outer.option.basetemp)) + live_outer_tree = outer_basetemp / "still-live" + live_outer_tree.mkdir(parents=True) + with _configured_pytest(middle): + with _configured_pytest(inner): + assert "POLYLOGUE_PYTEST_MANAGED_BASETEMP" not in os.environ + assert "POLYLOGUE_PYTEST_MANAGED_BASETEMP" not in os.environ + fixture_generator = cast(Any, conftest._reclaim_test_tmp_path).__wrapped__ + request = SimpleNamespace(config=SimpleNamespace(option=SimpleNamespace(basetemp=str(outer_basetemp)))) + cleanup = cast("Generator[None, BaseException, None]", fixture_generator(live_outer_tree, request)) + assert next(cleanup) is None + cleanup.close() + assert live_outer_tree.exists() + + def test_explicit_basetemp_claim_survives_real_pytest_basetemp_replacement(tmp_path: Path) -> None: """Exercise pytest's lazy TempPathFactory clearing against our real conftest.""" explicit = tmp_path / "pytest-polylogue-diagnostic" From aab4b5e278d570e8dd93a2dfa6f1b74deb31c8a1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 10:00:12 +0200 Subject: [PATCH 39/53] fix: harden nested verification harness evidence --- devtools/pytest_progress_plugin.py | 138 +++++++++--- devtools/run_tests.py | 41 +++- devtools/verify.py | 15 +- devtools/verify_runs.py | 196 ++++++++++++++++++ tests/conftest.py | 55 ++++- .../devtools/test_pytest_progress_plugin.py | 48 +++++ tests/unit/devtools/test_run_tests.py | 43 +++- tests/unit/devtools/test_verify.py | 98 +++++++++ tests/unit/test_pytest_temp_policy.py | 87 +++++++- 9 files changed, 678 insertions(+), 43 deletions(-) diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index 99dbec5c62..6abb073d46 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -12,6 +12,8 @@ import json import os import time +import uuid +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -34,6 +36,90 @@ _SLOW_REPORT_LIMIT = 20 _DEFAULT_SELECTION_NODEID_LIMIT = 500 _COLLECTION_FACT_SUFFIX = ".collection.json" +_ARTIFACT_ENV_NAMES = (_EVENTS_ENV, _EVENTS_DIR_ENV, _SELECTION_ENV, _SUMMARY_ENV) + + +@dataclass +class _SessionState: + deselected_nodeids_sample: list[str] + deselected_count: int + selected_count: int + slowest_reports: list[dict[str, Any]] + recorded_report_keys: set[tuple[int, str, str, str, float]] + collection_started_at: float | None + collection_duration_s: float | None + controller_collection_payload: dict[str, Any] | None + artifact_environment: dict[str, str | None] + + +_SESSION_STATE_STACK: list[_SessionState] = [] + + +def _capture_session_state() -> _SessionState: + return _SessionState( + deselected_nodeids_sample=list(_DESELECTED_NODEIDS_SAMPLE), + deselected_count=_DESELECTED_COUNT, + selected_count=_SELECTED_COUNT, + slowest_reports=list(_SLOWEST_REPORTS), + recorded_report_keys=set(_RECORDED_REPORT_KEYS), + collection_started_at=_COLLECTION_STARTED_AT, + collection_duration_s=_COLLECTION_DURATION_S, + controller_collection_payload=( + dict(_CONTROLLER_COLLECTION_PAYLOAD) if _CONTROLLER_COLLECTION_PAYLOAD else None + ), + artifact_environment={name: os.environ.get(name) for name in _ARTIFACT_ENV_NAMES}, + ) + + +def _restore_session_state(state: _SessionState) -> None: + global _COLLECTION_STARTED_AT, _COLLECTION_DURATION_S, _CONTROLLER_COLLECTION_PAYLOAD + global _DESELECTED_COUNT, _SELECTED_COUNT + _DESELECTED_NODEIDS_SAMPLE[:] = state.deselected_nodeids_sample + _DESELECTED_COUNT = state.deselected_count + _SELECTED_COUNT = state.selected_count + _SLOWEST_REPORTS[:] = state.slowest_reports + _RECORDED_REPORT_KEYS.clear() + _RECORDED_REPORT_KEYS.update(state.recorded_report_keys) + _COLLECTION_STARTED_AT = state.collection_started_at + _COLLECTION_DURATION_S = state.collection_duration_s + _CONTROLLER_COLLECTION_PAYLOAD = state.controller_collection_payload + for name, value in state.artifact_environment.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def _reset_session_state() -> None: + global _COLLECTION_STARTED_AT, _COLLECTION_DURATION_S, _CONTROLLER_COLLECTION_PAYLOAD + global _DESELECTED_COUNT, _SELECTED_COUNT + _DESELECTED_NODEIDS_SAMPLE.clear() + _DESELECTED_COUNT = 0 + _SELECTED_COUNT = 0 + _SLOWEST_REPORTS.clear() + _RECORDED_REPORT_KEYS.clear() + _COLLECTION_STARTED_AT = None + _COLLECTION_DURATION_S = None + _CONTROLLER_COLLECTION_PAYLOAD = None + + +def _isolate_nested_artifact_destinations() -> None: + """Give an in-process nested pytest invocation its own durable evidence.""" + raw_candidates = [os.environ.get(name) for name in _ARTIFACT_ENV_NAMES] + base = next((Path(value).parent for value in raw_candidates if value), None) + if base is None: + return + root = base / f"nested-pytest-{os.getpid()}-{uuid.uuid4().hex}" + if os.environ.get(_EVENTS_DIR_ENV): + os.environ[_EVENTS_DIR_ENV] = str(root / "events") + os.environ.pop(_EVENTS_ENV, None) + elif os.environ.get(_EVENTS_ENV): + os.environ[_EVENTS_ENV] = str(root / "events.jsonl") + os.environ.pop(_EVENTS_DIR_ENV, None) + if os.environ.get(_SELECTION_ENV): + os.environ[_SELECTION_ENV] = str(root / "selection.json") + if os.environ.get(_SUMMARY_ENV): + os.environ[_SUMMARY_ENV] = str(root / "summary.json") def _selection_nodeid_limit() -> int: @@ -191,16 +277,10 @@ def _durable_report_outcome(report: Any, outcome: str) -> str: def pytest_sessionstart(session: Any) -> None: """Reset per-session ledgers when tests invoke pytest in-process.""" del session - global _COLLECTION_STARTED_AT, _COLLECTION_DURATION_S, _CONTROLLER_COLLECTION_PAYLOAD - global _DESELECTED_COUNT, _SELECTED_COUNT - _DESELECTED_NODEIDS_SAMPLE.clear() - _DESELECTED_COUNT = 0 - _SELECTED_COUNT = 0 - _SLOWEST_REPORTS.clear() - _RECORDED_REPORT_KEYS.clear() - _COLLECTION_STARTED_AT = None - _COLLECTION_DURATION_S = None - _CONTROLLER_COLLECTION_PAYLOAD = None + _SESSION_STATE_STACK.append(_capture_session_state()) + _reset_session_state() + if len(_SESSION_STATE_STACK) > 1: + _isolate_nested_artifact_destinations() # The worker environment is assigned after process exec, so it is not # reliably visible through /proc//environ. Emit the identity from # inside the worker for the supervisor's process-state sampler. @@ -344,19 +424,25 @@ def pytest_runtest_logreport(report: Any) -> None: def pytest_sessionfinish(session: Any, exitstatus: int) -> None: """Write a compact post-run diagnosis artifact independent of pytest-json-report.""" del session - # Worker processes have their own in-memory slowest lists. The controller - # receives the forwarded timings and is the only writer for the shared - # summary path, so an empty worker summary cannot overwrite it. - if os.environ.get("PYTEST_XDIST_WORKER"): - return - collection_payload = merge_worker_collection_payloads() or _CONTROLLER_COLLECTION_PAYLOAD or _collection_payload() - _write_selection(collection_payload) - payload: dict[str, Any] = { - "exitstatus": int(exitstatus), - "selected_count": collection_payload["selected_count"], - "deselected_count": collection_payload["deselected_count"], - "slowest_reports": list(_SLOWEST_REPORTS), - } - if "collection_duration_s" in collection_payload: - payload["collection_duration_s"] = collection_payload["collection_duration_s"] - _write_summary(payload) + try: + # Worker processes have their own in-memory slowest lists. The controller + # receives the forwarded timings and is the only writer for the shared + # summary path, so an empty worker summary cannot overwrite it. + if os.environ.get("PYTEST_XDIST_WORKER"): + return + collection_payload = ( + merge_worker_collection_payloads() or _CONTROLLER_COLLECTION_PAYLOAD or _collection_payload() + ) + _write_selection(collection_payload) + payload: dict[str, Any] = { + "exitstatus": int(exitstatus), + "selected_count": collection_payload["selected_count"], + "deselected_count": collection_payload["deselected_count"], + "slowest_reports": list(_SLOWEST_REPORTS), + } + if "collection_duration_s" in collection_payload: + payload["collection_duration_s"] = collection_payload["collection_duration_s"] + _write_summary(payload) + finally: + if _SESSION_STATE_STACK: + _restore_session_state(_SESSION_STATE_STACK.pop()) diff --git a/devtools/run_tests.py b/devtools/run_tests.py index c378a05e6e..33b061b784 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -48,6 +48,7 @@ _run, ) from devtools.verify_runs import ( + CheckoutMutationMonitor, VerifyRun, append_verify_history, git_head, @@ -84,7 +85,10 @@ def _absolute_option_path( if expand_environment_variables: value = os.path.expandvars(value) path = Path(value) - return str(path if path.is_absolute() else (invocation_directory / path).resolve()) + # pytest deliberately uses ``os.path.abspath`` for command-line paths: + # resolving here would make ``-c config-link.ini`` select the linked + # target as its rootdir instead of preserving the caller's spelling. + return os.path.abspath(path if path.is_absolute() else invocation_directory / path) def _normalize_selection_paths(selection: list[str], *, invocation_directory: Path) -> list[str]: @@ -93,14 +97,28 @@ def _normalize_selection_paths(selection: list[str], *, invocation_directory: Pa pending_option: str | None = None for argument in selection: if pending_option is not None: + # pytest's --debug accepts an optional file name. A following + # option belongs to pytest, not to --debug's optional value. + if pending_option == "--debug" and argument.startswith("-"): + pending_option = None + else: + normalized.append( + _absolute_option_path( + argument, + invocation_directory=invocation_directory, + expand_environment_variables=pending_option in _ENV_EXPANDING_PATH_OPTIONS, + ) + ) + pending_option = None + continue + if argument.startswith("-c="): normalized.append( - _absolute_option_path( - argument, + "-c" + + _absolute_option_path( + argument[len("-c=") :], invocation_directory=invocation_directory, - expand_environment_variables=pending_option in _ENV_EXPANDING_PATH_OPTIONS, ) ) - pending_option = None continue option_name, equals, option_value = argument.partition("=") if option_name in _PATH_VALUE_OPTIONS: @@ -250,6 +268,8 @@ def main(argv: list[str] | None = None) -> int: with _run_lock(enabled=not no_lock): _clear_pytest_report(cmd) initial_worktree_fingerprint = worktree_fingerprint(ROOT) + mutation_monitor = CheckoutMutationMonitor(ROOT) + mutation_monitor.start() run = VerifyRun( tier="focused-test", argv=selection, @@ -267,13 +287,19 @@ def main(argv: list[str] | None = None) -> int: metadata = {"diagnosis": "pytest_interrupted", "termination_reason": "operator_interrupt"} run.finish_interrupted_steps(exit_code=rc, diagnosis=str(metadata["diagnosis"])) final_worktree_fingerprint = worktree_fingerprint(ROOT) - if "unavailable" in {initial_worktree_fingerprint, final_worktree_fingerprint}: + mutation_observation = mutation_monitor.finish() + if ( + "unavailable" in {initial_worktree_fingerprint, final_worktree_fingerprint} + or mutation_observation.unavailable + ): metadata["diagnosis"] = "checkout_fingerprint_unavailable" if rc == 0: rc = 125 sys.stderr.write("devtools test: checkout fingerprint unavailable; evidence is not exact-head.\n") - elif final_worktree_fingerprint != initial_worktree_fingerprint: + elif mutation_observation.changed or final_worktree_fingerprint != initial_worktree_fingerprint: metadata["diagnosis"] = "checkout_changed_during_focused_test" + metadata["transient_checkout_mutation"] = mutation_observation.changed + metadata["checkout_mutation_path"] = mutation_observation.observed_path if rc == 0: rc = 125 sys.stderr.write("devtools test: checkout contents changed during pytest; evidence is not exact-head.\n") @@ -284,6 +310,7 @@ def main(argv: list[str] | None = None) -> int: verification_scope="affected", release_baseline_allowed=False, final_worktree_fingerprint=final_worktree_fingerprint, + checkout_mutation_path=mutation_observation.observed_path, ) append_verify_history(payload) if use_json: diff --git a/devtools/verify.py b/devtools/verify.py index ef8506558f..36c545be1d 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -88,6 +88,7 @@ PYTEST_CANONICAL_REPORT_NAME, PYTEST_EXPLICIT_BASETEMP_ENV, VERIFY_HISTORY_PATH, + CheckoutMutationMonitor, PytestResourceError, PytestStepArtifacts, ResourceSampler, @@ -3556,7 +3557,7 @@ def main(argv: list[str] | None = None) -> int: head = _git_head() t0 = time.monotonic() - checkout_fingerprint = worktree_fingerprint() + checkout_fingerprint = worktree_fingerprint(ROOT) verify_run = VerifyRun( tier=tier, argv=list(sys.argv[1:] if argv is None else argv), @@ -3625,6 +3626,8 @@ def main(argv: list[str] | None = None) -> int: return 125 step_results: list[dict[str, Any]] = [] + mutation_monitor = CheckoutMutationMonitor(ROOT) + mutation_monitor.start() for label, cmd in steps: if label.startswith("pytest"): @@ -3777,8 +3780,9 @@ def main(argv: list[str] | None = None) -> int: f"inspect {TESTMON_SEED_ATTEMPT}.\n" ) - final_checkout_fingerprint = worktree_fingerprint() - if "unavailable" in {checkout_fingerprint, final_checkout_fingerprint}: + final_checkout_fingerprint = worktree_fingerprint(ROOT) + mutation_observation = mutation_monitor.finish() + if "unavailable" in {checkout_fingerprint, final_checkout_fingerprint} or mutation_observation.unavailable: step_results.append( { "name": "checkout stability", @@ -3792,7 +3796,7 @@ def main(argv: list[str] | None = None) -> int: if exit_code == 0: exit_code = 125 sys.stderr.write("verify: checkout fingerprint unavailable; evidence is not exact-head.\n") - elif final_checkout_fingerprint != checkout_fingerprint: + elif mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: step_results.append( { "name": "checkout stability", @@ -3801,6 +3805,8 @@ def main(argv: list[str] | None = None) -> int: "diagnosis": "checkout_changed_during_verification", "initial_worktree_fingerprint": checkout_fingerprint, "final_worktree_fingerprint": final_checkout_fingerprint, + "transient_checkout_mutation": mutation_observation.changed, + "checkout_mutation_path": mutation_observation.observed_path, } ) if exit_code == 0: @@ -3919,6 +3925,7 @@ def main(argv: list[str] | None = None) -> int: release_baseline_allowed=release_baseline_allowed, terminal_authorization=args.terminal_authorization, final_worktree_fingerprint=final_checkout_fingerprint, + checkout_mutation_path=mutation_observation.observed_path, ) if exit_code == 0: _stamp_head() diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 846d067604..54504e89b7 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -8,6 +8,7 @@ from __future__ import annotations import contextlib +import ctypes import fcntl import hashlib import json @@ -15,7 +16,9 @@ import re import shutil import stat +import struct import subprocess +import threading import time import uuid from collections.abc import Mapping, Sequence @@ -197,6 +200,196 @@ def worktree_fingerprint(root: Path | None = None) -> str: return digest.hexdigest() +@dataclass(frozen=True) +class CheckoutMutationObservation: + """Whether an exact-head verification interval observed a checkout write.""" + + changed: bool + unavailable: bool + observed_path: str | None = None + + +class CheckoutMutationMonitor: + """Fail closed when inotify cannot observe the checkout interval. + + Endpoint hashes establish the state of the checkout, while this monitor + records writes that occur and are later reverted before the final sample. + Watches exclude verifier-owned disposable directories so receipts do not + invalidate themselves. + """ + + _IGNORED_TOP_LEVEL = frozenset( + { + ".cache", + ".git", + ".hypothesis", + ".local", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".venv", + "__pycache__", + } + ) + _EVENT_MASK = ( + 0x00000002 + | 0x00000004 + | 0x00000008 + | 0x00000040 + | 0x00000080 + | 0x00000100 + | 0x00000200 + | 0x00000400 + | 0x00000800 + | 0x00002000 + ) + _INIT_FLAGS = 0x00000800 | 0x00080000 + + def __init__(self, root: Path) -> None: + self.root = root.resolve() + self._changed = False + self._observed_path: str | None = None + self._unavailable = False + self._descriptor: int | None = None + self._paths_by_watch: dict[int, Path] = {} + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._state_lock = threading.Lock() + + def start(self) -> None: + """Start the interval monitor, recording an unavailable monitor eagerly.""" + try: + descriptor = self._open_inotify() + self._descriptor = descriptor + for directory in self._watched_directories(): + watch = self._add_watch(descriptor, directory) + if watch < 0: + raise OSError(ctypes.get_errno(), "inotify_add_watch") + self._paths_by_watch[watch] = directory + except OSError: + self._unavailable = True + self._close_descriptor() + return + self._thread = threading.Thread(target=self._watch, name="checkout-mutation-monitor", daemon=True) + self._thread.start() + + def finish(self) -> CheckoutMutationObservation: + """Stop monitoring only after the caller took its final fingerprint.""" + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=1) + self._drain_events() + self._close_descriptor() + with self._state_lock: + return CheckoutMutationObservation( + changed=self._changed, + unavailable=self._unavailable, + observed_path=self._observed_path, + ) + + def _watched_directories(self) -> list[Path]: + directories: list[Path] = [] + for current, child_directories, _files in os.walk(self.root): + current_path = Path(current) + child_directories[:] = [child for child in child_directories if child not in self._IGNORED_TOP_LEVEL] + if current_path == self.root or not any( + part in self._IGNORED_TOP_LEVEL for part in current_path.relative_to(self.root).parts + ): + directories.append(current_path) + return directories + + @staticmethod + def _open_inotify() -> int: + libc = ctypes.CDLL("libc.so.6", use_errno=True) + init = libc.inotify_init1 + init.argtypes = [ctypes.c_int] + init.restype = ctypes.c_int + descriptor = int(init(CheckoutMutationMonitor._INIT_FLAGS)) + if descriptor < 0: + raise OSError(ctypes.get_errno(), "inotify_init1") + return descriptor + + @staticmethod + def _add_watch(descriptor: int, directory: Path) -> int: + libc = ctypes.CDLL("libc.so.6", use_errno=True) + add_watch = libc.inotify_add_watch + add_watch.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32] + add_watch.restype = ctypes.c_int + return int(add_watch(descriptor, os.fsencode(directory), CheckoutMutationMonitor._EVENT_MASK)) + + def _watch(self) -> None: + while not self._stop.wait(0.02): + self._drain_events() + + def _drain_events(self) -> None: + descriptor = self._descriptor + if descriptor is None: + return + while True: + try: + raw_events = os.read(descriptor, 64 * 1024) + except BlockingIOError: + return + except OSError: + if not self._stop.is_set(): + with self._state_lock: + self._unavailable = True + return + offset = 0 + while offset + 16 <= len(raw_events): + watch, mask, _cookie, name_length = struct.unpack_from("iIII", raw_events, offset) + offset += 16 + name = raw_events[offset : offset + name_length].rstrip(b"\0") + offset += name_length + if mask & 0x00004000: + with self._state_lock: + self._unavailable = True + return + directory = self._paths_by_watch.get(watch) + if directory is None: + continue + candidate = directory / os.fsdecode(name) if name else directory + try: + relative = candidate.relative_to(self.root) + except ValueError: + continue + if self._path_is_ignored(relative): + continue + with self._state_lock: + self._changed = True + self._observed_path = relative.as_posix() + return + + def _path_is_ignored(self, relative: Path) -> bool: + if any(part in self._IGNORED_TOP_LEVEL for part in relative.parts): + return True + try: + result = subprocess.run( + ["git", "check-ignore", "--quiet", "--no-index", "--", relative.as_posix()], + cwd=self.root, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=1, + ) + except (OSError, subprocess.TimeoutExpired): + with self._state_lock: + self._unavailable = True + return True + if result.returncode == 0: + return True + if result.returncode == 1: + return False + with self._state_lock: + self._unavailable = True + return True + + def _close_descriptor(self) -> None: + descriptor, self._descriptor = self._descriptor, None + if descriptor is not None: + with contextlib.suppress(OSError): + os.close(descriptor) + + @dataclass(frozen=True) class PytestRuntimePolicy: """One start-time resource decision for a managed pytest run.""" @@ -693,6 +886,7 @@ def finish( release_baseline_allowed: bool | None = None, terminal_authorization: str | None = None, final_worktree_fingerprint: str | None = None, + checkout_mutation_path: str | None = None, ) -> dict[str, Any]: self._payload["finished_at"] = utc_now() self._payload["duration_s"] = round(duration_s, 2) @@ -702,6 +896,8 @@ def finish( self._payload["diagnosis"] = diagnosis if final_worktree_fingerprint is not None: self._payload["final_worktree_fingerprint"] = final_worktree_fingerprint + if checkout_mutation_path is not None: + self._payload["checkout_mutation_path"] = checkout_mutation_path if verification_scope is not None: self._payload["verification_scope"] = verification_scope self._payload["release_baseline_allowed"] = release_baseline_allowed diff --git a/tests/conftest.py b/tests/conftest.py index ce28581624..21bfd9b0cc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -74,7 +74,11 @@ # A completed invocation therefore cannot authorize stale environment values, # and an unmanaged middle scope cannot resurrect a managed outer identity. _ACTIVE_PYTEST_SCOPES: list[tuple[str, str] | None] = [] +_ACTIVE_PYTEST_BASETEMPS: set[Path] = set() _PYTEST_SCOPE_ATTR = "_polylogue_pytest_scope" +_PYTEST_SCOPE_BASETEMP_ATTR = "_polylogue_pytest_scope_basetemp" +_PYTEST_SCOPE_ENV_ATTR = "_polylogue_pytest_scope_environment" +_NESTED_BASETEMP_POLICY_ENV = ("POLYLOGUE_PYTEST_BASETEMP_ROOT", "POLYLOGUE_PYTEST_TMPFS") if TYPE_CHECKING: from click.testing import CliRunner @@ -100,10 +104,40 @@ def _set_managed_pytest_identity(identity: tuple[str, str] | None) -> None: os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = basetemp -def _push_pytest_scope(config: pytest.Config, identity: tuple[str, str] | None) -> None: +def _push_pytest_scope( + config: pytest.Config, identity: tuple[str, str] | None, *, basetemp: Path | None = None +) -> None: """Register one controller invocation, managed or caller-owned.""" _ACTIVE_PYTEST_SCOPES.append(identity) setattr(config, _PYTEST_SCOPE_ATTR, identity) + if basetemp is not None: + claim_path = _basetemp_claim_path(basetemp, kind="lock") + _ACTIVE_PYTEST_BASETEMPS.add(claim_path) + setattr(config, _PYTEST_SCOPE_BASETEMP_ATTR, claim_path) + + +def _force_nested_pytest_scratch(config: pytest.Config) -> None: + """Keep an unsupervised nested controller out of an outer tmpfs budget.""" + previous = {name: os.environ.get(name) for name in _NESTED_BASETEMP_POLICY_ENV} + setattr(config, _PYTEST_SCOPE_ENV_ATTR, previous) + forced = verify_runs.force_managed_pytest_scratch(os.environ) + for name in _NESTED_BASETEMP_POLICY_ENV: + value = forced.get(name) + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def _restore_nested_pytest_scratch(config: pytest.Config) -> None: + previous = getattr(config, _PYTEST_SCOPE_ENV_ATTR, None) + if previous is None: + return + for name, value in previous.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value def pytest_configure(config: pytest.Config) -> None: @@ -140,8 +174,12 @@ def pytest_configure(config: pytest.Config) -> None: ) if supervised_managed and not hasattr(config, "workerinput"): assert run_id is not None + if _basetemp_claim_path(Path(configured_basetemp), kind="lock") in _ACTIVE_PYTEST_BASETEMPS: + raise pytest.UsageError( + f"pytest: explicit basetemp is already active in this pytest process: {configured_basetemp}" + ) identity = (run_id, configured_basetemp) - _push_pytest_scope(config, identity) + _push_pytest_scope(config, identity, basetemp=Path(configured_basetemp)) return # A second in-process pytest.main() inherits os.environ from the first # run. Explicit basetemp ownership is per invocation, so stale managed @@ -149,7 +187,7 @@ def pytest_configure(config: pytest.Config) -> None: # fodder at session finish. if not hasattr(config, "workerinput"): _mark_caller_owned_basetemp(Path(configured_basetemp)) - _push_pytest_scope(config, None) + _push_pytest_scope(config, None, basetemp=Path(configured_basetemp)) _set_managed_pytest_identity(None) return @@ -159,6 +197,7 @@ def pytest_configure(config: pytest.Config) -> None: ) if _ACTIVE_PYTEST_SCOPES and not hasattr(config, "workerinput"): _set_managed_pytest_identity(None) + _force_nested_pytest_scratch(config) normalized_basetemp_env = normalize_pytest_basetemp_env(os.environ) configured_root = normalized_basetemp_env.get("POLYLOGUE_PYTEST_BASETEMP_ROOT") unmanaged_tmpfs_root = configured_root is not None and verify_runs._is_beneath( @@ -186,6 +225,7 @@ def pytest_configure(config: pytest.Config) -> None: if not hasattr(config, "workerinput"): _mark_basetemp_owner(basetemp) except PytestResourceError as exc: + _restore_nested_pytest_scratch(config) _set_managed_pytest_identity(prior_scope) # Fail loudly and early: refuse before pytest starts collecting, # rather than crashing an unrelated command later with a bare @@ -195,7 +235,7 @@ def pytest_configure(config: pytest.Config) -> None: os.environ["POLYLOGUE_PYTEST_MANAGED_BASETEMP"] = str(basetemp) if not hasattr(config, "workerinput"): identity = (run_id, str(basetemp)) - _push_pytest_scope(config, identity) + _push_pytest_scope(config, identity, basetemp=basetemp) sys.stderr.write(f"pytest: basetemp → {config.option.basetemp} ({label})\n") @@ -207,6 +247,10 @@ def pytest_unconfigure(config: pytest.Config) -> None: if not _ACTIVE_PYTEST_SCOPES or _ACTIVE_PYTEST_SCOPES[-1] != scope: return _ACTIVE_PYTEST_SCOPES.pop() + active_basetemp = getattr(config, _PYTEST_SCOPE_BASETEMP_ATTR, None) + if active_basetemp is not None: + _ACTIVE_PYTEST_BASETEMPS.discard(active_basetemp) + _restore_nested_pytest_scratch(config) _set_managed_pytest_identity(_ACTIVE_PYTEST_SCOPES[-1] if _ACTIVE_PYTEST_SCOPES else None) @@ -255,6 +299,9 @@ def _mark_basetemp_owner(basetemp: Path) -> None: def _mark_caller_owned_basetemp(basetemp: Path) -> None: """Claim an explicit ``--basetemp`` before pytest may replace its tree.""" + lock_path = _basetemp_claim_path(basetemp, kind="lock") + if lock_path in _ACTIVE_PYTEST_BASETEMPS: + raise pytest.UsageError(f"pytest: explicit basetemp is already active in this pytest process: {basetemp}") handle = _acquire_basetemp_claim_lock(basetemp, blocking=True) if handle is None: raise pytest.UsageError(f"pytest: cannot claim the explicit basetemp: {basetemp}") diff --git a/tests/unit/devtools/test_pytest_progress_plugin.py b/tests/unit/devtools/test_pytest_progress_plugin.py index 6aabf59160..76c07aeaa7 100644 --- a/tests/unit/devtools/test_pytest_progress_plugin.py +++ b/tests/unit/devtools/test_pytest_progress_plugin.py @@ -34,7 +34,9 @@ def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: collection_duration_s = pytest_progress_plugin._COLLECTION_DURATION_S controller_collection_payload = pytest_progress_plugin._CONTROLLER_COLLECTION_PAYLOAD recorded_report_keys = set(pytest_progress_plugin._RECORDED_REPORT_KEYS) + session_state_stack = list(pytest_progress_plugin._SESSION_STATE_STACK) pytest_progress_plugin._RECORDED_REPORT_KEYS.clear() + pytest_progress_plugin._SESSION_STATE_STACK.clear() yield pytest_progress_plugin._SELECTED_COUNT = selected_count pytest_progress_plugin._DESELECTED_COUNT = deselected_count @@ -45,6 +47,7 @@ def _restore_plugin_state(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: pytest_progress_plugin._CONTROLLER_COLLECTION_PAYLOAD = controller_collection_payload pytest_progress_plugin._RECORDED_REPORT_KEYS.clear() pytest_progress_plugin._RECORDED_REPORT_KEYS.update(recorded_report_keys) + pytest_progress_plugin._SESSION_STATE_STACK[:] = session_state_stack @dataclass(frozen=True) @@ -421,6 +424,51 @@ def test_progress_plugin_retains_controller_selection_through_session_finish( assert selection["selected_nodeids_omitted"] == 0 +def test_nested_pytest_session_keeps_outer_progress_and_artifacts_isolated( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + outer_events = tmp_path / "outer-events.jsonl" + outer_selection = tmp_path / "outer-selection.json" + outer_summary = tmp_path / "outer-summary.json" + monkeypatch.setenv("POLYLOGUE_PYTEST_EVENTS_PATH", str(outer_events)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SELECTION_PATH", str(outer_selection)) + monkeypatch.setenv("POLYLOGUE_PYTEST_SUMMARY_PATH", str(outer_summary)) + + pytest_progress_plugin.pytest_sessionstart(object()) + pytest_progress_plugin.pytest_collection_modifyitems( + _Session(["tests/outer.py::test_outer"]), object(), [_Item("tests/outer.py::test_outer")] + ) + pytest_progress_plugin.pytest_runtest_logreport(_Report("tests/outer.py::test_outer", "call", "passed")) + + pytest_progress_plugin.pytest_sessionstart(object()) + nested_selection = Path(os.environ["POLYLOGUE_PYTEST_SELECTION_PATH"]) + nested_summary = Path(os.environ["POLYLOGUE_PYTEST_SUMMARY_PATH"]) + nested_events = Path(os.environ["POLYLOGUE_PYTEST_EVENTS_PATH"]) + assert nested_selection != outer_selection + assert nested_summary != outer_summary + assert nested_events != outer_events + pytest_progress_plugin.pytest_collection_modifyitems( + _Session(["tests/inner.py::test_inner"]), object(), [_Item("tests/inner.py::test_inner")] + ) + pytest_progress_plugin.pytest_runtest_logreport(_Report("tests/inner.py::test_inner", "call", "passed")) + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + assert os.environ["POLYLOGUE_PYTEST_SELECTION_PATH"] == str(outer_selection) + assert json.loads(outer_selection.read_text())["selected_nodeids"] == ["tests/outer.py::test_outer"] + assert json.loads(nested_selection.read_text())["selected_nodeids"] == ["tests/inner.py::test_inner"] + assert [json.loads(line)["nodeid"] for line in nested_events.read_text().splitlines() if "nodeid" in line] == [ + "tests/inner.py::test_inner" + ] + + pytest_progress_plugin.pytest_sessionfinish(object(), 0) + + outer_payload = json.loads(outer_summary.read_text()) + nested_payload = json.loads(nested_summary.read_text()) + assert [report["nodeid"] for report in outer_payload["slowest_reports"]] == ["tests/outer.py::test_outer"] + assert [report["nodeid"] for report in nested_payload["slowest_reports"]] == ["tests/inner.py::test_inner"] + + def test_progress_plugin_merges_xdist_collection_facts_without_double_counting( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index 1f6ebbc3a3..16a94ed13b 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -11,7 +11,23 @@ import pytest from devtools import run_tests, verify -from devtools.verify_runs import CURRENT_STATISTICS_PATH, git_head, pytest_command_worker_request +from devtools.verify_runs import ( + CURRENT_STATISTICS_PATH, + CheckoutMutationObservation, + git_head, + pytest_command_worker_request, +) + + +class _NoMutationMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) def test_build_pytest_cmd_defaults_to_single_process() -> None: @@ -243,6 +259,28 @@ def test_normalize_selection_paths_preserves_pytest_path_option_semantics( assert normalized[-1] == str(invocation / "$PYTEST_ROOT" / "literal.ini") +def test_normalize_selection_paths_preserves_pytest_symlinks_and_optional_debug( + tmp_path: Path, +) -> None: + invocation = tmp_path / "invocation" + invocation.mkdir() + target = invocation / "target.ini" + target.write_text("[pytest]\n", encoding="utf-8") + config_link = invocation / "config-link.ini" + config_link.symlink_to(target.name) + + normalized = run_tests._normalize_selection_paths( + ["-c", "config-link.ini", "-c=config-link.ini", "--debug", "-k", "focused"], + invocation_directory=invocation, + ) + + lexical_link = str(invocation / "config-link.ini") + assert normalized[:2] == ["-c", lexical_link] + assert normalized[2] == f"-c{lexical_link}" + assert normalized[3:] == ["--debug", "-k", "focused"] + assert str(target) not in normalized + + def test_main_withholds_success_when_checkout_changes_during_pytest( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -253,6 +291,7 @@ def test_main_withholds_success_when_checkout_changes_during_pytest( monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) monkeypatch.setattr(run_tests, "_run", lambda *_args, **_kwargs: (0, 0.01, {"diagnosis": "pytest_passed"})) monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprints)) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: captured.update(payload)) assert run_tests.main(["tests/unit/example.py"]) == 125 @@ -274,6 +313,7 @@ def test_main_withholds_success_when_checkout_fingerprint_is_unavailable( monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) monkeypatch.setattr(run_tests, "_run", lambda *_args, **_kwargs: (0, 0.01, {"diagnosis": "pytest_passed"})) monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprint_values)) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: captured.update(payload)) assert run_tests.main(["tests/unit/example.py"]) == 125 @@ -324,6 +364,7 @@ def fake_run(_label: str, _cmd: list[str], **kwargs: Any) -> tuple[int, float, d ) monkeypatch.setattr(run_tests, "git_head", lambda _root: "head") monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "fingerprint") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) monkeypatch.setattr(run_tests, "_run", fake_run) monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 7ae7d1605a..dca0f49b3c 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -78,6 +78,8 @@ main, ) from devtools.verify_runs import ( + CheckoutMutationMonitor, + CheckoutMutationObservation, PytestResourceError, PytestStepArtifacts, ResourceSampler, @@ -1796,6 +1798,56 @@ def test_worktree_fingerprint_hashes_untracked_file_contents(tmp_path: Path) -> assert before != after +def test_checkout_mutation_monitor_detects_a_change_that_reverts_before_the_final_fingerprint( + tmp_path: Path, +) -> None: + subprocess.run(["git", "init", "-q"], check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], check=True) + tracked = tmp_path / "tracked.py" + original = "VALUE = 1\n" + tracked.write_text(original, encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], check=True) + subprocess.run(["git", "commit", "-qm", "seed"], check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + tracked.write_text("VALUE = 2\n", encoding="utf-8") + tracked.write_text(original, encoding="utf-8") + observation = monitor.finish() + + assert observation.changed is True + assert observation.unavailable is False + assert observation.observed_path == "tracked.py" + + +def test_checkout_mutation_monitor_ignores_nested_disposable_cache_writes(tmp_path: Path) -> None: + package = tmp_path / "package" + package.mkdir() + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + cache_file = package / "__pycache__" / "module.pyc" + cache_file.parent.mkdir(parents=True) + cache_file.write_bytes(b"cache") + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + + +def test_checkout_mutation_monitor_uses_gitignore_for_verifier_task_history(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], check=True) + (tmp_path / ".gitignore").write_text(".agent/*\n", encoding="utf-8") + history = tmp_path / ".agent" / "task-history" / "tasks.jsonl" + history.parent.mkdir(parents=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + history.write_text('{"task": "verification"}\n', encoding="utf-8") + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + + def test_seed_receipt_classifies_every_node_terminal_outcome( tmp_path: Path, ) -> None: @@ -4379,6 +4431,52 @@ def test_checkout_stability_failure_controls_every_broad_run_receipt( assert durable_payload["final_worktree_fingerprint"] == fingerprints[1] +def test_transient_checkout_mutation_controls_every_broad_run_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class _ChangedMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=True, unavailable=False) + + history: dict[str, Any] = {} + receipt = tmp_path / "invocation" / "run.json" + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.setattr( + verify, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setattr(verify, "CheckoutMutationMonitor", _ChangedMonitor) + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "broad-invocation") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {"diagnosis": "pytest_passed"})), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + ): + assert main(["--quick", "--json"]) == 125 + + payload = json.loads(capsys.readouterr().out) + run_payload = json.loads(next((tmp_path / ".cache" / "verify" / "runs").glob("*/run.json")).read_text()) + current_payload = json.loads((tmp_path / ".cache" / "verify" / "current-run.json").read_text()) + receipt_payload = json.loads(receipt.read_text()) + for durable_payload in (history, payload, run_payload, current_payload, receipt_payload): + assert durable_payload["diagnosis"] == "checkout_changed_during_verification" + assert durable_payload["final_worktree_fingerprint"] == "stable" + + def test_verify_stops_after_failed_heavy_step(capsys: pytest.CaptureFixture[str]) -> None: calls: list[str] = [] diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index 76759f506a..d38e74ca43 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -370,7 +370,6 @@ def test_nested_explicit_basetemp_restores_active_outer_managed_identity( os.environ.get("POLYLOGUE_PYTEST_RUN_ID"), os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP"), ) - assert all(ambient_identity) _shm, _scratch = _make_real_candidates(monkeypatch, tmp_path) for name in ( "POLYLOGUE_VERIFY_RUN_ID", @@ -408,6 +407,92 @@ def test_nested_explicit_basetemp_restores_active_outer_managed_identity( assert os.environ.get("POLYLOGUE_PYTEST_MANAGED_BASETEMP") == ambient_identity[1] +@pytest.mark.parametrize("alias", [False, True]) +def test_nested_explicit_basetemp_reuse_is_rejected_before_the_nonreentrant_claim_lock( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + alias: bool, +) -> None: + active = tmp_path / "active-basetemp" + requested = active + if alias: + active.mkdir() + requested = tmp_path / "active-basetemp-alias" + requested.symlink_to(active, target_is_directory=True) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_BASETEMPS", set()) + outer = SimpleNamespace( + option=SimpleNamespace(basetemp=str(active)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + nested = SimpleNamespace( + option=SimpleNamespace(basetemp=str(requested)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(outer): + with pytest.raises(pytest.UsageError, match="already active in this pytest process"): + conftest.pytest_configure(cast("pytest.Config", nested)) + + +def test_nested_supervised_explicit_basetemp_reuse_is_rejected_before_claiming( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + active = tmp_path / "active-basetemp" + active.mkdir() + run_id = "supervised-run" + config = SimpleNamespace( + option=SimpleNamespace(basetemp=str(active)), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_BASETEMPS", set()) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_BASETEMPS", set()) + monkeypatch.setenv("POLYLOGUE_VERIFY_RUN_ID", run_id) + monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", run_id) + monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(active)) + conftest._mark_basetemp_owner(active) + try: + with _configured_pytest(config): + with pytest.raises(pytest.UsageError, match="already active in this pytest process"): + conftest.pytest_configure(cast("pytest.Config", config)) + finally: + conftest._release_basetemp_claim_lock(active) + + +def test_nested_managed_pytest_forces_scratch_outside_the_outer_tmpfs_budget( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + shm, scratch = _make_real_candidates(monkeypatch, tmp_path) + monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) + monkeypatch.setenv("POLYLOGUE_VERIFY_RUN_ID", "outer-supervisor") + monkeypatch.setenv("POLYLOGUE_PYTEST_BASETEMP_ROOT", str(shm)) + monkeypatch.setenv("POLYLOGUE_PYTEST_TMPFS", "1") + outer = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + nested = SimpleNamespace( + option=SimpleNamespace(basetemp=None), + addinivalue_line=lambda *args, **kwargs: None, + rootpath=tmp_path, + ) + + with _configured_pytest(outer): + assert Path(str(outer.option.basetemp)).parent == shm + with _configured_pytest(nested): + assert Path(str(nested.option.basetemp)).parent == scratch + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "0" + assert os.environ["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(shm) + assert os.environ["POLYLOGUE_PYTEST_TMPFS"] == "1" + + def test_nested_unmanaged_scopes_do_not_reclaim_the_live_outer_tree( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From a6deeb0bb341293facfdfe76644e366bb20fe76f Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 10:31:13 +0200 Subject: [PATCH 40/53] fix: use portable checkout mutation watcher --- devtools/verify_runs.py | 165 ++++++++++------------------- tests/unit/devtools/test_verify.py | 68 +++++++++++- 2 files changed, 123 insertions(+), 110 deletions(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 54504e89b7..283e5aca00 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -8,7 +8,6 @@ from __future__ import annotations import contextlib -import ctypes import fcntl import hashlib import json @@ -16,7 +15,6 @@ import re import shutil import stat -import struct import subprocess import threading import time @@ -27,6 +25,8 @@ from pathlib import Path from typing import Any, TextIO +import watchfiles + from polylogue.core.metrics import read_cgroup_memory_headroom_bytes VERIFY_CACHE = Path(".cache/verify") @@ -210,7 +210,7 @@ class CheckoutMutationObservation: class CheckoutMutationMonitor: - """Fail closed when inotify cannot observe the checkout interval. + """Fail closed when watchfiles cannot observe the checkout interval. Endpoint hashes establish the state of the checkout, while this monitor records writes that occur and are later reverted before the final sample. @@ -231,55 +231,42 @@ class CheckoutMutationMonitor: "__pycache__", } ) - _EVENT_MASK = ( - 0x00000002 - | 0x00000004 - | 0x00000008 - | 0x00000040 - | 0x00000080 - | 0x00000100 - | 0x00000200 - | 0x00000400 - | 0x00000800 - | 0x00002000 - ) - _INIT_FLAGS = 0x00000800 | 0x00080000 + _WATCH_START_TIMEOUT_S = 1.0 + _WATCH_SETTLE_S = 0.2 + _WATCH_RUST_TIMEOUT_MS = 25 def __init__(self, root: Path) -> None: self.root = root.resolve() self._changed = False self._observed_path: str | None = None self._unavailable = False - self._descriptor: int | None = None - self._paths_by_watch: dict[int, Path] = {} self._stop = threading.Event() + self._ready = threading.Event() self._thread: threading.Thread | None = None self._state_lock = threading.Lock() def start(self) -> None: - """Start the interval monitor, recording an unavailable monitor eagerly.""" - try: - descriptor = self._open_inotify() - self._descriptor = descriptor - for directory in self._watched_directories(): - watch = self._add_watch(descriptor, directory) - if watch < 0: - raise OSError(ctypes.get_errno(), "inotify_add_watch") - self._paths_by_watch[watch] = directory - except OSError: - self._unavailable = True - self._close_descriptor() - return + """Start and prove the portable interval watcher before verification.""" self._thread = threading.Thread(target=self._watch, name="checkout-mutation-monitor", daemon=True) self._thread.start() + if not self._ready.wait(timeout=self._WATCH_START_TIMEOUT_S): + with self._state_lock: + self._unavailable = True + self._stop.set() + self._thread.join(timeout=self._WATCH_START_TIMEOUT_S) def finish(self) -> CheckoutMutationObservation: """Stop monitoring only after the caller took its final fingerprint.""" + # The final fingerprint is already sampled. Give watchfiles one short + # backend turn to surface any event emitted before that sample, then + # stop the generator cleanly through its portable stop event. + self._stop.wait(self._WATCH_SETTLE_S) self._stop.set() if self._thread is not None: self._thread.join(timeout=1) - self._drain_events() - self._close_descriptor() + if self._thread.is_alive(): + with self._state_lock: + self._unavailable = True with self._state_lock: return CheckoutMutationObservation( changed=self._changed, @@ -287,78 +274,46 @@ def finish(self) -> CheckoutMutationObservation: observed_path=self._observed_path, ) - def _watched_directories(self) -> list[Path]: - directories: list[Path] = [] - for current, child_directories, _files in os.walk(self.root): - current_path = Path(current) - child_directories[:] = [child for child in child_directories if child not in self._IGNORED_TOP_LEVEL] - if current_path == self.root or not any( - part in self._IGNORED_TOP_LEVEL for part in current_path.relative_to(self.root).parts - ): - directories.append(current_path) - return directories - - @staticmethod - def _open_inotify() -> int: - libc = ctypes.CDLL("libc.so.6", use_errno=True) - init = libc.inotify_init1 - init.argtypes = [ctypes.c_int] - init.restype = ctypes.c_int - descriptor = int(init(CheckoutMutationMonitor._INIT_FLAGS)) - if descriptor < 0: - raise OSError(ctypes.get_errno(), "inotify_init1") - return descriptor - - @staticmethod - def _add_watch(descriptor: int, directory: Path) -> int: - libc = ctypes.CDLL("libc.so.6", use_errno=True) - add_watch = libc.inotify_add_watch - add_watch.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32] - add_watch.restype = ctypes.c_int - return int(add_watch(descriptor, os.fsencode(directory), CheckoutMutationMonitor._EVENT_MASK)) - def _watch(self) -> None: - while not self._stop.wait(0.02): - self._drain_events() + try: + for changes in watchfiles.watch( + self.root, + watch_filter=None, + debounce=0, + step=1, + stop_event=self._stop, + rust_timeout=self._WATCH_RUST_TIMEOUT_MS, + yield_on_timeout=True, + raise_interrupt=False, + ): + # An empty timeout batch proves the backend initialized before + # a verification command starts, closing the startup race. + self._ready.set() + for _change, raw_path in changes: + self._record_change(Path(raw_path)) + if self._changed or self._unavailable: + return + if not self._stop.is_set(): + with self._state_lock: + self._unavailable = True + except Exception: + with self._state_lock: + self._unavailable = True + finally: + self._ready.set() - def _drain_events(self) -> None: - descriptor = self._descriptor - if descriptor is None: + def _record_change(self, candidate: Path) -> None: + if not candidate.is_absolute(): + candidate = self.root / candidate + try: + relative = candidate.relative_to(self.root) + except ValueError: return - while True: - try: - raw_events = os.read(descriptor, 64 * 1024) - except BlockingIOError: - return - except OSError: - if not self._stop.is_set(): - with self._state_lock: - self._unavailable = True - return - offset = 0 - while offset + 16 <= len(raw_events): - watch, mask, _cookie, name_length = struct.unpack_from("iIII", raw_events, offset) - offset += 16 - name = raw_events[offset : offset + name_length].rstrip(b"\0") - offset += name_length - if mask & 0x00004000: - with self._state_lock: - self._unavailable = True - return - directory = self._paths_by_watch.get(watch) - if directory is None: - continue - candidate = directory / os.fsdecode(name) if name else directory - try: - relative = candidate.relative_to(self.root) - except ValueError: - continue - if self._path_is_ignored(relative): - continue - with self._state_lock: - self._changed = True - self._observed_path = relative.as_posix() - return + if self._path_is_ignored(relative): + return + with self._state_lock: + self._changed = True + self._observed_path = relative.as_posix() def _path_is_ignored(self, relative: Path) -> bool: if any(part in self._IGNORED_TOP_LEVEL for part in relative.parts): @@ -383,12 +338,6 @@ def _path_is_ignored(self, relative: Path) -> bool: self._unavailable = True return True - def _close_descriptor(self) -> None: - descriptor, self._descriptor = self._descriptor, None - if descriptor is not None: - with contextlib.suppress(OSError): - os.close(descriptor) - @dataclass(frozen=True) class PytestRuntimePolicy: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index dca0f49b3c..0b44266a75 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -8,6 +8,7 @@ import sqlite3 import subprocess import sys +import threading import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -16,6 +17,7 @@ from unittest.mock import MagicMock, patch import pytest +import watchfiles from devtools import run_tests, verify, verify_runs from devtools.testmon_state import ( @@ -1834,20 +1836,82 @@ def test_checkout_mutation_monitor_ignores_nested_disposable_cache_writes(tmp_pa assert observation == CheckoutMutationObservation(changed=False, unavailable=False) -def test_checkout_mutation_monitor_uses_gitignore_for_verifier_task_history(tmp_path: Path) -> None: +def test_checkout_mutation_monitor_uses_gitignore_for_verifier_task_history( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: subprocess.run(["git", "init", "-q"], check=True) (tmp_path / ".gitignore").write_text(".agent/*\n", encoding="utf-8") history = tmp_path / ".agent" / "task-history" / "tasks.jsonl" history.parent.mkdir(parents=True) + def portable_watch(*_paths: Path, **kwargs: object) -> object: + yield set() + yield {(watchfiles.Change.modified, str(history))} + stop_event = kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait() + + monkeypatch.setattr(watchfiles, "watch", portable_watch) monitor = CheckoutMutationMonitor(tmp_path) monitor.start() - history.write_text('{"task": "verification"}\n', encoding="utf-8") observation = monitor.finish() assert observation == CheckoutMutationObservation(changed=False, unavailable=False) +def test_checkout_mutation_monitor_uses_portable_watchfiles_events_without_linux_kernel( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("VALUE = 1\n", encoding="utf-8") + calls: dict[str, object] = {} + event_emitted = threading.Event() + + def portable_watch(*paths: Path, **kwargs: object) -> object: + calls["paths"] = paths + calls["kwargs"] = kwargs + yield set() + event_emitted.set() + yield {(watchfiles.Change.modified, str(tracked))} + + monkeypatch.setattr(watchfiles, "watch", portable_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + assert event_emitted.wait(timeout=1) + observation = monitor.finish() + + assert calls["paths"] == (tmp_path.resolve(),) + assert calls["kwargs"] == { + "watch_filter": None, + "debounce": 0, + "step": 1, + "stop_event": monitor._stop, + "rust_timeout": monitor._WATCH_RUST_TIMEOUT_MS, + "yield_on_timeout": True, + "raise_interrupt": False, + } + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="tracked.py") + + +def test_checkout_mutation_monitor_fails_closed_when_portable_watcher_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def broken_watch(*_paths: Path, **_kwargs: object) -> object: + raise OSError("watcher unavailable") + yield set() + + monkeypatch.setattr(watchfiles, "watch", broken_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + def test_seed_receipt_classifies_every_node_terminal_outcome( tmp_path: Path, ) -> None: From 524ffe03a36be8be7bd1334ea16c844089455cdd Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 10:53:27 +0200 Subject: [PATCH 41/53] fix: reject lossy checkout mutation watchers --- devtools/verify_runs.py | 29 ++++++++++- tests/unit/devtools/test_verify.py | 78 ++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 283e5aca00..90bbf21d19 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -12,6 +12,7 @@ import hashlib import json import os +import platform import re import shutil import stat @@ -234,6 +235,7 @@ class CheckoutMutationMonitor: _WATCH_START_TIMEOUT_S = 1.0 _WATCH_SETTLE_S = 0.2 _WATCH_RUST_TIMEOUT_MS = 25 + _POLLING_DISABLED_VALUES = frozenset({"false", "disable", "disabled"}) def __init__(self, root: Path) -> None: self.root = root.resolve() @@ -247,6 +249,11 @@ def __init__(self, root: Path) -> None: def start(self) -> None: """Start and prove the portable interval watcher before verification.""" + if self._polling_backend_requested(): + with self._state_lock: + self._unavailable = True + self._ready.set() + return self._thread = threading.Thread(target=self._watch, name="checkout-mutation-monitor", daemon=True) self._thread.start() if not self._ready.wait(timeout=self._WATCH_START_TIMEOUT_S): @@ -277,7 +284,7 @@ def finish(self) -> CheckoutMutationObservation: def _watch(self) -> None: try: for changes in watchfiles.watch( - self.root, + *self._watched_directories(), watch_filter=None, debounce=0, step=1, @@ -285,6 +292,8 @@ def _watch(self) -> None: rust_timeout=self._WATCH_RUST_TIMEOUT_MS, yield_on_timeout=True, raise_interrupt=False, + force_polling=False, + recursive=False, ): # An empty timeout batch proves the backend initialized before # a verification command starts, closing the startup race. @@ -302,6 +311,24 @@ def _watch(self) -> None: finally: self._ready.set() + @classmethod + def _polling_backend_requested(cls) -> bool: + """Reject watchfiles modes that cannot witness every interval mutation.""" + forced = os.getenv("WATCHFILES_FORCE_POLLING") + if forced: + return forced.lower() not in cls._POLLING_DISABLED_VALUES + uname = platform.uname() + return uname.system.lower() == "linux" and "microsoft-standard" in uname.release.lower() + + def _watched_directories(self) -> list[Path]: + """Watch existing source directories shallowly and omit disposable trees.""" + directories: list[Path] = [] + for current, child_directories, _files in os.walk(self.root): + current_path = Path(current) + child_directories[:] = [child for child in child_directories if child not in self._IGNORED_TOP_LEVEL] + directories.append(current_path) + return directories + def _record_change(self, candidate: Path) -> None: if not candidate.is_absolute(): candidate = self.root / candidate diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 0b44266a75..43edaad6a8 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -4,6 +4,7 @@ import hashlib import json import os +import platform import shutil import sqlite3 import subprocess @@ -1892,10 +1893,87 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: "rust_timeout": monitor._WATCH_RUST_TIMEOUT_MS, "yield_on_timeout": True, "raise_interrupt": False, + "force_polling": False, + "recursive": False, } assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="tracked.py") +def test_checkout_mutation_monitor_rejects_forced_polling_backend( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("WATCHFILES_FORCE_POLLING", "1") + + def unexpected_watch(*_paths: Path, **_kwargs: object) -> object: + raise AssertionError("polling mode must fail before watchfiles starts") + yield set() + + monkeypatch.setattr(watchfiles, "watch", unexpected_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + +def test_checkout_mutation_monitor_rejects_wsl_auto_polling_backend( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("WATCHFILES_FORCE_POLLING", raising=False) + monkeypatch.setattr( + platform, + "uname", + lambda: SimpleNamespace(system="Linux", release="6.6.0-microsoft-standard-WSL2"), + ) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + +def test_checkout_mutation_monitor_prunes_disposable_trees_and_observes_new_source_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], check=True) + source = tmp_path / "src" / "package" + source.mkdir(parents=True) + for disposable in (".venv", ".git", ".cache"): + (tmp_path / disposable / "nested").mkdir(parents=True, exist_ok=True) + calls: dict[str, object] = {} + allow_event = threading.Event() + new_source = tmp_path / "new_source" + + def portable_watch(*paths: Path, **kwargs: object) -> object: + calls["paths"] = paths + calls["kwargs"] = kwargs + yield set() + assert allow_event.wait(timeout=1) + yield {(watchfiles.Change.added, str(new_source))} + + monkeypatch.setattr(watchfiles, "watch", portable_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + new_source.mkdir() + allow_event.set() + observation = monitor.finish() + + raw_paths = calls["paths"] + raw_kwargs = calls["kwargs"] + assert isinstance(raw_paths, tuple) + assert isinstance(raw_kwargs, dict) + watched = {Path(path) for path in raw_paths} + assert tmp_path.resolve() in watched + assert source in watched + assert all(not any(part in {".venv", ".git", ".cache"} for part in path.parts) for path in watched) + assert raw_kwargs["recursive"] is False + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="new_source") + + def test_checkout_mutation_monitor_fails_closed_when_portable_watcher_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 3a8dbe1be9797b328f2c77bef20b181ad99ed048 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 11:12:42 +0200 Subject: [PATCH 42/53] fix(devtools): prune ignored watcher trees --- devtools/verify_runs.py | 48 +++++++++++++++++++++++++++++- tests/unit/devtools/test_verify.py | 13 ++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 90bbf21d19..6de85191e2 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -322,13 +322,59 @@ def _polling_backend_requested(cls) -> bool: def _watched_directories(self) -> list[Path]: """Watch existing source directories shallowly and omit disposable trees.""" + ignored_roots = self._ignored_directory_roots() directories: list[Path] = [] for current, child_directories, _files in os.walk(self.root): current_path = Path(current) - child_directories[:] = [child for child in child_directories if child not in self._IGNORED_TOP_LEVEL] + relative_current = current_path.relative_to(self.root) + child_directories[:] = [ + child + for child in child_directories + if child not in self._IGNORED_TOP_LEVEL + and not self._is_within_ignored_root(relative_current / child, ignored_roots) + ] directories.append(current_path) return directories + def _ignored_directory_roots(self) -> frozenset[Path]: + """Return existing ignored directory roots without traversing their contents.""" + try: + result = subprocess.run( + [ + "git", + "status", + "--porcelain=v1", + "-z", + "--ignored=matching", + "--untracked-files=normal", + ], + cwd=self.root, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=2, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + with self._state_lock: + self._unavailable = True + return frozenset() + if result.returncode != 0: + with self._state_lock: + self._unavailable = True + return frozenset() + ignored: set[Path] = set() + for record in result.stdout.split(b"\0"): + if not record.startswith(b"!! "): + continue + relative = Path(os.fsdecode(record[3:]).rstrip("/")) + if relative.parts and (self.root / relative).is_dir(): + ignored.add(relative) + return frozenset(ignored) + + @staticmethod + def _is_within_ignored_root(relative: Path, ignored_roots: frozenset[Path]) -> bool: + return any(relative == root or relative.is_relative_to(root) for root in ignored_roots) + def _record_change(self, candidate: Path) -> None: if not candidate.is_absolute(): candidate = self.root / candidate diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 43edaad6a8..edd7c4adfa 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1825,6 +1825,7 @@ def test_checkout_mutation_monitor_detects_a_change_that_reverts_before_the_fina def test_checkout_mutation_monitor_ignores_nested_disposable_cache_writes(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) package = tmp_path / "package" package.mkdir() monitor = CheckoutMutationMonitor(tmp_path) @@ -1940,10 +1941,18 @@ def test_checkout_mutation_monitor_prunes_disposable_trees_and_observes_new_sour monkeypatch: pytest.MonkeyPatch, ) -> None: subprocess.run(["git", "init", "-q"], check=True) + (tmp_path / ".gitignore").write_text( + "browser-extension/node_modules/\ncustom/generated-output/\n", + encoding="utf-8", + ) source = tmp_path / "src" / "package" source.mkdir(parents=True) for disposable in (".venv", ".git", ".cache"): (tmp_path / disposable / "nested").mkdir(parents=True, exist_ok=True) + ignored_dependency = tmp_path / "browser-extension" / "node_modules" / "dependency" + ignored_dependency.mkdir(parents=True) + ignored_build = tmp_path / "custom" / "generated-output" / "deep" / "tree" + ignored_build.mkdir(parents=True) calls: dict[str, object] = {} allow_event = threading.Event() new_source = tmp_path / "new_source" @@ -1970,6 +1979,10 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: assert tmp_path.resolve() in watched assert source in watched assert all(not any(part in {".venv", ".git", ".cache"} for part in path.parts) for path in watched) + assert all("node_modules" not in path.parts for path in watched) + assert all("generated-output" not in path.parts for path in watched) + assert tmp_path / "browser-extension" in watched + assert tmp_path / "custom" in watched assert raw_kwargs["recursive"] is False assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="new_source") From 95e0f55251fcc237aa1e0fcc3afeb273f43e7a59 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 11:45:18 +0200 Subject: [PATCH 43/53] fix(devtools): bind mutation watch to Git authority --- devtools/verify.py | 3 + devtools/verify_runs.py | 111 +++++++++++++++++++---- tests/unit/devtools/test_verify.py | 138 ++++++++++++++++++++++++++++- 3 files changed, 233 insertions(+), 19 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 36c545be1d..5a9c633ee6 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2020,6 +2020,9 @@ def _pytest_command_basetemp( def _subprocess_env() -> dict[str, str]: env = normalize_pytest_basetemp_env(os.environ) + # Tests and verification helpers may inspect Git, but observational reads + # must not refresh the index and invalidate the exact-head mutation watch. + env["GIT_OPTIONAL_LOCKS"] = "0" env["POLYLOGUE_ROOT"] = str(ROOT) env["POLYLOGUE_REPO_ROOT"] = str(ROOT) inherited_pythonpath = env.get("PYTHONPATH", "") diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 6de85191e2..673726a82c 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -151,6 +151,11 @@ def pytest_command_worker_request(cmd: Sequence[str]) -> str | None: return request +def _read_only_git_env() -> dict[str, str]: + """Prevent observational Git commands from refreshing checkout authority.""" + return {**os.environ, "GIT_OPTIONAL_LOCKS": "0"} + + def worktree_fingerprint(root: Path | None = None) -> str: """Fingerprint tracked changes plus exact non-ignored untracked content.""" checkout_root = (root or Path.cwd()).resolve() @@ -160,7 +165,13 @@ def worktree_fingerprint(root: Path | None = None) -> str: ["git", "diff", "--binary", "HEAD", "--"], ): try: - result = subprocess.run(command, capture_output=True, timeout=30, cwd=checkout_root) + result = subprocess.run( + command, + capture_output=True, + timeout=30, + cwd=checkout_root, + env=_read_only_git_env(), + ) except (OSError, subprocess.TimeoutExpired): return "unavailable" if result.returncode != 0: @@ -173,6 +184,7 @@ def worktree_fingerprint(root: Path | None = None) -> str: capture_output=True, timeout=30, cwd=checkout_root, + env=_read_only_git_env(), ) except (OSError, subprocess.TimeoutExpired): return "unavailable" @@ -246,6 +258,9 @@ def __init__(self, root: Path) -> None: self._ready = threading.Event() self._thread: threading.Thread | None = None self._state_lock = threading.Lock() + self._tracked_paths: frozenset[Path] = frozenset() + self._git_index_path: Path | None = None + self._git_index_fingerprint: str | None = None def start(self) -> None: """Start and prove the portable interval watcher before verification.""" @@ -283,8 +298,11 @@ def finish(self) -> CheckoutMutationObservation: def _watch(self) -> None: try: + watched_directories = self._watched_directories() + if self._unavailable: + return for changes in watchfiles.watch( - *self._watched_directories(), + *watched_directories, watch_filter=None, debounce=0, step=1, @@ -322,9 +340,15 @@ def _polling_backend_requested(cls) -> bool: def _watched_directories(self) -> list[Path]: """Watch existing source directories shallowly and omit disposable trees.""" + self._tracked_paths = self._git_tracked_paths() ignored_roots = self._ignored_directory_roots() directories: list[Path] = [] - for current, child_directories, _files in os.walk(self.root): + + def walk_error(_error: OSError) -> None: + with self._state_lock: + self._unavailable = True + + for current, child_directories, _files in os.walk(self.root, onerror=walk_error): current_path = Path(current) relative_current = current_path.relative_to(self.root) child_directories[:] = [ @@ -334,33 +358,56 @@ def _watched_directories(self) -> list[Path]: and not self._is_within_ignored_root(relative_current / child, ignored_roots) ] directories.append(current_path) + self._git_index_path = self._resolve_git_index_path() + if self._git_index_path is not None: + self._git_index_fingerprint = self._fingerprint_git_index() + if self._git_index_path.parent not in directories: + directories.append(self._git_index_path.parent) return directories - def _ignored_directory_roots(self) -> frozenset[Path]: - """Return existing ignored directory roots without traversing their contents.""" + def _git_tracked_paths(self) -> frozenset[Path]: + """Snapshot index membership so tracked paths never inherit ignore rules.""" + result = self._git_command(["ls-files", "-z"]) + if result is None: + return frozenset() + return frozenset(Path(os.fsdecode(raw)) for raw in result.stdout.split(b"\0") if raw) + + def _resolve_git_index_path(self) -> Path | None: + """Resolve the worktree-specific index whose writes can change path authority.""" + result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", "index"]) + if result is None: + return None + raw_path = os.fsdecode(result.stdout).strip() + if not raw_path: + with self._state_lock: + self._unavailable = True + return None + return Path(raw_path) + + def _git_command(self, args: list[str]) -> subprocess.CompletedProcess[bytes] | None: try: result = subprocess.run( - [ - "git", - "status", - "--porcelain=v1", - "-z", - "--ignored=matching", - "--untracked-files=normal", - ], + ["git", *args], cwd=self.root, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, + capture_output=True, timeout=2, check=False, + env=_read_only_git_env(), ) except (OSError, subprocess.TimeoutExpired): with self._state_lock: self._unavailable = True - return frozenset() - if result.returncode != 0: + return None + if result.returncode != 0 or result.stderr.strip(): with self._state_lock: self._unavailable = True + return None + return result + + def _ignored_directory_roots(self) -> frozenset[Path]: + """Return existing ignored directory roots without traversing their contents.""" + result = self._git_command(["status", "--porcelain=v1", "-z", "--ignored=matching", "--untracked-files=normal"]) + if result is None: return frozenset() ignored: set[Path] = set() for record in result.stdout.split(b"\0"): @@ -371,6 +418,19 @@ def _ignored_directory_roots(self) -> frozenset[Path]: ignored.add(relative) return frozenset(ignored) + def _fingerprint_git_index(self) -> str | None: + """Hash index authority so a stale backend event is not a mutation.""" + if self._git_index_path is None: + return None + try: + return hashlib.sha256(self._git_index_path.read_bytes()).hexdigest() + except FileNotFoundError: + return None + except OSError: + with self._state_lock: + self._unavailable = True + return None + @staticmethod def _is_within_ignored_root(relative: Path, ignored_roots: frozenset[Path]) -> bool: return any(relative == root or relative.is_relative_to(root) for root in ignored_roots) @@ -378,6 +438,20 @@ def _is_within_ignored_root(relative: Path, ignored_roots: frozenset[Path]) -> b def _record_change(self, candidate: Path) -> None: if not candidate.is_absolute(): candidate = self.root / candidate + if self._git_index_path is not None and candidate.parent == self._git_index_path.parent: + if candidate.name == f"{self._git_index_path.name}.lock": + # Read-only commands such as ``git describe --dirty`` create + # and discard an optional index lock. Authority changes are + # witnessed when a completed lock replaces the index itself. + return + if candidate.name == self._git_index_path.name: + current_fingerprint = self._fingerprint_git_index() + if self._unavailable or current_fingerprint == self._git_index_fingerprint: + return + with self._state_lock: + self._changed = True + self._observed_path = ".git/index" + return try: relative = candidate.relative_to(self.root) except ValueError: @@ -389,6 +463,8 @@ def _record_change(self, candidate: Path) -> None: self._observed_path = relative.as_posix() def _path_is_ignored(self, relative: Path) -> bool: + if relative in self._tracked_paths: + return False if any(part in self._IGNORED_TOP_LEVEL for part in relative.parts): return True try: @@ -398,6 +474,7 @@ def _path_is_ignored(self, relative: Path) -> bool: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=1, + env=_read_only_git_env(), ) except (OSError, subprocess.TimeoutExpired): with self._state_lock: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index edd7c4adfa..4687fb39ff 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1862,6 +1862,27 @@ def portable_watch(*_paths: Path, **kwargs: object) -> object: assert observation == CheckoutMutationObservation(changed=False, unavailable=False) +def test_checkout_mutation_monitor_observes_tracked_file_that_matches_gitignore(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / ".gitignore").write_text(".agent/*\n", encoding="utf-8") + tracked = tmp_path / ".agent" / "script.py" + tracked.parent.mkdir() + tracked.write_text("before\n", encoding="utf-8") + subprocess.run(["git", "add", "-f", ".agent/script.py"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + tracked.write_text("during\n", encoding="utf-8") + tracked.write_text("before\n", encoding="utf-8") + observation = monitor.finish() + + assert observation == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path=".agent/script.py", + ) + + def test_checkout_mutation_monitor_uses_portable_watchfiles_events_without_linux_kernel( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1885,7 +1906,7 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: assert event_emitted.wait(timeout=1) observation = monitor.finish() - assert calls["paths"] == (tmp_path.resolve(),) + assert calls["paths"] == (tmp_path.resolve(), (tmp_path / ".git").resolve()) assert calls["kwargs"] == { "watch_filter": None, "debounce": 0, @@ -1978,7 +1999,10 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: watched = {Path(path) for path in raw_paths} assert tmp_path.resolve() in watched assert source in watched - assert all(not any(part in {".venv", ".git", ".cache"} for part in path.parts) for path in watched) + assert all( + path == (tmp_path / ".git").resolve() or not any(part in {".venv", ".git", ".cache"} for part in path.parts) + for path in watched + ) assert all("node_modules" not in path.parts for path in watched) assert all("generated-output" not in path.parts for path in watched) assert tmp_path / "browser-extension" in watched @@ -1987,6 +2011,116 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="new_source") +@pytest.mark.uses_real_clock("waits for the real filesystem watcher to witness an index replacement") +def test_checkout_mutation_monitor_observes_transient_index_authority_change(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / ".gitignore").write_text("ignored/\n", encoding="utf-8") + baseline = tmp_path / "baseline.py" + baseline.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", ".gitignore", "baseline.py"], cwd=tmp_path, check=True) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + hidden = tmp_path / "ignored" / "hidden.py" + hidden.parent.mkdir() + hidden.write_text("secret authority\n", encoding="utf-8") + subprocess.run(["git", "add", "-f", "ignored/hidden.py"], cwd=tmp_path, check=True) + deadline = time.monotonic() + 1 + while not monitor._changed and time.monotonic() < deadline: + time.sleep(0.005) + subprocess.run(["git", "reset", "-q", "--", "ignored/hidden.py"], cwd=tmp_path, check=True) + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") + + +def test_checkout_mutation_monitor_ignores_read_only_git_index_lock(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + subprocess.run( + ["git", "describe", "--dirty", "--always", "--long", "--abbrev=40"], + cwd=tmp_path, + check=True, + capture_output=True, + ) + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + + +def test_checkout_mutation_monitor_ignores_stale_index_event( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / "tracked.py").write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + index = tmp_path / ".git" / "index" + + def portable_watch(*_paths: Path, **_kwargs: object) -> object: + yield set() + yield {(watchfiles.Change.modified, str(index))} + stop_event = _kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait(timeout=1) + + monkeypatch.setattr(watchfiles, "watch", portable_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + + +def test_checkout_mutation_monitor_rejects_partial_git_enumeration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + real_run = subprocess.run + + def warning_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[bytes]: + result = real_run(*args, **kwargs) + command = args[0] + if isinstance(command, list) and command[:2] == ["git", "status"]: + return subprocess.CompletedProcess(command, 0, result.stdout, b"warning: partial enumeration\n") + return result + + monkeypatch.setattr(subprocess, "run", warning_run) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + +def test_checkout_mutation_monitor_rejects_filesystem_walk_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + real_walk = os.walk + + def failing_walk(top: Path, *, onerror: object = None) -> object: + assert callable(onerror) + onerror(PermissionError("unreadable source directory")) + yield from real_walk(top) + + monkeypatch.setattr(os, "walk", failing_walk) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=False, unavailable=True) + + def test_checkout_mutation_monitor_fails_closed_when_portable_watcher_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From f59134071c962fc757238d51e2d29785f4ba850c Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 11:50:52 +0200 Subject: [PATCH 44/53] test(devtools): type partial enumeration probe --- tests/unit/devtools/test_verify.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 4687fb39ff..2b637a7459 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -14,7 +14,7 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace -from typing import Any +from typing import Any, cast from unittest.mock import MagicMock, patch import pytest @@ -2086,8 +2086,8 @@ def test_checkout_mutation_monitor_rejects_partial_git_enumeration( subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) real_run = subprocess.run - def warning_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[bytes]: - result = real_run(*args, **kwargs) + def warning_run(*args: Any, **kwargs: Any) -> subprocess.CompletedProcess[bytes]: + result = cast(subprocess.CompletedProcess[bytes], real_run(*args, **kwargs)) command = args[0] if isinstance(command, list) and command[:2] == ["git", "status"]: return subprocess.CompletedProcess(command, 0, result.stdout, b"warning: partial enumeration\n") From f57c739d5a87048d7d36bb85cff2c0772222d34a Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 12:32:44 +0200 Subject: [PATCH 45/53] fix(devtools): discard unstable verification authority --- devtools/evidence_dashboard.py | 8 +- devtools/verify.py | 87 ++++++++++---- devtools/verify_runs.py | 40 ++----- .../unit/devtools/test_evidence_dashboard.py | 36 ++++++ tests/unit/devtools/test_verify.py | 109 +++++++++++++++--- 5 files changed, 214 insertions(+), 66 deletions(-) diff --git a/devtools/evidence_dashboard.py b/devtools/evidence_dashboard.py index 348524b728..2f167bb49b 100644 --- a/devtools/evidence_dashboard.py +++ b/devtools/evidence_dashboard.py @@ -233,8 +233,14 @@ def _static_evidence_is_bound( worktree_fingerprint: str, ) -> bool: """Accept only evidence tied to the exact checkout contents being viewed.""" + steps = entry.get("steps") + stability_failed = isinstance(steps, list) and any( + isinstance(step, dict) and step.get("name") == "checkout stability" and step.get("exit") != 0 for step in steps + ) return ( - entry.get("checkout_root") == checkout_root + not stability_failed + and entry.get("diagnosis") not in {"checkout_changed_during_verification", "checkout_fingerprint_unavailable"} + and entry.get("checkout_root") == checkout_root and entry.get("git_head") == checkout_head and entry.get("worktree_fingerprint") == worktree_fingerprint and entry.get("final_worktree_fingerprint") == worktree_fingerprint diff --git a/devtools/verify.py b/devtools/verify.py index 5a9c633ee6..68ff392499 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -3470,6 +3470,19 @@ def _refresh_testmon_selection_attempt( _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) +def _discard_testmon_dependency_authority() -> None: + """Remove a dependency graph learned while checkout authority was unstable.""" + for path in ( + TESTMON_SEED_STAMP, + TESTMON_SEED_ATTEMPT, + TESTMON_DATA, + Path(f"{TESTMON_DATA}-wal"), + Path(f"{TESTMON_DATA}-shm"), + Path(f"{TESTMON_DATA}-journal"), + ): + path.unlink(missing_ok=True) + + # ── main ──────────────────────────────────────────────────────────── @@ -3629,6 +3642,10 @@ def main(argv: list[str] | None = None) -> int: return 125 step_results: list[dict[str, Any]] = [] + pending_testmon_stamp: TestmonSeedStamp | None = None + pending_affected_coverage: tuple[tuple[str, ...], int] | None = None + pending_selection_refresh: tuple[dict[str, Any], int] | None = None + testmon_graph_touched = False mutation_monitor = CheckoutMutationMonitor(ROOT) mutation_monitor.start() @@ -3636,6 +3653,8 @@ def main(argv: list[str] | None = None) -> int: if label.startswith("pytest"): _warn_low_memory() # check again right before the heavy step rc, elapsed, metadata = _run(label, cmd, run=verify_run) + if label in {"pytest testmon", "pytest testmon (broad)"} or label.startswith("pytest seed-testmon"): + testmon_graph_touched = True if rc == 0 and label in {"pytest testmon", "pytest testmon (broad)"}: raw_stamp = _read_json_artifact(TESTMON_SEED_STAMP) try: @@ -3649,7 +3668,7 @@ def main(argv: list[str] | None = None) -> int: if current_stamp is not None: refreshed_stamp = refresh_stamp(current_stamp, TESTMON_DATA) if refreshed_stamp is not None: - _atomic_write_json(TESTMON_SEED_STAMP, refreshed_stamp.as_dict()) + pending_testmon_stamp = refreshed_stamp executable_paths = _changed_executable_paths() selected_count = metadata.get("selected_count") if selected_count == 0 and executable_paths: @@ -3666,11 +3685,7 @@ def main(argv: list[str] | None = None) -> int: else: metadata["zero_selection_coverage"] = coverage elif isinstance(selected_count, int) and selected_count > 0: - _record_testmon_affected_coverage( - executable_paths=executable_paths, - selected_count=selected_count, - run_id=verify_run.run_id, - ) + pending_affected_coverage = (tuple(executable_paths), selected_count) step_result: dict[str, Any] = {"name": label, "duration_s": round(elapsed, 2), "exit": rc} step_result.update(metadata) step_results.append(step_result) @@ -3763,29 +3778,17 @@ def main(argv: list[str] | None = None) -> int: break continue if label in {"pytest testmon", "pytest testmon (broad)"} and not args.seed_testmon and not full_pytest: - _refresh_testmon_selection_attempt(step=step_result, run=verify_run, exit_code=rc) + pending_selection_refresh = (step_result, rc) if rc != 0: exit_code = rc if rc == 130 or _stop_after_failed_step(label): break - seed_receipt: dict[str, Any] | None = None - if prepared_seed_attempt is not None: - seed_receipt = _finalize_testmon_seed_attempt( - prepared=prepared_seed_attempt, - step_results=step_results, - exit_code=exit_code, - ) - if exit_code == 0 and seed_receipt["status"] != "complete": - exit_code = 5 - sys.stderr.write( - "verify: pytest passed but the testmon dependency baseline is incomplete; " - f"inspect {TESTMON_SEED_ATTEMPT}.\n" - ) - final_checkout_fingerprint = worktree_fingerprint(ROOT) mutation_observation = mutation_monitor.finish() + checkout_stable = True if "unavailable" in {checkout_fingerprint, final_checkout_fingerprint} or mutation_observation.unavailable: + checkout_stable = False step_results.append( { "name": "checkout stability", @@ -3800,6 +3803,7 @@ def main(argv: list[str] | None = None) -> int: exit_code = 125 sys.stderr.write("verify: checkout fingerprint unavailable; evidence is not exact-head.\n") elif mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: + checkout_stable = False step_results.append( { "name": "checkout stability", @@ -3816,6 +3820,47 @@ def main(argv: list[str] | None = None) -> int: exit_code = 125 sys.stderr.write("verify: checkout contents changed during verification; evidence is not exact-head.\n") + seed_receipt: dict[str, Any] | None = None + if checkout_stable: + if pending_testmon_stamp is not None: + _atomic_write_json(TESTMON_SEED_STAMP, pending_testmon_stamp.as_dict()) + if pending_affected_coverage is not None: + executable_paths, selected_count = pending_affected_coverage + _record_testmon_affected_coverage( + executable_paths=executable_paths, + selected_count=selected_count, + run_id=verify_run.run_id, + ) + if pending_selection_refresh is not None: + step_result, selection_exit_code = pending_selection_refresh + _refresh_testmon_selection_attempt( + step=step_result, + run=verify_run, + exit_code=selection_exit_code, + ) + if prepared_seed_attempt is not None: + seed_receipt = _finalize_testmon_seed_attempt( + prepared=prepared_seed_attempt, + step_results=step_results, + exit_code=exit_code, + ) + if exit_code == 0 and seed_receipt["status"] != "complete": + exit_code = 5 + sys.stderr.write( + "verify: pytest passed but the testmon dependency baseline is incomplete; " + f"inspect {TESTMON_SEED_ATTEMPT}.\n" + ) + elif testmon_graph_touched: + _discard_testmon_dependency_authority() + if prepared_seed_attempt is not None: + seed_receipt = { + "status": "discarded", + "outcome": SeedAttemptOutcome.INCOMPLETE.value, + "resume": False, + "expected_count": len(_testmon_seed_expected_nodeids(prepared_seed_attempt)), + "release_baseline_allowed": False, + } + total_duration = round(time.monotonic() - t0, 2) # Build history entry. diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 673726a82c..74d077b281 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -174,7 +174,7 @@ def worktree_fingerprint(root: Path | None = None) -> str: ) except (OSError, subprocess.TimeoutExpired): return "unavailable" - if result.returncode != 0: + if result.returncode != 0 or result.stderr.strip(): return "unavailable" digest.update(result.stdout) digest.update(b"\0") @@ -188,7 +188,7 @@ def worktree_fingerprint(root: Path | None = None) -> str: ) except (OSError, subprocess.TimeoutExpired): return "unavailable" - if untracked.returncode != 0: + if untracked.returncode != 0 or untracked.stderr.strip(): return "unavailable" for raw_path in sorted(path for path in untracked.stdout.split(b"\0") if path): try: @@ -259,8 +259,8 @@ def __init__(self, root: Path) -> None: self._thread: threading.Thread | None = None self._state_lock = threading.Lock() self._tracked_paths: frozenset[Path] = frozenset() + self._ignored_roots: frozenset[Path] = frozenset() self._git_index_path: Path | None = None - self._git_index_fingerprint: str | None = None def start(self) -> None: """Start and prove the portable interval watcher before verification.""" @@ -341,7 +341,7 @@ def _polling_backend_requested(cls) -> bool: def _watched_directories(self) -> list[Path]: """Watch existing source directories shallowly and omit disposable trees.""" self._tracked_paths = self._git_tracked_paths() - ignored_roots = self._ignored_directory_roots() + self._ignored_roots = self._ignored_directory_roots() directories: list[Path] = [] def walk_error(_error: OSError) -> None: @@ -355,14 +355,12 @@ def walk_error(_error: OSError) -> None: child for child in child_directories if child not in self._IGNORED_TOP_LEVEL - and not self._is_within_ignored_root(relative_current / child, ignored_roots) + and not self._is_within_ignored_root(relative_current / child, self._ignored_roots) ] directories.append(current_path) self._git_index_path = self._resolve_git_index_path() - if self._git_index_path is not None: - self._git_index_fingerprint = self._fingerprint_git_index() - if self._git_index_path.parent not in directories: - directories.append(self._git_index_path.parent) + if self._git_index_path is not None and self._git_index_path.parent not in directories: + directories.append(self._git_index_path.parent) return directories def _git_tracked_paths(self) -> frozenset[Path]: @@ -418,19 +416,6 @@ def _ignored_directory_roots(self) -> frozenset[Path]: ignored.add(relative) return frozenset(ignored) - def _fingerprint_git_index(self) -> str | None: - """Hash index authority so a stale backend event is not a mutation.""" - if self._git_index_path is None: - return None - try: - return hashlib.sha256(self._git_index_path.read_bytes()).hexdigest() - except FileNotFoundError: - return None - except OSError: - with self._state_lock: - self._unavailable = True - return None - @staticmethod def _is_within_ignored_root(relative: Path, ignored_roots: frozenset[Path]) -> bool: return any(relative == root or relative.is_relative_to(root) for root in ignored_roots) @@ -440,14 +425,11 @@ def _record_change(self, candidate: Path) -> None: candidate = self.root / candidate if self._git_index_path is not None and candidate.parent == self._git_index_path.parent: if candidate.name == f"{self._git_index_path.name}.lock": - # Read-only commands such as ``git describe --dirty`` create - # and discard an optional index lock. Authority changes are - # witnessed when a completed lock replaces the index itself. + # An uncommitted lock is not yet checkout authority. A + # completed transaction is witnessed when the lock replaces + # the index itself. return if candidate.name == self._git_index_path.name: - current_fingerprint = self._fingerprint_git_index() - if self._unavailable or current_fingerprint == self._git_index_fingerprint: - return with self._state_lock: self._changed = True self._observed_path = ".git/index" @@ -467,6 +449,8 @@ def _path_is_ignored(self, relative: Path) -> bool: return False if any(part in self._IGNORED_TOP_LEVEL for part in relative.parts): return True + if self._is_within_ignored_root(relative, self._ignored_roots): + return True try: result = subprocess.run( ["git", "check-ignore", "--quiet", "--no-index", "--", relative.as_posix()], diff --git a/tests/unit/devtools/test_evidence_dashboard.py b/tests/unit/devtools/test_evidence_dashboard.py index b03a6ae7f5..6b643d9d3f 100644 --- a/tests/unit/devtools/test_evidence_dashboard.py +++ b/tests/unit/devtools/test_evidence_dashboard.py @@ -201,3 +201,39 @@ def test_static_gates_reject_a_run_whose_checkout_changed_mid_verification( assert gates["available"] is False assert all(gate["available"] is False for gate in gates["gates"]) + + +def test_static_gates_reject_transient_checkout_mutation_with_matching_endpoint_fingerprints( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + history = tmp_path / "history.jsonl" + history.write_text( + json.dumps( + { + "checkout_root": str(tmp_path.resolve()), + "git_head": "current-head", + "worktree_fingerprint": "current-fingerprint", + "final_worktree_fingerprint": "current-fingerprint", + "diagnosis": "checkout_changed_during_verification", + "steps": [ + {"name": "ruff check", "exit": 0}, + { + "name": "checkout stability", + "exit": 125, + "diagnosis": "checkout_changed_during_verification", + }, + ], + } + ) + + "\n" + ) + monkeypatch.setattr(evidence_dashboard, "VERIFY_HISTORY_PATH", history) + monkeypatch.setattr(evidence_dashboard, "git_head", lambda _root: "current-head") + monkeypatch.setattr(evidence_dashboard, "git_dirty", lambda _root: False) + monkeypatch.setattr(evidence_dashboard, "_worktree_fingerprint", lambda _root: "current-fingerprint") + + gates = evidence_dashboard._static_gates(tmp_path, now=datetime(2026, 8, 12, tzinfo=timezone.utc)) + + assert gates["available"] is False + assert all(gate["available"] is False for gate in gates["gates"]) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 2b637a7459..8c0ff0059b 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1801,6 +1801,25 @@ def test_worktree_fingerprint_hashes_untracked_file_contents(tmp_path: Path) -> assert before != after +def test_worktree_fingerprint_rejects_partial_git_output( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + real_run = subprocess.run + + def warning_run(*args: Any, **kwargs: Any) -> subprocess.CompletedProcess[bytes]: + result = cast(subprocess.CompletedProcess[bytes], real_run(*args, **kwargs)) + command = args[0] + if isinstance(command, list) and command[:2] == ["git", "diff"]: + return subprocess.CompletedProcess(command, 0, result.stdout, b"warning: partial enumeration\n") + return result + + monkeypatch.setattr(subprocess, "run", warning_run) + + assert _worktree_fingerprint(tmp_path) == "unavailable" + + def test_checkout_mutation_monitor_detects_a_change_that_reverts_before_the_final_fingerprint( tmp_path: Path, ) -> None: @@ -2011,6 +2030,20 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="new_source") +def test_checkout_mutation_monitor_remembers_deleted_ignored_root(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / ".gitignore").write_text("browser-extension/node_modules/\n", encoding="utf-8") + ignored_root = tmp_path / "browser-extension" / "node_modules" + ignored_root.mkdir(parents=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + shutil.rmtree(ignored_root) + monitor._record_change(ignored_root) + + assert monitor.finish() == CheckoutMutationObservation(changed=False, unavailable=False) + + @pytest.mark.uses_real_clock("waits for the real filesystem watcher to witness an index replacement") def test_checkout_mutation_monitor_observes_transient_index_authority_change(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) @@ -2024,38 +2057,27 @@ def test_checkout_mutation_monitor_observes_transient_index_authority_change(tmp hidden.parent.mkdir() hidden.write_text("secret authority\n", encoding="utf-8") subprocess.run(["git", "add", "-f", "ignored/hidden.py"], cwd=tmp_path, check=True) - deadline = time.monotonic() + 1 - while not monitor._changed and time.monotonic() < deadline: - time.sleep(0.005) subprocess.run(["git", "reset", "-q", "--", "ignored/hidden.py"], cwd=tmp_path, check=True) observation = monitor.finish() assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") -def test_checkout_mutation_monitor_ignores_read_only_git_index_lock(tmp_path: Path) -> None: +def test_checkout_mutation_monitor_ignores_uncommitted_git_index_lock(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) - subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) - subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) tracked = tmp_path / "tracked.py" tracked.write_text("value = 1\n", encoding="utf-8") subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) - subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) monitor = CheckoutMutationMonitor(tmp_path) - monitor.start() - subprocess.run( - ["git", "describe", "--dirty", "--always", "--long", "--abbrev=40"], - cwd=tmp_path, - check=True, - capture_output=True, - ) + monitor._watched_directories() + monitor._record_change(tmp_path / ".git" / "index.lock") observation = monitor.finish() assert observation == CheckoutMutationObservation(changed=False, unavailable=False) -def test_checkout_mutation_monitor_ignores_stale_index_event( +def test_checkout_mutation_monitor_treats_every_ready_index_event_as_authority_change( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -2076,7 +2098,7 @@ def portable_watch(*_paths: Path, **_kwargs: object) -> object: monitor.start() observation = monitor.finish() - assert observation == CheckoutMutationObservation(changed=False, unavailable=False) + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") def test_checkout_mutation_monitor_rejects_partial_git_enumeration( @@ -4766,6 +4788,61 @@ def finish(self) -> CheckoutMutationObservation: assert durable_payload["final_worktree_fingerprint"] == "stable" +def test_transient_checkout_mutation_discards_testmon_graph_before_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class _ChangedMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=True, unavailable=False, observed_path="polylogue/example.py") + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.setattr(verify, "CheckoutMutationMonitor", _ChangedMonitor) + monkeypatch.setattr( + verify, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + TESTMON_DATA.parent.mkdir(parents=True) + TESTMON_DATA.write_bytes(b"transient dependency graph") + TESTMON_SEED_STAMP.write_text("{}", encoding="utf-8") + affected_publish = MagicMock() + selection_publish = MagicMock() + + with ( + patch("devtools.verify._anchor_verification_paths"), + patch("devtools.verify.maybe_bootstrap_testmon_seed", return_value=None), + patch("devtools.verify._testmon_preflight", return_value=None), + patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest", "--testmon"])]), + patch("devtools.verify._run", return_value=(0, 0.01, {"selected_count": 1})), + patch("devtools.verify._changed_executable_paths", return_value=("polylogue/example.py",)), + patch("devtools.verify._record_testmon_affected_coverage", affected_publish), + patch("devtools.verify._refresh_testmon_selection_attempt", selection_publish), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._testmon_release_baseline_permission", return_value=False), + patch("devtools.verify._warn_low_memory"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + ): + assert main(["--json"]) == 125 + + assert not TESTMON_DATA.exists() + assert not TESTMON_SEED_STAMP.exists() + affected_publish.assert_not_called() + selection_publish.assert_not_called() + assert json.loads(capsys.readouterr().out)["diagnosis"] == "checkout_changed_during_verification" + + def test_verify_stops_after_failed_heavy_step(capsys: pytest.CaptureFixture[str]) -> None: calls: list[str] = [] From 99e2219ab298f33ad6150fff17a7da326c37d571 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 12:35:14 +0200 Subject: [PATCH 46/53] test(devtools): isolate fingerprint failure receipts --- tests/unit/devtools/test_verify.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 8c0ff0059b..c8c348f4ce 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -4711,6 +4711,16 @@ def test_checkout_stability_failure_controls_every_broad_run_receipt( fingerprints: tuple[str, str], expected_diagnosis: str, ) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + history: dict[str, Any] = {} receipt = tmp_path / "invocation" / "run.json" monkeypatch.setattr(verify, "ROOT", tmp_path) @@ -4728,6 +4738,7 @@ def test_checkout_stability_failure_controls_every_broad_run_receipt( patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), patch("devtools.verify.worktree_fingerprint", side_effect=fingerprints), ): assert main(["--quick", "--json"]) == 125 From 34e9c08f2a09b3906cdb663325fca0868d157f16 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 13:11:24 +0200 Subject: [PATCH 47/53] fix(devtools): pin verification Git authority --- devtools/verify.py | 88 ++++++++++++--- .../devtools/test_testmon_seed_recovery.py | 27 ++++- tests/unit/devtools/test_verify.py | 103 +++++++++++++++++- 3 files changed, 198 insertions(+), 20 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 68ff392499..765770b194 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2374,6 +2374,25 @@ def _git_committed_tree() -> str | None: return None +def _git_commit(ref: str) -> str | None: + """Resolve a mutable Git ref once for an authority-sensitive run.""" + try: + result = subprocess.run( + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], + capture_output=True, + text=True, + timeout=5, + cwd=ROOT, + env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if result.returncode != 0 or result.stderr.strip(): + return None + commit = result.stdout.strip() + return commit or None + + def _stamp_head() -> None: head = _git_head() if head is None: @@ -2449,32 +2468,44 @@ def _pytest_uses_full_suite_basetemp(label: str) -> bool: } -def _changed_paths() -> set[str]: +def _changed_paths(base_commit: str, head_commit: str) -> set[str]: + """Return changes between immutable start-time Git authorities.""" changed: set[str] = set() commands = ( - ["git", "diff", "--name-only", "HEAD", "--"], - ["git", "diff", "--name-only", "origin/master...HEAD", "--"], + ["git", "diff", "--name-only", head_commit, "--"], + ["git", "diff", "--name-only", f"{base_commit}...{head_commit}", "--"], + ["git", "ls-files", "--others", "--exclude-standard", "--"], ) for command in commands: try: - result = subprocess.run(command, capture_output=True, text=True, timeout=5) - except (OSError, subprocess.TimeoutExpired): - continue - if result.returncode == 0: - changed.update(line.strip() for line in result.stdout.splitlines() if line.strip()) + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=5, + cwd=ROOT, + env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise PytestResourceError("testmon changed-path authority is unavailable") from exc + if result.returncode != 0 or result.stderr.strip(): + raise PytestResourceError("testmon changed-path authority is unavailable") + changed.update(line.strip() for line in result.stdout.splitlines() if line.strip()) return changed -def _default_testmon_is_broad_change() -> bool: +def _default_testmon_is_broad_change(base_commit: str, head_commit: str) -> bool: """Return true when affected-test selection should be treated as broad.""" - return bool(_changed_paths() & _BROAD_TESTMON_CHANGED_PATHS) + return bool(_changed_paths(base_commit, head_commit) & _BROAD_TESTMON_CHANGED_PATHS) -def _changed_executable_paths() -> tuple[str, ...]: +def _changed_executable_paths(base_commit: str, head_commit: str) -> tuple[str, ...]: """Return changed paths whose behavior should select at least one test.""" roots = ("polylogue/", "devtools/", "tests/", "packaging/") exact = {"pyproject.toml", "uv.lock"} - return tuple(sorted(path for path in _changed_paths() if path in exact or path.startswith(roots))) + return tuple( + sorted(path for path in _changed_paths(base_commit, head_commit) if path in exact or path.startswith(roots)) + ) def _testmon_coverage_identity(executable_paths: Sequence[str]) -> dict[str, Any]: @@ -3558,7 +3589,14 @@ def main(argv: list[str] | None = None) -> int: else: tier = "testmon" + head = _git_head() full_pytest = bool(args.all or args.full) + affected_testmon = not (args.quick or args.commit or args.seed_testmon or full_pytest) + testmon_base_commit = _git_commit("origin/master") if affected_testmon else None + testmon_head_commit = head if affected_testmon else None + if affected_testmon and (testmon_base_commit is None or testmon_head_commit is None): + sys.stderr.write("verify: cannot resolve immutable Git refs for affected-test authority.\n") + return 125 if args.terminal_authorization is not None and not ((full_pytest or args.seed_testmon) and args.skip_slow): parser.error("--terminal-authorization requires --all, --full, or --seed-testmon with --skip-slow") preflight_error = _testmon_preflight( @@ -3571,7 +3609,6 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.write(preflight_error) return 2 - head = _git_head() t0 = time.monotonic() checkout_fingerprint = worktree_fingerprint(ROOT) verify_run = VerifyRun( @@ -3629,7 +3666,11 @@ def main(argv: list[str] | None = None) -> int: seed_testmon=bool(args.seed_testmon), resume_testmon_seed=resume_testmon_seed, full_pytest=full_pytest, - broad_testmon=_default_testmon_is_broad_change(), + broad_testmon=( + _default_testmon_is_broad_change(testmon_base_commit, testmon_head_commit) + if testmon_base_commit is not None and testmon_head_commit is not None + else False + ), ) except PytestResourceError as exc: sys.stderr.write(f"verify: {exc}\n") @@ -3669,7 +3710,9 @@ def main(argv: list[str] | None = None) -> int: refreshed_stamp = refresh_stamp(current_stamp, TESTMON_DATA) if refreshed_stamp is not None: pending_testmon_stamp = refreshed_stamp - executable_paths = _changed_executable_paths() + assert testmon_base_commit is not None + assert testmon_head_commit is not None + executable_paths = _changed_executable_paths(testmon_base_commit, testmon_head_commit) selected_count = metadata.get("selected_count") if selected_count == 0 and executable_paths: coverage = _matching_testmon_coverage(executable_paths) @@ -3784,10 +3827,16 @@ def main(argv: list[str] | None = None) -> int: if rc == 130 or _stop_after_failed_step(label): break + final_head = _git_head() final_checkout_fingerprint = worktree_fingerprint(ROOT) mutation_observation = mutation_monitor.finish() checkout_stable = True - if "unavailable" in {checkout_fingerprint, final_checkout_fingerprint} or mutation_observation.unavailable: + if ( + head is None + or final_head is None + or "unavailable" in {checkout_fingerprint, final_checkout_fingerprint} + or mutation_observation.unavailable + ): checkout_stable = False step_results.append( { @@ -3795,6 +3844,8 @@ def main(argv: list[str] | None = None) -> int: "duration_s": 0.0, "exit": 125, "diagnosis": "checkout_fingerprint_unavailable", + "initial_git_head": head, + "final_git_head": final_head, "initial_worktree_fingerprint": checkout_fingerprint, "final_worktree_fingerprint": final_checkout_fingerprint, } @@ -3802,7 +3853,7 @@ def main(argv: list[str] | None = None) -> int: if exit_code == 0: exit_code = 125 sys.stderr.write("verify: checkout fingerprint unavailable; evidence is not exact-head.\n") - elif mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: + elif final_head != head or mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: checkout_stable = False step_results.append( { @@ -3810,6 +3861,8 @@ def main(argv: list[str] | None = None) -> int: "duration_s": 0.0, "exit": 125, "diagnosis": "checkout_changed_during_verification", + "initial_git_head": head, + "final_git_head": final_head, "initial_worktree_fingerprint": checkout_fingerprint, "final_worktree_fingerprint": final_checkout_fingerprint, "transient_checkout_mutation": mutation_observation.changed, @@ -3867,6 +3920,7 @@ def main(argv: list[str] | None = None) -> int: history_entry: dict[str, Any] = { "timestamp": datetime.now(timezone.utc).isoformat(), "git_head": head, + "final_git_head": final_head, "tier": tier, "run_id": verify_run.run_id, "checkout_root": str(ROOT.resolve()), diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py index 74699dd315..5968d77279 100644 --- a/tests/integration/devtools/test_testmon_seed_recovery.py +++ b/tests/integration/devtools/test_testmon_seed_recovery.py @@ -11,7 +11,8 @@ import pytest from devtools import testmon_bootstrap, testmon_state, verify -from devtools.testmon_state import file_fingerprint, inspect_testmon_database +from devtools.testmon_state import file_fingerprint, inspect_testmon_database, seed_shard_plan +from devtools.verify_runs import CheckoutMutationObservation def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( @@ -19,6 +20,16 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: + class _StableMutationMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + source = tmp_path / "source" source.mkdir() (source / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") @@ -44,6 +55,12 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( runtime_identity = testmon_state.testmon_runtime_identity(source) assert runtime_identity is not None dependency_environment, pytest_harness = runtime_identity + shards = seed_shard_plan(expected, shard_size=len(expected)) + shards[0]["status"] = "complete" + shards[0]["node_outcomes"] = [ + {"nodeid": expected[0], "outcome": "passed"}, + {"nodeid": expected[1], "outcome": "failed"}, + ] attempt = { "protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, "status": "reusable", @@ -65,6 +82,7 @@ def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( {"nodeid": expected[0], "outcome": "passed"}, {"nodeid": expected[1], "outcome": "failed"}, ], + "shards": shards, "exit_code": 1, "run_id": "real-testmon", "artifact_dir": ".cache/verify/runs/real-testmon", @@ -128,8 +146,13 @@ def fake_run(*_args: object, **_kwargs: object) -> tuple[int, float, dict[str, o return 0, 0.01, {"selected_count": 1} monkeypatch.setattr(verify, "_run", fake_run) - monkeypatch.setattr(verify, "_changed_executable_paths", lambda: ()) + monkeypatch.setattr(verify, "_git_head", lambda: "head") + monkeypatch.setattr(verify, "_git_commit", lambda _ref: "base") + monkeypatch.setattr(verify, "_default_testmon_is_broad_change", lambda _base_commit, _head_commit: False) + monkeypatch.setattr(verify, "_changed_executable_paths", lambda _base_commit, _head_commit: ()) monkeypatch.setattr(verify, "_stamp_head", lambda: None) + monkeypatch.setattr(verify, "worktree_fingerprint", lambda *_args: "stable") + monkeypatch.setattr(verify, "CheckoutMutationMonitor", _StableMutationMonitor) assert verify.main([]) == 1 result = json.loads(capsys.readouterr().out) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index c8c348f4ce..1cb00f209d 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1429,6 +1429,8 @@ def test_verify_main_records_containment_failure_as_terminal_history( with ( patch("devtools.verify._anchor_verification_paths"), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._testmon_preflight", return_value=None), patch("devtools.verify.build_verify_steps", return_value=[("pytest containment", ["pytest", "-n", "0"])]), patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), @@ -1820,6 +1822,59 @@ def warning_run(*args: Any, **kwargs: Any) -> subprocess.CompletedProcess[bytes] assert _worktree_fingerprint(tmp_path) == "unavailable" +def test_changed_paths_keep_start_time_base_when_remote_ref_advances( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + source = tmp_path / "polylogue" / "example.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "polylogue/example.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + base = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + subprocess.run(["git", "switch", "-qc", "feature"], cwd=tmp_path, check=True) + source.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "commit", "-qam", "feature"], cwd=tmp_path, check=True) + feature_head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + subprocess.run(["git", "update-ref", "refs/remotes/origin/master", base], cwd=tmp_path, check=True) + monkeypatch.setattr(verify, "ROOT", tmp_path) + + pinned_base = verify._git_commit("origin/master") + assert pinned_base == base + subprocess.run(["git", "update-ref", "refs/remotes/origin/master", "HEAD"], cwd=tmp_path, check=True) + + assert verify._changed_executable_paths(pinned_base, feature_head) == ("polylogue/example.py",) + + +def test_changed_paths_include_untracked_executable_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "README.md" + tracked.write_text("base\n", encoding="utf-8") + subprocess.run(["git", "add", "README.md"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + untracked = tmp_path / "devtools" / "new_command.py" + untracked.parent.mkdir() + untracked.write_text("value = 1\n", encoding="utf-8") + monkeypatch.setattr(verify, "ROOT", tmp_path) + + assert verify._changed_executable_paths(head, head) == ("devtools/new_command.py",) + + def test_checkout_mutation_monitor_detects_a_change_that_reverts_before_the_final_fingerprint( tmp_path: Path, ) -> None: @@ -4697,6 +4752,37 @@ def test_verify_withholds_success_when_checkout_fingerprint_is_unavailable( assert checkout_step["final_worktree_fingerprint"] == fingerprints[1] +def test_verify_rejects_git_head_change_with_matching_worktree_fingerprints( + capsys: pytest.CaptureFixture[str], +) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {})), + patch("devtools.verify._git_head", side_effect=("start-head", "different-head")), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + ): + assert main(["--quick", "--json"]) == 125 + + payload = json.loads(capsys.readouterr().out) + checkout_step = next(step for step in payload["steps"] if step["name"] == "checkout stability") + assert checkout_step["diagnosis"] == "checkout_changed_during_verification" + assert checkout_step["initial_git_head"] == "start-head" + assert checkout_step["final_git_head"] == "different-head" + + @pytest.mark.parametrize( ("fingerprints", "expected_diagnosis"), [ @@ -4838,6 +4924,8 @@ def finish(self) -> CheckoutMutationObservation: patch("devtools.verify._record_testmon_affected_coverage", affected_publish), patch("devtools.verify._refresh_testmon_selection_attempt", selection_publish), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._testmon_release_baseline_permission", return_value=False), patch("devtools.verify._warn_low_memory"), patch("devtools.verify._save_history"), @@ -4863,7 +4951,10 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo with ( patch("devtools.verify._run", side_effect=fake_run), + patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._save_history"), patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), @@ -5043,6 +5134,8 @@ def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.Ca with ( patch("devtools.verify.build_verify_steps", side_effect=PytestResourceError("only 0.50 GiB available")), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._testmon_preflight", return_value=None), patch("devtools.verify._run") as run, patch("devtools.verify._save_history") as save_history, @@ -5068,6 +5161,8 @@ def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirector def test_verify_rejects_zero_testmon_selection_for_executable_change( capsys: pytest.CaptureFixture[str], ) -> None: + changed_executable_paths = MagicMock(return_value=("polylogue/example.py",)) + def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: del command, kwargs return 0, 0.01, ({"selected_count": 0} if label.startswith("pytest") else {}) @@ -5075,11 +5170,13 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo with ( patch("devtools.verify._run", side_effect=fake_run), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="pinned-base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._save_history"), patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._changed_executable_paths", return_value=("polylogue/example.py",)), + patch("devtools.verify._changed_executable_paths", changed_executable_paths), patch("devtools.verify._matching_testmon_coverage", return_value=None), ): rc = main(["--json"]) @@ -5089,6 +5186,7 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo pytest_step = next(step for step in payload["steps"] if step["name"].startswith("pytest")) assert pytest_step["diagnosis"] == "zero_testmon_selection_for_executable_change" assert pytest_step["zero_selection_changed_paths"] == ["polylogue/example.py"] + changed_executable_paths.assert_called_once_with("pinned-base", "head") def test_verify_accepts_zero_testmon_selection_after_matching_coverage( @@ -5100,7 +5198,10 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo with ( patch("devtools.verify._run", side_effect=fake_run), + patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), patch("devtools.verify._save_history"), patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), From 995f7d1a4bf16e3977d66aa3d78d9245c4610703 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 13:31:15 +0200 Subject: [PATCH 48/53] fix(devtools): close Git authority races --- devtools/verify.py | 38 ++++++---- devtools/verify_runs.py | 59 +++++++++++++--- tests/unit/devtools/test_verify.py | 108 ++++++++++++++++++++++++++++- 3 files changed, 181 insertions(+), 24 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 765770b194..ca4fe44a24 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2353,14 +2353,8 @@ def _print_json(result: dict[str, Any]) -> None: def _git_head() -> str | None: - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - capture_output=True, - text=True, - ) - if result.returncode == 0: - return result.stdout.strip() - return None + """Resolve HEAD through the bounded authority-sensitive Git probe.""" + return _git_commit("HEAD") def _git_committed_tree() -> str | None: @@ -2472,8 +2466,8 @@ def _changed_paths(base_commit: str, head_commit: str) -> set[str]: """Return changes between immutable start-time Git authorities.""" changed: set[str] = set() commands = ( - ["git", "diff", "--name-only", head_commit, "--"], - ["git", "diff", "--name-only", f"{base_commit}...{head_commit}", "--"], + ["git", "diff", "--no-renames", "--name-only", head_commit, "--"], + ["git", "diff", "--no-renames", "--name-only", f"{base_commit}...{head_commit}", "--"], ["git", "ls-files", "--others", "--exclude-standard", "--"], ) for command in commands: @@ -3687,6 +3681,7 @@ def main(argv: list[str] | None = None) -> int: pending_affected_coverage: tuple[tuple[str, ...], int] | None = None pending_selection_refresh: tuple[dict[str, Any], int] | None = None testmon_graph_touched = False + changed_path_authority_failed = False mutation_monitor = CheckoutMutationMonitor(ROOT) mutation_monitor.start() @@ -3712,7 +3707,19 @@ def main(argv: list[str] | None = None) -> int: pending_testmon_stamp = refreshed_stamp assert testmon_base_commit is not None assert testmon_head_commit is not None - executable_paths = _changed_executable_paths(testmon_base_commit, testmon_head_commit) + try: + executable_paths = _changed_executable_paths(testmon_base_commit, testmon_head_commit) + except PytestResourceError as exc: + changed_path_authority_failed = True + executable_paths = () + rc = 125 + metadata["diagnosis"] = "testmon_changed_path_authority_unavailable" + metadata["error"] = str(exc) + pending_testmon_stamp = None + sys.stderr.write( + "verify: changed-path authority became unavailable after pytest; " + "discarding the affected dependency graph.\n" + ) selected_count = metadata.get("selected_count") if selected_count == 0 and executable_paths: coverage = _matching_testmon_coverage(executable_paths) @@ -3832,7 +3839,8 @@ def main(argv: list[str] | None = None) -> int: mutation_observation = mutation_monitor.finish() checkout_stable = True if ( - head is None + changed_path_authority_failed + or head is None or final_head is None or "unavailable" in {checkout_fingerprint, final_checkout_fingerprint} or mutation_observation.unavailable @@ -3843,7 +3851,11 @@ def main(argv: list[str] | None = None) -> int: "name": "checkout stability", "duration_s": 0.0, "exit": 125, - "diagnosis": "checkout_fingerprint_unavailable", + "diagnosis": ( + "testmon_changed_path_authority_unavailable" + if changed_path_authority_failed + else "checkout_fingerprint_unavailable" + ), "initial_git_head": head, "final_git_head": final_head, "initial_worktree_fingerprint": checkout_fingerprint, diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 74d077b281..cd2f4e0a02 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -261,6 +261,7 @@ def __init__(self, root: Path) -> None: self._tracked_paths: frozenset[Path] = frozenset() self._ignored_roots: frozenset[Path] = frozenset() self._git_index_path: Path | None = None + self._git_authority_paths: dict[Path, str] = {} def start(self) -> None: """Start and prove the portable interval watcher before verification.""" @@ -359,8 +360,12 @@ def walk_error(_error: OSError) -> None: ] directories.append(current_path) self._git_index_path = self._resolve_git_index_path() - if self._git_index_path is not None and self._git_index_path.parent not in directories: - directories.append(self._git_index_path.parent) + if self._git_index_path is not None: + self._git_authority_paths[self._git_index_path] = ".git/index" + self._git_authority_paths.update(self._resolve_git_head_paths()) + for authority_path in self._git_authority_paths: + if authority_path.parent not in directories: + directories.append(authority_path.parent) return directories def _git_tracked_paths(self) -> frozenset[Path]: @@ -382,7 +387,41 @@ def _resolve_git_index_path(self) -> Path | None: return None return Path(raw_path) - def _git_command(self, args: list[str]) -> subprocess.CompletedProcess[bytes] | None: + def _resolve_git_head_paths(self) -> dict[Path, str]: + """Resolve the worktree HEAD file and its current symbolic ref.""" + paths: dict[Path, str] = {} + head_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", "HEAD"]) + symbolic_result = self._git_command( + ["symbolic-ref", "--quiet", "HEAD"], + allowed_returncodes=frozenset({0, 1}), + ) + if head_result is None or symbolic_result is None: + return paths + raw_head_path = os.fsdecode(head_result.stdout).strip() + symbolic_ref = os.fsdecode(symbolic_result.stdout).strip() + if not raw_head_path or (symbolic_result.returncode == 0 and not symbolic_ref): + with self._state_lock: + self._unavailable = True + return paths + paths[Path(raw_head_path)] = ".git/HEAD" + if symbolic_result.returncode == 0: + ref_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", symbolic_ref]) + if ref_result is None: + return paths + raw_ref_path = os.fsdecode(ref_result.stdout).strip() + if not raw_ref_path: + with self._state_lock: + self._unavailable = True + return paths + paths[Path(raw_ref_path)] = f".git/{symbolic_ref}" + return paths + + def _git_command( + self, + args: list[str], + *, + allowed_returncodes: frozenset[int] = frozenset({0}), + ) -> subprocess.CompletedProcess[bytes] | None: try: result = subprocess.run( ["git", *args], @@ -396,7 +435,7 @@ def _git_command(self, args: list[str]) -> subprocess.CompletedProcess[bytes] | with self._state_lock: self._unavailable = True return None - if result.returncode != 0 or result.stderr.strip(): + if result.returncode not in allowed_returncodes or result.stderr.strip(): with self._state_lock: self._unavailable = True return None @@ -423,16 +462,18 @@ def _is_within_ignored_root(relative: Path, ignored_roots: frozenset[Path]) -> b def _record_change(self, candidate: Path) -> None: if not candidate.is_absolute(): candidate = self.root / candidate - if self._git_index_path is not None and candidate.parent == self._git_index_path.parent: - if candidate.name == f"{self._git_index_path.name}.lock": + for authority_path, label in self._git_authority_paths.items(): + if candidate.parent != authority_path.parent: + continue + if candidate.name == f"{authority_path.name}.lock": # An uncommitted lock is not yet checkout authority. A # completed transaction is witnessed when the lock replaces - # the index itself. + # its authority file. return - if candidate.name == self._git_index_path.name: + if candidate.name == authority_path.name: with self._state_lock: self._changed = True - self._observed_path = ".git/index" + self._observed_path = label return try: relative = candidate.relative_to(self.root) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 1cb00f209d..be438d7b6e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1875,6 +1875,40 @@ def test_changed_paths_include_untracked_executable_files( assert verify._changed_executable_paths(head, head) == ("devtools/new_command.py",) +def test_changed_paths_include_executable_rename_sources( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + source = tmp_path / "polylogue" / "example.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "polylogue/example.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + base = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + docs = tmp_path / "docs" + docs.mkdir() + subprocess.run(["git", "mv", "polylogue/example.py", "docs/example.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "move module"], cwd=tmp_path, check=True) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + monkeypatch.setattr(verify, "ROOT", tmp_path) + + assert verify._changed_executable_paths(base, head) == ("polylogue/example.py",) + + +def test_git_head_uses_bounded_authoritative_probe() -> None: + with patch("devtools.verify._git_commit", return_value="resolved-head") as resolve: + assert verify._git_head() == "resolved-head" + + resolve.assert_called_once_with("HEAD") + + def test_checkout_mutation_monitor_detects_a_change_that_reverts_before_the_final_fingerprint( tmp_path: Path, ) -> None: @@ -1980,7 +2014,11 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: assert event_emitted.wait(timeout=1) observation = monitor.finish() - assert calls["paths"] == (tmp_path.resolve(), (tmp_path / ".git").resolve()) + assert calls["paths"] == ( + tmp_path.resolve(), + (tmp_path / ".git").resolve(), + (tmp_path / ".git" / "refs" / "heads").resolve(), + ) assert calls["kwargs"] == { "watch_filter": None, "debounce": 0, @@ -2073,8 +2111,9 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: watched = {Path(path) for path in raw_paths} assert tmp_path.resolve() in watched assert source in watched + git_dir = (tmp_path / ".git").resolve() assert all( - path == (tmp_path / ".git").resolve() or not any(part in {".venv", ".git", ".cache"} for part in path.parts) + path.is_relative_to(git_dir) or not any(part in {".venv", ".git", ".cache"} for part in path.parts) for path in watched ) assert all("node_modules" not in path.parts for path in watched) @@ -2118,6 +2157,40 @@ def test_checkout_mutation_monitor_observes_transient_index_authority_change(tmp assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") +@pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a branch-ref replacement") +def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "first"], cwd=tmp_path, check=True) + first = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + tracked.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "commit", "-qam", "second"], cwd=tmp_path, check=True) + second = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + branch = subprocess.run( + ["git", "symbolic-ref", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + subprocess.run(["git", "update-ref", branch, first], cwd=tmp_path, check=True) + subprocess.run(["git", "update-ref", branch, second], cwd=tmp_path, check=True) + observation = monitor.finish() + + assert observation == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path=f".git/{branch}", + ) + + def test_checkout_mutation_monitor_ignores_uncommitted_git_index_lock(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) tracked = tmp_path / "tracked.py" @@ -5189,6 +5262,37 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo changed_executable_paths.assert_called_once_with("pinned-base", "head") +def test_verify_finalizes_and_discards_graph_when_post_pytest_path_authority_fails( + capsys: pytest.CaptureFixture[str], +) -> None: + monitor = MagicMock() + monitor.finish.return_value = CheckoutMutationObservation(changed=False, unavailable=False) + discard = MagicMock() + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {"selected_count": 1})), + patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._default_testmon_is_broad_change", return_value=False), + patch("devtools.verify._changed_executable_paths", side_effect=PytestResourceError("git unavailable")), + patch("devtools.verify._discard_testmon_dependency_authority", discard), + patch("devtools.verify.CheckoutMutationMonitor", return_value=monitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + patch("devtools.verify._testmon_preflight", return_value=None), + ): + rc = main(["--json"]) + + assert rc == 125 + monitor.finish.assert_called_once_with() + discard.assert_called_once_with() + payload = json.loads(capsys.readouterr().out) + assert payload["diagnosis"] == "testmon_changed_path_authority_unavailable" + + def test_verify_accepts_zero_testmon_selection_after_matching_coverage( capsys: pytest.CaptureFixture[str], ) -> None: From c0f0c3d97959ea37c2a5b5390384c3d02e85a0f3 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 13:55:52 +0200 Subject: [PATCH 49/53] fix(devtools): cover hidden Git authority --- devtools/verify.py | 16 ++--- devtools/verify_runs.py | 41 ++++++++++- tests/unit/devtools/test_verify.py | 107 +++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 10 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index ca4fe44a24..d68ec19ba3 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -2466,16 +2466,15 @@ def _changed_paths(base_commit: str, head_commit: str) -> set[str]: """Return changes between immutable start-time Git authorities.""" changed: set[str] = set() commands = ( - ["git", "diff", "--no-renames", "--name-only", head_commit, "--"], - ["git", "diff", "--no-renames", "--name-only", f"{base_commit}...{head_commit}", "--"], - ["git", "ls-files", "--others", "--exclude-standard", "--"], + ["git", "diff", "--no-renames", "--name-only", "-z", head_commit, "--"], + ["git", "diff", "--no-renames", "--name-only", "-z", f"{base_commit}...{head_commit}", "--"], + ["git", "ls-files", "--others", "--exclude-standard", "-z", "--"], ) for command in commands: try: result = subprocess.run( command, capture_output=True, - text=True, timeout=5, cwd=ROOT, env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}, @@ -2484,7 +2483,7 @@ def _changed_paths(base_commit: str, head_commit: str) -> set[str]: raise PytestResourceError("testmon changed-path authority is unavailable") from exc if result.returncode != 0 or result.stderr.strip(): raise PytestResourceError("testmon changed-path authority is unavailable") - changed.update(line.strip() for line in result.stdout.splitlines() if line.strip()) + changed.update(os.fsdecode(raw_path) for raw_path in result.stdout.split(b"\0") if raw_path) return changed @@ -3604,6 +3603,8 @@ def main(argv: list[str] | None = None) -> int: return 2 t0 = time.monotonic() + mutation_monitor = CheckoutMutationMonitor(ROOT) + mutation_monitor.start() checkout_fingerprint = worktree_fingerprint(ROOT) verify_run = VerifyRun( tier=tier, @@ -3626,6 +3627,7 @@ def main(argv: list[str] | None = None) -> int: terminal_authorization=args.terminal_authorization, ) except RuntimeError as exc: + mutation_monitor.finish() sys.stderr.write(f"verify: {exc}\n") early_payload = verify_run.finish( exit_code=125, @@ -3667,6 +3669,7 @@ def main(argv: list[str] | None = None) -> int: ), ) except PytestResourceError as exc: + mutation_monitor.finish() sys.stderr.write(f"verify: {exc}\n") early_payload = verify_run.finish( exit_code=125, @@ -3682,9 +3685,6 @@ def main(argv: list[str] | None = None) -> int: pending_selection_refresh: tuple[dict[str, Any], int] | None = None testmon_graph_touched = False changed_path_authority_failed = False - mutation_monitor = CheckoutMutationMonitor(ROOT) - mutation_monitor.start() - for label, cmd in steps: if label.startswith("pytest"): _warn_low_memory() # check again right before the heavy step diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index cd2f4e0a02..f9eeb16795 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -160,6 +160,26 @@ def worktree_fingerprint(root: Path | None = None) -> str: """Fingerprint tracked changes plus exact non-ignored untracked content.""" checkout_root = (root or Path.cwd()).resolve() digest = hashlib.sha256() + try: + tracked_flags = subprocess.run( + ["git", "ls-files", "-v", "-z"], + capture_output=True, + timeout=30, + cwd=checkout_root, + env=_read_only_git_env(), + ) + except (OSError, subprocess.TimeoutExpired): + return "unavailable" + if tracked_flags.returncode != 0 or tracked_flags.stderr.strip(): + return "unavailable" + for record in tracked_flags.stdout.split(b"\0"): + if not record: + continue + tag = record[:1] + if tag.islower() or tag == b"S": + # assume-unchanged and skip-worktree can hide worktree bytes from + # both status and diff, so Git cannot authorize exact evidence. + return "unavailable" for command in ( ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"], ["git", "diff", "--binary", "HEAD", "--"], @@ -364,8 +384,11 @@ def walk_error(_error: OSError) -> None: self._git_authority_paths[self._git_index_path] = ".git/index" self._git_authority_paths.update(self._resolve_git_head_paths()) for authority_path in self._git_authority_paths: - if authority_path.parent not in directories: - directories.append(authority_path.parent) + watched_parent = authority_path.parent + while not watched_parent.exists() and watched_parent != watched_parent.parent: + watched_parent = watched_parent.parent + if watched_parent not in directories: + directories.append(watched_parent) return directories def _git_tracked_paths(self) -> frozenset[Path]: @@ -404,6 +427,15 @@ def _resolve_git_head_paths(self) -> dict[Path, str]: self._unavailable = True return paths paths[Path(raw_head_path)] = ".git/HEAD" + packed_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", "packed-refs"]) + if packed_result is None: + return paths + raw_packed_path = os.fsdecode(packed_result.stdout).strip() + if not raw_packed_path: + with self._state_lock: + self._unavailable = True + return paths + paths[Path(raw_packed_path)] = ".git/packed-refs" if symbolic_result.returncode == 0: ref_result = self._git_command(["rev-parse", "--path-format=absolute", "--git-path", symbolic_ref]) if ref_result is None: @@ -463,6 +495,11 @@ def _record_change(self, candidate: Path) -> None: if not candidate.is_absolute(): candidate = self.root / candidate for authority_path, label in self._git_authority_paths.items(): + if candidate != authority_path and authority_path.is_relative_to(candidate): + with self._state_lock: + self._changed = True + self._observed_path = label + return if candidate.parent != authority_path.parent: continue if candidate.name == f"{authority_path.name}.lock": diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index be438d7b6e..18c2494764 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1902,6 +1902,27 @@ def test_changed_paths_include_executable_rename_sources( assert verify._changed_executable_paths(base, head) == ("polylogue/example.py",) +def test_changed_paths_parse_non_ascii_names_without_git_quoting( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + source = tmp_path / "polylogue" / "café.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "polylogue/café.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + base = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + source.write_text("value = 2\n", encoding="utf-8") + monkeypatch.setattr(verify, "ROOT", tmp_path) + + assert verify._changed_executable_paths(base, base) == ("polylogue/café.py",) + + def test_git_head_uses_bounded_authoritative_probe() -> None: with patch("devtools.verify._git_commit", return_value="resolved-head") as resolve: assert verify._git_head() == "resolved-head" @@ -2191,6 +2212,50 @@ def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: ) +@pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a loose ref created from packed authority") +def test_checkout_mutation_monitor_observes_packed_nested_branch_ref_change(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "first"], cwd=tmp_path, check=True) + first = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, capture_output=True, text=True + ).stdout.strip() + subprocess.run(["git", "switch", "-qc", "feature/nested"], cwd=tmp_path, check=True) + tracked.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "commit", "-qam", "second"], cwd=tmp_path, check=True) + subprocess.run(["git", "pack-refs", "--all", "--prune"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + subprocess.run(["git", "update-ref", "refs/heads/feature/nested", first], cwd=tmp_path, check=True) + observation = monitor.finish() + + assert observation == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path=".git/refs/heads/feature/nested", + ) + + +def test_worktree_fingerprint_rejects_assume_unchanged_tracked_content(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + source = tmp_path / "polylogue" / "hidden.py" + source.parent.mkdir() + source.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "polylogue/hidden.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + subprocess.run(["git", "update-index", "--assume-unchanged", "polylogue/hidden.py"], cwd=tmp_path, check=True) + source.write_text("value = 2\n", encoding="utf-8") + + assert _worktree_fingerprint(tmp_path) == "unavailable" + + def test_checkout_mutation_monitor_ignores_uncommitted_git_index_lock(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) tracked = tmp_path / "tracked.py" @@ -3976,6 +4041,7 @@ def test_run_records_pytest_count_metadata_from_terminal_fallback() -> None: ) with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), patch("devtools.verify._read_pytest_report", return_value=None), ): @@ -3998,6 +4064,7 @@ def test_run_records_managed_basetemp_cleanup_metadata(tmp_path: Path) -> None: cleaned = tmp_path / "pytest-polylogue-run-1" with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), patch("devtools.verify._read_pytest_report", return_value=None), patch("devtools.verify.cleanup_managed_pytest_basetemp", return_value=cleaned) as cleanup, @@ -5221,6 +5288,46 @@ def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.Ca assert "only 0.50 GiB available" in capsys.readouterr().err +def test_verify_starts_checkout_monitor_before_broad_change_classification( + capsys: pytest.CaptureFixture[str], +) -> None: + events: list[str] = [] + + class _OrderingMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("monitor-started") + + def finish(self) -> CheckoutMutationObservation: + events.append("monitor-finished") + return CheckoutMutationObservation(changed=False, unavailable=False) + + def classify(_base: str, _head: str) -> bool: + assert events == ["monitor-started"] + events.append("classified") + return False + + with ( + patch("devtools.verify.CheckoutMutationMonitor", _OrderingMonitor), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._default_testmon_is_broad_change", side_effect=classify), + patch("devtools.verify.build_verify_steps", return_value=[]), + patch("devtools.verify._testmon_preflight", return_value=None), + patch("devtools.verify._testmon_release_baseline_permission", return_value=False), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + ): + assert main(["--json"]) == 0 + + assert events == ["monitor-started", "classified", "monitor-finished"] + assert json.loads(capsys.readouterr().out)["exit_code"] == 0 + + def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( monkeypatch: pytest.MonkeyPatch, ) -> None: From 2579dc67b8b9899d35a34a8214ab1540b7409e96 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 14:20:06 +0200 Subject: [PATCH 50/53] fix(devtools): close verification monitor gaps --- devtools/run_tests.py | 10 +++- devtools/verify.py | 12 ++-- devtools/verify_runs.py | 81 ++++++++++++++++++++++++--- tests/unit/devtools/test_run_tests.py | 59 +++++++++++++++++++ tests/unit/devtools/test_verify.py | 48 ++++++++++++++++ 5 files changed, 195 insertions(+), 15 deletions(-) diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 33b061b784..fe4fa6c1a0 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -51,8 +51,11 @@ CheckoutMutationMonitor, VerifyRun, append_verify_history, + finalize_checkout_mutation_monitors, + finish_checkout_mutation_monitor, git_head, pytest_command_worker_request, + start_checkout_mutation_monitor, worktree_fingerprint, ) @@ -235,6 +238,7 @@ def _run_lock(*, enabled: bool) -> Iterator[None]: handle.truncate() +@finalize_checkout_mutation_monitors def main(argv: list[str] | None = None) -> int: invocation_directory = Path.cwd() selection = list(sys.argv[1:] if argv is None else argv) @@ -267,9 +271,9 @@ def main(argv: list[str] | None = None) -> int: no_lock = os.environ.get("POLYLOGUE_TEST_NO_LOCK") == "1" with _run_lock(enabled=not no_lock): _clear_pytest_report(cmd) - initial_worktree_fingerprint = worktree_fingerprint(ROOT) mutation_monitor = CheckoutMutationMonitor(ROOT) - mutation_monitor.start() + start_checkout_mutation_monitor(mutation_monitor) + initial_worktree_fingerprint = worktree_fingerprint(ROOT) run = VerifyRun( tier="focused-test", argv=selection, @@ -287,7 +291,7 @@ def main(argv: list[str] | None = None) -> int: metadata = {"diagnosis": "pytest_interrupted", "termination_reason": "operator_interrupt"} run.finish_interrupted_steps(exit_code=rc, diagnosis=str(metadata["diagnosis"])) final_worktree_fingerprint = worktree_fingerprint(ROOT) - mutation_observation = mutation_monitor.finish() + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) if ( "unavailable" in {initial_worktree_fingerprint, final_worktree_fingerprint} or mutation_observation.unavailable diff --git a/devtools/verify.py b/devtools/verify.py index d68ec19ba3..8fcde7d7e5 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -100,6 +100,8 @@ cleanup_managed_pytest_basetemp, copy_current_pytest_artifacts, env_for_pytest_step, + finalize_checkout_mutation_monitors, + finish_checkout_mutation_monitor, force_managed_pytest_scratch, latest_event_from_paths, normalize_pytest_basetemp_env, @@ -107,6 +109,7 @@ pytest_command_worker_request, pytest_tmpfs_budget_exceeded, pytest_tmpfs_budget_kb, + start_checkout_mutation_monitor, utc_now, worktree_fingerprint, xdist_uninterruptible_stall_reason, @@ -3510,6 +3513,7 @@ def _discard_testmon_dependency_authority() -> None: # ── main ──────────────────────────────────────────────────────────── +@finalize_checkout_mutation_monitors def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Run the local verification baseline.") parser.add_argument("--quick", action="store_true", help="Skip pytest and run only fast local gates.") @@ -3604,7 +3608,7 @@ def main(argv: list[str] | None = None) -> int: t0 = time.monotonic() mutation_monitor = CheckoutMutationMonitor(ROOT) - mutation_monitor.start() + start_checkout_mutation_monitor(mutation_monitor) checkout_fingerprint = worktree_fingerprint(ROOT) verify_run = VerifyRun( tier=tier, @@ -3627,7 +3631,7 @@ def main(argv: list[str] | None = None) -> int: terminal_authorization=args.terminal_authorization, ) except RuntimeError as exc: - mutation_monitor.finish() + finish_checkout_mutation_monitor(mutation_monitor) sys.stderr.write(f"verify: {exc}\n") early_payload = verify_run.finish( exit_code=125, @@ -3669,7 +3673,7 @@ def main(argv: list[str] | None = None) -> int: ), ) except PytestResourceError as exc: - mutation_monitor.finish() + finish_checkout_mutation_monitor(mutation_monitor) sys.stderr.write(f"verify: {exc}\n") early_payload = verify_run.finish( exit_code=125, @@ -3836,7 +3840,7 @@ def main(argv: list[str] | None = None) -> int: final_head = _git_head() final_checkout_fingerprint = worktree_fingerprint(ROOT) - mutation_observation = mutation_monitor.finish() + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) checkout_stable = True if ( changed_path_authority_failed diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index f9eeb16795..76d0d24154 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -9,6 +9,7 @@ import contextlib import fcntl +import functools import hashlib import json import os @@ -20,11 +21,11 @@ import threading import time import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, replace from datetime import UTC, datetime from pathlib import Path -from typing import Any, TextIO +from typing import Any, ParamSpec, TextIO, TypeVar import watchfiles @@ -279,6 +280,7 @@ def __init__(self, root: Path) -> None: self._thread: threading.Thread | None = None self._state_lock = threading.Lock() self._tracked_paths: frozenset[Path] = frozenset() + self._tracked_directories: frozenset[Path] = frozenset() self._ignored_roots: frozenset[Path] = frozenset() self._git_index_path: Path | None = None self._git_authority_paths: dict[Path, str] = {} @@ -362,6 +364,9 @@ def _polling_backend_requested(cls) -> bool: def _watched_directories(self) -> list[Path]: """Watch existing source directories shallowly and omit disposable trees.""" self._tracked_paths = self._git_tracked_paths() + self._tracked_directories = frozenset( + parent for tracked_path in self._tracked_paths for parent in tracked_path.parents if parent != Path(".") + ) self._ignored_roots = self._ignored_directory_roots() directories: list[Path] = [] @@ -372,12 +377,17 @@ def walk_error(_error: OSError) -> None: for current, child_directories, _files in os.walk(self.root, onerror=walk_error): current_path = Path(current) relative_current = current_path.relative_to(self.root) - child_directories[:] = [ - child - for child in child_directories - if child not in self._IGNORED_TOP_LEVEL - and not self._is_within_ignored_root(relative_current / child, self._ignored_roots) - ] + retained_children: list[str] = [] + for child in child_directories: + relative_child = relative_current / child + disposable = child in self._IGNORED_TOP_LEVEL or self._is_within_ignored_root( + relative_child, + self._ignored_roots, + ) + if disposable and relative_child not in self._tracked_directories: + continue + retained_children.append(child) + child_directories[:] = retained_children directories.append(current_path) self._git_index_path = self._resolve_git_index_path() if self._git_index_path is not None: @@ -551,6 +561,61 @@ def _path_is_ignored(self, relative: Path) -> bool: return True +_MonitorParams = ParamSpec("_MonitorParams") +_MonitorResult = TypeVar("_MonitorResult") +_MONITOR_STATE = threading.local() + + +def _checkout_monitor_stack() -> list[CheckoutMutationMonitor]: + stack = getattr(_MONITOR_STATE, "stack", None) + if stack is None: + stack = [] + _MONITOR_STATE.stack = stack + return stack + + +def start_checkout_mutation_monitor(monitor: CheckoutMutationMonitor) -> None: + """Start a runner-owned monitor and register it for exceptional cleanup.""" + stack = _checkout_monitor_stack() + stack.append(monitor) + try: + monitor.start() + except BaseException: + with contextlib.suppress(Exception): + finish_checkout_mutation_monitor(monitor) + raise + + +def finish_checkout_mutation_monitor(monitor: CheckoutMutationMonitor) -> CheckoutMutationObservation: + """Finish one monitor and retire its runner cleanup obligation.""" + stack = _checkout_monitor_stack() + try: + return monitor.finish() + finally: + for index in range(len(stack) - 1, -1, -1): + if stack[index] is monitor: + del stack[index] + break + + +def finalize_checkout_mutation_monitors( + function: Callable[_MonitorParams, _MonitorResult], +) -> Callable[_MonitorParams, _MonitorResult]: + """Guarantee that monitors started by a runner finish on every exit.""" + + @functools.wraps(function) + def wrapped(*args: _MonitorParams.args, **kwargs: _MonitorParams.kwargs) -> _MonitorResult: + stack = _checkout_monitor_stack() + baseline_depth = len(stack) + try: + return function(*args, **kwargs) + finally: + while len(stack) > baseline_depth: + finish_checkout_mutation_monitor(stack[-1]) + + return wrapped + + @dataclass(frozen=True) class PytestRuntimePolicy: """One start-time resource decision for a managed pytest run.""" diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index 16a94ed13b..bcd66f9e2f 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -301,6 +301,65 @@ def test_main_withholds_success_when_checkout_changes_during_pytest( assert captured["final_worktree_fingerprint"] == "changed" +def test_main_starts_checkout_monitor_before_initial_fingerprint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[str] = [] + + class _OrderingMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("monitor-started") + + def finish(self) -> CheckoutMutationObservation: + events.append("monitor-finished") + return CheckoutMutationObservation(changed=False, unavailable=False) + + def fingerprint(_root: Path) -> str: + assert events[0] == "monitor-started" + events.append("fingerprinted") + return "stable" + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", lambda *_args, **_kwargs: (0, 0.01, {"diagnosis": "pytest_passed"})) + monkeypatch.setattr(run_tests, "worktree_fingerprint", fingerprint) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _OrderingMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) + + assert run_tests.main(["tests/unit/example.py"]) == 0 + assert events == ["monitor-started", "fingerprinted", "fingerprinted", "monitor-finished"] + + +def test_main_finalizes_checkout_monitor_when_initial_fingerprint_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[str] = [] + + class _ExceptionalExitMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("monitor-started") + + def finish(self) -> CheckoutMutationObservation: + events.append("monitor-finished") + return CheckoutMutationObservation(changed=False, unavailable=False) + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _ExceptionalExitMonitor) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: (_ for _ in ()).throw(RuntimeError("boom"))) + + with pytest.raises(RuntimeError, match="boom"): + run_tests.main(["tests/unit/example.py"]) + + assert events == ["monitor-started", "monitor-finished"] + + @pytest.mark.parametrize("fingerprints", [("unavailable", "stable"), ("stable", "unavailable")]) def test_main_withholds_success_when_checkout_fingerprint_is_unavailable( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 18c2494764..168ac9337a 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1967,6 +1967,29 @@ def test_checkout_mutation_monitor_ignores_nested_disposable_cache_writes(tmp_pa assert observation == CheckoutMutationObservation(changed=False, unavailable=False) +def test_checkout_mutation_monitor_observes_tracked_file_inside_disposable_cache(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) + tracked = tmp_path / "package" / "__pycache__" / "authority.py" + tracked.parent.mkdir(parents=True) + tracked.write_text("before\n", encoding="utf-8") + subprocess.run(["git", "add", "-f", "package/__pycache__/authority.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed tracked cache path"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + tracked.write_text("during\n", encoding="utf-8") + tracked.write_text("before\n", encoding="utf-8") + observation = monitor.finish() + + assert observation == CheckoutMutationObservation( + changed=True, + unavailable=False, + observed_path="package/__pycache__/authority.py", + ) + + def test_checkout_mutation_monitor_uses_gitignore_for_verifier_task_history( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -5328,6 +5351,31 @@ def classify(_base: str, _head: str) -> bool: assert json.loads(capsys.readouterr().out)["exit_code"] == 0 +def test_verify_finalizes_checkout_monitor_when_startup_fingerprint_raises() -> None: + events: list[str] = [] + + class _ExceptionalExitMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("monitor-started") + + def finish(self) -> CheckoutMutationObservation: + events.append("monitor-finished") + return CheckoutMutationObservation(changed=False, unavailable=False) + + with ( + patch("devtools.verify.CheckoutMutationMonitor", _ExceptionalExitMonitor), + patch("devtools.verify._git_head", return_value="head"), + patch("devtools.verify.worktree_fingerprint", side_effect=RuntimeError("fingerprint failed")), + ): + with pytest.raises(RuntimeError, match="fingerprint failed"): + main(["--quick", "--json"]) + + assert events == ["monitor-started", "monitor-finished"] + + def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( monkeypatch: pytest.MonkeyPatch, ) -> None: From c66671d1d0663dece927f99a0a7b2915bc40e973 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 15:16:36 +0200 Subject: [PATCH 51/53] fix(devtools): bind verification recovery evidence Require merge-gate authority from the invocation-bound receipt, finalize unexpected verification runner failures as typed exit-125 history, preserve pytest expression values, and reject topology drift during watcher startup. --- devtools/merge_boundary.py | 7 +- devtools/merge_gate.py | 50 +++++------- devtools/run_tests.py | 91 +++++++++++++++++----- devtools/verify.py | 52 ++++++++++++- devtools/verify_runs.py | 34 +++++++- tests/unit/devtools/test_merge_boundary.py | 17 ++++ tests/unit/devtools/test_merge_gate.py | 46 ++++++++++- tests/unit/devtools/test_run_tests.py | 62 +++++++++++++++ tests/unit/devtools/test_verify.py | 59 ++++++++++++++ 9 files changed, 357 insertions(+), 61 deletions(-) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index 361bc0837d..7cf334198e 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -815,13 +815,14 @@ def cmd_record_full_verify( print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) return 2 duration_s = round(time.time() - started, 2) - release_allowed = merge_gate._release_baseline_permission(result.stdout) - verification_scope = merge_gate._verification_scope(result.stdout) - terminal_authorization = merge_gate._terminal_authorization(result.stdout) try: structured = json.loads(result.stdout) except (TypeError, json.JSONDecodeError): structured = None + receipt = structured if isinstance(structured, dict) else None + release_allowed = merge_gate._release_baseline_permission(receipt) + verification_scope = merge_gate._verification_scope(receipt) + terminal_authorization = merge_gate._terminal_authorization(receipt) verified_head = structured.get("git_head") if isinstance(structured, dict) else None accepted = ( result.returncode == 0 diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index 4255846694..427038007b 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -77,6 +77,7 @@ import tempfile import time import uuid +from collections.abc import Mapping from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any @@ -236,24 +237,6 @@ def _command_skips_tests(command: str) -> bool: return not any(marker in lowered for marker in _LOOKS_LIKE_TESTS_MARKERS) -def _structured_verification_receipt(stdout: str) -> dict[str, Any] | None: - """Read a final JSON receipt even when the verifier streamed progress first.""" - - candidate_start = len(stdout) - while candidate_start: - candidate_start = stdout.rfind("\n{", 0, candidate_start) - start = candidate_start + 1 if candidate_start >= 0 else 0 - candidate = stdout[start:].strip() - try: - payload = json.loads(candidate) - except (TypeError, json.JSONDecodeError): - if candidate_start < 0: - return None - continue - return payload if isinstance(payload, dict) else None - return None - - def _invocation_receipt( *, path: Path, @@ -281,30 +264,34 @@ def _invocation_receipt( strict=False ): return None + if _verification_scope(payload) is None or _release_baseline_permission(payload) is None: + return None + terminal_authorization = payload.get("terminal_authorization") + if terminal_authorization is not None and terminal_authorization not in { + authorization.value for authorization in TerminalAuthorization + }: + return None return payload -def _release_baseline_permission(stdout: str) -> bool | None: - """Read the structured verify decision when the command emitted one.""" - payload = _structured_verification_receipt(stdout) +def _release_baseline_permission(payload: Mapping[str, Any] | None) -> bool | None: + """Read the typed release decision from an invocation-bound receipt.""" if payload is None: return None value = payload.get("release_baseline_allowed") return value if isinstance(value, bool) else None -def _verification_scope(stdout: str) -> str | None: - """Read the typed verification scope from a structured verify receipt.""" - payload = _structured_verification_receipt(stdout) +def _verification_scope(payload: Mapping[str, Any] | None) -> str | None: + """Read the typed verification scope from an invocation-bound receipt.""" if payload is None: return None value = payload.get("verification_scope") return value if value in {scope.value for scope in VerificationScope} else None -def _terminal_authorization(stdout: str) -> str | None: - """Read the typed terminal authorization from a structured receipt.""" - payload = _structured_verification_receipt(stdout) +def _terminal_authorization(payload: Mapping[str, Any] | None) -> str | None: + """Read terminal authorization from an invocation-bound receipt.""" if payload is None: return None value = payload.get("terminal_authorization") @@ -397,7 +384,7 @@ def cmd_record(pr: int, command: str) -> int: except OSError as exc: print(f"REFUSING to record: could not run {command!r}: {exc}", file=sys.stderr) return 2 - verification_receipt = _structured_verification_receipt(result.stdout) or _invocation_receipt( + verification_receipt = _invocation_receipt( path=receipt_path, invocation_id=invocation_id, head_sha=head_sha, @@ -405,7 +392,6 @@ def cmd_record(pr: int, command: str) -> int: checkout_root=checkout_root, ) duration_s = round(time.time() - started, 2) - verification_payload = json.dumps(verification_receipt) if verification_receipt is not None else "" receipt = { "pr": pr, @@ -422,9 +408,9 @@ def cmd_record(pr: int, command: str) -> int: "branch": info["headRefName"], "command": command, "skips_tests": _command_skips_tests(command), - "verification_scope": _verification_scope(verification_payload), - "release_baseline_allowed": _release_baseline_permission(verification_payload), - "terminal_authorization": _terminal_authorization(verification_payload), + "verification_scope": _verification_scope(verification_receipt), + "release_baseline_allowed": _release_baseline_permission(verification_receipt), + "terminal_authorization": _terminal_authorization(verification_receipt), "exit_code": result.returncode, "duration_s": duration_s, "recorded_at": time.time(), diff --git a/devtools/run_tests.py b/devtools/run_tests.py index fe4fa6c1a0..4ec62ffaee 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -49,6 +49,7 @@ ) from devtools.verify_runs import ( CheckoutMutationMonitor, + CheckoutMutationObservation, VerifyRun, append_verify_history, finalize_checkout_mutation_monitors, @@ -77,6 +78,22 @@ } ) _ENV_EXPANDING_PATH_OPTIONS = frozenset({"--rootdir"}) +_NON_PATH_VALUE_OPTIONS = frozenset( + { + "-k", + "--keyword", + "-m", + "--mark", + "--deselect", + "--maxfail", + "--tb", + "--capture", + "--durations", + "--durations-min", + "--override-ini", + "-o", + } +) def _absolute_option_path( @@ -104,7 +121,7 @@ def _normalize_selection_paths(selection: list[str], *, invocation_directory: Pa # option belongs to pytest, not to --debug's optional value. if pending_option == "--debug" and argument.startswith("-"): pending_option = None - else: + elif pending_option in _PATH_VALUE_OPTIONS: normalized.append( _absolute_option_path( argument, @@ -114,6 +131,10 @@ def _normalize_selection_paths(selection: list[str], *, invocation_directory: Pa ) pending_option = None continue + else: + normalized.append(argument) + pending_option = None + continue if argument.startswith("-c="): normalized.append( "-c" @@ -136,6 +157,11 @@ def _normalize_selection_paths(selection: list[str], *, invocation_directory: Pa normalized.append(argument) pending_option = option_name continue + if option_name in _NON_PATH_VALUE_OPTIONS: + normalized.append(argument) + if not equals: + pending_option = option_name + continue if argument.startswith("-c") and len(argument) > len("-c"): normalized.append( "-c" @@ -284,29 +310,58 @@ def main(argv: list[str] | None = None) -> int: worktree_fingerprint=initial_worktree_fingerprint, ) started = time.monotonic() + final_worktree_fingerprint = "unavailable" + mutation_observation = CheckoutMutationObservation(changed=False, unavailable=True) + runner_exception = False try: rc, _elapsed, metadata = _run("pytest focused", cmd, cwd=str(ROOT), run=run) except KeyboardInterrupt: rc = 130 metadata = {"diagnosis": "pytest_interrupted", "termination_reason": "operator_interrupt"} run.finish_interrupted_steps(exit_code=rc, diagnosis=str(metadata["diagnosis"])) - final_worktree_fingerprint = worktree_fingerprint(ROOT) - mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) - if ( - "unavailable" in {initial_worktree_fingerprint, final_worktree_fingerprint} - or mutation_observation.unavailable - ): - metadata["diagnosis"] = "checkout_fingerprint_unavailable" - if rc == 0: - rc = 125 - sys.stderr.write("devtools test: checkout fingerprint unavailable; evidence is not exact-head.\n") - elif mutation_observation.changed or final_worktree_fingerprint != initial_worktree_fingerprint: - metadata["diagnosis"] = "checkout_changed_during_focused_test" - metadata["transient_checkout_mutation"] = mutation_observation.changed - metadata["checkout_mutation_path"] = mutation_observation.observed_path - if rc == 0: - rc = 125 - sys.stderr.write("devtools test: checkout contents changed during pytest; evidence is not exact-head.\n") + except Exception as exc: + runner_exception = True + rc = 125 + metadata = { + "diagnosis": "focused_test_runner_exception", + "exception_type": type(exc).__name__, + "error": str(exc), + "termination_reason": "runner_exception", + } + run.finish_interrupted_steps( + exit_code=rc, + diagnosis=str(metadata["diagnosis"]), + termination_reason="runner_exception", + ) + try: + final_worktree_fingerprint = worktree_fingerprint(ROOT) + except Exception: + final_worktree_fingerprint = "unavailable" + try: + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) + except Exception: + mutation_observation = CheckoutMutationObservation(changed=False, unavailable=True) + sys.stderr.write(f"devtools test: unexpected runner exception: {exc}\n") + if not runner_exception: + final_worktree_fingerprint = worktree_fingerprint(ROOT) + mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) + if ( + "unavailable" in {initial_worktree_fingerprint, final_worktree_fingerprint} + or mutation_observation.unavailable + ): + metadata["diagnosis"] = "checkout_fingerprint_unavailable" + if rc == 0: + rc = 125 + sys.stderr.write("devtools test: checkout fingerprint unavailable; evidence is not exact-head.\n") + elif mutation_observation.changed or final_worktree_fingerprint != initial_worktree_fingerprint: + metadata["diagnosis"] = "checkout_changed_during_focused_test" + metadata["transient_checkout_mutation"] = mutation_observation.changed + metadata["checkout_mutation_path"] = mutation_observation.observed_path + if rc == 0: + rc = 125 + sys.stderr.write( + "devtools test: checkout contents changed during pytest; evidence is not exact-head.\n" + ) payload = run.finish( exit_code=rc, duration_s=time.monotonic() - started, diff --git a/devtools/verify.py b/devtools/verify.py index 8fcde7d7e5..56de0edd4e 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -3513,8 +3513,11 @@ def _discard_testmon_dependency_authority() -> None: # ── main ──────────────────────────────────────────────────────────── -@finalize_checkout_mutation_monitors -def main(argv: list[str] | None = None) -> int: +_ACTIVE_VERIFY_RUN: VerifyRun | None = None + + +def _main(argv: list[str] | None = None) -> int: + global _ACTIVE_VERIFY_RUN parser = argparse.ArgumentParser(description="Run the local verification baseline.") parser.add_argument("--quick", action="store_true", help="Skip pytest and run only fast local gates.") parser.add_argument( @@ -3618,6 +3621,7 @@ def main(argv: list[str] | None = None) -> int: environment_fingerprint=environment_fingerprint, worktree_fingerprint=checkout_fingerprint, ) + _ACTIVE_VERIFY_RUN = verify_run seed_identity: dict[str, Any] | None = None resume_testmon_seed = False prepared_seed_attempt: dict[str, Any] | None = None @@ -4061,3 +4065,47 @@ def main(argv: list[str] | None = None) -> int: ) return exit_code + + +def _finalize_verify_runner_exception(run: VerifyRun, exc: Exception, *, use_json: bool) -> int: + """Leave typed, durable failed evidence when verification orchestration raises.""" + diagnosis = "verify_runner_exception" + run.finish_interrupted_steps( + exit_code=125, + diagnosis=diagnosis, + termination_reason="runner_exception", + ) + try: + final_worktree_fingerprint = worktree_fingerprint(ROOT) + except Exception: + final_worktree_fingerprint = "unavailable" + payload = run.finish( + exit_code=125, + duration_s=0.0, + diagnosis=diagnosis, + verification_scope=VerificationScope.AFFECTED.value, + release_baseline_allowed=False, + final_worktree_fingerprint=final_worktree_fingerprint, + ) + payload["exception_type"] = type(exc).__name__ + payload["error"] = str(exc) + _save_history(payload) + if use_json: + _print_json(payload) + sys.stderr.write(f"verify: unexpected runner exception: {exc}\n") + return 125 + + +@finalize_checkout_mutation_monitors +def main(argv: list[str] | None = None) -> int: + global _ACTIVE_VERIFY_RUN + _ACTIVE_VERIFY_RUN = None + try: + return _main(argv) + except Exception as exc: + if _ACTIVE_VERIFY_RUN is None: + raise + raw_argv = sys.argv[1:] if argv is None else argv + return _finalize_verify_runner_exception(_ACTIVE_VERIFY_RUN, exc, use_json="--json" in raw_argv) + finally: + _ACTIVE_VERIFY_RUN = None diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 76d0d24154..906e598fa0 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -284,6 +284,7 @@ def __init__(self, root: Path) -> None: self._ignored_roots: frozenset[Path] = frozenset() self._git_index_path: Path | None = None self._git_authority_paths: dict[Path, str] = {} + self._directory_topology_fingerprint: frozenset[str] | None = None def start(self) -> None: """Start and prove the portable interval watcher before verification.""" @@ -338,6 +339,10 @@ def _watch(self) -> None: ): # An empty timeout batch proves the backend initialized before # a verification command starts, closing the startup race. + if not self._ready.is_set() and not self._directory_topology_is_stable(watched_directories): + with self._state_lock: + self._unavailable = True + return self._ready.set() for _change, raw_path in changes: self._record_change(Path(raw_path)) @@ -399,8 +404,25 @@ def walk_error(_error: OSError) -> None: watched_parent = watched_parent.parent if watched_parent not in directories: directories.append(watched_parent) + self._directory_topology_fingerprint = self._directory_topology(directories) return directories + def _directory_topology(self, directories: Sequence[Path]) -> frozenset[str]: + """Fingerprint source directory membership without trusting pre-watch state.""" + return frozenset( + relative.as_posix() + for directory in directories + if directory.exists() + and (relative := directory.resolve(strict=False)).is_relative_to(self.root) + and (relative := relative.relative_to(self.root)) is not None + ) + + def _directory_topology_is_stable(self, initial_directories: Sequence[Path]) -> bool: + """Reject source directories that changed while the watcher initialized.""" + initial = self._directory_topology_fingerprint or self._directory_topology(initial_directories) + current = self._directory_topology(self._watched_directories()) + return not self._unavailable and current == initial + def _git_tracked_paths(self) -> frozenset[Path]: """Snapshot index membership so tracked paths never inherit ignore rules.""" result = self._git_command(["ls-files", "-z"]) @@ -1088,8 +1110,14 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> dict[str, Any] self.write() return next((dict(step) for step in self._payload["steps"] if step.get("step_id") == step_id), None) - def finish_interrupted_steps(self, *, exit_code: int, diagnosis: str) -> None: - """Close any open step when the outer runner receives Ctrl-C.""" + def finish_interrupted_steps( + self, + *, + exit_code: int, + diagnosis: str, + termination_reason: str = "operator_interrupt", + ) -> None: + """Close every open step when the outer runner cannot continue.""" for step in self._payload["steps"]: if step.get("status") == "running": self.finish_step( @@ -1098,7 +1126,7 @@ def finish_interrupted_steps(self, *, exit_code: int, diagnosis: str) -> None: "duration_s": None, "exit": exit_code, "diagnosis": diagnosis, - "termination_reason": "operator_interrupt", + "termination_reason": termination_reason, }, ) diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index 517e131d80..c45065828c 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -94,6 +94,23 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: return MagicMock(returncode=0, stdout="", stderr="") + env = kwargs.get("env") + if isinstance(env, dict) and merge_gate.VERIFICATION_RECEIPT_PATH_ENV in env: + cwd = kwargs.get("cwd") + checkout_root = Path(cwd) if isinstance(cwd, str) else Path.cwd() + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": local_head_sha, + "checkout_root": str(checkout_root), + "exit_code": local_exit, + "verification_scope": "affected", + "release_baseline_allowed": False, + "terminal_authorization": None, + } + ) + ) return MagicMock( returncode=local_exit, stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 339589f678..a619f53c30 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -83,6 +83,21 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: return MagicMock(returncode=0, stdout=local_head_sha + "\n", stderr="") if cmd[:2] == ["git", "status"]: return MagicMock(returncode=0, stdout=" M dirty.py\n" if dirty else "", stderr="") + env = kwargs.get("env") + if isinstance(env, dict) and merge_gate.VERIFICATION_RECEIPT_PATH_ENV in env: + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": local_head_sha, + "checkout_root": str(Path.cwd()), + "exit_code": local_exit, + "verification_scope": "affected", + "release_baseline_allowed": False, + "terminal_authorization": None, + } + ) + ) return MagicMock( returncode=local_exit, stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), @@ -145,7 +160,9 @@ def test_check_blocks_full_receipt_without_release_baseline_permission( assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 1 -def test_record_consumes_structured_verify_release_permission(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +def test_record_rejects_stale_plausible_stdout_without_bound_receipt( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: monkeypatch.chdir(tmp_path) pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) @@ -157,14 +174,25 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: return base(cmd, **kwargs) return MagicMock( returncode=0, - stdout=json.dumps({"verification_scope": "affected", "release_baseline_allowed": False}), + stdout=json.dumps( + { + "invocation_id": "stale-invocation", + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + "terminal_authorization": None, + } + ), stderr="", ) monkeypatch.setattr(subprocess, "run", _run) assert merge_gate.cmd_record(42, "devtools verify") == 0 receipt = json.loads(merge_gate._receipt_path(42).read_text()) - assert receipt["release_baseline_allowed"] is False + assert receipt["verification_scope"] is None + assert receipt["release_baseline_allowed"] is None def test_record_consumes_receipt_after_streamed_verifier_progress( @@ -184,6 +212,18 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: return base(cmd, **kwargs) if cmd[:3] == ["gh", "pr", "view"]: return base(cmd, **kwargs) + env = cast(dict[str, str], kwargs["env"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + **payload, + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": "abc123", + "checkout_root": str(tmp_path), + "exit_code": 0, + } + ) + ) return MagicMock(returncode=0, stdout=f"pytest progress\n{json.dumps(payload, indent=2)}\n", stderr="") monkeypatch.setattr(subprocess, "run", _run) diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index bcd66f9e2f..f8f9b5d4d9 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -281,6 +281,68 @@ def test_normalize_selection_paths_preserves_pytest_symlinks_and_optional_debug( assert str(target) not in normalized +def test_main_preserves_keyword_and_marker_values_from_tests_directory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[str] = [] + monkeypatch.chdir(run_tests.ROOT / "tests") + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + + def capture(_label: str, command: list[str], **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + captured.extend(command) + return 0, 0.01, {"diagnosis": "pytest_passed"} + + monkeypatch.setattr( + run_tests, + "_run", + capture, + ) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "stable") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) + + assert run_tests.main(["-k", "unit", "-m", "unit"]) == 0 + + keyword_index = captured.index("-k") + marker_index = next(index for index in range(keyword_index + 1, len(captured)) if captured[index] == "-m") + assert captured[keyword_index + 1] == "unit" + assert captured[marker_index + 1] == "unit" + + +def test_main_finalizes_runner_exception_after_open_step( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + history: dict[str, Any] = {} + monkeypatch.setattr(run_tests, "ROOT", tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + run_tests, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "git_head", lambda _root: "head") + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "stable") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: history.update(payload)) + + def explode(_label: str, command: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: + run = kwargs["run"] + run.start_step(label="pytest focused", cmd=command) + raise RuntimeError("focused runner exploded") + + monkeypatch.setattr(run_tests, "_run", explode) + + assert run_tests.main(["focused-selector", "--json"]) == 125 + assert history["exit_code"] == 125 + assert history["diagnosis"] == "focused_test_runner_exception" + assert history["steps"][0]["status"] == "failed" + assert history["steps"][0]["exit"] == 125 + + def test_main_withholds_success_when_checkout_changes_during_pytest( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 168ac9337a..81f89db0a4 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -2077,6 +2077,30 @@ def portable_watch(*paths: Path, **kwargs: object) -> object: assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path="tracked.py") +def test_checkout_mutation_monitor_rejects_source_topology_changed_during_startup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + source = tmp_path / "source" / "package" + source.mkdir(parents=True) + shutil.rmtree(source) + + def portable_watch(*paths: Path, **_kwargs: object) -> object: + assert source not in paths + source.mkdir(parents=True) + yield set() + stop_event = _kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait(timeout=1) + + monkeypatch.setattr(watchfiles, "watch", portable_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + + assert monitor.finish() == CheckoutMutationObservation(changed=False, unavailable=True) + + def test_checkout_mutation_monitor_rejects_forced_polling_backend( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -5376,6 +5400,41 @@ def finish(self) -> CheckoutMutationObservation: assert events == ["monitor-started", "monitor-finished"] +def test_verify_finalizes_runner_exception_after_open_step( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + history: dict[str, Any] = {} + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + verify, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setattr(verify, "maybe_bootstrap_testmon_seed", lambda *_args, **_kwargs: None) + monkeypatch.setattr(verify, "_git_head", lambda: "head") + monkeypatch.setattr(verify, "worktree_fingerprint", lambda _root: "stable") + monitor = MagicMock() + monitor.finish.return_value = CheckoutMutationObservation(changed=False, unavailable=False) + monkeypatch.setattr(verify, "CheckoutMutationMonitor", lambda _root: monitor) + monkeypatch.setattr(verify, "_save_history", lambda payload: history.update(payload)) + monkeypatch.setattr(verify, "build_verify_steps", lambda **_kwargs: [("ruff check", ["ruff", "check"])]) + + def explode(_label: str, command: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: + run = kwargs["run"] + run.start_step(label="ruff check", cmd=command) + raise RuntimeError("verification runner exploded") + + monkeypatch.setattr(verify, "_run", explode) + + assert verify.main(["--quick", "--json"]) == 125 + assert history["exit_code"] == 125 + assert history["diagnosis"] == "verify_runner_exception" + assert history["steps"][0]["status"] == "failed" + assert history["steps"][0]["exit"] == 125 + + def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( monkeypatch: pytest.MonkeyPatch, ) -> None: From 97bf6c75e445f6258fbcfbf0915a1001335bd477 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 16:05:38 +0200 Subject: [PATCH 52/53] fix(devtools): bind terminal verification evidence --- devtools/merge_boundary.py | 32 +++++--- devtools/run_tests.py | 15 +++- devtools/verify.py | 53 +++++++++---- tests/unit/devtools/test_merge_boundary.py | 90 +++++++++++++++++----- tests/unit/devtools/test_run_tests.py | 25 ++++++ tests/unit/devtools/test_verify.py | 30 ++++++-- tests/unit/test_pytest_temp_policy.py | 1 - 7 files changed, 189 insertions(+), 57 deletions(-) diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index 7cf334198e..b7f4f90b6d 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -73,6 +73,7 @@ import sys import tempfile import time +import uuid from collections.abc import Iterator, Mapping from pathlib import Path from typing import Any @@ -809,21 +810,30 @@ def cmd_record_full_verify( if execution_root is not None: argv = ["direnv", "exec", str(execution_root), *argv] started = verification_started_at - try: - result = subprocess.run(argv, capture_output=True, text=True, cwd=cwd) - except OSError as exc: - print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) - return 2 + checkout_root = Path(cwd) if cwd is not None else Path.cwd() + invocation_id = uuid.uuid4().hex + with tempfile.TemporaryDirectory(prefix="polylogue-terminal-verify-") as temp_dir: + receipt_path = Path(temp_dir) / "run.json" + env = dict(os.environ) + env[merge_gate.VERIFICATION_INVOCATION_ID_ENV] = invocation_id + env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV] = str(receipt_path) + try: + result = subprocess.run(argv, capture_output=True, text=True, cwd=checkout_root, env=env) + except OSError as exc: + print(f"REFUSING: could not run {command!r}: {exc}", file=sys.stderr) + return 2 + receipt = merge_gate._invocation_receipt( + path=receipt_path, + invocation_id=invocation_id, + head_sha=target_sha, + command_exit=result.returncode, + checkout_root=checkout_root, + ) duration_s = round(time.time() - started, 2) - try: - structured = json.loads(result.stdout) - except (TypeError, json.JSONDecodeError): - structured = None - receipt = structured if isinstance(structured, dict) else None release_allowed = merge_gate._release_baseline_permission(receipt) verification_scope = merge_gate._verification_scope(receipt) terminal_authorization = merge_gate._terminal_authorization(receipt) - verified_head = structured.get("git_head") if isinstance(structured, dict) else None + verified_head = receipt.get("git_head") if isinstance(receipt, dict) else None accepted = ( result.returncode == 0 and release_allowed is True diff --git a/devtools/run_tests.py b/devtools/run_tests.py index 4ec62ffaee..bebc34a82a 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -349,12 +349,20 @@ def main(argv: list[str] | None = None) -> int: "unavailable" in {initial_worktree_fingerprint, final_worktree_fingerprint} or mutation_observation.unavailable ): - metadata["diagnosis"] = "checkout_fingerprint_unavailable" + checkout_diagnosis = "checkout_fingerprint_unavailable" + if rc == 130: + metadata["checkout_diagnosis"] = checkout_diagnosis + else: + metadata["diagnosis"] = checkout_diagnosis if rc == 0: rc = 125 sys.stderr.write("devtools test: checkout fingerprint unavailable; evidence is not exact-head.\n") elif mutation_observation.changed or final_worktree_fingerprint != initial_worktree_fingerprint: - metadata["diagnosis"] = "checkout_changed_during_focused_test" + checkout_diagnosis = "checkout_changed_during_focused_test" + if rc == 130: + metadata["checkout_diagnosis"] = checkout_diagnosis + else: + metadata["diagnosis"] = checkout_diagnosis metadata["transient_checkout_mutation"] = mutation_observation.changed metadata["checkout_mutation_path"] = mutation_observation.observed_path if rc == 0: @@ -371,6 +379,9 @@ def main(argv: list[str] | None = None) -> int: final_worktree_fingerprint=final_worktree_fingerprint, checkout_mutation_path=mutation_observation.observed_path, ) + recorded_checkout_diagnosis = metadata.get("checkout_diagnosis") + if isinstance(recorded_checkout_diagnosis, str): + payload["checkout_diagnosis"] = recorded_checkout_diagnosis append_verify_history(payload) if use_json: print(json.dumps(payload, indent=2, ensure_ascii=False)) diff --git a/devtools/verify.py b/devtools/verify.py index 56de0edd4e..da4a268652 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -3513,7 +3513,23 @@ def _discard_testmon_dependency_authority() -> None: # ── main ──────────────────────────────────────────────────────────── -_ACTIVE_VERIFY_RUN: VerifyRun | None = None +_ACTIVE_VERIFY_RUN: tuple[VerifyRun, float, VerificationScope] | None = None + + +def _planned_verification_scope(args: argparse.Namespace, *, full_pytest: bool) -> VerificationScope: + """Return the immutable scope requested before the runner starts.""" + if args.quick or args.commit: + return VerificationScope.NON_TEST + if full_pytest or args.seed_testmon: + return VerificationScope.NARROW_TERMINAL if args.skip_slow else VerificationScope.RELEASE_BASELINE + return VerificationScope.AFFECTED + + +def _changed_paths_from_testmon_authority(base_commit: str | None, head_commit: str | None) -> tuple[str, ...]: + """Require immutable refs before deriving affected executable paths.""" + if base_commit is None or head_commit is None: + raise PytestResourceError("testmon changed-path authority is unavailable") + return _changed_executable_paths(base_commit, head_commit) def _main(argv: list[str] | None = None) -> int: @@ -3592,6 +3608,7 @@ def _main(argv: list[str] | None = None) -> int: head = _git_head() full_pytest = bool(args.all or args.full) affected_testmon = not (args.quick or args.commit or args.seed_testmon or full_pytest) + planned_verification_scope = _planned_verification_scope(args, full_pytest=full_pytest) testmon_base_commit = _git_commit("origin/master") if affected_testmon else None testmon_head_commit = head if affected_testmon else None if affected_testmon and (testmon_base_commit is None or testmon_head_commit is None): @@ -3621,7 +3638,7 @@ def _main(argv: list[str] | None = None) -> int: environment_fingerprint=environment_fingerprint, worktree_fingerprint=checkout_fingerprint, ) - _ACTIVE_VERIFY_RUN = verify_run + _ACTIVE_VERIFY_RUN = (verify_run, t0, planned_verification_scope) seed_identity: dict[str, Any] | None = None resume_testmon_seed = False prepared_seed_attempt: dict[str, Any] | None = None @@ -3713,10 +3730,8 @@ def _main(argv: list[str] | None = None) -> int: refreshed_stamp = refresh_stamp(current_stamp, TESTMON_DATA) if refreshed_stamp is not None: pending_testmon_stamp = refreshed_stamp - assert testmon_base_commit is not None - assert testmon_head_commit is not None try: - executable_paths = _changed_executable_paths(testmon_base_commit, testmon_head_commit) + executable_paths = _changed_paths_from_testmon_authority(testmon_base_commit, testmon_head_commit) except PytestResourceError as exc: changed_path_authority_failed = True executable_paths = () @@ -3989,8 +4004,8 @@ def _main(argv: list[str] | None = None) -> int: "release_baseline_allowed": seed_receipt["release_baseline_allowed"], } + verification_scope = planned_verification_scope if args.quick or args.commit: - verification_scope = VerificationScope.NON_TEST # Non-test verification is intentionally not release authority, but it # is still a typed verification receipt. ``None`` made merge-gate # treat an explicit quick receipt as malformed instead of as a valid @@ -3999,9 +4014,6 @@ def _main(argv: list[str] | None = None) -> int: elif full_pytest or args.seed_testmon: narrow_terminal = bool(args.skip_slow) authorized_narrow_terminal = args.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value - verification_scope = ( - VerificationScope.NARROW_TERMINAL if narrow_terminal else VerificationScope.RELEASE_BASELINE - ) if full_pytest: release_baseline_allowed = exit_code == 0 and (not narrow_terminal or authorized_narrow_terminal) else: @@ -4009,7 +4021,6 @@ def _main(argv: list[str] | None = None) -> int: not narrow_terminal or authorized_narrow_terminal ) else: - verification_scope = VerificationScope.AFFECTED release_baseline_allowed = _testmon_release_baseline_permission() history_entry["verification_scope"] = verification_scope.value history_entry["release_baseline_allowed"] = release_baseline_allowed @@ -4067,7 +4078,14 @@ def _main(argv: list[str] | None = None) -> int: return exit_code -def _finalize_verify_runner_exception(run: VerifyRun, exc: Exception, *, use_json: bool) -> int: +def _finalize_verify_runner_exception( + run: VerifyRun, + exc: Exception, + *, + run_started: float, + verification_scope: VerificationScope, + use_json: bool, +) -> int: """Leave typed, durable failed evidence when verification orchestration raises.""" diagnosis = "verify_runner_exception" run.finish_interrupted_steps( @@ -4081,9 +4099,9 @@ def _finalize_verify_runner_exception(run: VerifyRun, exc: Exception, *, use_jso final_worktree_fingerprint = "unavailable" payload = run.finish( exit_code=125, - duration_s=0.0, + duration_s=time.monotonic() - run_started, diagnosis=diagnosis, - verification_scope=VerificationScope.AFFECTED.value, + verification_scope=verification_scope.value, release_baseline_allowed=False, final_worktree_fingerprint=final_worktree_fingerprint, ) @@ -4106,6 +4124,13 @@ def main(argv: list[str] | None = None) -> int: if _ACTIVE_VERIFY_RUN is None: raise raw_argv = sys.argv[1:] if argv is None else argv - return _finalize_verify_runner_exception(_ACTIVE_VERIFY_RUN, exc, use_json="--json" in raw_argv) + run, run_started, verification_scope = _ACTIVE_VERIFY_RUN + return _finalize_verify_runner_exception( + run, + exc, + run_started=run_started, + verification_scope=verification_scope, + use_json="--json" in raw_argv, + ) finally: _ACTIVE_VERIFY_RUN = None diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index c45065828c..b4c41f1a0d 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -4,7 +4,7 @@ import os import subprocess import threading -from collections.abc import Callable +from collections.abc import Callable, Mapping from pathlib import Path from typing import Any from unittest.mock import MagicMock @@ -97,7 +97,7 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: env = kwargs.get("env") if isinstance(env, dict) and merge_gate.VERIFICATION_RECEIPT_PATH_ENV in env: cwd = kwargs.get("cwd") - checkout_root = Path(cwd) if isinstance(cwd, str) else Path.cwd() + checkout_root = Path(cwd) if cwd is not None else Path.cwd() Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( json.dumps( { @@ -120,6 +120,32 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: return _run +def _write_terminal_receipt( + kwargs: Mapping[str, Any], + *, + head: str = "merged-master", + scope: str = "release-baseline", + release_allowed: bool = True, + terminal_authorization: str | None = None, +) -> None: + env = kwargs["env"] + assert isinstance(env, dict) + cwd = Path(kwargs["cwd"]) + Path(env[merge_gate.VERIFICATION_RECEIPT_PATH_ENV]).write_text( + json.dumps( + { + "invocation_id": env[merge_gate.VERIFICATION_INVOCATION_ID_ENV], + "git_head": head, + "checkout_root": str(cwd), + "exit_code": 0, + "verification_scope": scope, + "release_baseline_allowed": release_allowed, + "terminal_authorization": terminal_authorization, + } + ) + ) + + # --------------------------------------------------------------------------- # clean_merge_title # --------------------------------------------------------------------------- @@ -503,6 +529,7 @@ def test_merge_with_verify_records_terminal_full_verify(monkeypatch: pytest.Monk def run(cmd: list[str], **kwargs: Any) -> MagicMock: if cmd[:3] == ["devtools", "verify", "--all"]: + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( @@ -596,7 +623,7 @@ def test_post_merge_terminal_verify_uses_target_checkout_devshell( monkeypatch.chdir(tmp_path) commands: list[list[str]] = [] - def run(cmd: list[str], **_kwargs: Any) -> MagicMock: + def run(cmd: list[str], **kwargs: Any) -> MagicMock: commands.append(cmd) if cmd[:3] == ["git", "worktree", "add"]: return MagicMock(returncode=0, stdout="", stderr="") @@ -615,6 +642,7 @@ def run(cmd: list[str], **_kwargs: Any) -> MagicMock: python_executable=target / ".venv" / "bin" / "python", ) assert fingerprint.clean + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( @@ -822,6 +850,7 @@ def test_record_full_verify_clears_pending_prs(monkeypatch: pytest.MonkeyPatch, merge_boundary._append_merge_entry(1, "sha1", "some title") def _run(cmd: list[str], **kwargs: Any) -> MagicMock: + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( @@ -842,6 +871,29 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: assert merge_boundary.cmd_train_status(as_json=False) == 0 +def test_record_full_verify_rejects_plausible_unbound_stdout(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + monkeypatch.setattr( + subprocess, + "run", + lambda _cmd, **_kwargs: MagicMock( + returncode=0, + stdout=json.dumps( + { + "git_head": "merged-master", + "verification_scope": "release-baseline", + "release_baseline_allowed": True, + } + ), + stderr="", + ), + ) + + assert merge_boundary.cmd_record_full_verify("devtools verify --all", target_sha="merged-master") == 1 + assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False + + def test_record_full_verify_propagates_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) merge_boundary._append_merge_entry(1, "sha1", "some title") @@ -903,22 +955,16 @@ def test_record_full_verify_accepts_explicit_typed_narrow_terminal_authorization ) -> None: monkeypatch.chdir(tmp_path) merge_boundary._append_merge_entry(1, "sha1", "some title") - monkeypatch.setattr( - subprocess, - "run", - lambda _cmd, **_kwargs: MagicMock( - returncode=0, - stdout=json.dumps( - { - "git_head": "merged-master", - "verification_scope": "narrow-terminal", - "terminal_authorization": "narrow-terminal", - "release_baseline_allowed": True, - } - ), - stderr="", - ), - ) + + def run(_cmd: list[str], **kwargs: Any) -> MagicMock: + _write_terminal_receipt( + kwargs, + scope="narrow-terminal", + terminal_authorization="narrow-terminal", + ) + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", run) assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 0 assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is True @@ -951,11 +997,12 @@ def test_concurrent_merge_during_terminal_verify_remains_pending( monkeypatch.chdir(tmp_path) inserted = False - def run(_cmd: list[str], **_kwargs: Any) -> MagicMock: + def run(_cmd: list[str], **kwargs: Any) -> MagicMock: nonlocal inserted if not inserted: inserted = True merge_boundary._append_merge_entry(99, "concurrent-sha", "concurrent merge") + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( @@ -982,7 +1029,7 @@ def test_concurrent_ledger_writer_cannot_lose_merge_entry(monkeypatch: pytest.Mo started = threading.Event() writer: threading.Thread | None = None - def run(_cmd: list[str], **_kwargs: Any) -> MagicMock: + def run(_cmd: list[str], **kwargs: Any) -> MagicMock: nonlocal writer def append() -> None: @@ -992,6 +1039,7 @@ def append() -> None: writer = threading.Thread(target=append) writer.start() assert started.wait(timeout=1) + _write_terminal_receipt(kwargs) return MagicMock( returncode=0, stdout=json.dumps( diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index f8f9b5d4d9..c8f6e896f1 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -150,6 +150,8 @@ def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") monkeypatch.setattr("devtools.run_tests._clear_pytest_report", lambda _cmd: None) monkeypatch.setattr("devtools.run_tests._run", _fake_run) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "stable") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) monkeypatch.setattr("devtools.run_tests.append_verify_history", lambda _payload: None) assert run_tests.main(["core/test_identity_law.py::test_session_id_is_origin_native_id"]) == 0 @@ -174,6 +176,8 @@ def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) monkeypatch.setattr(run_tests, "_run", _fake_run) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: "stable") + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) monkeypatch.setattr(run_tests, "append_verify_history", lambda _payload: None) assert ( @@ -202,6 +206,27 @@ def _fake_run(_label: str, cmd: list[str], **_kwargs: Any) -> tuple[int, float, assert command[command.index("--junit-xml") + 1] == str(invocation / "reports" / "results.xml") +def test_main_keeps_interruption_diagnosis_when_checkout_verification_finds_a_change( + monkeypatch: pytest.MonkeyPatch, +) -> None: + history: dict[str, Any] = {} + fingerprints = iter(("before", "after")) + + def interrupt(*_args: Any, **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + raise KeyboardInterrupt + + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", interrupt) + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprints)) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: history.update(payload)) + + assert run_tests.main(["tests/unit/example.py"]) == 130 + assert history["diagnosis"] == "pytest_interrupted" + assert history["checkout_diagnosis"] == "checkout_changed_during_focused_test" + + def test_normalize_selection_paths_preserves_pytest_path_option_semantics( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 81f89db0a4..829793d5d5 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1933,14 +1933,14 @@ def test_git_head_uses_bounded_authoritative_probe() -> None: def test_checkout_mutation_monitor_detects_a_change_that_reverts_before_the_final_fingerprint( tmp_path: Path, ) -> None: - subprocess.run(["git", "init", "-q"], check=True) - subprocess.run(["git", "config", "user.email", "tests@example.invalid"], check=True) - subprocess.run(["git", "config", "user.name", "Polylogue Tests"], check=True) + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "tests@example.invalid"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Polylogue Tests"], cwd=tmp_path, check=True) tracked = tmp_path / "tracked.py" original = "VALUE = 1\n" tracked.write_text(original, encoding="utf-8") - subprocess.run(["git", "add", "tracked.py"], check=True) - subprocess.run(["git", "commit", "-qm", "seed"], check=True) + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "seed"], cwd=tmp_path, check=True) monitor = CheckoutMutationMonitor(tmp_path) monitor.start() @@ -1994,7 +1994,7 @@ def test_checkout_mutation_monitor_uses_gitignore_for_verifier_task_history( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - subprocess.run(["git", "init", "-q"], check=True) + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) (tmp_path / ".gitignore").write_text(".agent/*\n", encoding="utf-8") history = tmp_path / ".agent" / "task-history" / "tasks.jsonl" history.parent.mkdir(parents=True) @@ -2039,7 +2039,7 @@ def test_checkout_mutation_monitor_uses_portable_watchfiles_events_without_linux tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - subprocess.run(["git", "init", "-q"], check=True) + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) tracked = tmp_path / "tracked.py" tracked.write_text("VALUE = 1\n", encoding="utf-8") calls: dict[str, object] = {} @@ -2141,7 +2141,7 @@ def test_checkout_mutation_monitor_prunes_disposable_trees_and_observes_new_sour tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - subprocess.run(["git", "init", "-q"], check=True) + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) (tmp_path / ".gitignore").write_text( "browser-extension/node_modules/\ncustom/generated-output/\n", encoding="utf-8", @@ -5427,10 +5427,14 @@ def explode(_label: str, command: list[str], **kwargs: Any) -> tuple[int, float, raise RuntimeError("verification runner exploded") monkeypatch.setattr(verify, "_run", explode) + monotonic_values = iter((100.0, 107.5)) + monkeypatch.setattr("devtools.verify.time.monotonic", lambda: next(monotonic_values)) assert verify.main(["--quick", "--json"]) == 125 assert history["exit_code"] == 125 assert history["diagnosis"] == "verify_runner_exception" + assert history["duration_s"] == 7.5 + assert history["verification_scope"] == "non-test" assert history["steps"][0]["status"] == "failed" assert history["steps"][0]["exit"] == 125 @@ -5507,6 +5511,16 @@ def test_verify_finalizes_and_discards_graph_when_post_pytest_path_authority_fai assert payload["diagnosis"] == "testmon_changed_path_authority_unavailable" +def test_testmon_changed_path_authority_refuses_missing_commit_binding() -> None: + changed_paths = MagicMock() + + with patch("devtools.verify._changed_executable_paths", changed_paths): + with pytest.raises(PytestResourceError, match="changed-path authority is unavailable"): + verify._changed_paths_from_testmon_authority(None, "head") + + changed_paths.assert_not_called() + + def test_verify_accepts_zero_testmon_selection_after_matching_coverage( capsys: pytest.CaptureFixture[str], ) -> None: diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index d38e74ca43..9b4242851e 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -451,7 +451,6 @@ def test_nested_supervised_explicit_basetemp_reuse_is_rejected_before_claiming( ) monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_SCOPES", []) monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_BASETEMPS", set()) - monkeypatch.setattr(conftest, "_ACTIVE_PYTEST_BASETEMPS", set()) monkeypatch.setenv("POLYLOGUE_VERIFY_RUN_ID", run_id) monkeypatch.setenv("POLYLOGUE_PYTEST_RUN_ID", run_id) monkeypatch.setenv("POLYLOGUE_PYTEST_MANAGED_BASETEMP", str(active)) From 51a32ef4a280392298ebb1051e6d81363a851c74 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 13 Aug 2026 16:21:34 +0200 Subject: [PATCH 53/53] fix(devtools): persist checkout interruption evidence --- devtools/run_tests.py | 6 ++-- devtools/verify_runs.py | 3 ++ tests/unit/devtools/test_run_tests.py | 42 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/devtools/run_tests.py b/devtools/run_tests.py index bebc34a82a..5c1ac8d8d7 100644 --- a/devtools/run_tests.py +++ b/devtools/run_tests.py @@ -378,10 +378,10 @@ def main(argv: list[str] | None = None) -> int: release_baseline_allowed=False, final_worktree_fingerprint=final_worktree_fingerprint, checkout_mutation_path=mutation_observation.observed_path, + checkout_diagnosis=( + metadata["checkout_diagnosis"] if isinstance(metadata.get("checkout_diagnosis"), str) else None + ), ) - recorded_checkout_diagnosis = metadata.get("checkout_diagnosis") - if isinstance(recorded_checkout_diagnosis, str): - payload["checkout_diagnosis"] = recorded_checkout_diagnosis append_verify_history(payload) if use_json: print(json.dumps(payload, indent=2, ensure_ascii=False)) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index 906e598fa0..30d46bfaf1 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -1141,6 +1141,7 @@ def finish( terminal_authorization: str | None = None, final_worktree_fingerprint: str | None = None, checkout_mutation_path: str | None = None, + checkout_diagnosis: str | None = None, ) -> dict[str, Any]: self._payload["finished_at"] = utc_now() self._payload["duration_s"] = round(duration_s, 2) @@ -1152,6 +1153,8 @@ def finish( self._payload["final_worktree_fingerprint"] = final_worktree_fingerprint if checkout_mutation_path is not None: self._payload["checkout_mutation_path"] = checkout_mutation_path + if checkout_diagnosis is not None: + self._payload["checkout_diagnosis"] = checkout_diagnosis if verification_scope is not None: self._payload["verification_scope"] = verification_scope self._payload["release_baseline_allowed"] = release_baseline_allowed diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index c8f6e896f1..c57c78538a 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import subprocess import sys from pathlib import Path @@ -12,7 +13,10 @@ from devtools import run_tests, verify from devtools.verify_runs import ( + CURRENT_RUN_PATH, CURRENT_STATISTICS_PATH, + VERIFICATION_INVOCATION_ID_ENV, + VERIFICATION_RECEIPT_PATH_ENV, CheckoutMutationObservation, git_head, pytest_command_worker_request, @@ -227,6 +231,44 @@ def interrupt(*_args: Any, **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: assert history["checkout_diagnosis"] == "checkout_changed_during_focused_test" +def test_main_persists_interrupted_checkout_diagnosis_to_all_run_artifacts( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + history: dict[str, Any] = {} + fingerprints = iter(("before", "after")) + receipt = tmp_path / "receipt.json" + + def interrupt(*_args: Any, **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: + raise KeyboardInterrupt + + monkeypatch.setattr(run_tests, "ROOT", tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + run_tests, + "assert_polylogue_matches_checkout", + lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), + ) + monkeypatch.setenv("POLYLOGUE_TEST_NO_LOCK", "1") + monkeypatch.setenv(VERIFICATION_INVOCATION_ID_ENV, "focused-interrupt") + monkeypatch.setenv(VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + monkeypatch.setattr(run_tests, "_clear_pytest_report", lambda _cmd: None) + monkeypatch.setattr(run_tests, "_run", interrupt) + monkeypatch.setattr(run_tests, "git_head", lambda _root: "head") + monkeypatch.setattr(run_tests, "worktree_fingerprint", lambda _root: next(fingerprints)) + monkeypatch.setattr(run_tests, "CheckoutMutationMonitor", _NoMutationMonitor) + monkeypatch.setattr(run_tests, "append_verify_history", lambda payload: history.update(payload)) + + assert run_tests.main(["tests/unit/example.py"]) == 130 + + run_payload = json.loads((tmp_path / history["artifact_dir"] / "run.json").read_text()) + current_payload = json.loads((tmp_path / CURRENT_RUN_PATH).read_text()) + receipt_payload = json.loads(receipt.read_text()) + for payload in (history, run_payload, current_payload, receipt_payload): + assert payload["diagnosis"] == "pytest_interrupted" + assert payload["checkout_diagnosis"] == "checkout_changed_during_focused_test" + + def test_normalize_selection_paths_preserves_pytest_path_option_semantics( monkeypatch: pytest.MonkeyPatch, tmp_path: Path,