diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d546397d2..6e4ce7d51 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -310,8 +310,8 @@ complete correctness corpus. The pytest step covers unit, property, fuzz, and integration tests while excluding the separately operated `tests/benchmarks` performance surface. It uses `--testmon-forceselect` for incremental selection, with one parallel `not -load_sensitive` lane and one serial `load_sensitive` lane over the same native -environment. `tui` is a category marker and remains parallel unless a test is +load_sensitive` lane and one bounded-concurrency `load_sensitive` lane over the +same native environment. `tui` is a category marker and remains parallel unless a test is also explicitly `load_sensitive`. Use There is no manual seed or repair command, and no separate full-corpus flag: every plain run is scoped to the complete corpus, executing what changed and diff --git a/TESTING.md b/TESTING.md index 466753bf6..eadb24716 100644 --- a/TESTING.md +++ b/TESTING.md @@ -84,10 +84,22 @@ concurrency when memory pressure is elevated. Every native run has exactly two semantic lanes over one environment and one database: a parallel lane for tests not marked `load_sensitive`, followed by a -serial lane for the load-sensitive set. Ordinary test failures in the parallel -lane do not suppress the serial lane. Typed collection, containment, resource, +bounded lane for the load-sensitive set. Ordinary test failures in the parallel +lane do not suppress the bounded lane. Typed collection, containment, resource, or timeout failures do. +The bounded lane is not strictly serial. `load_sensitive` marks a test whose +wall-clock deadlines the *parallel* lane's full worker count starves, which +bounds how much concurrency the lane may use — it does not establish that the +members contend with each other. Measured on the 7-test daemon-resilience +corpus (2026-08-19): 71.95s at one process, 35.08s at four workers, and a hard +cliff above that (5 workers 107.76s with one starved SIGTERM deadline, 7 workers +95.20s with two). The lane therefore runs at +`devtools.verify.SERIAL_LANE_MAX_WORKERS` (4) under `--dist=loadgroup`, with +members assigned to `xdist_group` bins packed longest-first so the makespan is +bounded by the largest bin rather than by the order xdist happens to dispatch +in. Raising the cap requires repeating that measurement. + The lane boundary is evidence-based. On 2026-08-13 the complete correctness corpus collected 20,447 nodes in 35.06s; only 17 were `load_sensitive`, while 16 were tagged `tui` with no overlap. A managed serial run retained 93.09s of diff --git a/devtools/pytest_supervisor.py b/devtools/pytest_supervisor.py index 42aeb1e6a..2e99a9b8b 100644 --- a/devtools/pytest_supervisor.py +++ b/devtools/pytest_supervisor.py @@ -911,20 +911,82 @@ def _restore_owner_write(root: Path) -> None: candidate.chmod(candidate.stat().st_mode | stat.S_IWUSR) +def force_rmtree(path: Path, *, ignore_errors: bool = False) -> None: + """Remove a harness-owned tree, repairing the permissions the harness revoked. + + The shared entry point for every reclaimer, so the repair above has exactly + one implementation. `verify_runs.cleanup_managed_pytest_basetemp` and the + stale-basetemp sweep both need it: the read-only artifact trees that caused + polylogue-b9yw7 defeat a plain rmtree wherever it is called from, and the + supervisor's exit pass was only one of those places. + + Repair is the recovery path, not the common one -- an ordinary tree is + removed by the first call and never walked twice -- and it only ever runs + against a tree the caller has already established is harness-owned. + """ + try: + shutil.rmtree(path) + return + except FileNotFoundError: + return + except OSError: + pass + _restore_owner_write(path) + try: + shutil.rmtree(path) + except FileNotFoundError: + return + except OSError: + if not ignore_errors: + raise + + def cleanup_managed_tmpfs_path(path: Path | None) -> bool: """Remove only a harness-owned direct child of the system tmpfs.""" - if path is None or not path.name.startswith("pytest-polylogue-"): - return False + return describe_managed_tmpfs_cleanup(path)[0] + + +def describe_managed_tmpfs_cleanup(path: Path | None) -> tuple[bool, str, list[str]]: + """Reclaim a harness-owned tmpfs basetemp and say what happened. + + Returns ``(complete, reason, residual)``. The reason and residual sample + exist because a bare ``False`` here is load-bearing and opaque: it becomes + the run's ``cleanup.complete``, which `_release_baseline_allowed` requires + to be True, so an otherwise-green complete-corpus run silently loses release + authority over a scratch directory that failed to unlink. Observed + 2026-08-19 (run 20260819T003921Z, exit 0, no signals, controller group + quiescent) with nothing in the receipt to say why (polylogue-b9yw7). + + The cause found for that receipt was a permission bit, not a race: + published seeded-archive artifacts chmod their DIRECTORIES non-writable, and + entries cannot be unlinked from a directory without its write bit, so the + tree was simply unremovable. `_restore_owner_write` above repairs exactly + that. The residual sample stays because it is what makes the NEXT such + failure legible in one receipt rather than a night of bisecting: it names + the surviving paths, which is how this one was ultimately identified. + """ + if path is None: + return False, "no managed tmpfs path for this run", [] + if not path.name.startswith("pytest-polylogue-"): + return False, f"not a harness-owned basetemp name: {path.name}", [] try: if path.parent.resolve() != Path("/dev/shm").resolve(): - return False - except OSError: - return False - shutil.rmtree(path, ignore_errors=True) - if path.exists(): - _restore_owner_write(path) - shutil.rmtree(path, ignore_errors=True) - return not path.exists() + return False, f"not a direct child of /dev/shm: {path.parent}", [] + except OSError as exc: + return False, f"could not resolve parent: {exc}", [] + if not path.exists(): + return True, "already reclaimed before this pass", [] + force_rmtree(path, ignore_errors=True) + if not path.exists(): + return True, "reclaimed by this pass", [] + residual: list[str] = [] + with contextlib.suppress(OSError): + for index, entry in enumerate(path.rglob("*")): + if index >= 20: + residual.append("... (truncated)") + break + residual.append(str(entry.relative_to(path))) + return False, "tree survived rmtree", residual def main(argv: Sequence[str] | None = None) -> int: @@ -945,7 +1007,14 @@ def main(argv: Sequence[str] | None = None) -> int: finally: receipt = read_receipt(args.receipt) receipt_quiescent = receipt is not None and receipt.get("controller_group_alive") is False - cleanup_complete = cleanup_managed_tmpfs_path(args.cleanup_path) if receipt_quiescent else False + if receipt_quiescent: + cleanup_complete, cleanup_reason, cleanup_residual = describe_managed_tmpfs_cleanup(args.cleanup_path) + else: + cleanup_complete, cleanup_reason, cleanup_residual = ( + False, + "controller process group still alive at supervisor exit", + [], + ) if args.cleanup_path is not None: with contextlib.suppress(OSError): update_receipt( @@ -953,6 +1022,8 @@ def main(argv: Sequence[str] | None = None) -> int: { "tmpfs_cleanup_path": str(args.cleanup_path), "tmpfs_cleanup_complete": cleanup_complete, + "tmpfs_cleanup_reason": cleanup_reason, + "tmpfs_cleanup_residual": cleanup_residual, }, ) diff --git a/devtools/verify.py b/devtools/verify.py index 02976b749..f52903f24 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -41,7 +41,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import Any +from typing import Any, Final from devtools.checkout_guard import ( CheckoutImportMismatchError, @@ -119,6 +119,7 @@ pytest_tmpfs_budget_exceeded, pytest_tmpfs_budget_kb, start_checkout_mutation_monitor, + sweep_stale_managed_basetemps, utc_now, worktree_fingerprint, xdist_uninterruptible_stall_reason, @@ -2109,6 +2110,7 @@ def _run_step( runtime_policy = None pytest_concurrency = 0 basetemp_cleanup: Path | None = None + swept: list[Path] = [] if is_pytest or has_managed_pytest_child: try: if has_managed_pytest_child: @@ -2120,6 +2122,13 @@ def _run_step( env = force_managed_pytest_scratch(env) if is_pytest: pytest_concurrency = _pytest_command_concurrency(cmd, env=env) + # Reclaim prior runs' leaked tmpfs trees BEFORE admission, not only + # at exit. Cleanup used to be purely trailing, so a tree that failed + # to unlink stayed resident until someone noticed -- 16 of them and + # two blocked merges on 2026-08-19 (polylogue-b9yw7). Sweeping here + # also feeds the decision immediately below: the space it returns is + # headroom the basetemp admission policy can then admit against. + swept = sweep_stale_managed_basetemps() env, runtime_policy = apply_managed_pytest_runtime_policy( env, worker_count=pytest_concurrency, @@ -2241,6 +2250,7 @@ def _run_step( metadata["postmortem_path"] = str(CURRENT_POSTMORTEM_PATH) metadata["containment_path"] = str(PYTEST_CONTAINMENT_PATH) metadata["basetemp_cleanup"] = str(basetemp_cleanup) if basetemp_cleanup is not None else None + metadata["basetemp_swept"] = [str(path) for path in swept] junit_paths = [ str(path) for path in _pytest_artifact_paths(cmd) if path.suffix == ".xml" or path.name.endswith(".xml") ] @@ -2597,6 +2607,7 @@ def _native_pytest_steps( testmon_mode: str, testmon_environment: str, parallel_worker_args: Sequence[str], + serial_worker_args: Sequence[str], ) -> list[tuple[str, list[str]]]: pytest_cmd = [ sys.executable, @@ -2655,8 +2666,7 @@ def _serial_report_arg(arg: str) -> str: *native_args, "-p", "no:randomly", - "-n", - "0", + *serial_worker_args, ] ) return [ @@ -2674,10 +2684,15 @@ def _native_pytest_command_is_closed_world(label: str, cmd: Sequence[str]) -> bo worker_request = pytest_command_worker_request(cmd) if len(environment_args) != 1 or worker_request is None or not worker_request.isdigit(): return False + # Only the labelled lane's command is compared below, so reconstructing both + # lanes from the observed worker request is exact for the lane under test and + # irrelevant for its sibling. + observed_worker_args = ("--dist=loadgroup", "-n", worker_request) expected_steps = _native_pytest_steps( testmon_mode=match.group(2), testmon_environment=environment_args[0].removeprefix("--testmon-env="), - parallel_worker_args=("--dist=loadgroup", "-n", worker_request), + parallel_worker_args=observed_worker_args, + serial_worker_args=observed_worker_args, ) expected = dict(expected_steps).get(label) return expected is not None and list(cmd) == expected @@ -2755,6 +2770,7 @@ def build_verify_steps( testmon_mode=testmon_mode, testmon_environment=testmon_environment, parallel_worker_args=_pytest_worker_args(), + serial_worker_args=_pytest_worker_args(maximum=SERIAL_LANE_MAX_WORKERS), ) ) @@ -2865,6 +2881,27 @@ def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: temporary.replace(path) +#: Worker ceiling for the `load_sensitive` lane. +#: +#: The lane exists because these tests drive real daemon subprocesses under +#: wall-clock deadlines, and the parallel lane's full worker count starves +#: interpreter startup badly enough to flake them. It does NOT follow that the +#: lane must be strictly serial: measured on the 7-test corpus (2026-08-19, +#: tests/integration/test_daemon_resilience.py, same host and containment), +#: +#: -n 0 71.95s green +#: -n 4 35.08s green +#: -n 5 107.76s 1 failed (SIGTERM deadline starved) +#: -n 7 95.20s 2 failed (both SIGTERM tests, 90s subprocess timeout blown) +#: +#: so 4 is the last concurrency the deadline-sensitive members survive, with a +#: clear cliff immediately above it. Raising this needs the same measurement, +#: repeated, not an assumption that a faster host has more headroom -- the +#: failures are starvation of an idle-scheduled containment slice, not a +#: shortage of cores. +SERIAL_LANE_MAX_WORKERS: Final = 4 + + def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: """Return the managed worker count, optionally capped for a bounded lane.""" workers = adaptive_pytest_worker_count(os.environ) diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index b64b69c30..110e4039a 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -30,6 +30,7 @@ import watchfiles from devtools.cloud_sentinels import CLOUD_SENTINELS, running_in_cloud_sandbox +from devtools.pytest_supervisor import force_rmtree from devtools.testmon_bootstrap import canonical_test_nodeid from polylogue.core.metrics import read_cgroup_memory_headroom_bytes @@ -1237,6 +1238,12 @@ def aggregate_pytest_statistics( "complete": True if isinstance(parent_cleanup, str) and parent_cleanup else containment.get("tmpfs_cleanup_complete"), + # Carried so an incomplete cleanup explains itself. `complete` gates + # release-baseline authority, and until 2026-08-19 a False here left + # no evidence of which reclaimer failed or what survived + # (polylogue-b9yw7). + "reason": containment.get("tmpfs_cleanup_reason"), + "residual": containment.get("tmpfs_cleanup_residual"), "termination_reason": containment.get("termination_reason"), "escalated_to_sigkill": containment.get("escalated_to_sigkill"), "exit_code": containment.get("exit_code", (step_result or {}).get("exit")), @@ -2807,7 +2814,10 @@ def cleanup_managed_pytest_basetemp(*, root: Path, run_id: str, env: dict[str, s return None with contextlib.suppress(OSError): if basetemp.exists(): - shutil.rmtree(basetemp) + # Permission-repairing: a published seeded-archive cache under + # this tree is deliberately read-only, directories included, and + # a plain rmtree cannot unlink through it (polylogue-b9yw7). + force_rmtree(basetemp) if not basetemp.exists(): clear_managed_pytest_basetemp_claim(basetemp) return basetemp @@ -2817,6 +2827,49 @@ def cleanup_managed_pytest_basetemp(*, root: Path, run_id: str, env: dict[str, s return None +def sweep_stale_managed_basetemps(*, limit: int = 64) -> list[Path]: + """Reclaim harness-owned tmpfs basetemps left behind by finished runs. + + Cleanup used to be purely trailing: each run removed its own tree at exit, + so anything that failed to unlink stayed until someone noticed. On + 2026-08-19 that was 16 leaked trees and two blocked merges, because the + removal itself could not succeed (see `force_rmtree`). Repairing removal + without also reclaiming the existing debt would leave those trees resident + on a 16 GiB tmpfs indefinitely, so the harness now sweeps at start as well + as at exit -- eager and self-healing rather than trailing only. + + Ownership is the same test the exit path applies, and it is conservative in + the same direction: a tree is reclaimed only when its claim lock is free + (nothing else is using it) AND its recorded owner is positively dead. An + unknown or live owner is left alone. The shared `-seeded-` corpus cache is + never a candidate; it is a deliberate cross-run artifact, not run debt. + """ + reclaimed: list[Path] = [] + shm = Path("/dev/shm") + try: + candidates = sorted(shm.glob("pytest-polylogue-*")) + except OSError: + return reclaimed + for candidate in candidates[:limit]: + if not candidate.is_dir() or "-seeded-" in candidate.name: + continue + claim_lock = _try_acquire_pytest_basetemp_claim_lock(candidate) + if claim_lock is None: + continue + try: + if managed_pytest_basetemp_owner_alive(candidate) is not False: + continue + with contextlib.suppress(OSError): + force_rmtree(candidate) + if not candidate.exists(): + clear_managed_pytest_basetemp_claim(candidate) + reclaimed.append(candidate) + finally: + with contextlib.suppress(OSError): + claim_lock.close() + return reclaimed + + def _pytest_event_worker_ids(events_dir: Path | None) -> dict[int, str]: """Recover xdist worker identities emitted after process exec. diff --git a/docs/plans/oracle-integrity-baseline.json b/docs/plans/oracle-integrity-baseline.json index 69e9ce3cf..ef9711c46 100644 --- a/docs/plans/oracle-integrity-baseline.json +++ b/docs/plans/oracle-integrity-baseline.json @@ -2,7 +2,7 @@ "entries": [ { "code": "ambient_path_call", - "detail": "line 136: resolves an ambient location via Path.home()", + "detail": "line 165: resolves an ambient location via Path.home()", "path": "tests/integration/test_daemon_resilience.py" }, { @@ -82,7 +82,7 @@ }, { "code": "ambient_path_literal", - "detail": "line 1456: reads ambient path '~/.codex'", + "detail": "line 1464: reads ambient path '~/.codex'", "path": "tests/unit/daemon/test_web_reader.py" }, { diff --git a/tests/integration/test_daemon_resilience.py b/tests/integration/test_daemon_resilience.py index b3267eda8..eaa802a39 100644 --- a/tests/integration/test_daemon_resilience.py +++ b/tests/integration/test_daemon_resilience.py @@ -41,13 +41,42 @@ ), pytest.mark.slow, pytest.mark.integration, - # Real daemon subprocesses started under a 15s deadline: xdist worker - # contention starves the interpreter startup and flakes the whole gate - # (failed twice in-lane on 2026-08-18, green standalone at ~2s both - # times). Wall-clock-bound -> the isolated serial lane owns it. + # Real daemon subprocesses started under a wall-clock deadline: the parallel + # lane's full worker count starves interpreter startup and flakes the whole + # gate (failed twice in-lane on 2026-08-18, green standalone at ~2s both + # times). Wall-clock-bound -> the bounded `load_sensitive` lane owns it, + # capped at devtools.verify.SERIAL_LANE_MAX_WORKERS. pytest.mark.load_sensitive, ] +# Bin-packing for the bounded lane. +# +# The lane runs under `--dist=loadgroup`, which keeps one group on one worker +# and hands whole groups out as workers free up. Left to xdist's dynamic +# scheduling these tests pack badly: the longest one (`test_large_session_file`, +# ~23s of real 50K-message ingest) is declared sixth of seven, so it starts last +# and the lane's makespan becomes "when the longest test happened to begin" +# rather than its duration -- 35.1s against a 23.2s floor when measured at four +# workers. +# +# These four groups are longest-processing-time bins over the measured per-test +# call durations (2026-08-19 receipt 20260818T184401Z-full-1494889-23438ba4): +# +# a large_session_file 23.18 = 23.18 +# b sigkill_recovery 16.30 + sigterm_with_locked_ops 7.27 = 23.57 +# c concurrent_access 11.55 + wal_checkpoint 8.05 = 19.60 +# d memory_pressure 8.80 + sigterm_read_only 1.27 = 10.07 +# +# so the makespan is bounded by the largest bin instead of by arrival order. +# The bins are a scheduling hint, not a correctness contract: every test here is +# independent (its own archive root via `workspace_env`, its own loopback port), +# so a wrong or missing group costs wall-clock, never a false result. Re-measure +# and rebalance when adding a test or when a member's cost moves materially. +_BIN_A = pytest.mark.xdist_group("daemon-resilience-a") +_BIN_B = pytest.mark.xdist_group("daemon-resilience-b") +_BIN_C = pytest.mark.xdist_group("daemon-resilience-c") +_BIN_D = pytest.mark.xdist_group("daemon-resilience-d") + # --------------------------------------------------------------------------- # Session file writer (matches test_daemon_convergence_evidence.py) # --------------------------------------------------------------------------- @@ -249,6 +278,45 @@ def _wait_for_messages( ) +def _wait_for_api_ready( + proc: subprocess.Popen[bytes], + port: int, + *, + timeout_s: float = 120.0, +) -> None: + """Wait until the daemon's HTTP surface answers, i.e. startup has finished. + + ``_wait_for_lifecycle_start`` is NOT a readiness signal. It returns as soon + as ``DaemonLifecycle.start`` has persisted its row, which happens early in + ``polylogue.daemon.cli``: signal handlers are installed just after it, and + the startup lifecycle event, source-root creation and the maintenance loops + all run later and all write the ops tier. A test that acts on the lifecycle + row alone is racing the rest of startup, and the race widens exactly when + the host is busy. + + Two observed consequences, both intermittent and both load-dependent: + signalling in that window can reach the process before its SIGTERM handler + exists, and taking an EXCLUSIVE ops-tier lock in that window parks a + startup write inside a blocking ``sqlite3_step``, where the interpreter + cannot run the Python signal handler until the busy timeout expires. The + second is what held a signalled daemon past the 90s ``wait`` bound in a + strictly serial run on 2026-08-19 (1 failure in 3 repeats under host load). + + Answering on the API port happens after that startup sequence, so it is the + signal these tests actually want. + """ + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + _assert_daemon_alive(proc) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.settimeout(1.0) + if probe.connect_ex(("127.0.0.1", port)) == 0: + return + time.sleep(0.1) + raise TimeoutError(f"timed out waiting for daemon API readiness on port {port}") + + +@_BIN_D def test_sigterm_read_only_daemon_records_forensics( workspace_env: dict[str, Path], ) -> None: @@ -277,6 +345,7 @@ def test_sigterm_read_only_daemon_records_forensics( ) try: _wait_for_lifecycle_start(daemon, archive_root / "ops.db") + _wait_for_api_ready(daemon, api_port) daemon.send_signal(signal.SIGTERM) assert daemon.wait(timeout=90) == 128 + signal.SIGTERM finally: @@ -301,6 +370,7 @@ def test_sigterm_read_only_daemon_records_forensics( assert "Current thread" in log_text +@_BIN_B def test_sigterm_with_locked_ops_exits_without_normal_sqlite_wait( workspace_env: dict[str, Path], ) -> None: @@ -326,6 +396,7 @@ def test_sigterm_with_locked_ops_exits_without_normal_sqlite_wait( lock = sqlite3.connect(archive_root / "ops.db") try: _wait_for_lifecycle_start(daemon, archive_root / "ops.db") + _wait_for_api_ready(daemon, api_port) lock.execute("BEGIN EXCLUSIVE") started = time.monotonic() daemon.send_signal(signal.SIGTERM) @@ -487,6 +558,7 @@ def _has_systemd_scope() -> bool: # --------------------------------------------------------------------------- +@_BIN_B def test_sigkill_recovery(workspace_env: dict[str, Path]) -> None: """Kill the daemon mid-ingest and verify clean recovery on restart. @@ -606,6 +678,7 @@ def test_sigkill_recovery(workspace_env: dict[str, Path]) -> None: # --------------------------------------------------------------------------- +@_BIN_C def test_wal_checkpoint_recovery(workspace_env: dict[str, Path]) -> None: """Verify WAL checkpoint succeeds, and large-WAL recovery is clean. @@ -729,6 +802,7 @@ def test_wal_checkpoint_recovery(workspace_env: dict[str, Path]) -> None: @pytest.mark.skipif(not _has_systemd_scope(), reason="systemd-run --user --scope not available") +@_BIN_D def test_daemon_memory_pressure(workspace_env: dict[str, Path]) -> None: """Start daemon under a cgroup memory limit and assert it stays within budget. @@ -803,6 +877,7 @@ def test_daemon_memory_pressure(workspace_env: dict[str, Path]) -> None: # --------------------------------------------------------------------------- +@_BIN_A def test_large_session_file(workspace_env: dict[str, Path]) -> None: """Generate a 50K-message JSONL file and verify the daemon ingests it. @@ -919,6 +994,7 @@ def test_large_session_file(workspace_env: dict[str, Path]) -> None: # --------------------------------------------------------------------------- +@_BIN_C def test_concurrent_access_safety(workspace_env: dict[str, Path]) -> None: """Verify WAL read-during-write safety and daemon pidfile locking. diff --git a/tests/unit/daemon/test_web_reader.py b/tests/unit/daemon/test_web_reader.py index 5a70c4761..f0522557f 100644 --- a/tests/unit/daemon/test_web_reader.py +++ b/tests/unit/daemon/test_web_reader.py @@ -770,6 +770,14 @@ def _request_json( return exc.code, json.loads(raw) +# A loopback-socket wall-clock probe: it asserts that `_socket_peer_disconnected` +# observes the peer's FIN within the call itself, with no retry or grace window. +# That is exactly the case pyproject's `load_sensitive` marker names, and this +# test was the one place in the file not covered by it. It passed in 11 retained +# receipts and then failed once (`assert False is True`) in a 1990-test batch run +# under host load on 2026-08-19, passing standalone at 0.68s immediately after -- +# the signature of xdist contention delaying FIN propagation, not a product bug. +@pytest.mark.load_sensitive def test_socket_peer_disconnected_detects_closed_loopback_peer() -> None: from polylogue.daemon.http import _socket_peer_disconnected diff --git a/tests/unit/devtools/test_pytest_collection_contract.py b/tests/unit/devtools/test_pytest_collection_contract.py index 11bb295c3..95113b001 100644 --- a/tests/unit/devtools/test_pytest_collection_contract.py +++ b/tests/unit/devtools/test_pytest_collection_contract.py @@ -31,6 +31,7 @@ def _commands() -> dict[str, list[str]]: testmon_mode="affected", testmon_environment="env-under-test", parallel_worker_args=["-n", "4"], + serial_worker_args=["--dist=loadgroup", "-n", "2"], ) return dict(steps) diff --git a/tests/unit/devtools/test_pytest_supervisor.py b/tests/unit/devtools/test_pytest_supervisor.py index 9356dc6fa..31b7f1e91 100644 --- a/tests/unit/devtools/test_pytest_supervisor.py +++ b/tests/unit/devtools/test_pytest_supervisor.py @@ -7,6 +7,7 @@ import os import shutil import signal +import stat import subprocess import sys import threading @@ -24,6 +25,7 @@ SupervisorLaunch, build_supervisor_launch, cleanup_managed_tmpfs_path, + describe_managed_tmpfs_cleanup, read_receipt, signal_process_identity, termination_request_path, @@ -81,6 +83,95 @@ def test_cleanup_managed_tmpfs_path_removes_read_only_artifact_trees(tmp_path: P for candidate in sorted(run_root.rglob("*"), reverse=True): with contextlib.suppress(OSError): candidate.chmod(candidate.stat().st_mode | 0o200) + + +def test_managed_tmpfs_cleanup_explains_each_outcome(tmp_path: Path) -> None: + """An incomplete cleanup must say why: it silently gates release authority.""" + run_root = Path("/dev/shm") / f"pytest-polylogue-explain-{os.getpid()}-{time.monotonic_ns()}" + try: + run_root.mkdir() + (run_root / "payload").write_text("temporary", encoding="utf-8") + + complete, reason, residual = describe_managed_tmpfs_cleanup(run_root) + assert (complete, residual) == (True, []) + assert "reclaimed" in reason + + # Already gone is a completed cleanup, not a failed one. + assert describe_managed_tmpfs_cleanup(run_root)[0] is True + + complete, reason, residual = describe_managed_tmpfs_cleanup(None) + assert complete is False + assert reason and residual == [] + + complete, reason, residual = describe_managed_tmpfs_cleanup(tmp_path / "pytest-polylogue-not-tmpfs") + assert complete is False + assert "/dev/shm" in reason + finally: + shutil.rmtree(run_root, ignore_errors=True) + + +def test_managed_tmpfs_cleanup_removes_a_read_only_seeded_cache(tmp_path: Path) -> None: + """The b9yw7 red twin: a published read-only artifact tree must still reclaim. + + `tests/infra/workload_artifacts._make_read_only` strips write bits from + directories as well as files, and a directory without its write bit cannot + have entries unlinked from it. Tests that build such a cache under the run + basetemp (`query_cardinality_archive` at `work/"seeded-cache"`) therefore + left trees a plain rmtree could not remove, which set `cleanup.complete=false` + and withheld release-baseline authority from green runs. Reproduced live + 2026-08-19: 16 leaked /dev/shm trees, `rm` refusing with Permission denied. + """ + run_root = Path("/dev/shm") / f"pytest-polylogue-readonly-{os.getpid()}-{time.monotonic_ns()}" + try: + wire = run_root / "work" / "seeded-cache" / "artifacts" / "abc123" / "wire" + wire.mkdir(parents=True) + (wire / "sessions.jsonl").write_text('{"id": 1}\n', encoding="utf-8") + + # Exactly what _make_read_only does: bottom-up, directories included. + for path in sorted(run_root.rglob("*"), reverse=True): + mode = path.stat().st_mode + path.chmod(mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH) + + # Precondition: this is genuinely unremovable the ordinary way. + with pytest.raises(OSError): + shutil.rmtree(run_root) + assert run_root.exists() + + complete, reason, residual = describe_managed_tmpfs_cleanup(run_root) + assert (complete, residual) == (True, []) + assert "reclaimed" in reason + assert not run_root.exists() + finally: + if run_root.exists(): + for path in sorted(run_root.rglob("*"), reverse=True): + with contextlib.suppress(OSError): + path.chmod(path.stat().st_mode | stat.S_IWUSR | stat.S_IXUSR) + shutil.rmtree(run_root, ignore_errors=True) + + +def test_managed_tmpfs_cleanup_reports_survivors_when_the_tree_persists( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A surviving tree names its survivors, so a leak is separable from a race.""" + run_root = Path("/dev/shm") / f"pytest-polylogue-survive-{os.getpid()}-{time.monotonic_ns()}" + try: + run_root.mkdir() + (run_root / "nested").mkdir() + (run_root / "nested" / "held-open").write_text("still here", encoding="utf-8") + + # Stand in for a tree that resists removal for a reason the permission + # repair cannot fix, so the residual sample is what explains it. + monkeypatch.setattr("devtools.pytest_supervisor.shutil.rmtree", lambda *a, **k: None) + + complete, reason, residual = describe_managed_tmpfs_cleanup(run_root) + assert complete is False + assert reason == "tree survived rmtree" + assert "nested/held-open" in residual + finally: + # Undo before cleaning up: monkeypatch is still active inside this + # finally, so an un-undone patch would no-op the removal below and leak + # the fixture into /dev/shm on every run. + monkeypatch.undo() shutil.rmtree(run_root, ignore_errors=True) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 22a5e023a..c0bb1d02f 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1,15 +1,18 @@ from __future__ import annotations +import contextlib import fcntl import itertools import json import os import platform import shutil +import stat import subprocess import sys import threading import time +import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace @@ -31,6 +34,7 @@ PYTEST_PROGRESS_PATH, PYTEST_REPORT_PATH, ROOT, + SERIAL_LANE_MAX_WORKERS, _anchor_verification_paths, _format_completion_notification, _native_lane_failure_requires_stop, @@ -102,6 +106,7 @@ def test_quick_verify_omits_pytest() -> None: "verify doc-commands", "lab schema roundtrip", "lab policy schema-versioning", + "lab policy oracle-integrity", "schema promotion audit", ] @@ -136,7 +141,10 @@ def test_native_testmon_uses_exactly_two_semantic_lanes( assert _pytest_marker_expr(parallel) == "not load_sensitive" assert _pytest_marker_expr(serial) == "load_sensitive" assert parallel[parallel.index("-n") + 1] == "8" - assert serial[serial.index("-n") + 1] == "0" + # The load_sensitive lane is bounded, not serial: it is capped at + # SERIAL_LANE_MAX_WORKERS rather than pinned to a single process. + assert serial[serial.index("-n") + 1] == str(min(8, SERIAL_LANE_MAX_WORKERS)) + assert "--dist=loadgroup" in serial for _label, command in pytest_steps: assert "--testmon" in command assert "--testmon-env=env-digest" in command @@ -2735,6 +2743,50 @@ def test_cleanup_managed_pytest_basetemp_removes_run_root(tmp_path: Path) -> Non assert not basetemp.exists() +def test_sweep_reclaims_a_dead_runs_read_only_tree_but_spares_live_and_shared_ones() -> None: + """Eager sweep: reclaim finished runs' debt, never a live run or the shared cache. + + Cleanup used to run only at a run's own exit, so a tree that failed to + unlink stayed resident until a human noticed -- 16 leaked trees and two + blocked merges on 2026-08-19 (polylogue-b9yw7). + """ + stamp = uuid.uuid4().hex + dead = Path("/dev/shm") / f"pytest-polylogue-sweepdead-{stamp}" + shared = Path("/dev/shm") / f"pytest-polylogue-seeded-{stamp}" + unclaimed = Path("/dev/shm") / f"pytest-polylogue-sweepunclaimed-{stamp}" + created = [dead, shared, unclaimed] + try: + for root in created: + (root / "work").mkdir(parents=True) + (root / "work" / "payload").write_text("x", encoding="utf-8") + # Read-only, exactly as a published seeded-archive artifact is left. + for path in sorted((dead / "work").rglob("*"), reverse=True): + path.chmod(path.stat().st_mode & ~stat.S_IWUSR) + (dead / "work").chmod((dead / "work").stat().st_mode & ~stat.S_IWUSR) + + # A claim naming a pid that cannot be alive => positively dead owner. + verify_runs.pytest_basetemp_claim_path(dead, kind="managed").write_text("999999:1", encoding="utf-8") + # `unclaimed` carries no claim at all: owner unknown, so it is left alone. + + reclaimed = verify_runs.sweep_stale_managed_basetemps() + + assert dead in reclaimed + assert not dead.exists() + assert shared.exists(), "the shared -seeded- corpus cache is not run debt" + assert unclaimed.exists(), "an unknown owner must not be reclaimed" + finally: + for root in created: + if root.exists(): + for path in sorted(root.rglob("*"), reverse=True): + with contextlib.suppress(OSError): + path.chmod(path.stat().st_mode | stat.S_IWUSR | stat.S_IXUSR) + shutil.rmtree(root, ignore_errors=True) + with contextlib.suppress(OSError): + verify_runs.pytest_basetemp_claim_path(root, kind="managed").unlink() + with contextlib.suppress(OSError): + verify_runs.pytest_basetemp_claim_path(root, kind="lock").unlink() + + def test_pytest_basetemp_claim_path_canonicalizes_symlink_aliases(tmp_path: Path) -> None: real_root = tmp_path / "real-root" real_root.mkdir()