diff --git a/.github/workflows/nightly-scale.yml b/.github/workflows/nightly-scale.yml index ff0ea0f2f2..d38f5e3daf 100644 --- a/.github/workflows/nightly-scale.yml +++ b/.github/workflows/nightly-scale.yml @@ -1,6 +1,6 @@ name: Nightly Scale -# Runs the large-tier scale fixtures (issue #1183) outside the normal +# Runs the explicit large benchmark nodes outside the normal # CI critical path, captures benchmark results as structured JSON, and # compares against the committed baseline to detect regressions (#1220). # @@ -41,16 +41,19 @@ jobs: python-version: "3.14" - run: uv sync --extra dev --frozen - - name: Run large-tier scale tests + - name: Run large benchmark campaign env: POLYLOGUE_FORCE_PLAIN: "1" HYPOTHESIS_PROFILE: ci run: | - uv run pytest -q --tb=short --ignore=tests/integration \ - -m "scale_large" -p no:randomly -n 0 \ + uv run pytest -q --tb=short -p no:randomly -n 0 \ + --benchmark-enable \ --benchmark-json=nightly-results.json \ --benchmark-group-by=group \ - tests/benchmarks + 'tests/benchmarks/test_daemon_convergence.py::test_convergence_scale_tier[xxl-mega-session]' \ + tests/benchmarks/test_daemon_convergence.py::test_convergence_huge_session_memory_bounded \ + tests/benchmarks/test_scale_tiers.py::test_bench_50k_fts_search_returns_results \ + tests/benchmarks/test_scale_tiers.py::test_bench_50k_list_sessions - name: Upload benchmark results uses: actions/upload-artifact@v7 diff --git a/CLAUDE.md b/CLAUDE.md index f3a596762f..51ec00ca16 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -336,12 +336,13 @@ testmon-affected set. - `mypy --strict` (via `devtools verify`) is the primary net for type/identifier refactors — trust it. Config in `pyproject.toml`, no exclude list. -- Seed testmon on a fresh checkout / after harness or dependency changes: - `devtools verify --seed-testmon --skip-slow`. A linked worktree auto-bootstraps - its testmon cache from the main checkout's valid seed instead (`devtools/testmon_bootstrap.py`), - so this is only needed when the main checkout itself has no seed yet. -- Reserve `devtools verify --all` (full non-integration run) for - harness/dependency changes or a final pre-PR diagnostic. +- Plain `devtools verify` owns the native pytest-testmon lifecycle. It repairs + invalid local state, optionally copies a matching main-checkout database, + and automatically runs the complete correctness corpus when no valid native + environment exists. Never ask an operator or agent to seed or repair it. +- Reserve `devtools verify --all` (complete unit/property/fuzz/integration + correctness corpus; performance benchmarks excluded) for harness/dependency + changes or a final pre-PR diagnostic. - `devtools verify --quick` = format + lint + mypy + `render all --check` (no tests); it runs on `git push` via the pre-push hook. It is a fast gate, not a substitute for the default baseline before a PR. @@ -522,7 +523,7 @@ isolated XDG paths + archive root. without a managed run identity or an explicit basetemp root is forced to `/realm/tmp/polylogue-pytest` (NVMe). Managed `devtools test` and `devtools verify` runs may use bounded `/dev/shm` tmpfs only after the - runtime policy admits the requested demand; full-suite and seed-testmon + runtime policy admits the requested demand; full/bootstrap native runs default to NVMe because their aggregate fixture tree can exceed the supervised tmpfs ceiling. `POLYLOGUE_PYTEST_BASETEMP_MIN_FREE_MB` overrides required headroom; an @@ -565,7 +566,7 @@ Core loop: changing docs, CLI help, or schema. **Gotcha:** `render all --check` can print per-surface `sync OK` yet still exit 1 — grep the output for `out of sync`, don't trust the tail line. -- `devtools verify [--quick|--all|--lab|--seed-testmon]` — see +- `devtools verify [--quick|--all|--lab]` — see [Verification](#verification--testmon-inner-loop-never-blanket-run). - `devtools test ` — focused pytest through the managed harness. - `devtools lab …` — executable schema/provider/pipeline/lane checks. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87c535c270..e3b9dffc71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,9 +55,8 @@ targeting `master`. 2. Create a branch from `origin/master`. 3. Work on the branch. Git hooks enforce format and lint on commit, and run `devtools verify --quick` on push. -4. Run `devtools verify` before creating the PR. The default pytest step uses - pytest-testmon affected-test selection; run `devtools verify --seed-testmon` - first if the dependency database is not seeded. +4. Run `devtools verify` before creating the PR. It automatically builds or + repairs native pytest-testmon state, then uses affected-test selection. 5. Open a pull request. The template has required sections — fill them all in. The PR title becomes the squash-merge subject on `master`. 6. CI must pass. Fix failures on the branch, do not merge with red CI. @@ -295,27 +294,29 @@ local pytest selection is accelerated by pytest-testmon. ```bash devtools verify # static/generated gates + pytest-testmon affected tests -devtools verify --seed-testmon --skip-slow # seed/update affected-test DB -devtools verify --all # explicit full non-integration pytest diagnostic +devtools verify --all # complete correctness corpus; benchmarks excluded devtools verify --quick # format + lint + mypy + render all --check (skip tests) devtools verify --lab # explicit lab checks beyond the quick/default loop ``` The quick gate runs on `git push` via the active pre-push hook. It's a fast -check, not a substitute for the default baseline. The default command fails -fast when `.cache/testmon/testmondata` and `.cache/testmon/seed.json` are -missing; do not rely on silent full-suite fallback. +check, not a substitute for the default baseline. The default command safely +repairs missing or invalid native testmon state and automatically builds a new +environment when collection semantics change. `devtools verify` does not replay a prior verify result. It always runs the -static gates and then invokes pytest-testmon for affected-test selection from -the current source, dependency, and Python-version state. The default pytest -step combines marker filters with `--testmon-forceselect` so scale-tier -deselection does not silently expand the run back to the whole suite; affected -testmon runs are single-process by default to avoid xdist collection skew. -Polylogue does not maintain a parallel changed-file router for helper/config -paths; use `devtools verify --seed-testmon` when you intentionally want to -refresh the dependency database and `devtools verify --all` for an explicit full -diagnostic. +static gates. With a valid native environment, it invokes pytest-testmon for +affected-test selection from the current source, dependency, and Python-version +state. When it bootstraps a missing or invalid native environment, it runs the +complete correctness corpus. The pytest step covers unit, property, fuzz, and +integration tests while excluding the separately operated `tests/benchmarks` +performance surface. It uses +`--testmon-forceselect` for affected selection, with one parallel `not +load_sensitive` lane and one serial `load_sensitive` lane over the same native +environment. `tui` is a category marker and remains parallel unless a test is +also explicitly `load_sensitive`. Use +`devtools verify --all` for the complete correctness corpus; there is no +manual seed or repair command. Add `devtools release build-package` or `nix flake check` when touching packaging or Nix expressions. See [TESTING.md](TESTING.md) and [docs/devtools.md](docs/devtools.md) diff --git a/TESTING.md b/TESTING.md index 3973786889..ab6b6ace45 100644 --- a/TESTING.md +++ b/TESTING.md @@ -9,10 +9,6 @@ All commands below assume you are inside the project devshell. See # Normal repository verification devtools verify -# First run after checkout, or when you intentionally want to refresh -# pytest-testmon's dependency database -devtools verify --seed-testmon --skip-slow - # Focused inner-loop runs — prefer `devtools test` over raw pytest. It runs the # selection through the managed harness (repo env, single-process by default, # live output, current-node progress artifacts, stall/runtime timeouts) and @@ -24,10 +20,9 @@ devtools test tests/unit/pipeline -x POLYLOGUE_PYTEST_WORKERS=8 devtools test tests/unit/storage # override workers # Raw pytest still works for ad-hoc needs the wrapper does not cover: -pytest -x --ignore=tests/integration -pytest tests/unit/storage/test_hybrid_laws.py +pytest -x tests/unit/storage/test_hybrid_laws.py -# Explicit full non-integration pytest diagnostic +# Complete correctness corpus (unit/property/fuzz/integration; benchmarks excluded) devtools verify --all # Full Nix/CI parity @@ -53,31 +48,58 @@ CI runs this journey in the `web-first-party-auth` job. Local NixOS development uses the system Chrome path discovered by `webui/playwright.config.ts`, so the browser install step is normally unnecessary after `npm ci`. -`devtools verify` uses pytest-testmon for per-test affected selection. The -seed command records `.cache/testmon/testmondata` plus -`.cache/testmon/seed.json`; those files are local generated state and are not -committed. If the seed is missing, the default command fails with setup -guidance instead of silently running the whole suite. Every seed writes -`.cache/testmon/seed-attempt.json` before work begins. An interrupted attempt -recovers its node ledger from the immutable run artifact when the outer process -could not finalize the receipt, then resumes only its unseen, failed, or changed -tests. Corrective code commits do not invalidate that attempt: pytest-testmon -owns dependency-change selection, while the Python and marker-policy identity -still prevents resuming against a different test corpus. `.cache/testmon/seed.json` -is published only after every originally selected node has a failure-free row -in the dependency database. +`devtools verify` owns the complete pytest-testmon lifecycle. A missing or +invalid `.cache/testmon/testmondata` is repaired automatically. A linked +worktree may copy a valid matching database from the main checkout through a +SQLite online backup; otherwise the same plain command runs the complete +correctness corpus and builds native state. Interrupted runs need no separate +receipt or resume command: pytest-testmon keeps failed, unfinished, and new +tests selected on the next plain invocation. + +The native environment key includes Python, active distributions, lock and +project metadata, pytest configuration, the repository-root and `tests/**` +`conftest.py` files, and collection-active local pytest plugins. Ordinary +`tests/infra` helpers stay incremental: import +them from an executing fixture or test when their module initialization is +dependency-bearing, so pytest-testmon observes that initialization and helper +execution instead of forcing a complete-corpus bootstrap for every helper edit. +Product source changes likewise stay incremental. A genuine collection or +dependency-semantics change starts a fresh native environment. +The 2026-08-13 audit reduced this key from all 83 `tests/infra` Python files to +23 actual inputs (configuration, conftests, and active plugins); discovery took +0.95s. A real isolated mutation of a runtime-imported helper retained the same +environment and selected both owning tests through native dependency edges. Plain focused `pytest` runs are single-process by default so small inner-loop checks do not spawn a worker pool. `devtools verify` keeps pytest-testmon as the affected-test selector and runs the selected default lane with an adaptive worker pool (up to 12, override with `POLYLOGUE_PYTEST_WORKERS`) so a stale or genuinely broad affected set cannot spend the full timeout in one -multi-GiB Python process. Because the default gate also applies marker filters -for scale tiers, it passes `--testmon-forceselect` so pytest-testmon still -selects affected tests instead of letting pytest marker selection expand the -run. Full diagnostic and seed runs use the same policy, which budgets roughly -768 MiB per worker, reserves host and tmpfs headroom, and reduces concurrency -when memory pressure is elevated. +multi-GiB Python process. It passes `--testmon-forceselect` so pytest-testmon +selects affected tests within the two semantic lanes. Bootstrap and full runs +cover unit, property, fuzz, and integration correctness tests while excluding +the separately operated `tests/benchmarks` performance surface. They budget +roughly 768 MiB per worker, reserve host and tmpfs headroom, and reduce +concurrency when memory pressure is elevated. + +Every native run has exactly two semantic lanes over one environment and one +database: a parallel lane for tests not marked `load_sensitive`, followed by a +serial lane for the load-sensitive set. Ordinary test failures in the parallel +lane do not suppress the serial lane. Typed collection, containment, resource, +or timeout failures do. + +The lane boundary is evidence-based. On 2026-08-13 the complete correctness +corpus collected 20,447 nodes in 35.06s; only 17 were `load_sensitive`, while +16 were tagged `tui` with no overlap. A managed serial run retained 93.09s of +call time for the load-sensitive set (real PTYs, loopback servers, timing SLAs, +and process/cgroup teardown), versus 7.10s for TUI (0.74s maximum). The same 16 +TUI nodes passed under two xdist workers in 45.22s including duplicate worker +collection, so `tui` remains a useful category but is not a serial-execution +boundary. The captured serial run passed 32 nodes; one process-owner SIGKILL +probe missed its five-second readiness poll twice, then passed a diagnostic +live-output rerun in 6.28s. That host-timing sensitivity supports retaining the +probe in the serial lane. Performance benchmarks remain outside this +correctness corpus. Every collected test has a 120-second `pytest-timeout` budget. A test that genuinely needs longer must declare the exception at the test site with @@ -96,7 +118,7 @@ independent placement policy that can silently disagree with this one: still headroom-checked (see below), never silently downgraded. 2. `/dev/shm` (tmpfs) — the focused-run default, because measured SQLite fsync traffic makes it substantially faster when it clears the free-space - requirement. Full-suite and seed-testmon runs use it only when + requirement. Full/bootstrap native runs use it only when `POLYLOGUE_PYTEST_TMPFS=1` is explicit. 3. `/realm/tmp/polylogue-pytest` (NVMe scratch) — the broad-run default, and the fallback when `/dev/shm` lacks headroom. Broad fixture trees have @@ -143,15 +165,10 @@ external supervisor and parent runner independently remove the whole run root on completion or termination, with startup stale-root cleanup as recovery after an uncatchable process kill or reboot. -An affected run that selects zero tests is accepted only when no executable, -test, dependency, or harness path changed. A zero selection after such a change -fails loudly with the changed paths instead of granting an empty green check. - The default path does not replay cached verify results. Every invocation runs the static gates and then invokes pytest-testmon for affected-test selection. -Polylogue does not maintain a parallel changed-file router for helper/config -paths; explicit full collection is limited to `devtools verify --seed-testmon` -for dependency-database refreshes and `devtools verify --all` for diagnostics. +There is no seed, repair, shard, or registry command. `devtools verify --all` +forces a complete diagnostic in the current native environment. `devtools verify` and `devtools test` treat pytest as a bounded, supervised child workload, not an unowned shell. Each pytest step gets a run directory @@ -175,9 +192,11 @@ and a postmortem diagnosis. The latest run is mirrored to The devtools process drains pytest output, prints periodic heartbeat lines, and samples the pytest process tree and host memory/pressure state. A separate -supervisor owns the pytest controller's process group, watches the devtools -owner process, and enforces `POLYLOGUE_VERIFY_PYTEST_TIMEOUT_S` (default 45 -minutes). Termination sends SIGTERM to that exact group, then SIGKILL after +supervisor owns the pytest controller's process group and watches the devtools +owner process. One 3600-second deadline covers the complete `devtools verify` +invocation. Each step, including both pytest lanes, receives only the time +remaining from that same budget. Termination sends SIGTERM to that exact group, +then SIGKILL after `POLYLOGUE_VERIFY_PYTEST_TERM_GRACE_S` (default 5 seconds). On Sinnix, the supervisor runs in a unique transient scope under the configured build slice; `KillMode=control-group` and a slightly later `RuntimeMaxSec` are the final @@ -194,9 +213,9 @@ pytest produces no output for `POLYLOGUE_VERIFY_RESOURCE_INTERVAL_S` controls resource sampling cadence (default 2 seconds). Basetemp size is a recursive filesystem walk, so it is sampled less frequently; `POLYLOGUE_VERIFY_BASETEMP_SIZE_INTERVAL_S` controls -that cadence (default 15 seconds, `0` disables the size walk). Set timeout -variables to `0` only for an explicit diagnostic run where an unusually long -full-suite pass is expected and supervised. +that cadence (default 15 seconds, `0` disables the size walk). Focused +`devtools test` runs retain their command-specific timeout control; it does not +compose with or extend the verify invocation deadline. Selection artifacts preserve exact selected/deselected counts but sample node IDs by default (`POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT`, default 500) so @@ -208,8 +227,12 @@ verify` or `devtools test` invocation automatically appends its compact run summary to `$XDG_STATE_HOME/polylogue/devtools/verify-history.jsonl` (or the corresponding `~/.local/state` path), shared across linked worktrees without a separate recording command. `devtools verify --history` prints the recent -cross-worktree runs. Setup, call, and teardown timings come only from pytest -reports in the event stream. +cross-worktree runs. A native verify record also carries one run-level pytest +aggregate: environment and corpus digest/count, selection mode, outcomes +across both lanes, wall and collection time, peak RSS/PSS/swap/storage, +read/write totals, cleanup and containment, and the 3600-second budget result. +Setup, call, and teardown timings come only from pytest reports in the event +stream. `devtools test` uses the same pytest progress plugin and process supervisor for focused selections. During or after a run, inspect @@ -226,43 +249,21 @@ Optional lane, mutation-campaign, and benchmark commands remain discoverable through `devtools --help`; pytest and the concrete commands are the behavioral authority. -### Known limitation: collection-time-only imports are invisible to testmon - -`pytest-testmon` only builds a file-to-test dependency edge while a specific -test is *running* (its `pytest_runtest_protocol` hookwrapper opens the tracing -window). Anything a test module or `conftest.py` executes at **collection -time** — a bare `from polylogue.x import Y` at the top of a test file, before -any test in that file has started — falls outside every test's tracing -window and is never recorded, even though the coverage.py summary for a -normal `--cov` run legitimately counts those lines as executed. The result: -declarative-only modules (`TypedDict`/dataclass/`Protocol`/enum/Pydantic -model definitions, no behavior beyond class/field statements) that are only -ever referenced via a top-level import in test files show **zero** rows in -`.cache/testmon/testmondata`'s `file_fp` table, no matter how much of the -file's statements a full-suite coverage run reports as covered. This is -inherent to how testmon (and coverage-context-based selective testing in -general) works — it is **not** dependency-graph staleness, and running -`devtools verify --seed-testmon` does not fix it. - -**Blast radius:** the default `devtools verify` gate (`--testmon ---testmon-forceselect`) is the only local pre-merge signal for a change -scoped to one of these files — `devtools test ` forwards a literal -pytest selection and is not testmon-aware, so it does not share this gap -(point it at the file's *owning test module*, not the changed source file). -A change confined to one of these files can select zero tests locally and -still report a clean `devtools verify`. The heavy full-suite `devtools verify -coverage` CI job (`.github/workflows/ci.yml`) does not use testmon selection -and still catches such a regression, but only **post-merge** (it is -intentionally off the per-PR gate) — so the exposure window is "merged before -caught," not "never caught." - -**Mitigation:** there is no testmon configuration knob for this — it is -upstream tool behavior. When changing a file that is purely declarative -(only type/model/protocol definitions, no function bodies with real logic), -do not trust "0 tests selected" from the default `devtools verify` gate as -proof of safety; run the file's owning test module directly with `devtools -test `, and rely on `mypy --strict` (already in the default gate) -to catch structural regressions in `TypedDict`/protocol shapes. +### Native graph validation for collection-time imports + +`pytest-testmon` records dependencies only while a specific test is running. +A product import executed at test-module or `conftest.py` collection time can +therefore be absent from every test's native dependency graph. + +The automatic lifecycle validates this condition without an always-run +registry. Changed Python modules are classified from their AST. Executable +modules must occur in the native `file_fp` graph; if one is missing, plain +verify invalidates the derived database, rebuilds it, and still fails if the +edge remains absent. Move such an import into an executing fixture or test so +testmon can observe the dependency. + +Declaration-only modules need no cohort or allowlist. Their structural +contracts remain protected by `mypy --strict`, which runs in every verify. ## Test Suite Layout diff --git a/devtools/checkout_guard.py b/devtools/checkout_guard.py index 313512aaef..f4da98f490 100644 --- a/devtools/checkout_guard.py +++ b/devtools/checkout_guard.py @@ -68,8 +68,6 @@ import tomllib -from devtools.testmon_state import attempt_is_checkout_bound, seed_marker_is_checkout_bound - class CheckoutImportMismatchError(RuntimeError): """``import polylogue`` resolved to a package outside the invoking checkout.""" @@ -106,7 +104,6 @@ class CheckoutEnvironmentFingerprint: python_executable: Path python_environment_root: Path | None linked_worktree: bool - testmon_state_origin: Path | None verify_state_origin: Path | None artifacts: tuple[EnvironmentArtifact, ...] @@ -123,16 +120,11 @@ def as_dict(self) -> dict[str, object]: str(self.python_environment_root) if self.python_environment_root is not None else None ), "linked_worktree": self.linked_worktree, - "testmon_state_origin": str(self.testmon_state_origin) if self.testmon_state_origin else None, "verify_state_origin": str(self.verify_state_origin) if self.verify_state_origin else None, "artifacts": [artifact.as_dict() for artifact in self.artifacts], } -_TESTMON_STATE_DIR = Path(".cache/testmon") -_TESTMON_STATE_MARKER = _TESTMON_STATE_DIR / "seed.json" -_TESTMON_SEED_ATTEMPT = _TESTMON_STATE_DIR / "seed-attempt.json" -_TESTMON_SEED_PROTOCOL_VERSION = 7 _VERIFY_STATE_DIR = Path(".cache/verify") _VERIFY_STATE_MARKER = _VERIFY_STATE_DIR / "current-run.json" @@ -265,76 +257,6 @@ def _marker_origin(marker: Path) -> Path | None: return Path(raw).resolve() -def _is_valid_in_progress_testmon_seed_attempt(attempt: Path, *, checkout_root: Path) -> bool: - """Recognize the live seed ledger before its completion marker exists. - - ``verify --seed-testmon`` writes this receipt before pytest starts and - rewrites it after the run. It is the only unmarked testmon state that a - linked worktree may trust. Keep this contract local to the guard so a - random JSON file cannot turn an inherited cache into accepted state. - """ - try: - payload = json.loads(attempt.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - return False - if not isinstance(payload, Mapping) or payload.get("status") not in { - "running", - "incomplete", - "reusable", - "complete", - }: - return False - if payload.get("status") == "complete": - return attempt_is_checkout_bound( - payload, - checkout_root=checkout_root, - protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, - ) - if payload.get("status") == "reusable": - return attempt_is_checkout_bound( - payload, - checkout_root=checkout_root, - protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, - ) - protocol_version = payload.get("protocol_version") - if not isinstance(protocol_version, int) or isinstance(protocol_version, bool) or protocol_version <= 0: - return False - identity = payload.get("identity") - if not isinstance(identity, Mapping): - return False - if not isinstance(identity.get("worktree_fingerprint"), str) or not identity["worktree_fingerprint"]: - return False - if not isinstance(identity.get("python"), str) or not identity["python"]: - return False - if not isinstance(identity.get("skip_slow"), bool) or not isinstance(identity.get("lab"), bool): - return False - git_head = identity.get("git_head") - if git_head is not None and (not isinstance(git_head, str) or not git_head): - return False - if "checkout_root" in payload: - return False - expected_nodeids = payload.get("expected_nodeids") - if not isinstance(expected_nodeids, list) or any( - not isinstance(nodeid, str) or not nodeid for nodeid in expected_nodeids - ): - return False - expected_count = payload.get("expected_count") - if ( - not isinstance(expected_count, int) - or isinstance(expected_count, bool) - or expected_count != len(expected_nodeids) - ): - return False - if not isinstance(payload.get("resume"), bool): - return False - for key in ("started_at", "run_id", "artifact_dir", "testmon_data_before"): - value = payload.get(key) - if not isinstance(value, str) or not value: - return False - artifact_dir = Path(payload["artifact_dir"]) - return not artifact_dir.is_absolute() and artifact_dir.parts[:3] == (".cache", "verify", "runs") - - def _cache_artifact( *, repo_root: Path, @@ -355,34 +277,7 @@ def _cache_artifact( marker_path = repo_root / marker origin = _marker_origin(marker_path) if origin == repo_root: - if state_dir == _TESTMON_STATE_DIR and not seed_marker_is_checkout_bound( - marker_path, - checkout_root=repo_root, - protocol_version=_TESTMON_SEED_PROTOCOL_VERSION, - ): - return ( - origin, - EnvironmentArtifact( - kind="invalid_testmon_seed", - path=marker_path, - detail="testmon seed marker is stale, malformed, or its SQLite graph is incomplete", - remediation=( - f"remove {state_path} and run `devtools verify --seed-testmon` " - "to rebuild the typed testmon state" - ), - ), - ) return origin, None - if ( - origin is None - and not marker_path.exists() - and state_dir == _TESTMON_STATE_DIR - and _is_valid_in_progress_testmon_seed_attempt( - repo_root / _TESTMON_SEED_ATTEMPT, - checkout_root=repo_root, - ) - ): - return None, None if origin is None: detail = f"{state_path} has no verifiable checkout-root marker" else: @@ -418,7 +313,6 @@ def checkout_environment_fingerprint( executable = executable_input.resolve() linked = _is_linked_worktree(resolved_root) artifacts: list[EnvironmentArtifact] = [] - testmon_origin: Path | None = None verify_origin: Path | None = None if linked: @@ -457,16 +351,6 @@ def checkout_environment_fingerprint( remediation=f"remove {node_modules} before running the lane verification", ) ) - testmon_origin, testmon_artifact = _cache_artifact( - repo_root=resolved_root, - state_dir=_TESTMON_STATE_DIR, - marker=_TESTMON_STATE_MARKER, - kind="inherited_testmon_cache", - remediation=( - f"remove {resolved_root / _TESTMON_STATE_DIR} and let `devtools verify --seed-testmon` " - "or the managed bootstrap recreate it" - ), - ) verify_origin, verify_artifact = _cache_artifact( repo_root=resolved_root, state_dir=_VERIFY_STATE_DIR, @@ -474,8 +358,6 @@ def checkout_environment_fingerprint( kind="inherited_verify_cache", remediation=f"remove {resolved_root / _VERIFY_STATE_DIR} and rerun the managed devtools command", ) - if testmon_artifact is not None: - artifacts.append(testmon_artifact) if verify_artifact is not None: artifacts.append(verify_artifact) @@ -485,7 +367,6 @@ def checkout_environment_fingerprint( python_executable=executable, python_environment_root=environment_root, linked_worktree=linked, - testmon_state_origin=testmon_origin, verify_state_origin=verify_origin, artifacts=tuple(artifacts), ) diff --git a/devtools/command_catalog.py b/devtools/command_catalog.py index a4708a83b8..f060cac46d 100644 --- a/devtools/command_catalog.py +++ b/devtools/command_catalog.py @@ -306,18 +306,6 @@ def to_dict(self) -> dict[str, object]: "devtools verify mutation-freshness --strict --default-freshness-days 30", ), ), - CommandSpec( - "lab testmon-proof", - "verification lab", - "Prove real testmon affected selection against a semantic production mutation.", - "devtools.testmon_mutation_proof", - use_when=( - "Validate the affected-test harness itself: a disposable copy of a real Polylogue module " - "and existing route test is seeded, semantically mutated, edge-severed, restored, and checked " - "for bounded unrelated-change selection." - ), - examples=("devtools lab testmon-proof", "devtools lab testmon-proof --json"), - ), CommandSpec( "bench ingest-amplification", "benchmarking", diff --git a/devtools/merge_boundary.py b/devtools/merge_boundary.py index 656b4bc46d..7bf07f30f1 100644 --- a/devtools/merge_boundary.py +++ b/devtools/merge_boundary.py @@ -37,8 +37,8 @@ 6. Runs the actual ``gh pr merge --squash``. 7. Appends a merge-train ledger entry (``.cache/verify/merge-gate/merge-train-ledger.json``) and, unless ``--with-verify`` was given, prints a reminder that the - ledger's terminal step -- one full-suite ``devtools verify --all`` (or - narrower agreed selection) since the last one -- has not yet been + ledger's terminal step -- one release-baseline ``devtools verify --all`` + since the last one -- has not yet been recorded for this train. ``devtools workspace merge train-status`` inspects the ledger and reports @@ -79,7 +79,7 @@ from typing import Any from devtools import merge_gate, pr_scope -from devtools.testmon_state import TerminalAuthorization, VerificationScope +from devtools.verification_contracts import VerificationScope _LEDGER_PATH = Path(".cache/verify/merge-gate/merge-train-ledger.json") _LEDGER_PENDING_PATH = _LEDGER_PATH.with_name(f"{_LEDGER_PATH.name}.pending") @@ -207,12 +207,12 @@ def _validate_ledger(data: object) -> dict[str, Any]: ): raise LedgerStateError("merge-train terminal receipt has malformed status fields") scope = receipt.get("verification_scope") - if scope is not None and scope not in {item.value for item in VerificationScope}: - raise LedgerStateError("merge-train terminal receipt has an invalid verification scope") + if scope is not None and not isinstance(scope, str): + raise LedgerStateError("merge-train terminal receipt has a malformed verification scope") permission = receipt.get("release_baseline_allowed") if permission is not None and not isinstance(permission, bool): raise LedgerStateError("merge-train terminal receipt has malformed release permission") - for key in ("terminal_authorization", "verified_head_sha", "target_sha", "merged_master_sha"): + for key in ("verified_head_sha", "target_sha", "merged_master_sha"): value = receipt.get(key) if value is not None and (not isinstance(value, str) or not value): raise LedgerStateError(f"merge-train terminal receipt has malformed {key!r}") @@ -396,13 +396,7 @@ def _pending_prs_since_last_full_verify(ledger: dict[str, Any]) -> list[dict[str if last_verify.get("accepted") is True and last_verify.get("exit_code") == 0 and last_verify.get("release_baseline_allowed") is True - and ( - scope == VerificationScope.RELEASE_BASELINE.value - or ( - scope == VerificationScope.NARROW_TERMINAL.value - and last_verify.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value - ) - ) + and scope == VerificationScope.RELEASE_BASELINE.value else 0.0 ) snapshot_sequence = last_verify.get("merge_sequence") @@ -781,7 +775,8 @@ def cmd_train_status(as_json: bool) -> int: print(f" PR #{entry['pr']} @ {entry['head_sha'][:8]}: {entry['title']}") print( 'Run `devtools workspace merge record-full-verify --command "devtools verify --all"` ' - "(or the narrower agreed selection) before declaring this merge-train session done -- " + "before declaring this merge-train session done. A narrower successful selection does not " + "grant the release-baseline authority this ledger requires. " "per-PR CI skips the heavy suite, so nothing else will catch a master-red class only " "visible on the merged whole." ) @@ -834,18 +829,11 @@ def cmd_record_full_verify( duration_s = round(time.time() - started, 2) release_allowed = merge_gate._release_baseline_permission(receipt) verification_scope = merge_gate._verification_scope(receipt) - terminal_authorization = merge_gate._terminal_authorization(receipt) verified_head = receipt.get("git_head") if isinstance(receipt, dict) else None accepted = ( result.returncode == 0 and release_allowed is True - and ( - verification_scope == VerificationScope.RELEASE_BASELINE.value - or ( - verification_scope == VerificationScope.NARROW_TERMINAL.value - and terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value - ) - ) + and verification_scope == VerificationScope.RELEASE_BASELINE.value and verified_head == target_sha ) @@ -860,7 +848,6 @@ def cmd_record_full_verify( "verification_started_at": verification_started_at, "verification_scope": verification_scope, "release_baseline_allowed": release_allowed, - "terminal_authorization": terminal_authorization, "verified_head_sha": verified_head, "target_sha": target_sha, "merged_master_sha": target_sha, diff --git a/devtools/merge_gate.py b/devtools/merge_gate.py index 06870e491a..c8c30ee255 100644 --- a/devtools/merge_gate.py +++ b/devtools/merge_gate.py @@ -75,7 +75,7 @@ from typing import Any from devtools import pr_scope -from devtools.testmon_state import TerminalAuthorization, VerificationScope +from devtools.verification_contracts import VerificationScope from devtools.verify_runs import VERIFICATION_INVOCATION_ID_ENV as VERIFICATION_INVOCATION_ID_ENV from devtools.verify_runs import VERIFICATION_RECEIPT_PATH_ENV as VERIFICATION_RECEIPT_PATH_ENV @@ -86,7 +86,6 @@ _MERGE_AUTHORIZING_VERIFICATION_SCOPES = frozenset( { VerificationScope.AFFECTED.value, - VerificationScope.NARROW_TERMINAL.value, VerificationScope.RELEASE_BASELINE.value, } ) @@ -227,11 +226,6 @@ def _invocation_receipt( return None if _verification_scope(payload) is None or _release_baseline_permission(payload) is None: return None - terminal_authorization = payload.get("terminal_authorization") - if terminal_authorization is not None and terminal_authorization not in { - authorization.value for authorization in TerminalAuthorization - }: - return None return payload @@ -251,14 +245,6 @@ def _verification_scope(payload: Mapping[str, Any] | None) -> str | None: return value if value in {scope.value for scope in VerificationScope} else None -def _terminal_authorization(payload: Mapping[str, Any] | None) -> str | None: - """Read terminal authorization from an invocation-bound receipt.""" - if payload is None: - return None - value = payload.get("terminal_authorization") - return value if value in {authorization.value for authorization in TerminalAuthorization} else None - - def _base_sha(info: dict[str, Any]) -> str | None: """Read the PR base commit SHA when GitHub reported one.""" value = info.get("baseRefOid") @@ -370,7 +356,6 @@ def cmd_record(pr: int, command: str) -> int: "command": command, "verification_scope": _verification_scope(verification_receipt), "release_baseline_allowed": _release_baseline_permission(verification_receipt), - "terminal_authorization": _terminal_authorization(verification_receipt), "exit_code": result.returncode, "duration_s": duration_s, "recorded_at": time.time(), diff --git a/devtools/project_motd.py b/devtools/project_motd.py index bf51e5a982..5916ca5385 100644 --- a/devtools/project_motd.py +++ b/devtools/project_motd.py @@ -158,7 +158,7 @@ def status_snapshot(cwd: Path, *, verify_generated: bool = False) -> StatusSnaps "render_all_check": control_plane_command("render all", "--check"), "verify_quick": control_plane_command("verify", "--quick"), "build_package": control_plane_command("release build-package"), - "test_baseline": "pytest -q --ignore=tests/integration", + "test_baseline": control_plane_command("verify"), }, "local_state": { "cache": ".cache/", diff --git a/devtools/pytest_progress_plugin.py b/devtools/pytest_progress_plugin.py index 6abb073d46..87cf24fbf3 100644 --- a/devtools/pytest_progress_plugin.py +++ b/devtools/pytest_progress_plugin.py @@ -213,7 +213,6 @@ def _collection_payload() -> dict[str, Any]: "selected_count": _SELECTED_COUNT, "deselected_count": _DESELECTED_COUNT, "selected_nodeids": [], - "selected_node_markers": {}, "selected_nodeids_omitted": _SELECTED_COUNT, "deselected_nodeids": list(_DESELECTED_NODEIDS_SAMPLE), "deselected_nodeids_omitted": max(0, _DESELECTED_COUNT - len(_DESELECTED_NODEIDS_SAMPLE)), @@ -317,20 +316,10 @@ def pytest_collection_modifyitems(session: Any, config: Any, items: list[Any]) - _SELECTED_COUNT = len(items) limit = _selection_nodeid_limit() selected_nodeids = [str(getattr(item, "nodeid", item)) for item in items[:limit]] - # Marker metadata is a compact routing index, not a node-id sample. Keep - # it complete so seed sharding can isolate load-sensitive/TUI nodes even - # when the human-readable node-id sample is capped at 500 entries. - selected_node_markers = { - str(getattr(item, "nodeid", item)): sorted( - {str(mark.name) for mark in getattr(item, "iter_markers", lambda: ())()} - ) - for item in items - } payload = _collection_payload() payload.update( { "selected_nodeids": selected_nodeids, - "selected_node_markers": selected_node_markers, "selected_nodeids_omitted": max(0, _SELECTED_COUNT - len(selected_nodeids)), } ) @@ -412,7 +401,7 @@ def pytest_runtest_makereport(item: Any, call: Any) -> Any: def pytest_runtest_logreport(report: Any) -> None: """Retain the direct/log-hook fallback used by older pytest plugins/tests.""" # xdist forwards each worker's report to the controller. The worker has - # already written the authoritative shard event through makereport. Keep + # already written the authoritative worker event through makereport. Keep # its timing in the controller's summary, but do not duplicate the ledger. if not os.environ.get("PYTEST_XDIST_WORKER") and getattr(report, "worker_id", None): _record_phase_report(report, write_event=False) diff --git a/devtools/testmon_bootstrap.py b/devtools/testmon_bootstrap.py index 7397bbe294..d8204d5d7e 100644 --- a/devtools/testmon_bootstrap.py +++ b/devtools/testmon_bootstrap.py @@ -1,527 +1,811 @@ -"""Bootstrap a fresh worktree's pytest-testmon cache from the main checkout. - -The hazard this closes (polylogue-mq4vx): every fresh agent worktree lane -re-seeds pytest-testmon from scratch. `devtools verify --seed-testmon` seeds -the affected-selection dependency database with a full non-integration pytest -run; that run costs real wall-clock (the main checkout's -``.cache/testmon/testmondata`` is ~28MB, built from the whole non-integration -suite). A linked worktree with no local seed either pays that cost again or -hits the unseeded-refusal preflight in ``devtools/verify.py`` -(``_testmon_preflight``) and blocks entirely. - -But the seed database is copyable. ``pytest-testmon`` records ``file_fp`` -entries keyed by path **relative to the invoking repo root**, each with a -per-file content checksum (``fsha``). A worktree is a distinct working tree -sharing the same relative layout as the main checkout, so a testmondata file -copied verbatim from main is immediately meaningful there: any file that -differs between the worktree and the main checkout at copy time -self-invalidates (its ``fsha`` won't match), and testmon correctly treats the -tests that depend on it as affected on the very next run. No merge or rewrite -is needed for the relative file fingerprints. - -The reusable stamp is typed. It records collection completeness, graph -completeness, baseline color, and whether the graph is exact or rebound to a -new checkout. A red graph is allowed for affected selection only. Bootstrap -revalidates the SQLite graph after the online backup and recomputes its file -fingerprint because SQLite backup can produce a byte-different equivalent -database. - -This module owns exactly one decision and one action: - -- :func:`decide_testmon_bootstrap` -- pure decision, no subprocess beyond the - caller. It validates the main stamp or a complete red seed attempt. -- :func:`bootstrap_testmon_seed_files` -- the copy action once bootstrapping - has been decided. A red attempt is copied as a rebound attempt receipt and - never synthesized into ``seed.json``. -- :func:`maybe_bootstrap_testmon_seed` -- the orchestrator `devtools verify` - calls: detects whether ``repo_root`` is a linked worktree (via - ``git rev-parse --absolute-git-dir --git-common-dir``, the same mechanism - ``devtools/verify_worktree.py`` uses), finds the main checkout, and wires - the decision to the action. - -Concurrency: the main checkout may be mid-seed (a live ``--seed-testmon`` run -appending to its own testmondata) at the exact moment a worktree bootstraps -from it. ``testmondata`` is a real sqlite database, so a naive byte copy of an -open, actively-written file can capture a torn, inconsistent snapshot. This -copies it through :meth:`sqlite3.Connection.backup`, sqlite's own online-backup -API -- built for copying a live database without an exclusive lock, immune to -concurrent writers by design. ``seed.json`` is a small file written atomically -by ``verify.py`` (write-temp-then-rename), so bootstrap writes its newly bound -stamp atomically after the copied graph has been revalidated. - -This module NEVER writes to the main checkout's copy of either file -- -only reads from main, only writes to ``repo_root``. +"""Prepare checkout-local pytest-testmon state for plain ``devtools verify``. + +Pytest-testmon already owns interrupted-run recovery, failing/new test +selection, node deletion, and dependency replacement. This module therefore +does only the checkout boundary work that the plugin cannot do itself: + +* derive the native ``--testmon-env`` name from collection semantics; +* validate that the local SQLite database contains that environment; +* require changed executable modules to occur in its dependency graph; +* remove only an invalid checkout-owned database and its SQLite sidecars; +* optionally copy a matching main-checkout database into a linked worktree by + SQLite online backup plus atomic rename. + +There are no seed markers, completion stamps, shard ledgers, or release grants. +An absent main database is normal. The next plain verify invocation builds +the current environment by running the ordinary correctness corpus. """ from __future__ import annotations +import ast +import contextlib +import hashlib +import importlib +import importlib.metadata import json import os -import shutil +import platform import sqlite3 +import stat import subprocess -import tempfile -from collections.abc import Mapping +import sys +import time +import uuid +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from devtools.testmon_state import ( - TestmonSeedStamp, - refresh_stamp, - stamp_from_attempt, - validate_stamp, +from pathlib import Path, PurePosixPath +from typing import Literal + +TESTMON_DATA_RELPATH = Path(".cache/testmon/testmondata") +TESTMON_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal") + +NativeStateStatus = Literal["absent", "valid", "invalid"] +NativeSelectionMode = Literal["bootstrap", "affected"] +ASTClassification = Literal["declaration-only", "executable", "source-unreadable"] + +_ENVIRONMENT_INPUTS = ( + "uv.lock", + "pyproject.toml", + "pytest.ini", + "tox.ini", + "setup.cfg", + # Keep the conventional repository-root hook as an absent-path sentinel: + # creating it changes collection even though it was not present when the + # previous environment was named. + "conftest.py", + "devtools/checkout_guard.py", + "devtools/testmon_bootstrap.py", + "devtools/verify.py", + "devtools/verify_runs.py", ) +_PYTEST_ENVIRONMENT_KEYS = ( + "HYPOTHESIS_PROFILE", + "POLYLOGUE_CI", +) + + +def _is_ignored_native_testmon_path(relative: str) -> bool: + return relative == "tests/benchmarks" or relative.startswith("tests/benchmarks/") -TESTMON_DATA_RELPATH = ".cache/testmon/testmondata" -TESTMON_SEED_STAMP_RELPATH = ".cache/testmon/seed.json" -TESTMON_SEED_ATTEMPT_RELPATH = ".cache/testmon/seed-attempt.json" +@dataclass(frozen=True, slots=True) +class NativeTestmonEnvironment: + name: str + corpus_count: int + corpus_digest: str + nodeids: tuple[str, ...] + fingerprinted_files: frozenset[str] -@dataclass(frozen=True) -class BootstrapDecision: - """Whether a worktree's testmon cache should be bootstrapped from main, and why.""" - should_bootstrap: bool +@dataclass(frozen=True, slots=True) +class NativeTestmonState: + status: NativeStateStatus reason: str - main_testmon_data: Path | None = None - main_seed_stamp: Path | None = None - main_seed_attempt: Path | None = None - main_checkout_root: Path | None = None - protocol_version: int = 4 - selection_only: bool = False + environment: NativeTestmonEnvironment | None = None + missing_executable_paths: tuple[str, ...] = () + + @property + def valid(self) -> bool: + return self.status == "valid" and self.environment is not None + + +@dataclass(frozen=True, slots=True) +class NativeTestmonPreparation: + environment_name: str + selection_mode: NativeSelectionMode + local_state: NativeTestmonState + copied_from: Path | None + removed_paths: tuple[Path, ...] + linked_worktree: bool + main_checkout: Path | None + + +@dataclass(frozen=True, slots=True) +class NativeTestmonChangeImpact: + """Changed inputs that native Python tracing can and cannot select.""" + + executable_paths: tuple[str, ...] + runtime_data_paths: tuple[str, ...] + + +class NativeTestmonRepairError(RuntimeError): + """The exact derived testmon state could not be repaired safely.""" -def _checkout_root_for_data(data_path: Path) -> Path: - """Resolve the checkout root for canonical and test-local cache layouts.""" - resolved = data_path.resolve() - if resolved.parent.name == "testmon" and resolved.parent.parent.name == ".cache": - return resolved.parents[2] - return resolved.parent +class NativeTestmonDeadlineError(NativeTestmonRepairError): + """The verify invocation deadline expired during native-state preparation.""" -def _is_valid_complete_seed_stamp( - seed_stamp: Path, - testmon_data: Path, +def _ensure_deadline(deadline_monotonic: float | None) -> None: + if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic: + raise NativeTestmonDeadlineError("verify invocation deadline expired during native testmon preparation") + + +def _remaining_timeout(deadline_monotonic: float | None, maximum: float) -> float: + _ensure_deadline(deadline_monotonic) + if deadline_monotonic is None: + return maximum + return max(0.001, min(maximum, deadline_monotonic - time.monotonic())) + + +def _fingerprint_inputs( + root: Path, + relative_paths: Sequence[str], *, - protocol_version: int, - checkout_root: Path, -) -> bool: - """Validate both the typed stamp and the real SQLite graph it describes.""" + deadline_monotonic: float | None = None, +) -> str: + digest = hashlib.sha256() + for relative in relative_paths: + _ensure_deadline(deadline_monotonic) + digest.update(relative.encode()) + digest.update(b"\0") + try: + with (root / relative).open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + _ensure_deadline(deadline_monotonic) + except OSError: + digest.update(b"missing") + digest.update(b"\0") + return digest.hexdigest() + + +def _pytest_plugins_assignment(node: ast.stmt) -> ast.expr | None: + match node: + case ast.Assign(targets=targets, value=value) if any( + isinstance(target, ast.Name) and target.id == "pytest_plugins" for target in targets + ): + return value + case ast.AnnAssign(target=ast.Name(id="pytest_plugins"), value=value): + return value + case _: + return None + + +def _indirect_pytest_plugins_declaration(node: ast.stmt) -> bool: + for child in ast.walk(node): + if ( + isinstance(child, ast.Subscript) + and isinstance(child.slice, ast.Constant) + and child.slice.value == "pytest_plugins" + ): + return True + if isinstance(child, ast.Call): + if ( + isinstance(child.func, ast.Name) + and child.func.id == "setattr" + and len(child.args) >= 2 + and isinstance(child.args[1], ast.Constant) + and child.args[1].value == "pytest_plugins" + ): + return True + if any(keyword.arg == "pytest_plugins" for keyword in child.keywords): + return True + return False + + +def _declared_pytest_plugin_names( + root: Path, + *, + deadline_monotonic: float | None = None, +) -> set[str]: + """Read static local plugin declarations that pytest loads at collection.""" + names: set[str] = set() + candidates: set[Path] = set() + candidates.add(root / "conftest.py") + for path in root.glob("tests/**/*.py"): + _ensure_deadline(deadline_monotonic) + if _is_ignored_native_testmon_path(path.relative_to(root).as_posix()): + continue + try: + source = path.read_text(encoding="utf-8") + except OSError: + continue + if "pytest_plugins" not in source: + continue + candidates.add(path) + for path in root.glob("tests/**/conftest.py"): + _ensure_deadline(deadline_monotonic) + if _is_ignored_native_testmon_path(path.relative_to(root).as_posix()): + continue + candidates.add(path) + for path in sorted(candidates): + _ensure_deadline(deadline_monotonic) + try: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + except (OSError, SyntaxError, UnicodeDecodeError): + continue + declaration_count = 0 + for node in tree.body: + value = _pytest_plugins_assignment(node) + if value is None: + dynamic_reference = ( + any(isinstance(child, ast.Name) and child.id == "pytest_plugins" for child in ast.walk(node)) + or any( + alias.name == "pytest_plugins" or alias.asname == "pytest_plugins" + for child in ast.walk(node) + if isinstance(child, ast.Import | ast.ImportFrom) + for alias in child.names + ) + or _indirect_pytest_plugins_declaration(node) + ) + if dynamic_reference: + raise NativeTestmonRepairError( + f"repository pytest_plugins declaration must be one literal assignment: {path}" + ) + continue + declaration_count += 1 + if declaration_count != 1: + raise NativeTestmonRepairError( + f"repository pytest_plugins declaration must be one literal assignment: {path}" + ) + try: + declared = ast.literal_eval(value) + except (ValueError, TypeError) as exc: + raise NativeTestmonRepairError( + f"repository pytest_plugins declaration must be a literal string/list/tuple: {path}" + ) from exc + if isinstance(declared, str): + names.add(declared) + elif isinstance(declared, tuple | list) and all(isinstance(name, str) for name in declared): + names.update(declared) + else: + raise NativeTestmonRepairError( + f"repository pytest_plugins declaration must contain only literal plugin names: {path}" + ) + return names + + +def _active_local_pytest_plugin_paths( + root: Path, + *, + deadline_monotonic: float | None = None, +) -> set[str]: + """Resolve collection-active local pytest plugins regardless of filename.""" + paths: set[str] = set() + plugin_names = _declared_pytest_plugin_names(root, deadline_monotonic=deadline_monotonic) + for raw_name in plugin_names: + _ensure_deadline(deadline_monotonic) + module_name = raw_name.strip() + if not module_name or any(part in {"", ".", ".."} for part in module_name.split(".")): + continue + module_path = Path(*module_name.split(".")) + module_file = root / module_path.with_suffix(".py") + if module_file.is_file() and not _is_ignored_native_testmon_path(module_file.relative_to(root).as_posix()): + paths.add(module_file.relative_to(root).as_posix()) + package = root / module_path + if (package / "__init__.py").is_file(): + for path in package.rglob("*.py"): + _ensure_deadline(deadline_monotonic) + if path.is_file() and not _is_ignored_native_testmon_path(path.relative_to(root).as_posix()): + paths.add(path.relative_to(root).as_posix()) + return paths + + +def _environment_input_paths( + root: Path, + *, + deadline_monotonic: float | None = None, +) -> tuple[str, ...]: + """Discover collection and managed-pytest harness inputs.""" + paths = set(_ENVIRONMENT_INPUTS) + patterns = ( + "devtools/pytest*.py", + "tests/**/conftest.py", + ) + for pattern in patterns: + for path in root.glob(pattern): + _ensure_deadline(deadline_monotonic) + if path.is_file() and not _is_ignored_native_testmon_path(path.relative_to(root).as_posix()): + paths.add(path.relative_to(root).as_posix()) + paths.update(_active_local_pytest_plugin_paths(root, deadline_monotonic=deadline_monotonic)) + return tuple(sorted(paths)) + + +def _installed_distributions() -> tuple[tuple[str, str], ...]: + distributions: list[tuple[str, str]] = [] + for distribution in importlib.metadata.distributions(): + name = distribution.metadata["Name"] + version = distribution.version + if not name or not version: + raise NativeTestmonRepairError("active Python distributions are not fully identifiable") + distributions.append((name.casefold(), version)) + return tuple(sorted(distributions)) + + +def testmon_environment_digest( + repo_root: Path, + *, + pytest_profile: str = "default", + pytest_environment: Mapping[str, str | None] | None = None, + deadline_monotonic: float | None = None, +) -> str: + """Return the native testmon environment name for collection semantics.""" + root = repo_root.resolve() + _ensure_deadline(deadline_monotonic) + payload = { + "protocol": 1, + "python": { + "implementation": sys.implementation.name, + "cache_tag": sys.implementation.cache_tag, + "version": platform.python_version(), + "abi_flags": getattr(sys, "abiflags", ""), + "platform": platform.platform(), + }, + "distributions": _installed_distributions(), + "inputs": _fingerprint_inputs( + root, + _environment_input_paths(root, deadline_monotonic=deadline_monotonic), + deadline_monotonic=deadline_monotonic, + ), + "pytest_environment": { + key: (os.environ.get(key) if pytest_environment is None else pytest_environment.get(key)) + for key in _PYTEST_ENVIRONMENT_KEYS + }, + "pytest_profile": pytest_profile, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return f"polylogue-{hashlib.sha256(encoded).hexdigest()}" + + +def _is_docstring(node: ast.stmt, *, first: bool) -> bool: return ( - validate_stamp( - seed_stamp, - testmon_data, - checkout_root=checkout_root, - protocol_version=protocol_version, - ) - is not None + first + and isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) ) -def decide_testmon_bootstrap( - *, - is_linked_worktree: bool, - local_testmon_data: Path, - local_seed_stamp: Path, - main_testmon_data: Path, - main_seed_stamp: Path, - protocol_version: int, - main_seed_attempt: Path | None = None, - main_checkout_root: Path | None = None, - local_checkout_root: Path | None = None, - local_seed_attempt: Path | None = None, -) -> BootstrapDecision: - """Decide whether to copy the main checkout's testmon seed into a worktree. - - Pure with respect to process state (no subprocess, no git): every input is - an already-resolved path or flag, so this is directly unit-testable with - tmp dirs standing in for "local worktree" and "main checkout". - """ - if not is_linked_worktree: - return BootstrapDecision(False, "repo_root is not a linked worktree; nothing to bootstrap") - local_root = (local_checkout_root or _checkout_root_for_data(local_testmon_data)).resolve() - if ( - local_testmon_data.is_file() - and local_seed_stamp.is_file() - and _is_valid_complete_seed_stamp( - local_seed_stamp, - local_testmon_data, - protocol_version=protocol_version, - checkout_root=local_root, +def _is_type_checking_guard(node: ast.expr) -> bool: + return ( + isinstance(node, ast.Name) + and node.id == "TYPE_CHECKING" + or ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "typing" + and node.attr == "TYPE_CHECKING" ) - ): - return BootstrapDecision(False, "local .cache/testmon already has a validated testmondata + seed stamp") - if local_testmon_data.is_file() and local_seed_attempt is not None and local_seed_attempt.is_file(): - try: - local_attempt = json.loads(local_seed_attempt.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - local_attempt = None + ) + + +def _body_is_executable(body: list[ast.stmt]) -> bool: + for index, node in enumerate(body): + if _is_docstring(node, first=index == 0): + continue + if isinstance(node, ast.Pass): + continue if ( - isinstance(local_attempt, Mapping) - and stamp_from_attempt( - local_attempt, - local_testmon_data, - checkout_root=local_root, - protocol_version=protocol_version, - published_marker=False, - ) - is not None + isinstance(node, ast.ImportFrom) + and node.module == "typing" + and all(alias.name == "TYPE_CHECKING" for alias in node.names) ): - return BootstrapDecision(False, "local .cache/testmon already has a checkout-bound selection attempt") - if not main_testmon_data.is_file(): - return BootstrapDecision( - False, - "main checkout has no valid testmon graph because its testmondata file is missing", - ) - root = main_checkout_root or _checkout_root_for_data(main_testmon_data) - root = root.resolve() - try: - main_testmon_data.resolve().relative_to(root) - main_seed_stamp.resolve().relative_to(root) - if main_seed_attempt is not None: - main_seed_attempt.resolve().relative_to(root) - except ValueError: - return BootstrapDecision(False, "main testmon paths are not bound to the declared checkout root") - if _is_valid_complete_seed_stamp( - main_seed_stamp, - main_testmon_data, - protocol_version=protocol_version, - checkout_root=root, - ): - return BootstrapDecision( - True, - f"main checkout has a validated testmon graph ({main_seed_stamp}); bootstrapping worktree cache", - main_testmon_data=main_testmon_data, - main_seed_stamp=main_seed_stamp, - main_checkout_root=root, - protocol_version=protocol_version, - ) - if main_seed_attempt is not None and main_seed_attempt.is_file(): - try: - attempt = json.loads(main_seed_attempt.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - attempt = None + # Importing the sentinel only enables a declaration-only guard. + # Imports elsewhere execute at module import time and therefore + # remain executable graph inputs. + continue if ( - isinstance(attempt, dict) - and ( - attempt_stamp := stamp_from_attempt( - attempt, - main_testmon_data, - checkout_root=root, - protocol_version=protocol_version, - published_marker=False, - ) - ) - is not None + isinstance(node, ast.ImportFrom) + and node.module == "__future__" + and all(alias.name == "annotations" for alias in node.names) ): - return BootstrapDecision( - True, - "main checkout has a validated complete graph from a red seed attempt; bootstrapping worktree cache", - main_testmon_data=main_testmon_data, - main_seed_attempt=main_seed_attempt, - main_checkout_root=root, - protocol_version=protocol_version, - selection_only=not attempt_stamp.release_baseline_allowed, - ) - if main_seed_stamp.is_file(): - return BootstrapDecision(False, "main checkout seed stamp is stale, malformed, or graph-incomplete") - return BootstrapDecision( - False, - "main checkout has no validated reusable testmon state", - ) + continue + if ( + isinstance(node, ast.ImportFrom) + and node.module == "enum" + and all(alias.name in {"Enum", "IntEnum", "StrEnum"} for alias in node.names) + ): + continue + if ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and (isinstance(node.value.value, str) or node.value.value is Ellipsis) + ): + continue + if isinstance(node, ast.AnnAssign) and node.value is None: + continue + if isinstance(node, ast.If) and _is_type_checking_guard(node.test): + # The guarded body is deliberately invisible at runtime. An else + # branch does execute and therefore retains ordinary classification. + if _body_is_executable(node.orelse): + return True + continue + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if node.decorator_list or node.args.defaults or any(value is not None for value in node.args.kw_defaults): + return True + if _body_is_executable(node.body): + return True + continue + if isinstance(node, ast.ClassDef): + if _is_pure_enum_declaration(node): + continue + if node.decorator_list or node.bases or node.keywords or _body_is_executable(node.body): + return True + continue + if isinstance(node, ast.Assign): + return True + if isinstance(node, ast.AnnAssign): + if node.value is not None: + return True + continue + return True + return False -def _atomic_write_json(path: Path, payload: dict[str, object]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") +def _is_pure_enum_declaration(node: ast.ClassDef) -> bool: + """Recognize enum value declarations that tracing cannot observe usefully.""" + bases = {base.id for base in node.bases if isinstance(base, ast.Name)} + if not bases.intersection({"Enum", "IntEnum", "StrEnum"}): + return False + for member in node.body: + if ( + isinstance(member, ast.Expr) + and isinstance(member.value, ast.Constant) + and isinstance(member.value.value, str) + ): + continue + if isinstance(member, (ast.Assign, ast.AnnAssign, ast.Pass)): + continue + return False + return True + + +def classify_source_ast(source_path: Path) -> ASTClassification: + """Classify whether a module contains executable runtime behavior.""" try: - tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - tmp.replace(path) - finally: - tmp.unlink(missing_ok=True) + tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path)) + except OSError: + return "source-unreadable" + except (SyntaxError, UnicodeDecodeError): + return "executable" + return "executable" if _body_is_executable(tree.body) else "declaration-only" -def _atomic_write_stamp(seed_stamp: Path, stamp: TestmonSeedStamp) -> None: - _atomic_write_json(seed_stamp, stamp.as_dict()) +def _safe_relative_path(raw: str) -> str | None: + normalized = PurePosixPath(raw.replace("\\", "/")) + if normalized.is_absolute() or not normalized.parts or ".." in normalized.parts: + return None + return str(normalized) -def _rebind_run_receipt( - *, source: Path, destination: Path, checkout_root: Path, run_id: str, current_run_path: Path | None = None -) -> bool: - """Copy the run receipt while rebinding its checkout-local provenance.""" - try: - payload = json.loads((source / "run.json").read_text(encoding="utf-8")) - if not isinstance(payload, Mapping) or payload.get("run_id") != run_id: - return False - source_root = payload.get("checkout_root") - if not isinstance(source_root, str) or Path(source_root).resolve() != source.parents[3].resolve(): - return False - payload_dict: dict[str, Any] = dict(payload) - payload_dict["checkout_root"] = str(checkout_root.resolve()) - payload_dict["artifact_dir"] = str(Path(".cache") / "verify" / "runs" / run_id) - environment = payload_dict.get("environment_fingerprint") - if isinstance(environment, dict): - environment["checkout_root"] = str(checkout_root.resolve()) - environment["verify_state_origin"] = str(checkout_root.resolve()) - _atomic_write_json(destination / "run.json", payload_dict) - _atomic_write_json( - current_run_path or checkout_root / ".cache" / "verify" / "current-run.json", - payload_dict, - ) - return True - except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): - return False +def executable_python_paths(repo_root: Path, paths: Iterable[str]) -> tuple[str, ...]: + """Return changed Python paths whose runtime behavior needs graph edges. + + A deleted module cannot be parsed, but it still invalidates any graph edge + that pointed at it. Keep that path in the required set so inspection fails + closed and the verifier rebuilds the complete native corpus. + """ + root = repo_root.resolve() + executable: list[str] = [] + for raw in sorted(set(paths)): + relative = _safe_relative_path(raw) + if relative is None or not relative.endswith(".py"): + continue + source = root / relative + if source.exists() and source.is_file() and classify_source_ast(source) == "declaration-only": + continue + executable.append(relative) + return tuple(executable) -def _atomic_copy_sqlite_db(src: Path, dst: Path) -> None: - """Copy a (possibly concurrently-written) sqlite db via the online backup API. +def classify_native_testmon_changes(repo_root: Path, paths: Iterable[str]) -> NativeTestmonChangeImpact: + """Classify changed product inputs against native testmon's trace boundary. - `sqlite3.Connection.backup` is designed to snapshot a live database without - requiring an exclusive lock on the source, so this tolerates the main - checkout mid-write. The destination is built at a temp path and only - `rename`d into place once the backup completes, so a reader never observes - a partially-copied file at `dst`. + Python tracing does not observe non-Python runtime data. Non-Python files + under the shipped ``polylogue`` package or the test runtime tree are + therefore outside the native graph and cannot safely use affected + selection. The caller must run the complete native corpus for those + changes. This convention covers additions, deletions, and all data formats + without a filename registry. """ - dst.parent.mkdir(parents=True, exist_ok=True) - tmp = dst.with_name(f"{dst.name}.{os.getpid()}.tmp") - tmp.unlink(missing_ok=True) + normalized = tuple(relative for raw in sorted(set(paths)) if (relative := _safe_relative_path(raw)) is not None) + native_paths = tuple(relative for relative in normalized if not _is_ignored_native_testmon_path(relative)) + runtime_data = tuple( + relative + for relative in native_paths + if (not relative.endswith(".py") and relative.startswith(("polylogue/", "tests/"))) + or ( + relative.endswith(".py") + and relative.startswith(("devtools/", "polylogue/", "tests/")) + and classify_source_ast(repo_root / relative) == "declaration-only" + ) + or relative.startswith("packaging/") + ) + return NativeTestmonChangeImpact( + executable_paths=executable_python_paths( + repo_root, + (relative for relative in native_paths if relative not in runtime_data), + ), + runtime_data_paths=runtime_data, + ) + + +def _readonly_uri(path: Path) -> str: + return f"{path.absolute().as_uri()}?mode=ro" + + +def _testmon_schema_version() -> int: + module = importlib.import_module("testmon.db") + value = getattr(module, "DATA_VERSION", None) + if not isinstance(value, int): + raise NativeTestmonRepairError("pytest-testmon does not expose an integer database version") + return value + + +def _digest_nodeids(nodeids: Sequence[str]) -> str: + return hashlib.sha256("\n".join(nodeids).encode()).hexdigest() + + +def inspect_native_testmon_environment( + data_path: Path, + *, + environment_name: str, + required_executable_paths: Sequence[str] = (), + deadline_monotonic: float | None = None, +) -> NativeTestmonState: + """Validate one native environment without interpreting plugin internals.""" + _ensure_deadline(deadline_monotonic) + sidecars = tuple(Path(f"{data_path}{suffix}") for suffix in TESTMON_SIDECAR_SUFFIXES) + if not data_path.exists(): + if any(path.exists() or path.is_symlink() for path in sidecars): + return NativeTestmonState("invalid", "SQLite sidecars exist without the owned database") + return NativeTestmonState("absent", "native testmon database is absent") try: - src_conn = sqlite3.connect(f"{src.resolve().as_uri()}?mode=ro", uri=True) + state = data_path.lstat() + except OSError as exc: + return NativeTestmonState("invalid", f"cannot inspect native testmon database: {exc}") + if not stat.S_ISREG(state.st_mode) or state.st_nlink != 1: + return NativeTestmonState("invalid", "native testmon database is not a single-link regular file") + for sidecar in sidecars: try: - dst_conn = sqlite3.connect(tmp) - try: - src_conn.backup(dst_conn) - finally: - dst_conn.close() - finally: - src_conn.close() - tmp.replace(dst) - finally: - tmp.unlink(missing_ok=True) + sidecar_state = sidecar.lstat() + except FileNotFoundError: + continue + except OSError as exc: + return NativeTestmonState("invalid", f"cannot inspect native testmon sidecar {sidecar}: {exc}") + if not stat.S_ISREG(sidecar_state.st_mode) or sidecar_state.st_nlink != 1: + return NativeTestmonState("invalid", f"native testmon sidecar is not a single-link regular file: {sidecar}") + try: + with ( + contextlib.closing( + sqlite3.connect( + _readonly_uri(data_path), + uri=True, + timeout=_remaining_timeout(deadline_monotonic, 10), + ) + ) as connection, + connection, + ): + if deadline_monotonic is not None: + connection.set_progress_handler(lambda: int(time.monotonic() >= deadline_monotonic), 1_000) + quick_check = connection.execute("PRAGMA quick_check").fetchone() + _ensure_deadline(deadline_monotonic) + if quick_check is None or quick_check[0] != "ok": + return NativeTestmonState("invalid", "SQLite quick_check failed") + version_row = connection.execute("PRAGMA user_version").fetchone() + if version_row is None or version_row[0] != _testmon_schema_version(): + return NativeTestmonState("invalid", "pytest-testmon database schema version changed") + environment_rows = connection.execute( + "SELECT id FROM environment WHERE environment_name = ? ORDER BY id DESC", + (environment_name,), + ).fetchall() + _ensure_deadline(deadline_monotonic) + if len(environment_rows) != 1: + reason = "native environment is absent" if not environment_rows else "native environment is ambiguous" + return NativeTestmonState("invalid", reason) + environment_id = int(environment_rows[0][0]) + nodeids = tuple( + row[0] + for row in connection.execute( + "SELECT test_name FROM test_execution WHERE environment_id = ? ORDER BY test_name", + (environment_id,), + ).fetchall() + if isinstance(row[0], str) and row[0] + ) + _ensure_deadline(deadline_monotonic) + if not nodeids or len(nodeids) != len(set(nodeids)): + return NativeTestmonState("invalid", "native environment has no unique collected corpus") + uncovered = connection.execute( + """ + SELECT COUNT(*) + FROM test_execution AS execution + LEFT JOIN test_execution_file_fp AS edge ON edge.test_execution_id = execution.id + WHERE execution.environment_id = ? AND edge.test_execution_id IS NULL + """, + (environment_id,), + ).fetchone() + _ensure_deadline(deadline_monotonic) + if uncovered is None or int(uncovered[0]) != 0: + return NativeTestmonState("invalid", "native environment has tests without dependency placeholders") + raw_files = connection.execute( + """ + SELECT DISTINCT fingerprint.filename + FROM test_execution AS execution + JOIN test_execution_file_fp AS edge ON edge.test_execution_id = execution.id + JOIN file_fp AS fingerprint ON fingerprint.id = edge.fingerprint_id + WHERE execution.environment_id = ? + """, + (environment_id,), + ).fetchall() + _ensure_deadline(deadline_monotonic) + except NativeTestmonDeadlineError: + raise + except (NativeTestmonRepairError, OSError, sqlite3.Error, TypeError, ValueError) as exc: + _ensure_deadline(deadline_monotonic) + return NativeTestmonState("invalid", f"native testmon database is unreadable: {exc}") + fingerprinted = frozenset( + relative + for row in raw_files + if row and isinstance(row[0], str) + if (relative := _safe_relative_path(row[0])) is not None + ) + required = tuple(sorted(set(required_executable_paths))) + missing = tuple(path for path in required if path not in fingerprinted) + environment = NativeTestmonEnvironment( + name=environment_name, + corpus_count=len(nodeids), + corpus_digest=_digest_nodeids(nodeids), + nodeids=nodeids, + fingerprinted_files=fingerprinted, + ) + if missing: + return NativeTestmonState( + "invalid", + "changed executable modules are absent from the native dependency graph", + environment, + missing, + ) + return NativeTestmonState("valid", "native environment is current", environment) + + +def _owned_paths(repo_root: Path) -> tuple[Path, ...]: + root = repo_root.resolve() + _validate_owned_state_parents(root) + data = root / TESTMON_DATA_RELPATH + return (data, *(Path(f"{data}{suffix}") for suffix in TESTMON_SIDECAR_SUFFIXES)) + + +def _validate_owned_state_parents(repo_root: Path) -> None: + """Reject state paths that escape the checkout through a symlink parent.""" + parent = repo_root.resolve() + for part in TESTMON_DATA_RELPATH.parent.parts: + parent /= part + try: + mode = parent.lstat().st_mode + except FileNotFoundError: + return + except OSError as exc: + raise NativeTestmonRepairError(f"cannot inspect owned testmon parent {parent}: {exc}") from exc + if stat.S_ISLNK(mode): + raise NativeTestmonRepairError(f"refusing symlinked owned testmon parent {parent}") + if not stat.S_ISDIR(mode): + raise NativeTestmonRepairError(f"owned testmon parent is not a directory: {parent}") + + +def validate_native_testmon_state_ownership(repo_root: Path) -> None: + """Reject parent or file replacement before managed SQLite access.""" + for path in _owned_paths(repo_root): + try: + state = path.lstat() + except FileNotFoundError: + continue + except OSError as exc: + raise NativeTestmonRepairError(f"cannot inspect owned testmon path {path}: {exc}") from exc + if not stat.S_ISREG(state.st_mode) or state.st_nlink != 1: + raise NativeTestmonRepairError(f"owned testmon path is not a single-link regular file: {path}") -def _publish_staged_bootstrap_files(*, staging_dir: Path, files: list[tuple[Path, Path | None]]) -> None: - """Publish a validated bootstrap as one rollback-capable file set.""" - backup_dir = staging_dir / "backups" - backup_dir.mkdir(parents=True, exist_ok=True) - backups: list[tuple[Path, Path]] = [] - published: list[Path] = [] +def remove_invalid_native_testmon_state(repo_root: Path) -> tuple[Path, ...]: + """Remove only the exact checkout-owned SQLite file and known sidecars.""" + removed: list[Path] = [] + for path in _owned_paths(repo_root): + try: + state = path.lstat() + except FileNotFoundError: + continue + except OSError as exc: + raise NativeTestmonRepairError(f"cannot inspect owned testmon path {path}: {exc}") from exc + if stat.S_ISDIR(state.st_mode): + raise NativeTestmonRepairError(f"refusing to remove directory at owned SQLite path {path}") + if stat.S_ISREG(state.st_mode) and state.st_nlink != 1: + raise NativeTestmonRepairError(f"refusing to remove hard-linked owned SQLite path {path}") + try: + path.unlink() + except OSError as exc: + raise NativeTestmonRepairError(f"cannot remove invalid owned testmon path {path}: {exc}") from exc + removed.append(path) + return tuple(removed) + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: - for index, (destination, staged) in enumerate(files): - destination.parent.mkdir(parents=True, exist_ok=True) - backup = backup_dir / str(index) - if destination.exists(): - os.replace(destination, backup) - backups.append((destination, backup)) - if staged is not None: - os.replace(staged, destination) - published.append(destination) - except (OSError, ValueError): - for destination in reversed(published): - destination.unlink(missing_ok=True) - for destination, backup in reversed(backups): - if backup.exists(): - os.replace(backup, destination) - raise + os.fsync(descriptor) finally: - shutil.rmtree(backup_dir, ignore_errors=True) - - -def _copy_runtime_identity_inputs(*, source_root: Path, destination_root: Path) -> None: - """Mirror the inputs used to validate a staged testmon receipt.""" - for relative_path in ( - "uv.lock", - "pyproject.toml", - "pytest.ini", - "tox.ini", - "setup.cfg", - "tests/conftest.py", - ): - source = source_root / relative_path - if not source.is_file(): - continue - destination = destination_root / relative_path - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, destination) + os.close(descriptor) -def bootstrap_testmon_seed_files( - decision: BootstrapDecision, +def _atomic_copy_sqlite_database( + source: Path, + destination: Path, *, - local_testmon_data: Path, - local_seed_stamp: Path, - local_seed_attempt: Path | None = None, - checkout_root: Path | None = None, - inherited_from: Path | None = None, -) -> bool: - """Perform the copy `decision` describes and report whether it was stamped.""" - if not decision.should_bootstrap: - return True - assert decision.main_testmon_data is not None - if decision.main_seed_stamp is None and decision.main_seed_attempt is None: - return False - if decision.main_seed_attempt is not None and local_seed_attempt is None: - return False - if checkout_root is None or inherited_from is None: - return False - stamp: TestmonSeedStamp | None = None + environment_name: str, + required_executable_paths: Sequence[str], + deadline_monotonic: float | None, +) -> None: + _ensure_deadline(deadline_monotonic) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f".{destination.name}.copy-{os.getpid()}-{uuid.uuid4().hex}.tmp") try: - source_root = (decision.main_checkout_root or inherited_from).resolve() - destination_root = checkout_root.resolve() - if source_root == destination_root: - return False - if inherited_from.resolve() != source_root: - return False - if decision.main_testmon_data.resolve() == local_testmon_data.resolve(): - return False - decision.main_testmon_data.resolve().relative_to(source_root) - local_testmon_data.resolve().relative_to(destination_root) - local_seed_stamp.resolve().relative_to(destination_root) - if local_seed_stamp.resolve() == local_testmon_data.resolve(): - return False - if local_seed_attempt is not None: - local_seed_attempt.resolve().relative_to(destination_root) - if local_seed_attempt.resolve() in {local_testmon_data.resolve(), local_seed_stamp.resolve()}: - return False - if decision.main_seed_stamp is not None: - decision.main_seed_stamp.resolve().relative_to(source_root) - if decision.main_seed_stamp.resolve() == decision.main_testmon_data.resolve(): - return False - if decision.main_seed_attempt is not None: - decision.main_seed_attempt.resolve().relative_to(source_root) - if decision.main_seed_attempt.resolve() == decision.main_testmon_data.resolve(): - return False - if decision.main_seed_stamp is not None: - stamp = validate_stamp( - decision.main_seed_stamp, - decision.main_testmon_data, - checkout_root=source_root, - protocol_version=decision.protocol_version, - ) - else: - assert decision.main_seed_attempt is not None - source = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) - if not isinstance(source, dict): - return False - stamp = stamp_from_attempt( - source, - decision.main_testmon_data, - checkout_root=source_root, - protocol_version=decision.protocol_version, - published_marker=False, - ) - if stamp is None: - return False - except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError, TypeError, sqlite3.Error): - return False - if stamp is None: - return False - staging_dir: Path | None = None - try: - local_testmon_data.parent.mkdir(parents=True, exist_ok=True) - staging_dir = Path(tempfile.mkdtemp(prefix=".bootstrap-", dir=str(local_testmon_data.parent))) - staged_data = staging_dir / "testmondata" - staged_stamp = staging_dir / "seed.json" - staged_attempt = staging_dir / "seed-attempt.json" - staged_artifact = staging_dir / "artifact" - staged_current_run = staging_dir / "current-run.json" - - _atomic_copy_sqlite_db(decision.main_testmon_data, staged_data) - rebound = stamp.rebound(checkout_root=destination_root, inherited_from=source_root) - refreshed = refresh_stamp(rebound, staged_data) - if refreshed is None or refreshed.graph != rebound.graph: - return False - source_artifact = source_root / Path(stamp.artifact_dir) - destination_artifact = (destination_root / Path(refreshed.artifact_dir)).resolve() - destination_artifact.relative_to(destination_root) - if not _rebind_run_receipt( - source=source_artifact, - destination=staged_artifact, - checkout_root=destination_root, - run_id=refreshed.run_id, - current_run_path=staged_current_run, + with ( + contextlib.closing( + sqlite3.connect( + _readonly_uri(source), + uri=True, + timeout=_remaining_timeout(deadline_monotonic, 60), + ) + ) as source_connection, + contextlib.closing( + sqlite3.connect(temporary, timeout=_remaining_timeout(deadline_monotonic, 60)) + ) as destination_connection, + source_connection, + destination_connection, ): - return False - staged_attempt_path: Path | None = None - publishes_selection_attempt = decision.main_seed_attempt is not None and decision.selection_only - if publishes_selection_attempt: - assert decision.main_seed_attempt is not None - assert local_seed_attempt is not None - source_attempt = json.loads(decision.main_seed_attempt.read_text(encoding="utf-8")) - if not isinstance(source_attempt, dict): - return False - rebound_attempt = dict(source_attempt) - rebound_attempt["testmon_data"] = refreshed.testmon_data - rebound_attempt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" - rebound_attempt["binding"] = refreshed.binding.as_dict() - rebound_attempt["release_baseline_allowed"] = False - rebound_attempt["verification_scope"] = "affected" - validation_root = staging_dir / "validation" - validation_receipt = json.loads(staged_current_run.read_text(encoding="utf-8")) - if not isinstance(validation_receipt, dict): - return False - validation_receipt["checkout_root"] = str(validation_root.resolve()) - validation_receipt["artifact_dir"] = f".cache/verify/runs/{refreshed.run_id}" - _copy_runtime_identity_inputs(source_root=destination_root, destination_root=validation_root) - _atomic_write_json( - validation_root / ".cache" / "verify" / "runs" / refreshed.run_id / "run.json", - validation_receipt, + source_connection.backup( + destination_connection, + pages=256, + progress=lambda _status, _remaining, _total: _ensure_deadline(deadline_monotonic), + sleep=0.05, ) - validation_attempt = dict(rebound_attempt) - raw_binding = validation_attempt.get("binding") - if not isinstance(raw_binding, Mapping): - return False - validation_binding = dict(raw_binding) - validation_binding["checkout_root"] = str(validation_root.resolve()) - validation_attempt["binding"] = validation_binding - if ( - stamp_from_attempt( - validation_attempt, - staged_data, - checkout_root=validation_root, - protocol_version=decision.protocol_version, - ) - is None - ): - return False - _atomic_write_json(staged_attempt, rebound_attempt) - staged_attempt_path = staged_attempt - else: - _atomic_write_stamp(staged_stamp, refreshed) - publication_files: list[tuple[Path, Path | None]] = [ - (local_testmon_data, staged_data), - (destination_artifact / "run.json", staged_artifact / "run.json"), - (destination_root / ".cache" / "verify" / "current-run.json", staged_current_run), - (local_seed_stamp, None if publishes_selection_attempt else staged_stamp), - ] - if local_seed_attempt is not None: - publication_files.append((local_seed_attempt, staged_attempt_path)) - _publish_staged_bootstrap_files(staging_dir=staging_dir, files=publication_files) - return True - except (OSError, sqlite3.Error, TypeError, ValueError): - return False + _ensure_deadline(deadline_monotonic) + copied = inspect_native_testmon_environment( + temporary, + environment_name=environment_name, + required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, + ) + if not copied.valid: + raise NativeTestmonRepairError(f"copied main-checkout database failed validation: {copied.reason}") + descriptor = os.open(temporary, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, destination) + _fsync_directory(destination.parent) + _ensure_deadline(deadline_monotonic) + except NativeTestmonDeadlineError: + raise + except (OSError, sqlite3.Error) as exc: + raise NativeTestmonRepairError(f"SQLite online backup failed: {exc}") from exc finally: - if staging_dir is not None: - shutil.rmtree(staging_dir, ignore_errors=True) - + with contextlib.suppress(FileNotFoundError): + temporary.unlink() + for suffix in TESTMON_SIDECAR_SUFFIXES: + with contextlib.suppress(FileNotFoundError): + Path(f"{temporary}{suffix}").unlink() -def _git_worktree_info(repo_root: Path) -> tuple[bool, Path] | None: - """Return `(is_linked_worktree, main_checkout_path)`, or None if undeterminable. - Same `git rev-parse --absolute-git-dir --git-common-dir` mechanism - `devtools/verify_worktree.py:inspect_worktree` uses: a linked worktree's - git-dir (`.git/worktrees/`) differs from the shared - git-common-dir; a main checkout's git-dir *is* the common-dir. - """ +def linked_worktree_info( + repo_root: Path, + *, + deadline_monotonic: float | None = None, +) -> tuple[bool, Path] | None: + """Return linked-worktree status and the main checkout path.""" try: result = subprocess.run( ["git", "-C", str(repo_root), "rev-parse", "--absolute-git-dir", "--git-common-dir"], capture_output=True, text=True, - timeout=10, + timeout=_remaining_timeout(deadline_monotonic, 10), + env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}, ) - except (OSError, subprocess.TimeoutExpired): + except subprocess.TimeoutExpired: + _ensure_deadline(deadline_monotonic) + return None + except OSError: return None if result.returncode != 0: return None @@ -531,83 +815,109 @@ def _git_worktree_info(repo_root: Path) -> tuple[bool, Path] | None: git_dir = Path(lines[0]).resolve() raw_common = Path(lines[1]) common_dir = raw_common.resolve() if raw_common.is_absolute() else (repo_root / raw_common).resolve() - is_linked = git_dir != common_dir - main_checkout = common_dir.parent - return is_linked, main_checkout + return git_dir != common_dir, common_dir.parent -def maybe_bootstrap_testmon_seed( +def prepare_native_testmon_environment( repo_root: Path, *, - testmon_data_relpath: str = TESTMON_DATA_RELPATH, - seed_stamp_relpath: str = TESTMON_SEED_STAMP_RELPATH, - seed_attempt_relpath: str = TESTMON_SEED_ATTEMPT_RELPATH, - protocol_version: int, -) -> str | None: - """Bootstrap `repo_root`'s testmon seed from its main checkout if warranted. - - Returns a one-line message to log on success, or ``None`` when no - bootstrap happened (not a linked worktree, already seeded locally, or the - main checkout has nothing valid to offer). Called from - `devtools/verify.py` before `_testmon_preflight`, so a freshly-bootstrapped - worktree passes that preflight instead of refusing. - """ - info = _git_worktree_info(repo_root) - if info is None: - return None - is_linked_worktree, main_checkout = info - if main_checkout == repo_root.resolve(): - return None - local_testmon_data = repo_root / testmon_data_relpath - local_seed_stamp = repo_root / seed_stamp_relpath - local_seed_attempt = repo_root / seed_attempt_relpath - main_testmon_data = main_checkout / testmon_data_relpath - main_seed_stamp = main_checkout / seed_stamp_relpath - main_seed_attempt = main_checkout / seed_attempt_relpath - decision = decide_testmon_bootstrap( - is_linked_worktree=is_linked_worktree, - local_testmon_data=local_testmon_data, - local_seed_stamp=local_seed_stamp, - main_testmon_data=main_testmon_data, - main_seed_stamp=main_seed_stamp, - protocol_version=protocol_version, - main_seed_attempt=main_seed_attempt, - main_checkout_root=main_checkout, - local_checkout_root=repo_root, - local_seed_attempt=local_seed_attempt, + required_executable_paths: Sequence[str] = (), + pytest_profile: str = "default", + pytest_environment: Mapping[str, str | None] | None = None, + deadline_monotonic: float | None = None, +) -> NativeTestmonPreparation: + """Repair derived local state and optionally reuse a matching main graph.""" + root = repo_root.resolve() + _validate_owned_state_parents(root) + environment_name = testmon_environment_digest( + root, + pytest_profile=pytest_profile, + pytest_environment=pytest_environment, + deadline_monotonic=deadline_monotonic, ) - if not decision.should_bootstrap: - return None - stamped = bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_testmon_data, - local_seed_stamp=local_seed_stamp, - local_seed_attempt=local_seed_attempt, - checkout_root=repo_root, - inherited_from=main_checkout, + local_data = root / TESTMON_DATA_RELPATH + local_data.parent.mkdir(parents=True, exist_ok=True) + _validate_owned_state_parents(root) + local = inspect_native_testmon_environment( + local_data, + environment_name=environment_name, + required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, ) - if not stamped: - return ( - f"verify: refused pytest-testmon bootstrap into {local_testmon_data.parent}; " - "no local state was published because provenance validation failed" + missing_checkout_paths = tuple( + path for path in sorted(set(required_executable_paths)) if not (root / path).is_file() + ) + if local.valid and missing_checkout_paths: + local = NativeTestmonState( + "invalid", + "changed executable modules are absent from the current checkout", + local.environment, + missing_checkout_paths, ) - if decision.main_seed_attempt is not None and decision.selection_only: - return ( - f"verify: bootstrapped pytest-testmon graph from main checkout {main_checkout} " - f"into {local_testmon_data.parent} as a selection-only attempt receipt (no seed.json)" + info = linked_worktree_info(root, deadline_monotonic=deadline_monotonic) + linked = bool(info and info[0]) + main_checkout = info[1] if linked and info is not None else None + if local.valid: + return NativeTestmonPreparation(environment_name, "affected", local, None, (), linked, main_checkout) + + removed = remove_invalid_native_testmon_state(root) + _ensure_deadline(deadline_monotonic) + copied_from: Path | None = None + if main_checkout is not None and main_checkout != root and not missing_checkout_paths: + _validate_owned_state_parents(main_checkout) + main_data = main_checkout / TESTMON_DATA_RELPATH + main = inspect_native_testmon_environment( + main_data, + environment_name=environment_name, + required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, ) - return ( - f"verify: bootstrapped pytest-testmon seed from main checkout {main_checkout} " - f"into {local_testmon_data.parent} (worktree had no local seed)" - ) + if main.valid: + _atomic_copy_sqlite_database( + main_data, + local_data, + environment_name=environment_name, + required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, + ) + copied_from = main_data + local = inspect_native_testmon_environment( + local_data, + environment_name=environment_name, + required_executable_paths=required_executable_paths, + deadline_monotonic=deadline_monotonic, + ) + if not local.valid: + raise NativeTestmonRepairError(f"published native testmon copy is invalid: {local.reason}") + return NativeTestmonPreparation( + environment_name, + "affected", + local, + copied_from, + removed, + linked, + main_checkout, + ) + + return NativeTestmonPreparation(environment_name, "bootstrap", local, copied_from, removed, linked, main_checkout) __all__ = [ + "ASTClassification", + "NativeTestmonEnvironment", + "NativeTestmonDeadlineError", + "NativeTestmonChangeImpact", + "NativeTestmonPreparation", + "NativeTestmonRepairError", + "NativeTestmonState", "TESTMON_DATA_RELPATH", - "TESTMON_SEED_STAMP_RELPATH", - "TESTMON_SEED_ATTEMPT_RELPATH", - "BootstrapDecision", - "decide_testmon_bootstrap", - "bootstrap_testmon_seed_files", - "maybe_bootstrap_testmon_seed", + "classify_source_ast", + "classify_native_testmon_changes", + "executable_python_paths", + "inspect_native_testmon_environment", + "linked_worktree_info", + "prepare_native_testmon_environment", + "remove_invalid_native_testmon_state", + "testmon_environment_digest", + "validate_native_testmon_state_ownership", ] diff --git a/devtools/testmon_mutation_proof.py b/devtools/testmon_mutation_proof.py deleted file mode 100644 index 3e80bc20ba..0000000000 --- a/devtools/testmon_mutation_proof.py +++ /dev/null @@ -1,288 +0,0 @@ -"""Executable anti-vacuity proof for the pytest-testmon affected gate. - -The proof deliberately operates on a disposable copy of an actual Polylogue -production module and its existing production-route test. It does not mock -pytest, testmon, changed paths, selection, or the verdict: pytest-testmon -records the dependency graph and the real test must fail after a semantic -source mutation. -""" - -from __future__ import annotations - -import argparse -import json -import os -import shutil -import signal -import sqlite3 -import subprocess -import sys -import tempfile -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import asdict, dataclass, replace -from pathlib import Path -from typing import Any - -from devtools import repo_root - -_SOURCE = Path("polylogue/core/web_urls.py") -_UNRELATED_SOURCE = Path("polylogue/core/stats.py") -_TEST = Path("tests/unit/core/test_web_urls.py") -_UNRELATED_TEST = Path("tests/unit/core/test_percentile.py") -_TARGET_NODEID = "tests/unit/core/test_web_urls.py::test_chatgpt_url_bare" -_MUTATION_FROM = 'return f"https://chatgpt.com/c/{native_id}"' -_MUTATION_TO = 'return f"https://chatgpt.invalid/c/{native_id}"' -_TIMEOUT_S = 60.0 - - -@dataclass(frozen=True, slots=True) -class ProofResult: - ok: bool - target_nodeid: str - selected_nodeids: tuple[str, ...] - selected_count: int - total_seeded_nodes: int - mutation_exit_code: int - restored_exit_code: int - severed_edge_rejected: bool - unrelated_selected_count: int - cleanup_complete: bool - failure: str | None = None - - def to_dict(self) -> dict[str, object]: - return asdict(self) - - -def _copy_required_tree(source_root: Path, scratch: Path) -> None: - shutil.copytree(source_root / "polylogue" / "core", scratch / "polylogue" / "core") - for relative in ( - Path("polylogue/__init__.py"), - # polylogue/__init__.py unconditionally imports this before any - # sqlite3 import (must run first to swap in a modern bundled - # SQLite); omitting it breaks collection of every copied test. - Path("polylogue/_sqlite_compat.py"), - _TEST, - _UNRELATED_TEST, - Path("devtools/__init__.py"), - Path("devtools/pytest_progress_plugin.py"), - ): - destination = scratch / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source_root / relative, destination) - - -def _pytest_env(scratch: Path, name: str) -> dict[str, str]: - env = os.environ.copy() - env["PYTHONPATH"] = str(scratch) - # The project devshell normally points testmon at .cache/testmon/testmondata. - # Keep this proof's graph inside its disposable copy instead. - env["TESTMON_DATAFILE"] = str(scratch / ".testmondata") - env["POLYLOGUE_PYTEST_SELECTION_PATH"] = str(scratch / ".artifacts" / f"{name}-selection.json") - env["POLYLOGUE_PYTEST_EVENTS_PATH"] = str(scratch / ".artifacts" / f"{name}-events.jsonl") - env["POLYLOGUE_PYTEST_SUMMARY_PATH"] = str(scratch / ".artifacts" / f"{name}-summary.json") - env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "100" - return env - - -def _run_pytest(scratch: Path, *, name: str, args: list[str]) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "-q", - "-p", - "devtools.pytest_progress_plugin", - # The scratch tree deliberately omits pyproject.toml (a disposable - # copy of only the files this proof needs), so pytest-randomly's - # default random ordering is uncontrolled here. That randomness - # is real: two trivial same-file tests whose production-module - # coverage happens to fingerprint identically can, depending on - # which runs first, leave one test's file_fp edge unrecorded -- - # not a testmon defect, just order nondeterminism this proof - # must not have. Force a fixed order so the seed graph (and the - # dependency-edge assertions that follow) are reproducible. - "-p", - "no:randomly", - *args, - ], - cwd=scratch, - env=_pytest_env(scratch, name), - text=True, - capture_output=True, - timeout=_TIMEOUT_S, - check=False, - ) - - -def _selection(scratch: Path, name: str) -> tuple[int, tuple[str, ...]]: - path = scratch / ".artifacts" / f"{name}-selection.json" - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return 0, () - selected = payload.get("selected_nodeids") if isinstance(payload, dict) else None - count = payload.get("selected_count") if isinstance(payload, dict) else None - if not isinstance(selected, list) or not all(isinstance(item, str) for item in selected): - selected = [] - return (count if isinstance(count, int) else len(selected), tuple(selected)) - - -def _mutate(path: Path, before: str, after: str) -> None: - source = path.read_text(encoding="utf-8") - if source.count(before) != 1: - raise RuntimeError(f"expected one mutation anchor in {path}") - path.write_text(source.replace(before, after), encoding="utf-8") - - -def _sever_target_edge(testmon_db: Path) -> None: - with sqlite3.connect(testmon_db) as connection: - cursor = connection.execute( - """ - DELETE FROM test_execution_file_fp - WHERE test_execution_id IN ( - SELECT id FROM test_execution WHERE test_name = ? - ) - AND fingerprint_id IN ( - SELECT id FROM file_fp WHERE filename LIKE ? - ) - """, - (_TARGET_NODEID, "%polylogue/core/web_urls.py"), - ) - if cursor.rowcount < 1: - raise RuntimeError("could not sever the named testmon dependency edge") - - -def _target_edge_present(testmon_db: Path) -> bool: - with sqlite3.connect(testmon_db) as connection: - row = connection.execute( - """ - SELECT 1 - FROM test_execution_file_fp AS edge - JOIN test_execution AS execution ON execution.id = edge.test_execution_id - JOIN file_fp AS fingerprint ON fingerprint.id = edge.fingerprint_id - WHERE execution.test_name = ? AND fingerprint.filename LIKE ? - LIMIT 1 - """, - (_TARGET_NODEID, "%polylogue/core/web_urls.py"), - ).fetchone() - return row is not None - - -@contextmanager -def _cleanup_on_interrupt() -> Iterator[None]: - # typeshed models the previous handler returned by ``signal.signal`` more - # narrowly than the handler value it accepts on restore. - previous: dict[int, Any] = {} - - def interrupt(signum: int, _frame: object) -> None: - raise InterruptedError(f"received signal {signum}") - - for signum in (int(signal.SIGINT), int(signal.SIGTERM)): - previous[signum] = signal.signal(signum, interrupt) - try: - yield - finally: - for restored_signal, handler in previous.items(): - signal.signal(restored_signal, handler) - - -def run_proof(*, source_root: Path | None = None) -> ProofResult: - """Run the real, bounded testmon dependency proof in a disposable copy.""" - source_root = (source_root or repo_root()).resolve() - scratch: Path | None = None - try: - with _cleanup_on_interrupt(), tempfile.TemporaryDirectory(prefix="polylogue-testmon-proof-") as temporary: - scratch = Path(temporary) - _copy_required_tree(source_root, scratch) - seed = _run_pytest(scratch, name="seed", args=["--testmon", "--testmon-noselect"]) - if seed.returncode != 0: - raise RuntimeError(f"real-route seed failed: {seed.stderr or seed.stdout}") - total_seeded_nodes, _ = _selection(scratch, "seed") - if total_seeded_nodes < 2: - raise RuntimeError("seed did not record a meaningful real-route test set") - if not _target_edge_present(scratch / ".testmondata"): - raise RuntimeError("seed did not record the named real-route dependency edge") - seed_graph = scratch / ".testmondata.seed" - shutil.copy2(scratch / ".testmondata", seed_graph) - - _mutate(scratch / _SOURCE, _MUTATION_FROM, _MUTATION_TO) - affected = _run_pytest(scratch, name="mutation", args=["--testmon", "--testmon-forceselect", "-n", "0"]) - selected_count, selected_nodeids = _selection(scratch, "mutation") - if ( - affected.returncode == 0 - or _TARGET_NODEID not in selected_nodeids - or _TARGET_NODEID not in (affected.stdout + affected.stderr) - ): - raise RuntimeError("semantic mutation was not selected and failed by the named real-route test") - - _mutate(scratch / _SOURCE, _MUTATION_TO, _MUTATION_FROM) - restored = _run_pytest(scratch, name="restored", args=[]) - if restored.returncode != 0: - raise RuntimeError("removing the semantic mutation did not return the real route to green") - - shutil.copy2(seed_graph, scratch / ".testmondata") - _sever_target_edge(scratch / ".testmondata") - if _target_edge_present(scratch / ".testmondata"): - raise RuntimeError("severed testmon dependency edge remained readable") - severed_edge_rejected = True - - shutil.copy2(seed_graph, scratch / ".testmondata") - # A byte-preserving rewrite is not a source change, so append a comment - # outside executable behavior solely to make testmon inspect an unrelated file. - unrelated = scratch / _UNRELATED_SOURCE - unrelated.write_text( - unrelated.read_text(encoding="utf-8") + "\n# testmon-proof unrelated change\n", encoding="utf-8" - ) - unrelated_run = _run_pytest( - scratch, name="unrelated", args=["--testmon", "--testmon-forceselect", "-n", "0"] - ) - del unrelated_run - unrelated_selected_count, _ = _selection(scratch, "unrelated") - if unrelated_selected_count >= total_seeded_nodes: - raise RuntimeError("unrelated production change fell back to the complete seeded route") - - proof = ProofResult( - ok=True, - target_nodeid=_TARGET_NODEID, - selected_nodeids=selected_nodeids, - selected_count=selected_count, - total_seeded_nodes=total_seeded_nodes, - mutation_exit_code=affected.returncode, - restored_exit_code=restored.returncode, - severed_edge_rejected=severed_edge_rejected, - unrelated_selected_count=unrelated_selected_count, - cleanup_complete=False, - ) - return replace(proof, cleanup_complete=not scratch.exists()) - except (InterruptedError, subprocess.TimeoutExpired, OSError, RuntimeError, sqlite3.Error) as exc: - return ProofResult( - ok=False, - target_nodeid=_TARGET_NODEID, - selected_nodeids=(), - selected_count=0, - total_seeded_nodes=0, - mutation_exit_code=-1, - restored_exit_code=-1, - severed_edge_rejected=False, - unrelated_selected_count=0, - cleanup_complete=scratch is not None and not scratch.exists(), - failure=str(exc), - ) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description="Prove real pytest-testmon affected selection with a semantic mutation." - ) - parser.add_argument("--json", action="store_true", help="Emit the complete proof receipt as JSON.") - args = parser.parse_args(argv) - result = run_proof() - if args.json: - print(json.dumps(result.to_dict(), indent=2)) - elif result.ok: - print(f"testmon mutation proof passed: {result.target_nodeid}") - else: - print(f"testmon mutation proof failed: {result.failure}", file=sys.stderr) - return 0 if result.ok else 1 diff --git a/devtools/testmon_state.py b/devtools/testmon_state.py deleted file mode 100644 index 8ac4bdaa14..0000000000 --- a/devtools/testmon_state.py +++ /dev/null @@ -1,1154 +0,0 @@ -"""Typed safety contract for reusable pytest-testmon state. - -The testmon database answers two different questions which must not share a -boolean marker: - -* did collection and dependency capture cover every promised node? -* did that run establish a green release baseline? - -A failed test can still have a complete dependency graph. Such a graph is -usable for affected-test selection, but it is never evidence that the suite -is releasable. This module is the single parser and SQLite validator used by -verification, worktree bootstrap, and the checkout guard. -""" - -from __future__ import annotations - -import contextlib -import hashlib -import importlib.metadata -import json -import os -import sqlite3 -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, replace -from enum import StrEnum -from pathlib import Path -from typing import Any - - -class CollectionStatus(StrEnum): - COMPLETE = "complete" - INCOMPLETE = "incomplete" - - -class GraphStatus(StrEnum): - COMPLETE = "complete" - INCOMPLETE = "incomplete" - INVALID = "invalid" - - -class BaselineStatus(StrEnum): - GREEN = "green" - RED = "red" - - -class SeedAttemptOutcome(StrEnum): - """Terminal result of a seed attempt, separate from graph reusability.""" - - GREEN_RELEASE_BASELINE = "green-release-baseline" - RED_BASELINE = "red-baseline" - SELECTION_ONLY = "selection-only" - INCOMPLETE = "incomplete" - RESOURCE_TIMEOUT = "resource-timeout" - - -class SeedShardStatus(StrEnum): - """Durable state of one sequential pytest-testmon seed shard.""" - - PENDING = "pending" - RUNNING = "running" - COMPLETE = "complete" - INCOMPLETE = "incomplete" - - -class BindingMode(StrEnum): - EXACT = "exact" - RELATIVE_FILE_FINGERPRINTS = "relative-file-fingerprints" - - -class VerificationScope(StrEnum): - AFFECTED = "affected" - RELEASE_BASELINE = "release-baseline" - NARROW_TERMINAL = "narrow-terminal" - NON_TEST = "non-test" - - -class TerminalAuthorization(StrEnum): - NARROW_TERMINAL = "narrow-terminal" - - -# Pytest reports expected failures separately from ordinary skips/passes. They -# still finish the selected node and therefore make its dependency capture -# reusable. ``xpassed`` remains subject to pytest's configured strictness via -# the process exit code; it is not independently recast as a failure here. -TERMINAL_NODE_OUTCOMES = frozenset({"passed", "failed", "error", "skipped", "xfailed", "xpassed"}) -SUCCESSFUL_NODE_OUTCOMES = frozenset({"passed", "skipped", "xfailed", "xpassed"}) - - -def seed_shard_plan( - nodeids: Sequence[str], - *, - shard_size: int, - serial_nodeids: Sequence[str] = (), -) -> list[dict[str, Any]]: - """Partition a complete node set into stable, contiguous, serial shards.""" - if shard_size <= 0: - raise ValueError("testmon seed shard_size must be positive") - if not nodeids or any(not nodeid for nodeid in nodeids): - raise ValueError("testmon seed nodeids must be non-empty strings") - if len(set(nodeids)) != len(nodeids): - raise ValueError("testmon seed nodeids must be unique") - ordered = tuple(sorted(nodeids)) - serial = set(serial_nodeids) - if not serial.issubset(ordered): - raise ValueError("testmon serial shard nodes must belong to the seed corpus") - chunks: list[tuple[str, list[str]]] = [] - for offset in range(0, len(ordered), shard_size): - chunk = list(ordered[offset : offset + shard_size]) - parallel = [nodeid for nodeid in chunk if nodeid not in serial] - isolated = [nodeid for nodeid in chunk if nodeid in serial] - if parallel: - chunks.append(("parallel", parallel)) - if isolated: - chunks.append(("serial", isolated)) - return [ - { - "index": index, - "nodeids": chunk, - "nodeid_count": len(chunk), - "nodeid_digest": hashlib.sha256("\n".join(chunk).encode()).hexdigest(), - "execution_mode": mode, - "status": SeedShardStatus.PENDING.value, - "node_outcomes": [], - } - for index, (mode, chunk) in enumerate(chunks, start=1) - ] - - -def validate_seed_shard_ledger( - shards: object, - *, - expected_nodeids: Sequence[str], -) -> list[dict[str, Any]] | None: - """Validate the full shard ledger without granting release authority. - - Every shard owns a disjoint contiguous part of the sorted expected node - set. A completed shard carries an explicit terminal result for every node; - interrupted shards remain visible and are eligible for resume. - """ - if not isinstance(shards, list) or not shards: - return None - expected = tuple(sorted(expected_nodeids)) - if not expected or len(set(expected)) != len(expected): - return None - normalized: list[dict[str, Any]] = [] - observed: set[str] = set() - for index, raw in enumerate(shards, start=1): - if not isinstance(raw, Mapping) or raw.get("index") != index: - return None - nodeids = raw.get("nodeids") - if ( - not isinstance(nodeids, list) - or not nodeids - or any(not isinstance(nodeid, str) or not nodeid for nodeid in nodeids) - or nodeids != sorted(nodeids) - ): - return None - if raw.get("nodeid_count") != len(nodeids): - return None - if raw.get("nodeid_digest") != hashlib.sha256("\n".join(nodeids).encode()).hexdigest(): - return None - raw_status = raw.get("status") - if not isinstance(raw_status, str): - return None - try: - status = SeedShardStatus(raw_status) - except (TypeError, ValueError): - return None - outcomes = raw.get("node_outcomes") - if not isinstance(outcomes, list): - return None - outcome_by_node: dict[str, dict[str, Any]] = {} - for outcome in outcomes: - if not isinstance(outcome, Mapping): - return None - nodeid = outcome.get("nodeid") - state = outcome.get("outcome") - if not isinstance(nodeid, str) or nodeid not in nodeids or not isinstance(state, str): - return None - if nodeid in outcome_by_node: - return None - outcome_by_node[nodeid] = dict(outcome) - if status is SeedShardStatus.PENDING and outcomes: - return None - if status is SeedShardStatus.COMPLETE and ( - set(outcome_by_node) != set(nodeids) - or any(item.get("outcome") not in TERMINAL_NODE_OUTCOMES for item in outcome_by_node.values()) - ): - return None - if ( - status in {SeedShardStatus.RUNNING, SeedShardStatus.INCOMPLETE} - and outcomes - and set(outcome_by_node) != set(nodeids) - ): - return None - if observed.intersection(nodeids): - return None - normalized.append(dict(raw)) - observed.update(nodeids) - if observed != set(expected): - return None - return normalized - - -def seed_shard_ledger_is_terminal(shards: Sequence[Mapping[str, Any]]) -> bool: - """Return whether every planned shard completed with explicit node results.""" - return all(shard.get("status") == SeedShardStatus.COMPLETE.value for shard in shards) - - -@dataclass(frozen=True, slots=True) -class TestmonIdentity: - git_head: str | None - worktree_fingerprint: str - python: str - skip_slow: bool - lab: bool - git_tree: str | None = None - terminal_authorization: str | None = None - dependency_environment: str = "" - pytest_harness: str = "" - - @classmethod - def from_mapping(cls, value: Mapping[str, Any]) -> TestmonIdentity: - git_head = value.get("git_head") - if git_head is not None and (not isinstance(git_head, str) or not git_head): - raise ValueError("identity.git_head must be a non-empty string or null") - git_tree = value.get("git_tree") - if git_tree is not None and (not isinstance(git_tree, str) or not git_tree): - raise ValueError("identity.git_tree must be a non-empty string or null") - worktree = value.get("worktree_fingerprint") - python = value.get("python") - if not isinstance(worktree, str) or not worktree: - raise ValueError("identity.worktree_fingerprint must be a non-empty string") - if not isinstance(python, str) or not python: - raise ValueError("identity.python must be a non-empty string") - dependency_environment = value.get("dependency_environment") - pytest_harness = value.get("pytest_harness") - if dependency_environment is None: - dependency_environment = "" - if pytest_harness is None: - pytest_harness = "" - if not isinstance(dependency_environment, str): - raise ValueError("identity.dependency_environment must be a string") - if not isinstance(pytest_harness, str): - raise ValueError("identity.pytest_harness must be a string") - if not isinstance(value.get("skip_slow"), bool) or not isinstance(value.get("lab"), bool): - raise ValueError("identity selection flags must be booleans") - terminal_authorization = value.get("terminal_authorization") - if terminal_authorization is not None and terminal_authorization not in { - authorization.value for authorization in TerminalAuthorization - }: - raise ValueError("identity.terminal_authorization is invalid") - return cls( - git_head, - worktree, - python, - value["skip_slow"], - value["lab"], - git_tree, - terminal_authorization, - dependency_environment, - pytest_harness, - ) - - def as_dict(self) -> dict[str, Any]: - return { - "git_head": self.git_head, - "worktree_fingerprint": self.worktree_fingerprint, - "python": self.python, - "skip_slow": self.skip_slow, - "lab": self.lab, - "git_tree": self.git_tree, - "terminal_authorization": self.terminal_authorization, - "dependency_environment": self.dependency_environment, - "pytest_harness": self.pytest_harness, - } - - -def _fingerprint_files(checkout_root: Path, relative_paths: Sequence[str]) -> str: - """Hash named checkout inputs, preserving absent inputs as typed state.""" - digest = hashlib.sha256() - for relative_path in relative_paths: - digest.update(relative_path.encode()) - digest.update(b"\0") - try: - contents = (checkout_root / relative_path).read_bytes() - except OSError: - digest.update(b"missing") - else: - digest.update(contents) - digest.update(b"\0") - return digest.hexdigest() - - -def _installed_distributions() -> tuple[tuple[str, str], ...] | None: - """Return the active environment's normalized installed distributions.""" - try: - distributions = [] - for distribution in importlib.metadata.distributions(): - try: - name = distribution.metadata["Name"] - except KeyError: - return None - version = distribution.version - if not name or not version: - return None - distributions.append((name.casefold(), version)) - except (OSError, TypeError, ValueError, importlib.metadata.PackageNotFoundError): - return None - return tuple(sorted(distributions)) - - -def testmon_runtime_identity(checkout_root: Path) -> tuple[str, str] | None: - """Identify the lock, installed dependencies, and pytest execution harness. - - A testmon graph is reusable only under this exact dependency environment. - The application lock catches declared changes; installed distributions and - pytest-specific configuration catch a stale or differently provisioned - virtual environment even when ``sys.version`` is unchanged. - """ - distributions = _installed_distributions() - if distributions is None: - return None - normalized_root = checkout_root.resolve() - dependency_payload = { - "lock_inputs": _fingerprint_files(normalized_root, ("uv.lock", "pyproject.toml")), - "distributions": distributions, - } - harness_payload = { - "configuration": _fingerprint_files( - normalized_root, - ("pyproject.toml", "pytest.ini", "tox.ini", "setup.cfg", "tests/conftest.py"), - ), - "environment": { - key: os.environ.get(key) - for key in ( - "PYTEST_ADDOPTS", - "PYTEST_DISABLE_PLUGIN_AUTOLOAD", - "PYTEST_PLUGINS", - "HYPOTHESIS_PROFILE", - "POLYLOGUE_CI", - ) - }, - "pytest_distributions": tuple( - item for item in distributions if item[0] in {"pytest", "pytest-testmon", "pytest-xdist", "pluggy"} - ), - } - return ( - hashlib.sha256(json.dumps(dependency_payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), - hashlib.sha256(json.dumps(harness_payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), - ) - - -def _identity_matches_runtime(identity: TestmonIdentity, *, checkout_root: Path, protocol_version: int) -> bool: - """Keep pre-binding protocol receipts parseable but never reusable today.""" - if protocol_version < 5: - return True - runtime_identity = testmon_runtime_identity(checkout_root) - return ( - runtime_identity is not None - and ( - identity.dependency_environment, - identity.pytest_harness, - ) - == runtime_identity - ) - - -@dataclass(frozen=True, slots=True) -class TestmonBinding: - mode: BindingMode - checkout_root: str - source_checkout_root: str | None = None - - @classmethod - def from_mapping(cls, value: Mapping[str, Any]) -> TestmonBinding: - raw_mode = value.get("mode") - if not isinstance(raw_mode, str): - raise ValueError("binding.mode is invalid") - try: - mode = BindingMode(raw_mode) - except ValueError as exc: - raise ValueError("binding.mode is invalid") from exc - checkout_root = value.get("checkout_root") - source = value.get("source_checkout_root") - if not isinstance(checkout_root, str) or not checkout_root: - raise ValueError("binding.checkout_root must be a non-empty string") - if not Path(checkout_root).is_absolute(): - raise ValueError("binding.checkout_root must be absolute") - if source is not None and (not isinstance(source, str) or not source): - raise ValueError("binding.source_checkout_root must be a non-empty string or null") - if source is not None and not Path(source).is_absolute(): - raise ValueError("binding.source_checkout_root must be absolute") - if mode is BindingMode.EXACT and source is not None: - raise ValueError("exact bindings cannot have a source checkout") - if mode is BindingMode.RELATIVE_FILE_FINGERPRINTS: - if source is None: - raise ValueError("rebound bindings require a source checkout") - if Path(source).resolve() == Path(checkout_root).resolve(): - raise ValueError("rebound binding source and destination must differ") - return cls(mode, checkout_root, source) - - def as_dict(self) -> dict[str, Any]: - return { - "mode": self.mode.value, - "checkout_root": self.checkout_root, - "source_checkout_root": self.source_checkout_root, - } - - -@dataclass(frozen=True, slots=True) -class GraphInspection: - status: GraphStatus - recorded_count: int - dependency_edge_count: int - missing_nodeids: tuple[str, ...] - orphan_execution_edges: int - orphan_fingerprint_edges: int - error: str | None - failed_nodeids: tuple[str, ...] - - @property - def usable_for_selection(self) -> bool: - return self.status is GraphStatus.COMPLETE - - def as_dict(self) -> dict[str, Any]: - return { - "status": self.status.value, - "recorded_count": self.recorded_count, - "dependency_edge_count": self.dependency_edge_count, - "missing_nodeids": list(self.missing_nodeids), - "orphan_execution_edges": self.orphan_execution_edges, - "orphan_fingerprint_edges": self.orphan_fingerprint_edges, - "error": self.error, - "failed_nodeids": list(self.failed_nodeids), - } - - -@dataclass(frozen=True, slots=True) -class TestmonSeedStamp: - protocol_version: int - collection_status: CollectionStatus - expected_nodeids: tuple[str, ...] - selected_nodeids_omitted: int - baseline_status: BaselineStatus - release_baseline_allowed: bool - baseline_exit_code: int - graph: GraphInspection - identity: TestmonIdentity - binding: TestmonBinding - testmon_data: str - run_id: str - artifact_dir: str - - @property - def affected_selection_allowed(self) -> bool: - return ( - self.collection_status is CollectionStatus.COMPLETE - and self.selected_nodeids_omitted == 0 - and self.graph.usable_for_selection - ) - - @property - def expected_digest(self) -> str: - return hashlib.sha256("\n".join(sorted(self.expected_nodeids)).encode()).hexdigest() - - def as_dict(self) -> dict[str, Any]: - return { - "protocol_version": self.protocol_version, - "status": "usable", - "collection": { - "status": self.collection_status.value, - "expected_count": len(self.expected_nodeids), - "expected_digest": self.expected_digest, - "selected_nodeids": list(self.expected_nodeids), - "selected_nodeids_omitted": self.selected_nodeids_omitted, - }, - "baseline": { - "status": self.baseline_status.value, - "exit_code": self.baseline_exit_code, - "release_baseline_allowed": self.release_baseline_allowed, - }, - "graph": self.graph.as_dict(), - "identity": self.identity.as_dict(), - "binding": self.binding.as_dict(), - "testmon_data": self.testmon_data, - "run_id": self.run_id, - "artifact_dir": self.artifact_dir, - } - - @classmethod - def from_mapping(cls, value: Mapping[str, Any], *, protocol_version: int) -> TestmonSeedStamp: - if value.get("protocol_version") != protocol_version or value.get("status") != "usable": - raise ValueError("seed stamp is not a current usable testmon stamp") - collection = value.get("collection") - baseline = value.get("baseline") - graph = value.get("graph") - identity = value.get("identity") - binding = value.get("binding") - if not all(isinstance(item, Mapping) for item in (collection, baseline, graph, identity, binding)): - raise ValueError("seed stamp has incomplete typed state") - assert isinstance(collection, Mapping) - assert isinstance(baseline, Mapping) - assert isinstance(graph, Mapping) - assert isinstance(identity, Mapping) - assert isinstance(binding, Mapping) - if collection.get("status") != CollectionStatus.COMPLETE.value: - raise ValueError("seed stamp collection is not complete") - nodeids = collection.get("selected_nodeids") - if ( - not isinstance(nodeids, list) - or not nodeids - or any(not isinstance(item, str) or not item for item in nodeids) - ): - raise ValueError("seed stamp selected nodeids are missing or malformed") - if len(set(nodeids)) != len(nodeids): - raise ValueError("seed stamp selected nodeids are not unique") - omitted = collection.get("selected_nodeids_omitted") - if not isinstance(omitted, int) or isinstance(omitted, bool) or omitted != 0: - raise ValueError("seed stamp has controlled collection omissions") - if collection.get("expected_count") != len(nodeids): - raise ValueError("seed stamp expected count does not match selected nodeids") - expected_digest = hashlib.sha256("\n".join(sorted(nodeids)).encode()).hexdigest() - if collection.get("expected_digest") != expected_digest: - raise ValueError("seed stamp expected nodeid digest is stale") - raw_baseline_status = baseline.get("status") - if not isinstance(raw_baseline_status, str): - raise ValueError("seed stamp baseline status is invalid") - try: - baseline_status = BaselineStatus(raw_baseline_status) - except ValueError as exc: - raise ValueError("seed stamp baseline status is invalid") from exc - exit_code = baseline.get("exit_code") - release_allowed = baseline.get("release_baseline_allowed") - if not isinstance(exit_code, int) or isinstance(exit_code, bool) or not isinstance(release_allowed, bool): - raise ValueError("seed stamp baseline fields are malformed") - if release_allowed != (baseline_status is BaselineStatus.GREEN): - raise ValueError("release permission does not match baseline status") - if baseline_status is BaselineStatus.GREEN and exit_code != 0: - raise ValueError("green seed stamp must have a zero exit code") - graph_status = graph.get("status") - if not isinstance(graph_status, str): - raise ValueError("seed stamp graph status is invalid") - try: - status = GraphStatus(graph_status) - except ValueError as exc: - raise ValueError("seed stamp graph status is invalid") from exc - if status is not GraphStatus.COMPLETE: - raise ValueError("seed stamp graph is not complete") - graph_expected = [ - "recorded_count", - "dependency_edge_count", - "orphan_execution_edges", - "orphan_fingerprint_edges", - ] - graph_counts = {key: graph.get(key) for key in graph_expected} - if any(not isinstance(item, int) or isinstance(item, bool) or item < 0 for item in graph_counts.values()): - raise ValueError("seed stamp graph counts are malformed") - dependency_edge_count = graph.get("dependency_edge_count") - if not isinstance(dependency_edge_count, int) or isinstance(dependency_edge_count, bool): - raise ValueError("seed stamp dependency edge count is malformed") - if ( - graph.get("recorded_count") != len(nodeids) - or dependency_edge_count < len(nodeids) - or graph.get("orphan_execution_edges") != 0 - or graph.get("orphan_fingerprint_edges") != 0 - ): - raise ValueError("seed stamp graph coverage is incomplete") - missing_nodeids = graph.get("missing_nodeids") - if ( - not isinstance(missing_nodeids, list) - or any(not isinstance(item, str) or not item for item in missing_nodeids) - or not set(missing_nodeids).issubset(nodeids) - ): - raise ValueError("seed stamp missing-node ledger is malformed") - if graph.get("error") is not None or missing_nodeids: - raise ValueError("seed stamp graph has missing or erroneous nodes") - graph_nodeids = graph.get("failed_nodeids", []) - if ( - not isinstance(graph_nodeids, list) - or any(not isinstance(item, str) or not item for item in graph_nodeids) - or not set(graph_nodeids).issubset(nodeids) - or len(set(graph_nodeids)) != len(graph_nodeids) - ): - raise ValueError("seed stamp graph failure ledger is malformed") - if baseline_status is BaselineStatus.GREEN and graph_nodeids: - raise ValueError("green seed stamp cannot contain failed graph nodes") - testmon_data = value.get("testmon_data") - run_id = value.get("run_id") - artifact_dir = value.get("artifact_dir") - if not all(isinstance(item, str) and item for item in (testmon_data, run_id, artifact_dir)): - raise ValueError("seed stamp provenance is incomplete") - assert isinstance(testmon_data, str) - assert isinstance(run_id, str) - assert isinstance(artifact_dir, str) - typed_binding = TestmonBinding.from_mapping(binding) - if not _is_bound_run_artifact( - artifact_dir, - checkout_root=Path(typed_binding.checkout_root), - run_id=run_id, - ): - raise ValueError("seed stamp artifact directory is not checkout-bound") - return cls( - protocol_version, - CollectionStatus.COMPLETE, - tuple(nodeids), - 0, - baseline_status, - release_allowed, - exit_code, - GraphInspection( - status, - graph["recorded_count"], - dependency_edge_count, - tuple(graph.get("missing_nodeids", [])), - graph["orphan_execution_edges"], - graph["orphan_fingerprint_edges"], - graph.get("error"), - tuple(graph_nodeids), - ), - TestmonIdentity.from_mapping(identity), - typed_binding, - testmon_data, - run_id, - artifact_dir, - ) - - def rebound(self, *, checkout_root: Path, inherited_from: Path) -> TestmonSeedStamp: - return replace( - self, - binding=TestmonBinding( - BindingMode.RELATIVE_FILE_FINGERPRINTS, - str(checkout_root.resolve()), - str(inherited_from.resolve()), - ), - ) - - -def file_fingerprint(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _is_bound_run_artifact(raw: object, *, checkout_root: Path, run_id: str) -> bool: - if not isinstance(raw, str) or not raw or not run_id: - return False - path = Path(raw) - if path.is_absolute() or path.parts[:3] != (".cache", "verify", "runs"): - return False - if path.parts[3:] != (run_id,): - return False - try: - artifact_dir = (checkout_root / path).resolve() - artifact_dir.relative_to((checkout_root / ".cache" / "verify" / "runs" / run_id).resolve()) - receipt = json.loads((artifact_dir / "run.json").read_text(encoding="utf-8")) - if not isinstance(receipt, Mapping): - return False - return ( - receipt.get("run_id") == run_id - and isinstance(receipt.get("checkout_root"), str) - and Path(receipt["checkout_root"]).resolve() == checkout_root.resolve() - and receipt.get("artifact_dir") == str(Path(".cache") / "verify" / "runs" / run_id) - ) - except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): - return False - - -def seed_marker_is_checkout_bound( - marker_path: Path, - *, - checkout_root: Path, - protocol_version: int, -) -> bool: - """Validate only the typed ownership envelope of a seed marker. - - This intentionally does not open or fingerprint SQLite. The checkout guard - uses this cheap predicate for every entrypoint; verify preflight performs - the exhaustive graph validation before authorizing selection. - """ - try: - payload = json.loads(marker_path.read_text(encoding="utf-8")) - if not isinstance(payload, Mapping): - return False - stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) - return Path(stamp.binding.checkout_root).resolve() == checkout_root.resolve() - except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError): - return False - - -def attempt_is_checkout_bound( - attempt: Mapping[str, Any], - *, - checkout_root: Path, - protocol_version: int, - reusable_only: bool = True, -) -> bool: - """Check a seed-attempt receipt without inspecting its SQLite graph.""" - allowed_statuses = {"reusable", "complete"} if reusable_only else {"running", "incomplete", "reusable", "complete"} - if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in allowed_statuses: - return False - identity = attempt.get("identity") - expected = attempt.get("expected_nodeids") - selection = attempt.get("selection") - if not isinstance(identity, Mapping) or not isinstance(expected, list) or not isinstance(selection, Mapping): - return False - if not expected or any(not isinstance(nodeid, str) or not nodeid for nodeid in expected): - return False - if len(set(expected)) != len(expected): - return False - if ( - not isinstance(attempt.get("expected_count"), int) - or isinstance(attempt.get("expected_count"), bool) - or attempt.get("expected_count") != len(expected) - ): - return False - expected_digest = hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() - if attempt.get("expected_digest") != expected_digest: - return False - try: - typed_identity = TestmonIdentity.from_mapping(identity) - except ValueError: - return False - if not _identity_matches_runtime(typed_identity, checkout_root=checkout_root, protocol_version=protocol_version): - return False - omitted = selection.get("selected_nodeids_omitted") - selected_count = selection.get("selected_count") - if ( - not isinstance(omitted, int) - or isinstance(omitted, bool) - or omitted != 0 - or not isinstance(selected_count, int) - or isinstance(selected_count, bool) - or selected_count != len(expected) - ): - return False - recorded_data = attempt.get("testmon_data") - run_id = attempt.get("run_id") - artifact_dir = attempt.get("artifact_dir") - if ( - not isinstance(recorded_data, str) - or not recorded_data - or not isinstance(run_id, str) - or not run_id - or not isinstance(artifact_dir, str) - or not artifact_dir - or not _is_bound_run_artifact(artifact_dir, checkout_root=checkout_root, run_id=run_id) - ): - return False - raw_binding = attempt.get("binding") - if raw_binding is None: - binding = TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())) - elif isinstance(raw_binding, Mapping): - try: - binding = TestmonBinding.from_mapping(raw_binding) - except ValueError: - return False - else: - return False - if Path(binding.checkout_root).resolve() != checkout_root.resolve(): - return False - raw_permission = attempt.get("release_baseline_allowed") - if raw_permission is not None and not isinstance(raw_permission, bool): - return False - raw_scope = attempt.get("verification_scope") - if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: - return False - if reusable_only and raw_permission is not False: - return False - if reusable_only: - outcomes = attempt.get("node_outcomes") - if not isinstance(outcomes, list) or len(outcomes) != len(expected): - return False - nodeids = [item.get("nodeid") for item in outcomes if isinstance(item, Mapping)] - if len(nodeids) != len(outcomes) or set(nodeids) != set(expected) or len(set(nodeids)) != len(nodeids): - return False - if any(item.get("outcome") not in TERMINAL_NODE_OUTCOMES for item in outcomes): - return False - return True - - -def inspect_testmon_database(path: Path, expected_nodeids: Sequence[str]) -> GraphInspection: - """Validate the real testmon schema and every expected dependency edge.""" - expected = tuple(expected_nodeids) - if not path.is_file() or not expected or len(set(expected)) != len(expected): - return GraphInspection( - GraphStatus.INCOMPLETE, 0, 0, expected, 0, 0, "missing or malformed expected nodeids", () - ) - try: - with contextlib.closing(sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)) as connection: - if connection.execute("PRAGMA integrity_check").fetchone() != ("ok",): - return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "sqlite integrity check failed", ()) - required = {"test_execution", "test_execution_file_fp", "file_fp"} - tables = {str(row[0]) for row in connection.execute("select name from sqlite_master where type='table'")} - if not required <= tables: - return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon schema is incomplete", ()) - required_columns = { - "test_execution": {"id", "test_name", "failed"}, - "test_execution_file_fp": {"test_execution_id", "fingerprint_id"}, - "file_fp": {"id", "filename", "fsha"}, - } - for table, columns in required_columns.items(): - actual = {str(row[1]) for row in connection.execute(f"pragma table_info({table})")} - if not columns <= actual: - return GraphInspection( - GraphStatus.INVALID, - 0, - 0, - expected, - 0, - 0, - f"testmon schema is missing columns from {table}", - (), - ) - executions = connection.execute( - "select id, test_name, failed from test_execution where test_name is not null" - ).fetchall() - latest: dict[str, tuple[int, bool]] = {} - execution_ids: set[int] = set() - for execution_id, test_name, failed in executions: - if ( - not isinstance(execution_id, int) - or isinstance(execution_id, bool) - or execution_id <= 0 - or not isinstance(test_name, str) - or not test_name - or not isinstance(failed, int) - or isinstance(failed, bool) - or failed not in (0, 1) - ): - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon execution row is malformed", () - ) - if execution_id in execution_ids: - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon execution ids are not unique", () - ) - execution_ids.add(execution_id) - name = test_name - if name not in expected: - grouped = [nodeid for nodeid in expected if name.startswith(nodeid + "@")] - name = max(grouped, key=len, default=name) - prior = latest.get(name) - if prior is None or execution_id > prior[0]: - latest[name] = (execution_id, failed == 1) - missing = tuple(sorted(set(expected) - latest.keys())) - expected_ids = {latest[nodeid][0] for nodeid in expected if nodeid in latest} - edge_rows = connection.execute( - "select test_execution_id, fingerprint_id from test_execution_file_fp" - ).fetchall() - fingerprints = connection.execute("select id, filename, fsha from file_fp").fetchall() - fingerprint_ids: set[int] = set() - for fingerprint_id, filename, fsha in fingerprints: - if ( - not isinstance(fingerprint_id, int) - or isinstance(fingerprint_id, bool) - or fingerprint_id <= 0 - or not isinstance(filename, str) - or not filename - or Path(filename).is_absolute() - or ".." in Path(filename).parts - or not isinstance(fsha, str) - or not fsha - ): - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon fingerprint row is malformed", () - ) - if fingerprint_id in fingerprint_ids: - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon fingerprint ids are not unique", () - ) - fingerprint_ids.add(fingerprint_id) - for execution_id, fingerprint_id in edge_rows: - if ( - not isinstance(execution_id, int) - or isinstance(execution_id, bool) - or execution_id <= 0 - or not isinstance(fingerprint_id, int) - or isinstance(fingerprint_id, bool) - or fingerprint_id <= 0 - ): - return GraphInspection( - GraphStatus.INVALID, 0, 0, expected, 0, 0, "testmon dependency edge is malformed", () - ) - orphan_execution_edges = sum(1 for row in edge_rows if row[0] not in execution_ids) - orphan_fingerprint_edges = sum(1 for row in edge_rows if row[1] not in fingerprint_ids) - edge_counts: dict[int, int] = {} - for execution_id, _fingerprint_id in edge_rows: - edge_counts[execution_id] = edge_counts.get(execution_id, 0) + 1 - uncovered = tuple( - sorted(nodeid for nodeid in expected if nodeid in latest and edge_counts.get(latest[nodeid][0], 0) == 0) - ) - missing = tuple(sorted(set(missing) | set(uncovered))) - failed = tuple(sorted(nodeid for nodeid in expected if nodeid in latest and latest[nodeid][1])) - edge_count = sum(edge_counts.get(execution_id, 0) for execution_id in expected_ids) - status = ( - GraphStatus.COMPLETE - if not missing and not orphan_execution_edges and not orphan_fingerprint_edges - else GraphStatus.INCOMPLETE - ) - return GraphInspection( - status, - len(expected) - len(missing), - edge_count, - missing, - orphan_execution_edges, - orphan_fingerprint_edges, - None, - failed, - ) - except (OSError, sqlite3.Error, UnicodeError, TypeError, ValueError, OverflowError) as exc: - return GraphInspection(GraphStatus.INVALID, 0, 0, expected, 0, 0, str(exc), ()) - - -def validate_stamp( - stamp_path: Path, - data_path: Path, - *, - checkout_root: Path, - protocol_version: int, -) -> TestmonSeedStamp | None: - """Parse and re-check a stamp against its current SQLite graph.""" - try: - payload = json.loads(stamp_path.read_text(encoding="utf-8")) - if not isinstance(payload, Mapping): - return None - stamp = TestmonSeedStamp.from_mapping(payload, protocol_version=protocol_version) - if not stamp.release_baseline_allowed: - return None - if ( - stamp.identity.skip_slow - and stamp.identity.terminal_authorization != TerminalAuthorization.NARROW_TERMINAL.value - ): - return None - if not _identity_matches_runtime( - stamp.identity, checkout_root=checkout_root, protocol_version=protocol_version - ): - return None - if Path(stamp.binding.checkout_root).resolve() != checkout_root.resolve(): - return None - if file_fingerprint(data_path) != stamp.testmon_data: - return None - graph = inspect_testmon_database(data_path, stamp.expected_nodeids) - if graph != stamp.graph: - return None - return stamp - except (OSError, TypeError, ValueError, json.JSONDecodeError): - return None - - -def refresh_stamp(stamp: TestmonSeedStamp, data_path: Path) -> TestmonSeedStamp | None: - """Refresh mutable SQLite provenance after a successful affected run.""" - graph = inspect_testmon_database(data_path, stamp.expected_nodeids) - if not graph.usable_for_selection: - return None - try: - return replace(stamp, graph=graph, testmon_data=file_fingerprint(data_path)) - except OSError: - return None - - -def stamp_from_attempt( - attempt: Mapping[str, Any], - data_path: Path, - *, - checkout_root: Path, - protocol_version: int, - published_marker: bool = True, -) -> TestmonSeedStamp | None: - """Parse a complete attempt, withholding release authority until publication.""" - if attempt.get("protocol_version") != protocol_version or attempt.get("status") not in {"reusable", "complete"}: - return None - if protocol_version >= 6: - raw_outcome = attempt.get("outcome") - if not isinstance(raw_outcome, str): - return None - try: - outcome = SeedAttemptOutcome(raw_outcome) - except ValueError: - return None - if attempt.get("status") == "complete" and outcome is not SeedAttemptOutcome.GREEN_RELEASE_BASELINE: - return None - if attempt.get("status") == "reusable" and outcome not in { - SeedAttemptOutcome.RED_BASELINE, - SeedAttemptOutcome.SELECTION_ONLY, - }: - return None - selection = attempt.get("selection") - expected = attempt.get("expected_nodeids") - identity = attempt.get("identity") - if not isinstance(selection, Mapping) or not isinstance(expected, list) or not isinstance(identity, Mapping): - return None - assert isinstance(selection, Mapping) - assert isinstance(identity, Mapping) - omitted = selection.get("selected_nodeids_omitted") - selected_count = selection.get("selected_count") - if ( - not isinstance(omitted, int) - or isinstance(omitted, bool) - or omitted != 0 - or not isinstance(selected_count, int) - or isinstance(selected_count, bool) - or selected_count != len(expected) - or not expected - or any(not isinstance(nodeid, str) or not nodeid for nodeid in expected) - or len(set(expected)) != len(expected) - ): - return None - expected_count = attempt.get("expected_count") - if not isinstance(expected_count, int) or isinstance(expected_count, bool) or expected_count != len(expected): - return None - if protocol_version >= 7: - shards = validate_seed_shard_ledger(attempt.get("shards"), expected_nodeids=expected) - if shards is None or not seed_shard_ledger_is_terminal(shards): - return None - expected_digest = attempt.get("expected_digest") - if ( - not isinstance(expected_digest, str) - or expected_digest != hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() - ): - return None - recorded_data = attempt.get("testmon_data") - if not isinstance(recorded_data, str) or not recorded_data or not data_path.is_file(): - return None - try: - if file_fingerprint(data_path) != recorded_data: - return None - except OSError: - return None - run_id = attempt.get("run_id") - artifact_dir = attempt.get("artifact_dir") - if not isinstance(run_id, str) or not run_id or not isinstance(artifact_dir, str) or not artifact_dir: - return None - if not _is_bound_run_artifact(artifact_dir, checkout_root=checkout_root, run_id=run_id): - return None - outcomes = attempt.get("node_outcomes") - if not isinstance(outcomes, list) or len(outcomes) != len(expected): - return None - if any(not isinstance(item, Mapping) for item in outcomes): - return None - outcome_items = [item for item in outcomes if isinstance(item, Mapping)] - if any( - not isinstance(item.get("nodeid"), str) or not item.get("nodeid") or item.get("nodeid") not in expected - for item in outcome_items - ): - return None - outcome_by_node = {item["nodeid"]: item.get("outcome") for item in outcome_items} - if set(outcome_by_node) != set(expected): - return None - if len(outcome_by_node) != len(outcomes) or any( - not isinstance(nodeid, str) or not nodeid for nodeid in outcome_by_node - ): - return None - if any(outcome not in TERMINAL_NODE_OUTCOMES for outcome in outcome_by_node.values()): - return None - exit_code = attempt.get("exit_code") - if not isinstance(exit_code, int) or isinstance(exit_code, bool): - return None - graph = inspect_testmon_database(data_path, [str(nodeid) for nodeid in expected]) - if not graph.usable_for_selection: - return None - try: - typed_identity = TestmonIdentity.from_mapping(identity) - except ValueError: - return None - if not _identity_matches_runtime(typed_identity, checkout_root=checkout_root, protocol_version=protocol_version): - return None - baseline = ( - BaselineStatus.GREEN - if attempt.get("status") == "complete" - and exit_code == 0 - and all(outcome in SUCCESSFUL_NODE_OUTCOMES for outcome in outcome_by_node.values()) - and not graph.failed_nodeids - else BaselineStatus.RED - ) - raw_scope = attempt.get("verification_scope") - if raw_scope is not None and raw_scope not in {scope.value for scope in VerificationScope}: - return None - terminal_authorized = ( - typed_identity.skip_slow is True - and raw_scope == VerificationScope.NARROW_TERMINAL.value - and typed_identity.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value - ) - if baseline is BaselineStatus.GREEN and typed_identity.skip_slow and not terminal_authorized: - baseline = BaselineStatus.RED - if not published_marker: - baseline = BaselineStatus.RED - raw_permission = attempt.get("release_baseline_allowed") - if baseline is BaselineStatus.GREEN and ( - raw_scope != VerificationScope.NARROW_TERMINAL.value - if typed_identity.skip_slow - else raw_scope != VerificationScope.RELEASE_BASELINE.value - ): - baseline = BaselineStatus.RED - if baseline is BaselineStatus.GREEN and raw_permission is not True: - baseline = BaselineStatus.RED - if raw_permission is not None and not isinstance(raw_permission, bool): - return None - if published_marker and raw_permission is not None and raw_permission != (baseline is BaselineStatus.GREEN): - return None - raw_binding = attempt.get("binding") - if raw_binding is None: - typed_binding = TestmonBinding(BindingMode.EXACT, str(checkout_root.resolve())) - elif isinstance(raw_binding, Mapping): - try: - typed_binding = TestmonBinding.from_mapping(raw_binding) - except ValueError: - return None - if Path(typed_binding.checkout_root).resolve() != checkout_root.resolve(): - return None - else: - return None - return TestmonSeedStamp( - protocol_version, - CollectionStatus.COMPLETE, - tuple(str(nodeid) for nodeid in expected), - 0, - baseline, - baseline is BaselineStatus.GREEN, - exit_code, - graph, - typed_identity, - typed_binding, - recorded_data, - run_id, - artifact_dir, - ) - - -__all__ = [ - "BaselineStatus", - "BindingMode", - "CollectionStatus", - "GraphInspection", - "GraphStatus", - "SeedAttemptOutcome", - "TestmonBinding", - "TestmonIdentity", - "SUCCESSFUL_NODE_OUTCOMES", - "TERMINAL_NODE_OUTCOMES", - "TestmonSeedStamp", - "TerminalAuthorization", - "VerificationScope", - "attempt_is_checkout_bound", - "file_fingerprint", - "inspect_testmon_database", - "refresh_stamp", - "seed_marker_is_checkout_bound", - "stamp_from_attempt", - "testmon_runtime_identity", - "validate_stamp", -] diff --git a/devtools/verification_contracts.py b/devtools/verification_contracts.py new file mode 100644 index 0000000000..757f0f190a --- /dev/null +++ b/devtools/verification_contracts.py @@ -0,0 +1,14 @@ +"""Typed scope fields shared by verification and merge receipts.""" + +from __future__ import annotations + +from enum import StrEnum + + +class VerificationScope(StrEnum): + AFFECTED = "affected" + RELEASE_BASELINE = "release-baseline" + NON_TEST = "non-test" + + +__all__ = ["VerificationScope"] diff --git a/devtools/verify.py b/devtools/verify.py index b7c4171d2d..720aa90f0b 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -7,10 +7,9 @@ --commit Pre-commit tier: ruff format + check + mypy (~3s warm). --quick Pre-push tier: all non-pytest gates (~15s warm). (default) Baseline with pytest-testmon affected tests. - --seed-testmon - Full non-integration pytest run that seeds/updates .cache/testmon/testmondata. --all/--full - Explicit full non-integration pytest diagnostic. + Complete pytest correctness corpus in the current native + testmon environment (performance benchmarks excluded). --lab Default testmon baseline plus lab smoke and SLO checks. Output formats: @@ -22,6 +21,7 @@ import argparse import contextlib +import fcntl import hashlib import json import math @@ -31,11 +31,13 @@ import shlex import shutil import signal +import stat import subprocess import sys import time import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -58,27 +60,18 @@ update_receipt, write_termination_request, ) -from devtools.testmon_bootstrap import maybe_bootstrap_testmon_seed -from devtools.testmon_state import ( - SUCCESSFUL_NODE_OUTCOMES, - TERMINAL_NODE_OUTCOMES, - BindingMode, - GraphStatus, - SeedAttemptOutcome, - SeedShardStatus, - TerminalAuthorization, - TestmonBinding, - TestmonSeedStamp, - VerificationScope, - inspect_testmon_database, - refresh_stamp, - seed_shard_ledger_is_terminal, - seed_shard_plan, - stamp_from_attempt, - testmon_runtime_identity, - validate_seed_shard_ledger, - validate_stamp, +from devtools.testmon_bootstrap import ( + NativeTestmonDeadlineError, + NativeTestmonPreparation, + NativeTestmonRepairError, + NativeTestmonState, + classify_native_testmon_changes, + inspect_native_testmon_environment, + prepare_native_testmon_environment, + remove_invalid_native_testmon_state, + validate_native_testmon_state_ownership, ) +from devtools.verification_contracts import VerificationScope from devtools.verify_runs import ( CURRENT_CONTAINMENT_PATH, CURRENT_EVENTS_DIR, @@ -89,11 +82,13 @@ PYTEST_EXPLICIT_BASETEMP_ENV, VERIFY_HISTORY_PATH, CheckoutMutationMonitor, + CheckoutMutationObservation, PytestResourceError, PytestStepArtifacts, ResourceSampler, VerifyRun, adaptive_pytest_worker_count, + aggregate_native_testmon_run, append_verify_history, apply_managed_pytest_runtime_policy, classify_pytest_result, @@ -127,6 +122,185 @@ ) ROOT = Path(__file__).resolve().parents[1] +_PYTEST_CLEAR_CONFIGURED_ADDOPTS = "--override-ini=addopts=" +_PYTEST_MANAGED_PLUGIN_NAMES = ( + "anyio", + "asyncio", + "hypothesispytest", + "benchmark", + "pytest_cov", + "pytest_jsonreport", + "randomly", + "syrupy", + "timeout", + "xdist", + "pytest-testmon", +) +_PYTEST_MANAGED_PLUGIN_ARGS = tuple(argument for name in _PYTEST_MANAGED_PLUGIN_NAMES for argument in ("-p", name)) +_PYTEST_CLOSED_WORLD_COLLECTION_ARGS = ( + _PYTEST_CLEAR_CONFIGURED_ADDOPTS, + "--override-ini=python_files=test_*.py *_test.py fuzz_*.py", + "--override-ini=python_classes=Test", + "--override-ini=python_functions=test", + "--override-ini=norecursedirs=", + "tests", +) +NATIVE_TESTMON_LIFECYCLE_LOCK_TIMEOUT_S = 60.0 + + +def _normalize_managed_pytest_environment( + env: dict[str, str], + *, + disable_plugin_autoload: bool = True, +) -> None: + """Remove ambient pytest options and extensions from a managed child.""" + env.pop("PYTEST_ADDOPTS", None) + env.pop("PYTEST_PLUGINS", None) + if disable_plugin_autoload: + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + else: + env.pop("PYTEST_DISABLE_PLUGIN_AUTOLOAD", None) + + +def _python_optimization_level() -> int: + """Return the active interpreter optimization level.""" + return int(sys.flags.optimize) + + +@dataclass(slots=True) +class _OwnedNativeTestmonState: + descriptor: int + data_path: Path + + def close(self) -> None: + os.close(self.descriptor) + + +def _open_owned_native_testmon_state(repo_root: Path) -> _OwnedNativeTestmonState: + """Bind managed SQLite access to one no-follow checkout directory.""" + validate_native_testmon_state_ownership(repo_root) + raw_data = TESTMON_DATA if TESTMON_DATA.is_absolute() else repo_root.resolve() / TESTMON_DATA + parent = raw_data.parent + parent.mkdir(parents=True, exist_ok=True) + validate_native_testmon_state_ownership(repo_root) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor: int | None = None + try: + descriptor = os.open(parent, flags) + opened = os.fstat(descriptor) + current = parent.lstat() + except OSError as exc: + if descriptor is not None: + with contextlib.suppress(OSError): + os.close(descriptor) + raise NativeTestmonRepairError(f"cannot bind owned testmon directory {parent}: {exc}") from exc + assert descriptor is not None + if not stat.S_ISDIR(opened.st_mode) or (opened.st_dev, opened.st_ino) != (current.st_dev, current.st_ino): + os.close(descriptor) + raise NativeTestmonRepairError(f"owned testmon directory changed while binding: {parent}") + bound = Path(f"/proc/{os.getpid()}/fd/{descriptor}") / raw_data.name + return _OwnedNativeTestmonState(descriptor=descriptor, data_path=bound) + + +@contextlib.contextmanager +def _native_testmon_lifecycle_lock( + repo_root: Path, + *, + timeout_s: float = NATIVE_TESTMON_LIFECYCLE_LOCK_TIMEOUT_S, +) -> Iterator[None]: + """Serialize one checkout's native testmon preparation, lanes, and inspection.""" + cache = repo_root.resolve() / ".cache" + try: + mode = cache.lstat().st_mode + except FileNotFoundError: + cache.mkdir(exist_ok=True) + mode = cache.lstat().st_mode + except OSError as exc: + raise NativeTestmonRepairError(f"cannot inspect native testmon lock parent {cache}: {exc}") from exc + if not stat.S_ISDIR(mode): + raise NativeTestmonRepairError(f"native testmon lock parent is not an owned directory: {cache}") + lock_path = cache / "native-testmon-lifecycle.lock" + directory_descriptor: int | None = None + lock_descriptor: int | None = None + try: + directory_descriptor = os.open( + cache, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + opened_directory = os.fstat(directory_descriptor) + current_directory = cache.lstat() + if not stat.S_ISDIR(opened_directory.st_mode) or (opened_directory.st_dev, opened_directory.st_ino) != ( + current_directory.st_dev, + current_directory.st_ino, + ): + raise NativeTestmonRepairError(f"native testmon lock parent changed while binding: {cache}") + lock_descriptor = os.open( + lock_path.name, + os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), + 0o600, + dir_fd=directory_descriptor, + ) + opened_lock = os.fstat(lock_descriptor) + current_lock = os.stat(lock_path.name, dir_fd=directory_descriptor, follow_symlinks=False) + if ( + not stat.S_ISREG(opened_lock.st_mode) + or not stat.S_ISREG(current_lock.st_mode) + or opened_lock.st_nlink != 1 + or current_lock.st_nlink != 1 + or (opened_lock.st_dev, opened_lock.st_ino) != (current_lock.st_dev, current_lock.st_ino) + ): + raise NativeTestmonRepairError( + f"native testmon lifecycle lock is not an owned single-link regular file: {lock_path}" + ) + except OSError as exc: + if lock_descriptor is not None: + with contextlib.suppress(OSError): + os.close(lock_descriptor) + lock_descriptor = None + if directory_descriptor is not None: + with contextlib.suppress(OSError): + os.close(directory_descriptor) + directory_descriptor = None + raise NativeTestmonRepairError(f"cannot bind native testmon lifecycle lock {lock_path}: {exc}") from exc + try: + assert lock_descriptor is not None + with os.fdopen(lock_descriptor, "r+", encoding="utf-8") as handle: + lock_descriptor = None + deadline = time.monotonic() + max(0.0, timeout_s) + announced_wait = False + while True: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PytestResourceError( + f"timed out waiting for native testmon lifecycle lock after {timeout_s:.1f}s" + ) from exc + if not announced_wait: + handle.seek(0) + holder = handle.read().strip() or "another verify invocation" + sys.stderr.write(f"verify: waiting for native testmon lifecycle lock ({holder})\n") + sys.stderr.flush() + announced_wait = True + time.sleep(min(0.05, remaining)) + handle.seek(0) + handle.truncate() + handle.write(f"pid={os.getpid()}") + handle.flush() + try: + yield + finally: + handle.seek(0) + handle.truncate() + finally: + if lock_descriptor is not None: + with contextlib.suppress(OSError): + os.close(lock_descriptor) + if directory_descriptor is not None: + with contextlib.suppress(OSError): + os.close(directory_descriptor) def _anchor_verification_paths() -> None: @@ -150,6 +324,8 @@ def _mypy_cmd() -> list[str]: capture_output=True, text=True, timeout=5, + cwd=ROOT, + env=_subprocess_env(), ) if result.returncode == 0: return ["dmypy", "run", "--", "--no-error-summary"] @@ -231,15 +407,6 @@ def _format_completion_notification( HISTORY_PATH = VERIFY_HISTORY_PATH TESTMON_DATA = Path(".cache/testmon/testmondata") -TESTMON_SEED_STAMP = Path(".cache/testmon/seed.json") -TESTMON_SEED_ATTEMPT = Path(".cache/testmon/seed-attempt.json") -TESTMON_AFFECTED_STAMP = Path(".cache/testmon/affected.json") -TESTMON_SEED_PROTOCOL_VERSION = 7 -# Keep resumable checkpoints coarse enough that controller startup and -# per-shard testmon initialization do not dominate the seed. The seed still -# records every node outcome, so a failed shard remains retryable at node -# resolution; this size yields six shards for the current correctness corpus. -TESTMON_SEED_SHARD_SIZE = 4096 PYTEST_REPORT_DIR = Path(".cache/verify") PYTEST_REPORT_PATH = PYTEST_REPORT_DIR / "last-pytest.json" PYTEST_JUNIT_REPORT_DIR = Path(".cache/test-reports") @@ -258,6 +425,7 @@ def _format_completion_notification( PYTEST_RESOURCE_INTERVAL_ENV = "POLYLOGUE_VERIFY_RESOURCE_INTERVAL_S" DEFAULT_PYTEST_HEARTBEAT_S = 30.0 DEFAULT_PYTEST_TIMEOUT_S = 45 * 60.0 +VERIFY_INVOCATION_BUDGET_S = 3600.0 DEFAULT_PYTEST_STALL_TIMEOUT_S = 10 * 60.0 DEFAULT_PYTEST_TERM_GRACE_S = 5.0 DEFAULT_PYTEST_RESOURCE_INTERVAL_S = 2.0 @@ -449,13 +617,14 @@ def _pytest_workload_receipt( tmpfs_budget_mb: float | None, basetemp_cleanup: Path | None, concurrency: int, + timeout_s: float, ) -> dict[str, Any]: """Adapt managed-pytest accounting to the shared workload receipt.""" input_digest = hashlib.sha256( json.dumps(cmd, ensure_ascii=False, separators=(",", ":")).encode("utf-8") ).hexdigest() budgets: list[WorkloadBudget] = [] - if (timeout_s := _pytest_timeout_s()) > 0: + if timeout_s > 0: budgets.append( WorkloadBudget( BudgetMeasure.WALL_MS, @@ -1027,9 +1196,10 @@ def _run_pytest_with_heartbeat( t0: float, run: VerifyRun | None = None, artifacts: PytestStepArtifacts | None = None, + timeout_override_s: float | None = None, ) -> subprocess.CompletedProcess[str]: heartbeat_s = _pytest_heartbeat_interval() - timeout_s = _pytest_timeout_s() + timeout_s = _pytest_timeout_s() if timeout_override_s is None else max(0.0, timeout_override_s) stall_timeout_s = _pytest_stall_timeout_s() term_grace_s = _pytest_term_grace_s() resource_interval_s = _pytest_resource_interval_s() @@ -1608,27 +1778,85 @@ def _refresh_progress_marker(at: float, latest: dict[str, Any] | None = None) -> return subprocess.CompletedProcess(cmd, returncode, stdout, stderr) +def _recover_worker_collection_facts(*, events_dir: Path, selection_path: Path) -> bool: + """Recover xdist collection evidence if interruption skips sessionfinish.""" + merged = merge_worker_collection_payloads(events_dir) + if merged is None: + return False + selection = { + **merged, + "updated_at": datetime.now(timezone.utc).isoformat(), + "worker_id": "runner", + "pid": os.getpid(), + "recovered_after_interruption": True, + } + try: + _atomic_write_json(selection_path, selection) + except OSError: + return False + return True + + def _run( label: str, cmd: list[str], *, cwd: str | None = None, run: VerifyRun | None = None, + timeout_s: float | None = None, +) -> tuple[int, float, dict[str, Any]]: + if not label.startswith("pytest native"): + return _run_step(label, cmd, cwd=cwd, run=run, timeout_s=timeout_s) + state = _ACTIVE_VERIFY_RUN.owned_native_testmon_state if _ACTIVE_VERIFY_RUN is not None else None + temporary_state = state is None + if state is None: + state = _open_owned_native_testmon_state(ROOT) + try: + return _run_step( + label, + cmd, + cwd=cwd, + run=run, + timeout_s=timeout_s, + native_testmon_data=state.data_path, + ) + finally: + if temporary_state: + state.close() + + +def _run_step( + label: str, + cmd: list[str], + *, + cwd: str | None = None, + run: VerifyRun | None = None, + timeout_s: float | None = None, + native_testmon_data: Path | None = None, ) -> tuple[int, float, dict[str, Any]]: t0 = time.monotonic() sys.stderr.write(f" {label} ... ") sys.stderr.flush() is_pytest = label.startswith("pytest") + managed_native_lane = label.startswith("pytest native") + closed_world_command = _native_pytest_command_is_closed_world(label, cmd) # ``bench slo`` starts pytest-benchmark itself, so it needs the same # bounded temp policy and run marker as a direct pytest step. has_managed_pytest_child = label == "bench slo" + owns_pytest_environment = managed_native_lane or has_managed_pytest_child if is_pytest and run is not None: isolated_report = run.run_dir / f"pytest-report-{uuid.uuid4().hex}.json" cmd = [f"--json-report-file={isolated_report}" if arg.startswith("--json-report-file=") else arg for arg in cmd] if is_pytest: _clear_pytest_report(cmd) artifacts = run.start_step(label=label, cmd=cmd) if run is not None else None - env = _subprocess_env() + env = _subprocess_env(native_testmon_data=native_testmon_data) + external_addopts_neutralized = False + external_plugins_neutralized = False + if owns_pytest_environment: + _normalize_managed_pytest_environment(env, disable_plugin_autoload=managed_native_lane) + if managed_native_lane and _pytest_uses_full_suite_basetemp(label): + env["HYPOTHESIS_PROFILE"] = "default" explicit_basetemp = _pytest_command_basetemp(cmd, cwd=cwd, env=env) if explicit_basetemp is not None: env[PYTEST_EXPLICIT_BASETEMP_ENV] = str(explicit_basetemp) @@ -1660,7 +1888,6 @@ def _run( "diagnosis": "pytest_resource_preflight_failed", "error": str(exc), "termination_reason": "pytest resource preflight refused basetemp admission", - "verification_scope": "narrow-terminal", "release_baseline_allowed": False, } if run is not None and artifacts is not None: @@ -1676,20 +1903,36 @@ def _run( pytest_tmpfs = env.get("POLYLOGUE_PYTEST_TMPFS") == "1" budget_kb = pytest_tmpfs_budget_kb(env) pytest_tmpfs_budget_mb = budget_kb / 1024 if budget_kb is not None else None - if label.startswith("pytest seed-testmon"): - # A complete corpus is currently ~16K nodes. Preserve the whole - # selection in the attempt receipt so interrupted seeds can prove - # eventual coverage instead of relying on a 500-node sample. + if label.startswith("pytest native"): + # The invocation aggregate compares the exact two-lane collection + # with the native environment corpus before granting release + # authority. Keep the complete node set in these bounded artifacts. env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" if run is not None and artifacts is not None: env = env_for_pytest_step(env, run=run, artifacts=artifacts) + if owns_pytest_environment: + _normalize_managed_pytest_environment(env, disable_plugin_autoload=managed_native_lane) + if managed_native_lane: + external_addopts_neutralized = _PYTEST_CLEAR_CONFIGURED_ADDOPTS in cmd + external_plugins_neutralized = ( + "PYTEST_PLUGINS" not in env and env.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") == "1" + ) + closed_world_collection = closed_world_command and external_addopts_neutralized and external_plugins_neutralized interrupted = False pytest_containment_quiescent = True containment_error: str | None = None if is_pytest: try: try: - result = _run_pytest_with_heartbeat(cmd, cwd=cwd, env=env, t0=t0, run=run, artifacts=artifacts) + result = _run_pytest_with_heartbeat( + cmd, + cwd=cwd, + env=env, + t0=t0, + run=run, + artifacts=artifacts, + timeout_override_s=timeout_s, + ) except PytestContainmentError as exc: pytest_containment_quiescent = False containment_error = str(exc) @@ -1706,7 +1949,16 @@ def _run( ) else: try: - result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env, timeout=timeout_s) + except subprocess.TimeoutExpired as exc: + captured_stdout = exc.stdout.decode(errors="replace") if isinstance(exc.stdout, bytes) else "" + captured_stderr = exc.stderr.decode(errors="replace") if isinstance(exc.stderr, bytes) else "" + result = subprocess.CompletedProcess( + args=cmd, + returncode=124, + stdout=captured_stdout, + stderr=captured_stderr + "\nverify: invocation deadline exhausted\n", + ) except KeyboardInterrupt: interrupted = True result = subprocess.CompletedProcess(args=cmd, returncode=130, stdout="", stderr="") @@ -1720,8 +1972,11 @@ def _run( metadata["diagnosis"] = "pytest_containment_unproven" metadata["termination_reason"] = f"pytest containment did not quiesce: {containment_error}" metadata.update(_pytest_command_metadata(cmd)) + metadata["external_addopts_neutralized"] = external_addopts_neutralized + metadata["external_plugins_neutralized"] = external_plugins_neutralized + metadata["closed_world_collection"] = closed_world_collection metadata["heartbeat_s"] = _pytest_heartbeat_interval() - metadata["timeout_s"] = _pytest_timeout_s() + metadata["timeout_s"] = _pytest_timeout_s() if timeout_s is None else timeout_s metadata["stall_timeout_s"] = _pytest_stall_timeout_s() metadata["term_grace_s"] = _pytest_term_grace_s() metadata["resource_interval_s"] = _pytest_resource_interval_s() @@ -1810,6 +2065,19 @@ def _run( slowest_reports = summary.get("slowest_reports") if isinstance(slowest_reports, list): metadata["slowest_report_count"] = len(slowest_reports) + if ( + label.startswith("pytest native") + and result.returncode == 5 + and metadata.get("selected_count") == 0 + and metadata.get("report_status") == "present" + and isinstance(summary, Mapping) + and summary.get("exitstatus") == 5 + ): + # Either semantic partition may legitimately be empty. Collection + # still synchronized the complete native corpus, so an empty lane + # is successful evidence rather than a pytest usage failure. + result.returncode = 0 + metadata["empty_semantic_lane"] = True containment_path = artifacts.containment_path if artifacts is not None else PYTEST_CONTAINMENT_PATH containment = _read_json_artifact(containment_path) if containment is not None: @@ -1944,6 +2212,7 @@ def _run( tmpfs_budget_mb=pytest_tmpfs_budget_mb, basetemp_cleanup=basetemp_cleanup, concurrency=max(1, pytest_concurrency), + timeout_s=_pytest_timeout_s() if timeout_s is None else timeout_s, ) metadata["workload_receipt"] = workload_receipt if artifacts is not None: @@ -2021,18 +2290,22 @@ def _pytest_command_basetemp( return (Path(cwd) if cwd is not None else Path.cwd()) / path -def _subprocess_env() -> dict[str, str]: +def _subprocess_env(*, native_testmon_data: Path | None = None) -> dict[str, str]: env = normalize_pytest_basetemp_env(os.environ) # Tests and verification helpers may inspect Git, but observational reads # must not refresh the index and invalidate the exact-head mutation watch. env["GIT_OPTIONAL_LOCKS"] = "0" env["POLYLOGUE_ROOT"] = str(ROOT) env["POLYLOGUE_REPO_ROOT"] = str(ROOT) - inherited_pythonpath = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(ROOT) if not inherited_pythonpath else f"{ROOT}{os.pathsep}{inherited_pythonpath}" + # Managed verification owns Python startup. A checkout path on PYTHONPATH + # lets sitecustomize alter pytest controls before the managed command runs. + env.pop("PYTHONPATH", None) + env.pop("PYTHONOPTIMIZE", None) + env.pop("PYTHONHOME", None) + env.pop("PYTHONUSERBASE", None) + env["PYTHONNOUSERSITE"] = "1" env["PYTHONPYCACHEPREFIX"] = str(ROOT / ".cache" / "pycache") - TESTMON_DATA.parent.mkdir(parents=True, exist_ok=True) - env["TESTMON_DATAFILE"] = str(TESTMON_DATA) + env["TESTMON_DATAFILE"] = str(native_testmon_data or TESTMON_DATA) env["POLYLOGUE_PYTEST_EVENTS_DIR"] = str(ROOT / PYTEST_EVENTS_DIR) env["POLYLOGUE_PYTEST_EVENTS_PATH"] = str(ROOT / PYTEST_EVENTS_PATH) env["POLYLOGUE_PYTEST_SELECTION_PATH"] = str(ROOT / PYTEST_SELECTION_PATH) @@ -2041,37 +2314,104 @@ def _subprocess_env() -> dict[str, str]: def _stop_after_failed_step(label: str) -> bool: - return label.startswith("pytest") or label in {"lab smoke", "bench slo"} - + return label in {"lab smoke", "bench slo"} -def _seed_shard_failure_requires_stop(step: Mapping[str, Any], *, shard_complete: bool) -> bool: - """Stop shard admission after harness failure while retaining red-test evidence. - A normal pytest exit 1 with a structured ``pytest_failed`` diagnosis is - useful seed evidence: later shards can still populate the resumable - dependency graph. Timeouts, resource refusals, worker/internal errors, - usage errors, and unclassified failures mean the harness is no longer - healthy enough to admit another expensive shard. - """ - exit_code = step.get("exit") - if exit_code == 0: - return False - return not (exit_code == 1 and step.get("diagnosis") == "pytest_failed" and shard_complete) +def _native_lane_failure_requires_stop(step: Mapping[str, Any]) -> bool: + """Continue the serial lane only after an ordinary test failure.""" + return not (step.get("exit") == 1 and step.get("diagnosis") == "pytest_failed") # ── step builder ──────────────────────────────────────────────────── +def _native_pytest_steps( + *, + testmon_mode: str, + testmon_environment: str, + parallel_worker_args: Sequence[str], +) -> list[tuple[str, list[str]]]: + pytest_cmd = [ + sys.executable, + "-m", + "pytest", + "-q", + "--tb=short", + "--ignore=tests/benchmarks", + "--durations=10", + f"--junitxml={PYTEST_JUNIT_REPORT_DIR}/verify-latest.xml", + "--json-report", + "--json-report-omit=collectors,log,streams,warnings", + f"--json-report-file={PYTEST_REPORT_PATH}", + "-p", + "devtools.pytest_progress_plugin", + ] + pytest_cmd.extend(_PYTEST_MANAGED_PLUGIN_ARGS) + pytest_cmd.extend(_PYTEST_CLOSED_WORLD_COLLECTION_ARGS) + native_args = ["--testmon", f"--testmon-env={testmon_environment}"] + if testmon_mode == "affected": + native_args.append("--testmon-forceselect") + else: + native_args.append("--testmon-noselect") + + parallel_cmd = [ + *pytest_cmd, + "-m", + "not load_sensitive", + *native_args, + *parallel_worker_args, + ] + + def _serial_report_arg(arg: str) -> str: + if arg.startswith("--junitxml="): + return f"--junitxml={PYTEST_JUNIT_REPORT_DIR}/verify-latest-serial.xml" + if arg.startswith("--json-report-file="): + return f"--json-report-file={PYTEST_REPORT_DIR / 'last-pytest-serial.json'}" + return arg + + serial_cmd = [_serial_report_arg(arg) for arg in pytest_cmd] + serial_cmd.extend( + [ + "-m", + "load_sensitive", + *native_args, + "-p", + "no:randomly", + "-n", + "0", + ] + ) + return [ + (f"pytest native parallel ({testmon_mode})", parallel_cmd), + (f"pytest native serial ({testmon_mode})", serial_cmd), + ] + + +def _native_pytest_command_is_closed_world(label: str, cmd: Sequence[str]) -> bool: + """Accept only a command produced by the managed native-lane builder.""" + match = re.fullmatch(r"pytest native (parallel|serial) \((affected|bootstrap|full)\)", label) + if match is None: + return False + environment_args = [arg for arg in cmd if arg.startswith("--testmon-env=")] + worker_request = pytest_command_worker_request(cmd) + if len(environment_args) != 1 or worker_request is None or not worker_request.isdigit(): + return False + expected_steps = _native_pytest_steps( + testmon_mode=match.group(2), + testmon_environment=environment_args[0].removeprefix("--testmon-env="), + parallel_worker_args=("--dist=loadgroup", "-n", worker_request), + ) + expected = dict(expected_steps).get(label) + return expected is not None and list(cmd) == expected + + def build_verify_steps( *, quick: bool, lab: bool, - skip_slow: bool, commit: bool = False, - seed_testmon: bool = False, - resume_testmon_seed: bool = False, - full_pytest: bool = False, - broad_testmon: bool = False, + testmon_mode: str = "affected", + testmon_environment: str = "", ) -> list[tuple[str, list[str]]]: steps: list[tuple[str, list[str]]] = [ ("ruff format", ["ruff", "format", "--check", "polylogue/", "tests/", "devtools/"]), @@ -2117,80 +2457,19 @@ def build_verify_steps( _report_dir = PYTEST_JUNIT_REPORT_DIR _report_dir.mkdir(parents=True, exist_ok=True) PYTEST_REPORT_DIR.mkdir(parents=True, exist_ok=True) - # Scale-tier policy (issue #1183): default verify includes - # ``scale_small`` but excludes ``scale_medium`` / ``scale_large``. - # ``--lab`` lets the medium tier in; the large tier is reserved - # for nightly CI's direct pytest-benchmark execution. - scale_marker_expr = "not scale_large" if lab else "not scale_medium and not scale_large" - pytest_cmd = [ - sys.executable, - "-m", - "pytest", - "-q", - "--tb=short", - "--ignore=tests/integration", - # Benchmark files are an explicit campaign surface. A number of - # them are correctness-shaped and lack the benchmark marker, so a - # marker expression alone cannot keep performance probes out of - # the correctness/testmon corpus. - "--ignore=tests/benchmarks", - "--durations=10", - f"--junitxml={_report_dir}/verify-latest.xml", - "--json-report", - "--json-report-omit=collectors,log,streams,warnings", - f"--json-report-file={PYTEST_REPORT_PATH}", - "-p", - "devtools.pytest_progress_plugin", - ] - # Benchmark cases are an explicit campaign surface, not part of the - # correctness/testmon seed. Keeping them out here is important: a - # benchmark marker is not necessarily paired with ``slow`` or a scale - # marker, and a serial shard would otherwise spend minutes executing a - # performance probe before it can checkpoint any correctness nodes. - base_marker = f"not benchmark and {scale_marker_expr}" - if skip_slow: - base_marker = f"not slow and {base_marker}" - if seed_testmon: - # Collection produces the exact corpus contract before any testmon - # write. Shards below are generated from this ledger and run one - # at a time, so pytest-testmon has exactly one SQLite writer. - pytest_cmd.extend(["-m", base_marker, "--collect-only", "-n", "0"]) - label = "pytest seed-testmon collect (resume)" if resume_testmon_seed else "pytest seed-testmon collect" - steps.append((label, pytest_cmd)) - elif full_pytest: - # #1775: the full diagnostic runs as two lanes. The bulk lane keeps - # xdist parallelism but deselects wall-clock-bound tests; the - # isolated lane reruns those (``load_sensitive``/``tui`` — timing - # budgets, loopback-socket timeouts, TUI render timing) single- - # process with a stable order, so worker contention can no longer - # flake them. Both lanes are correctness blockers; the split only - # removes the scheduling jitter that made ``--all`` an unreliable - # completion gate. - bulk_cmd = [ - *pytest_cmd, - "-m", - f"({base_marker}) and not load_sensitive and not tui", - *_pytest_worker_args(), - ] - steps.append((BROAD_PYTEST_STEP_LABELS["full_parallel"], bulk_cmd)) - - def _isolated_report_arg(arg: str) -> str: - # Keep the bulk lane's canonical report artifacts intact for - # _compare_against_last; the isolated lane writes its own files. - if arg.startswith("--junitxml="): - return f"--junitxml={_report_dir}/verify-latest-isolated.xml" - if arg.startswith("--json-report-file="): - return f"--json-report-file={PYTEST_REPORT_DIR / 'last-pytest-isolated.json'}" - return arg - - isolated_cmd = [_isolated_report_arg(arg) for arg in pytest_cmd] - isolated_cmd.extend(["-m", f"({base_marker}) and (load_sensitive or tui)", "-p", "no:randomly", "-n", "0"]) - steps.append((BROAD_PYTEST_STEP_LABELS["load_sensitive"], isolated_cmd)) - else: - pytest_cmd.extend(["-m", base_marker, "--testmon", *_pytest_worker_args()]) - pytest_cmd.append("--testmon-forceselect") - label = BROAD_PYTEST_STEP_LABELS["testmon_broad"] if broad_testmon else "pytest testmon" - steps.append((label, pytest_cmd)) + if testmon_mode not in {"affected", "bootstrap", "full"}: + raise ValueError(f"unknown native testmon mode: {testmon_mode}") + if not testmon_environment: + raise ValueError("native testmon environment is required for pytest verification") + # Every native command owns its collection, option, and plugin surface; + # the benchmark root remains the one explicit non-correctness corpus. + steps.extend( + _native_pytest_steps( + testmon_mode=testmon_mode, + testmon_environment=testmon_environment, + parallel_worker_args=_pytest_worker_args(), + ) + ) if lab: steps.append(("lab smoke", _devtools_cmd("lab smoke", "run", "archive-smoke", "--tier", "0"))) @@ -2292,17 +2571,11 @@ def _stamp_head() -> None: (stamp_dir / "last-verify-head").write_text(head + "\n") -def _file_fingerprint(path: Path) -> str: - if not path.exists() or not path.is_file(): - return "missing" - h = hashlib.sha256() - try: - with path.open("rb") as f: - for chunk in iter(lambda: f.read(1024 * 1024), b""): - h.update(chunk) - except OSError: - return "unreadable" - return h.hexdigest() +def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + temporary.replace(path) def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: @@ -2313,34 +2586,18 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: return ["--dist=loadgroup", "-n", str(workers)] -BROAD_PYTEST_STEP_LABELS = { - "seed": "pytest seed-testmon", - "seed_resume": "pytest seed-testmon (resume)", - "full_parallel": "pytest full (parallel)", - "load_sensitive": "pytest load-sensitive (isolated)", - "testmon_broad": "pytest testmon (broad)", -} - - def _pytest_command_concurrency(cmd: Sequence[str], *, env: Mapping[str, str] | None = None) -> int: - """Return a fail-closed reservation for the final pytest command. - - ``-n auto`` can launch one worker per logical CPU. Reserve that maximum - instead of guessing one worker; an unrecognised xdist value is treated the - same way so malformed or future values cannot weaken admission. - """ + """Return a fail-closed reservation for the final pytest command.""" request = pytest_command_worker_request(cmd) if request is None: return 0 if request == "auto": auto_workers = (env if env is not None else os.environ).get("PYTEST_XDIST_AUTO_NUM_WORKERS", "").strip() if auto_workers: - try: + with contextlib.suppress(ValueError): configured = int(auto_workers) - except ValueError: - configured = 0 - if configured > 0: - return configured + if configured > 0: + return configured try: return max(0, int(request)) except ValueError: @@ -2348,14 +2605,8 @@ def _pytest_command_concurrency(cmd: Sequence[str], *, env: Mapping[str, str] | def _pytest_uses_full_suite_basetemp(label: str) -> bool: - """Whether this pytest step can materialize the measured full-suite tree.""" - return label in BROAD_PYTEST_STEP_LABELS.values() or label.startswith("pytest seed-testmon shard ") - - -_BROAD_TESTMON_CHANGED_PATHS = { - "pyproject.toml", - "tests/conftest.py", -} + """Whether this semantic lane may materialize the complete corpus tree.""" + return label.startswith("pytest native") and ("(bootstrap)" in label or "(full)" in label) def _changed_paths(base_commit: str, head_commit: str) -> set[str]: @@ -2383,1376 +2634,522 @@ def _changed_paths(base_commit: str, head_commit: str) -> set[str]: return changed -def _default_testmon_is_broad_change(base_commit: str, head_commit: str) -> bool: - """Return true when affected-test selection should be treated as broad.""" - return bool(_changed_paths(base_commit, head_commit) & _BROAD_TESTMON_CHANGED_PATHS) - - -def _changed_executable_paths(base_commit: str, head_commit: str) -> tuple[str, ...]: - """Return changed paths whose behavior should select at least one test.""" +def _changed_test_relevant_paths(base_commit: str, head_commit: str) -> tuple[str, ...]: roots = ("polylogue/", "devtools/", "tests/", "packaging/") - exact = {"pyproject.toml", "uv.lock"} + exact = {"pyproject.toml", "uv.lock", "pytest.ini", "tox.ini", "setup.cfg"} return tuple( sorted(path for path in _changed_paths(base_commit, head_commit) if path in exact or path.startswith(roots)) ) -def _testmon_coverage_identity(executable_paths: Sequence[str]) -> dict[str, Any]: - """Identify the exact worktree contents covered by an affected/full run.""" - return { - "worktree_fingerprint": worktree_fingerprint(), - "executable_paths": list(executable_paths), - } +@dataclass(slots=True) +class _ActiveVerifyRun: + run: VerifyRun + started_at: float + verification_scope: VerificationScope + head: str | None + mutation_monitor: CheckoutMutationMonitor | None = None + initial_worktree_fingerprint: str | None = None + owned_native_testmon_state: _OwnedNativeTestmonState | None = None -def _matching_testmon_coverage(executable_paths: Sequence[str]) -> str | None: - """Return the receipt kind proving that zero new selection is legitimate.""" - identity = _testmon_coverage_identity(executable_paths) - affected = _read_json_artifact(TESTMON_AFFECTED_STAMP) - selected_count = affected.get("selected_count") if isinstance(affected, dict) else None - if ( - isinstance(affected, dict) - and affected.get("protocol_version") == 1 - and affected.get("status") == "complete" - and isinstance(affected.get("timestamp"), str) - and bool(affected.get("timestamp")) - and isinstance(affected.get("run_id"), str) - and bool(affected.get("run_id")) - and isinstance(selected_count, int) - and not isinstance(selected_count, bool) - and selected_count > 0 - and affected.get("identity") == identity - ): - return "successful_affected_run" - return None +_ACTIVE_VERIFY_RUN: _ActiveVerifyRun | None = None -def _record_testmon_affected_coverage(*, executable_paths: Sequence[str], selected_count: int, run_id: str) -> None: - """Persist proof that testmon exercised dependencies for these contents.""" - _atomic_write_json( - TESTMON_AFFECTED_STAMP, - { - "protocol_version": 1, - "status": "complete", - "timestamp": datetime.now(timezone.utc).isoformat(), - "run_id": run_id, - "selected_count": selected_count, - "identity": _testmon_coverage_identity(executable_paths), - }, - ) +def _start_active_checkout_mutation_monitor(monitor: CheckoutMutationMonitor) -> None: + start_checkout_mutation_monitor(monitor) + if _ACTIVE_VERIFY_RUN is not None: + _ACTIVE_VERIFY_RUN.mutation_monitor = monitor -def _testmon_preflight(*, seed_testmon: bool, full_pytest: bool, quick: bool, commit: bool) -> str | None: - if quick or commit or seed_testmon or full_pytest: - return None - seed_message = ( - "verify: pytest-testmon is not seeded; run `devtools verify --seed-testmon` " - "to create .cache/testmon/testmondata and .cache/testmon/seed.json " - "before using the default affected-test path.\n" - ) - if not TESTMON_DATA.exists(): - return seed_message - if not TESTMON_SEED_STAMP.exists(): - attempt = _read_testmon_seed_attempt() - if ( - attempt is not None - and stamp_from_attempt( - attempt, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - published_marker=False, - ) - is not None - ): - sys.stderr.write( - "verify: using a validated complete pytest-testmon graph from a red seed attempt; " - "the release baseline remains red.\n" - ) - return None - return seed_message - stamp = validate_stamp( - TESTMON_SEED_STAMP, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - ) - if stamp is None: - return ( - "verify: pytest-testmon seed state is unreadable, stale, malformed, or not graph-complete; run " - "`devtools verify --seed-testmon` to rebuild the dependency baseline.\n" - ) - return None +def _finish_active_checkout_mutation_monitor(monitor: CheckoutMutationMonitor) -> CheckoutMutationObservation: + try: + return finish_checkout_mutation_monitor(monitor) + finally: + if _ACTIVE_VERIFY_RUN is not None and _ACTIVE_VERIFY_RUN.mutation_monitor is monitor: + _ACTIVE_VERIFY_RUN.mutation_monitor = None -def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp") - temporary.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - temporary.replace(path) +def _close_active_native_testmon_state() -> None: + if _ACTIVE_VERIFY_RUN is None or _ACTIVE_VERIFY_RUN.owned_native_testmon_state is None: + return + state = _ACTIVE_VERIFY_RUN.owned_native_testmon_state + _ACTIVE_VERIFY_RUN.owned_native_testmon_state = None + state.close() -def _testmon_seed_identity( +def _planned_verification_scope( + args: argparse.Namespace, *, - git_head: str | None, - git_tree: str | None = None, - skip_slow: bool, - lab: bool, - terminal_authorization: str | None = None, -) -> dict[str, Any]: - runtime_identity = testmon_runtime_identity(ROOT) - if runtime_identity is None: - raise RuntimeError("could not identify the active dependency environment and pytest harness") - dependency_environment, pytest_harness = runtime_identity - return { - "git_head": git_head, - "git_tree": git_tree, - "worktree_fingerprint": worktree_fingerprint(), - "python": sys.version, - "skip_slow": skip_slow, - "lab": lab, - "terminal_authorization": terminal_authorization, - "dependency_environment": dependency_environment, - "pytest_harness": pytest_harness, - } - + testmon_mode: str | None, +) -> VerificationScope: + if args.quick or args.commit: + return VerificationScope.NON_TEST + if testmon_mode in {"bootstrap", "full"}: + return VerificationScope.RELEASE_BASELINE + return VerificationScope.AFFECTED -def _read_testmon_seed_attempt() -> dict[str, Any] | None: - payload = _read_json_artifact(TESTMON_SEED_ATTEMPT) - return payload if isinstance(payload, dict) else None +def _pytest_profile() -> str: + return "correctness=complete" -def _recover_worker_collection_facts(*, events_dir: Path, selection_path: Path) -> bool: - """Publish xdist worker collection facts when its controller never finishes. - The progress plugin normally merges these facts during controller - ``pytest_sessionfinish``. Interrupted containment bypasses that hook, so - the runner recovers the same canonical worker fact before it terminalizes - the durable step record. - """ - merged = merge_worker_collection_payloads(events_dir) - if merged is None: - return False - selection = dict(merged) - selection.update( - { - "updated_at": datetime.now(timezone.utc).isoformat(), - "worker_id": "runner", - "pid": os.getpid(), - "recovered_after_interruption": True, - } - ) - try: - selection_path.parent.mkdir(parents=True, exist_ok=True) - temporary = selection_path.with_name(f"{selection_path.name}.{os.getpid()}.tmp") - temporary.write_text(json.dumps(selection, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - temporary.replace(selection_path) - except OSError: - return False - return True +def _native_pytest_environment(*, force_release_profile: bool) -> dict[str, str | None]: + environment = { + # Hypothesis uses its default profile when the variable is absent. + # Record that effective value in the testmon environment identity so a + # bootstrap graph is reusable by the following affected invocation. + "HYPOTHESIS_PROFILE": os.environ.get("HYPOTHESIS_PROFILE") or "default", + "POLYLOGUE_CI": os.environ.get("POLYLOGUE_CI"), + } + if force_release_profile: + environment["HYPOTHESIS_PROFILE"] = "default" + return environment -def _flatten_seed_outcomes(attempt: Mapping[str, Any] | None) -> list[dict[str, Any]]: - """Flatten outcomes from every interrupted attempt, newest result winning.""" - if attempt is None: - return [] - flattened: dict[str, dict[str, Any]] = {} - for field in ("prior_node_outcomes", "node_outcomes"): - raw = attempt.get(field) - if not isinstance(raw, list): - continue - for item in raw: - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) and item["nodeid"]: - flattened[item["nodeid"]] = dict(item) - return [flattened[nodeid] for nodeid in sorted(flattened)] - - -def _testmon_release_baseline_permission() -> bool | None: - """Return release permission for current testmon state, or ``None`` when not applicable.""" - if TESTMON_SEED_STAMP.exists(): - stamp = validate_stamp( - TESTMON_SEED_STAMP, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - ) - return stamp.release_baseline_allowed if stamp is not None else False - attempt = _read_testmon_seed_attempt() - if attempt is None: - return False - stamp = stamp_from_attempt( - attempt, - TESTMON_DATA, - checkout_root=ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - published_marker=False, - ) - return stamp.release_baseline_allowed if stamp is not None else False +def _remaining_invocation_budget(started_at: float) -> float: + return max(0.0, VERIFY_INVOCATION_BUDGET_S - (time.monotonic() - started_at)) -def _safe_testmon_artifact_dir(raw: object, *, require_run_root: bool = False) -> Path | None: - if not isinstance(raw, str) or not raw: - return None - path = Path(raw) - checkout_root = Path.cwd().resolve() - if require_run_root and path.is_absolute(): - return None - resolved = (path if path.is_absolute() else checkout_root / path).resolve() +def _native_environment_after_run( + preparation: NativeTestmonPreparation, + *, + required_executable_paths: Sequence[str], +) -> NativeTestmonState: + state = _ACTIVE_VERIFY_RUN.owned_native_testmon_state if _ACTIVE_VERIFY_RUN is not None else None + temporary_state = state is None + if state is None: + state = _open_owned_native_testmon_state(ROOT) try: - resolved.relative_to(checkout_root) - if require_run_root: - resolved.relative_to((checkout_root / ".cache" / "verify" / "runs").resolve()) - except ValueError: - return None - return resolved - - -def _testmon_seed_expected_nodeids(attempt: Mapping[str, Any]) -> list[str]: - """Recover the seed ledger, including after an abrupt outer-run exit.""" - expected = attempt.get("expected_nodeids") - if isinstance(expected, list) and expected: - if ( - any(not isinstance(nodeid, str) or not nodeid for nodeid in expected) - or len(set(expected)) != len(expected) - or not isinstance(attempt.get("expected_count"), int) - or isinstance(attempt.get("expected_count"), bool) - or attempt.get("expected_count") != len(expected) - or not isinstance(attempt.get("expected_digest"), str) - or attempt.get("expected_digest") != hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() - ): - return [] - return list(expected) - - artifact_dir = _safe_testmon_artifact_dir(attempt.get("artifact_dir"), require_run_root=True) - if artifact_dir is None: - return [] - for selection_path in sorted(artifact_dir.glob("steps/*/selection.json")): - selection = _read_json_artifact(selection_path) - if not isinstance(selection, dict): - continue - omitted = selection.get("selected_nodeids_omitted") - selected_count = selection.get("selected_count") - if ( - not isinstance(omitted, int) - or isinstance(omitted, bool) - or omitted != 0 - or not isinstance(selected_count, int) - or isinstance(selected_count, bool) - ): - continue - selected = selection.get("selected_nodeids") - if ( - isinstance(selected, list) - and selected - and all(isinstance(nodeid, str) and nodeid for nodeid in selected) - and len(set(selected)) == len(selected) - and selected_count == len(selected) - ): - return list(selected) - return [] - - -def _testmon_seed_resume_contract(identity: Mapping[str, Any]) -> dict[str, Any]: - """Return inputs that change which corpus a seed promises to cover.""" - return { - key: identity.get(key) - for key in ( - "git_tree", - "worktree_fingerprint", - "python", - "skip_slow", - "lab", - "terminal_authorization", - "dependency_environment", - "pytest_harness", + return inspect_native_testmon_environment( + state.data_path, + environment_name=preparation.environment_name, + required_executable_paths=required_executable_paths, ) - } + finally: + if temporary_state: + state.close() -def _testmon_seed_can_resume(identity: Mapping[str, Any]) -> bool: - attempt = _read_testmon_seed_attempt() - if attempt is None or not TESTMON_DATA.exists(): +def _release_baseline_allowed( + *, + selection_mode: str | None, + verification_scope: VerificationScope, + exit_code: int, + checkout_stable: bool, + aggregate: Mapping[str, Any] | None, +) -> bool: + if selection_mode not in {"bootstrap", "full"} or exit_code != 0 or not checkout_stable or aggregate is None: return False - prior_identity = attempt.get("identity") - contract = _testmon_seed_resume_contract(identity) - return ( - attempt.get("protocol_version") == TESTMON_SEED_PROTOCOL_VERSION - and attempt.get("status") in {"running", "incomplete"} - and isinstance(prior_identity, dict) - and isinstance(contract["git_tree"], str) - and bool(contract["git_tree"]) - and _testmon_seed_resume_contract(prior_identity) == contract - and bool(_testmon_seed_expected_nodeids(attempt)) + if verification_scope != VerificationScope.RELEASE_BASELINE: + return False + cleanup = aggregate.get("cleanup") + containment = aggregate.get("containment") + deadline = aggregate.get("deadline") + return bool( + aggregate.get("complete_corpus_covered") is True + and aggregate.get("terminal_green") is True + and aggregate.get("external_addopts_neutralized") is True + and aggregate.get("external_plugins_neutralized") is True + and aggregate.get("closed_world_collection") is True + and isinstance(cleanup, Mapping) + and cleanup.get("complete") is True + and isinstance(containment, Mapping) + and containment.get("complete") is True + and isinstance(deadline, Mapping) + and deadline.get("met") is True ) -def _prepare_testmon_seed_attempt( - *, - identity: Mapping[str, Any], +def _finalize_preflight_failure( run: VerifyRun, - resume: bool, -) -> dict[str, Any]: - prior = _read_testmon_seed_attempt() if resume else None - expected = sorted(_testmon_seed_expected_nodeids(prior)) if prior is not None else [] - prior_outcomes = _flatten_seed_outcomes(prior) - payload = { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": dict(identity), - "resume": resume, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, - "prior_node_outcomes": prior_outcomes, - "shards": list(prior.get("shards", [])) if prior is not None and isinstance(prior.get("shards"), list) else [], - "started_at": datetime.now(timezone.utc).isoformat(), - "run_id": run.run_id, - "artifact_dir": str(run.relative_run_dir), - "testmon_data_before": _file_fingerprint(TESTMON_DATA), - "binding": TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict(), - } - TESTMON_SEED_STAMP.unlink(missing_ok=True) - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - return payload - - -def _seed_selection_nodeids(selection: Mapping[str, Any]) -> list[str] | None: - """Accept only a complete, untruncated collection ledger.""" - nodeids = selection.get("selected_nodeids") - selected_count = selection.get("selected_count") - omitted = selection.get("selected_nodeids_omitted") - if ( - not isinstance(nodeids, list) - or not nodeids - or any(not isinstance(nodeid, str) or not nodeid for nodeid in nodeids) - or len(set(nodeids)) != len(nodeids) - or not isinstance(selected_count, int) - or isinstance(selected_count, bool) - or selected_count != len(nodeids) - or not isinstance(omitted, int) - or isinstance(omitted, bool) - or omitted != 0 - ): - return None - return sorted(nodeids) - - -def _prepare_testmon_seed_shards( - prepared: Mapping[str, Any], *, - selection: Mapping[str, Any] | None, -) -> dict[str, Any]: - """Persist the full planned corpus before the first testmon DB mutation.""" - expected = sorted(_testmon_seed_expected_nodeids(prepared)) if prepared.get("resume") else [] - if not expected: - expected = _seed_selection_nodeids(selection or {}) or [] - prior_shards = validate_seed_shard_ledger(prepared.get("shards"), expected_nodeids=expected) - shards = ( - prior_shards - if prior_shards is not None - else ( - seed_shard_plan( - expected, - shard_size=TESTMON_SEED_SHARD_SIZE, - serial_nodeids=[ - nodeid - for nodeid, markers in (selection or {}).get("selected_node_markers", {}).items() - if "load_sensitive" in markers or "tui" in markers - ], - ) - if expected - else [] - ) + started_at: float, + tier: str, + head: str | None, + verification_scope: VerificationScope, + diagnosis: str, + exit_code: int, + message: str, + use_json: bool, + mutation_monitor: CheckoutMutationMonitor | None = None, + initial_worktree_fingerprint: str | None = None, +) -> int: + """Persist one normalized failed invocation before pytest can start.""" + final_head = _git_head() + try: + final_worktree_fingerprint = worktree_fingerprint(ROOT) if mutation_monitor is not None else "unavailable" + except Exception: + final_worktree_fingerprint = "unavailable" + mutation_observation = ( + _finish_active_checkout_mutation_monitor(mutation_monitor) if mutation_monitor is not None else None + ) + checkout_diagnosis: str | None = None + if mutation_monitor is None: + checkout_diagnosis = "preflight_failed_before_checkout_monitor" + elif ( + head is None + or final_head is None + or initial_worktree_fingerprint in {None, "unavailable"} + or final_worktree_fingerprint == "unavailable" + or mutation_observation is None + or mutation_observation.unavailable + ): + checkout_diagnosis = "checkout_fingerprint_unavailable" + elif ( + final_head != head or mutation_observation.changed or final_worktree_fingerprint != initial_worktree_fingerprint + ): + checkout_diagnosis = "checkout_changed_during_verification" + + duration_s = round(time.monotonic() - started_at, 2) + artifacts = run.start_step(label="verify preflight", cmd=[]) + step = run.finish_step( + step_id=artifacts.step_id, + result={ + "duration_s": duration_s, + "exit": exit_code, + "diagnosis": diagnosis, + "error": message, + "checkout_diagnosis": checkout_diagnosis, + }, ) - payload = { - **dict(prepared), - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(expected).encode()).hexdigest() if expected else None, - "selection": dict(selection or {}), - "shard_size": TESTMON_SEED_SHARD_SIZE, - "shards": shards, + payload = run.finish( + exit_code=exit_code, + duration_s=duration_s, + diagnosis=diagnosis, + verification_scope=verification_scope.value, + release_baseline_allowed=False, + final_git_head=final_head, + final_worktree_fingerprint=final_worktree_fingerprint, + checkout_mutation_path=(mutation_observation.observed_path if mutation_observation is not None else None), + checkout_diagnosis=checkout_diagnosis, + invocation_budget_s=VERIFY_INVOCATION_BUDGET_S, + ) + history_entry = { + **payload, + "timestamp": datetime.now(timezone.utc).isoformat(), + "git_head": head, + "final_git_head": final_head, + "tier": tier, + "checkout_root": str(ROOT.resolve()), + "worktree_fingerprint": initial_worktree_fingerprint, + "final_worktree_fingerprint": final_worktree_fingerprint, + "steps": [step] if step is not None else [], + "total_duration_s": duration_s, + "invocation_budget_s": VERIFY_INVOCATION_BUDGET_S, + "exit_code": exit_code, + "verification_scope": verification_scope.value, + "release_baseline_allowed": False, + "diagnosis": diagnosis, } - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - return payload - - -def _seed_shard_command( - collection_command: Sequence[str], - shard: Mapping[str, Any], - *, - nodeids_file: Path, -) -> list[str]: - """Build a bounded-argv, dynamically balanced pytest-testmon invocation. - - A full shard's node IDs can exceed the host's ``execve`` argument budget - once ``systemd-run`` and the managed environment are included. Pytest's - response-file syntax keeps the authoritative node list in the run - artifact while making the child command size independent of shard size. - """ - nodeids = shard.get("nodeids") - if not isinstance(nodeids, list) or not nodeids: - raise ValueError("testmon seed shard is missing nodeids") - nodeids_file.parent.mkdir(parents=True, exist_ok=True) - nodeids_file.write_text("\n".join(nodeids) + "\n", encoding="utf-8") - command: list[str] = [] - skip_next = False - for argument in collection_command: - if skip_next: - skip_next = False - continue - if argument == "--collect-only": - continue - if argument in {"-n", "--numprocesses"}: - skip_next = True - continue - if argument.startswith("--numprocesses=") or (argument.startswith("-n") and len(argument) > 2): - continue - command.append(argument) - # Collection is deliberately serial, but execution is not. pytest-testmon - # has an xdist-aware controller database; retaining the managed worker pool - # here avoids turning a 20k-node seed into hours of serial fixture setup. - if shard.get("execution_mode") == "serial": - command.extend(["-n", "0", "--testmon", "--testmon-noselect", f"@{nodeids_file}"]) - else: - command.extend( - [ - *_pytest_worker_args(maximum=10), - "--testmon", - "--testmon-noselect", - f"@{nodeids_file}", - ] + _save_history(history_entry) + if use_json: + _print_json(history_entry) + sys.stderr.write(f"verify: {message}\n") + _notify( + _format_completion_notification( + exit_code=exit_code, + total_duration=duration_s, + step_results=history_entry["steps"], ) - return command - - -def _canonical_seed_nodeid(nodeid: str, expected_nodeids: Sequence[str]) -> str: - """Map xdist's ``nodeid@group`` reports back to the collected node ID.""" - if nodeid in expected_nodeids: - return nodeid - candidates = [expected for expected in expected_nodeids if nodeid.startswith(expected + "@")] - return max(candidates, key=len, default=nodeid) - - -def _seed_shard_outcomes(shards: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: - """Flatten the shard ledger in canonical node order for legacy readers.""" - outcomes: dict[str, dict[str, Any]] = {} - for shard in shards: - raw_outcomes = shard.get("node_outcomes") - if not isinstance(raw_outcomes, list): - continue - for item in raw_outcomes: - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str): - outcomes[str(item["nodeid"])] = dict(item) - return [outcomes[nodeid] for nodeid in sorted(outcomes)] - - -def _checkpoint_testmon_seed_shard( - *, - prepared: Mapping[str, Any], - shard_index: int, - step: Mapping[str, Any], -) -> dict[str, Any]: - """Record one shard's result atomically before another shard may start.""" - expected = sorted(_testmon_seed_expected_nodeids(prepared)) - shards = validate_seed_shard_ledger(prepared.get("shards"), expected_nodeids=expected) - if shards is None or shard_index < 1 or shard_index > len(shards): - raise ValueError("testmon seed shard ledger is malformed") - shard = dict(shards[shard_index - 1]) - nodeids = shard["nodeids"] - artifact_dir = _safe_testmon_artifact_dir(step.get("artifact_dir")) - selection = _read_json_artifact(artifact_dir / "selection.json") if artifact_dir is not None else None - selected_raw = _seed_selection_nodeids(selection) if isinstance(selection, Mapping) else None - selected = ( - sorted(_canonical_seed_nodeid(nodeid, nodeids) for nodeid in selected_raw) if selected_raw is not None else None - ) - database = _testmon_database_state(nodeids) - prior = { - str(item["nodeid"]): item - for item in shard.get("node_outcomes", []) - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) - } - outcomes = _seed_node_outcomes_from_events( - artifact_dir / "events.jsonl" if artifact_dir is not None else Path(".missing-testmon-events"), - expected_nodeids=nodeids, - database=database, - pytest_step=step, - prior_node_outcomes=prior, - use_database_fallback=False, - ) - terminal = all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in outcomes) - selection_matches = selected == nodeids - shard.update( - { - "status": SeedShardStatus.COMPLETE.value - if selection_matches and terminal - else SeedShardStatus.INCOMPLETE.value, - "started_at": shard.get("started_at") or datetime.now(timezone.utc).isoformat(), - "finished_at": datetime.now(timezone.utc).isoformat(), - "exit_code": step.get("exit"), - "artifact_dir": step.get("artifact_dir"), - "selection": dict(selection) if isinstance(selection, Mapping) else None, - "database": database, - "node_outcomes": outcomes, - "pytest_step": dict(step), - } - ) - shards[shard_index - 1] = shard - payload = { - **dict(prepared), - "status": "running", - "shards": shards, - "node_outcomes": _seed_shard_outcomes(shards), - "testmon_data": _file_fingerprint(TESTMON_DATA), - } - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - return payload - - -def _testmon_seed_terminal_authorized(prepared: Mapping[str, Any]) -> bool: - identity = prepared.get("identity") - return ( - isinstance(identity, Mapping) - and identity.get("skip_slow") is True - and identity.get("terminal_authorization") == TerminalAuthorization.NARROW_TERMINAL.value - ) - - -def _testmon_database_state(expected_nodeids: Sequence[str]) -> dict[str, Any]: - graph = inspect_testmon_database(TESTMON_DATA, expected_nodeids) - expected = set(expected_nodeids) - failed = list(graph.failed_nodeids) - return { - "recorded_count": graph.recorded_count, - "failed_count": len(failed), - "dependency_edge_count": graph.dependency_edge_count, - "missing_nodeids": list(graph.missing_nodeids), - "failed_nodeids": failed, - "node_outcomes": { - nodeid: ("failed" if nodeid in failed else "passed" if nodeid not in graph.missing_nodeids else "missing") - for nodeid in sorted(expected) - }, - "error": graph.error, - "graph_status": graph.status.value, - "orphan_execution_edges": graph.orphan_execution_edges, - "orphan_fingerprint_edges": graph.orphan_fingerprint_edges, - } - - -def _seed_node_outcomes_from_events( - path: Path, - *, - expected_nodeids: Sequence[str], - database: Mapping[str, Any], - pytest_step: Mapping[str, Any] | None, - use_database_fallback: bool = True, - prior_node_outcomes: Mapping[str, Mapping[str, Any]] | None = None, -) -> list[dict[str, Any]]: - """Classify every promised seed node into one explicit terminal state.""" - reports: dict[str, list[dict[str, Any]]] = {} - started: set[str] = set() - finished: set[str] = set() - try: - with path.open(encoding="utf-8") as handle: - for line in handle: - if not line.strip(): - continue - with contextlib.suppress(json.JSONDecodeError): - event = json.loads(line) - nodeid = event.get("nodeid") - if not isinstance(nodeid, str) or not nodeid: - continue - nodeid = _canonical_seed_nodeid(nodeid, expected_nodeids) - if event.get("event") == "test_started": - started.add(nodeid) - elif event.get("event") == "test_finished": - finished.add(nodeid) - elif event.get("event") == "test_report": - reports.setdefault(nodeid, []).append(event) - except OSError: - pass - - database_outcomes = database.get("node_outcomes") - recorded = database_outcomes if isinstance(database_outcomes, dict) else {} - diagnosis = str((pytest_step or {}).get("diagnosis") or "").lower() - results: list[dict[str, Any]] = [] - for nodeid in expected_nodeids: - node_reports = reports.get(nodeid, []) - failed_reports = [report for report in node_reports if report.get("outcome") == "failed"] - call_reports = [report for report in node_reports if report.get("when") == "call"] - longrepr = "\n".join(str(report.get("longrepr") or "") for report in failed_reports).lower() - outcome: str - reason: str - if "timeout" in longrepr: - outcome, reason = "timeout", "pytest-timeout report" - elif any(report.get("when") in {"setup", "teardown"} for report in failed_reports): - outcome, reason = "error", "fixture setup/teardown failed" - elif any(report.get("outcome") == "xfailed" for report in node_reports): - outcome, reason = "xfailed", "pytest expected failure" - elif any(report.get("outcome") == "xpassed" for report in node_reports): - outcome, reason = "xpassed", "pytest unexpected pass" - elif any(report.get("outcome") == "failed" for report in call_reports): - outcome, reason = "failed", "test call failed" - elif any(report.get("outcome") == "passed" for report in call_reports): - outcome, reason = "passed", "test call passed" - elif any(report.get("outcome") == "skipped" for report in call_reports): - outcome, reason = "skipped", "test call skipped" - elif any(report.get("outcome") == "skipped" for report in node_reports): - outcome, reason = "skipped", "test setup or teardown skipped" - elif nodeid in finished and any( - report.get("when") == "teardown" and report.get("outcome") == "passed" for report in node_reports - ): - # Teardown describes fixture cleanup, not the test body. It may - # corroborate a terminal testmon row, but it cannot replace a - # missing call report: a failed call can still end with a passing - # teardown, and an unrecorded call must remain resumable. - if recorded.get(nodeid) == "passed": - outcome, reason = "passed", "passing teardown corroborated by testmon success" - elif recorded.get(nodeid) == "failed": - outcome, reason = "failed", "passing teardown contradicted by testmon failure" - else: - outcome, reason = "missing", "passing teardown without call report or testmon result" - elif nodeid in started and nodeid not in finished and "timeout" in diagnosis: - outcome, reason = "timeout", "supervisor timed out while node was active" - elif nodeid in started and nodeid not in finished and "worker" in diagnosis: - outcome, reason = "worker_crash", "worker exited while node was active" - elif ( - nodeid in started - and nodeid not in finished - and any(marker in diagnosis for marker in ("interrupt", "signal", "terminated")) - ): - outcome, reason = "interrupted", "run ended while node was active" - elif use_database_fallback and recorded.get(nodeid) == "passed": - outcome, reason = "passed", "testmon database recorded success" - elif use_database_fallback and recorded.get(nodeid) == "failed": - outcome, reason = "failed", "testmon database recorded failure" - elif prior_node_outcomes is not None and nodeid in prior_node_outcomes: - prior = prior_node_outcomes[nodeid] - prior_outcome = prior.get("outcome") - if prior_outcome in TERMINAL_NODE_OUTCOMES: - outcome, reason = str(prior_outcome), "terminal outcome carried from the prior seed attempt" - else: - outcome, reason = "missing", "prior seed attempt has no terminal outcome" - else: - outcome, reason = "missing", "no terminal report or testmon execution row" - results.append( - { - "nodeid": nodeid, - "outcome": outcome, - "reason": reason, - "started": nodeid in started, - "finished": nodeid in finished, - "phases": [ - { - "when": report.get("when"), - "outcome": report.get("outcome"), - "duration_s": report.get("duration_s"), - } - for report in node_reports - ], - } - ) - return results - - -def _seed_attempt_outcome( - *, - release_eligible: bool, - terminal_graph: bool, - exit_code: int, - pytest_step: Mapping[str, Any] | None, -) -> SeedAttemptOutcome: - """Classify the terminal seed result without hiding a bounded resource stop.""" - if release_eligible: - return SeedAttemptOutcome.GREEN_RELEASE_BASELINE - if terminal_graph: - return SeedAttemptOutcome.RED_BASELINE if exit_code != 0 else SeedAttemptOutcome.SELECTION_ONLY - diagnosis = str((pytest_step or {}).get("diagnosis") or "").casefold() - termination_reason = str((pytest_step or {}).get("termination_reason") or "").casefold() - if diagnosis in {"pytest_timeout", "pytest_stall_timeout", "pytest_resource_preflight_failed"} or any( - marker in termination_reason - for marker in ("runtime exceeded", "tmpfs budget exceeded", "resource budget", "resource limit") - ): - return SeedAttemptOutcome.RESOURCE_TIMEOUT - return SeedAttemptOutcome.INCOMPLETE - - -def _finalize_testmon_seed_attempt( - *, - prepared: Mapping[str, Any], - step_results: Sequence[Mapping[str, Any]], - exit_code: int, -) -> dict[str, Any]: - pytest_step = next( - (step for step in step_results if str(step.get("name", "")).startswith("pytest seed-testmon")), None - ) - selection: dict[str, Any] = {} - events_path: Path | None = None - if pytest_step is not None: - artifact_dir = _safe_testmon_artifact_dir(pytest_step.get("artifact_dir")) - if artifact_dir is not None: - selection_payload = _read_json_artifact(artifact_dir / "selection.json") - if isinstance(selection_payload, dict): - selection = selection_payload - events_path = artifact_dir / "events.jsonl" - - raw_omitted = selection.get("selected_nodeids_omitted") - raw_selected_count = selection.get("selected_count") - selected_nodeids = selection.get("selected_nodeids") - selection_valid = ( - isinstance(raw_omitted, int) - and not isinstance(raw_omitted, bool) - and raw_omitted >= 0 - and isinstance(raw_selected_count, int) - and not isinstance(raw_selected_count, bool) - and isinstance(selected_nodeids, list) - and all(isinstance(nodeid, str) and nodeid for nodeid in selected_nodeids) - and len(set(selected_nodeids)) == len(selected_nodeids) - and raw_selected_count == len(selected_nodeids) - ) - prepared_expected = prepared.get("expected_nodeids") - expected_raw = prepared_expected if isinstance(prepared_expected, list) and prepared_expected else selected_nodeids - expected = list(expected_raw) if isinstance(expected_raw, list) else [] - shards = validate_seed_shard_ledger(prepared.get("shards"), expected_nodeids=expected) - sharded = shards is not None - if sharded: - assert shards is not None - selection_valid = seed_shard_ledger_is_terminal(shards) - omitted = 0 - database = _testmon_database_state(expected) - outcome_by_node = {item["nodeid"]: item for item in _seed_shard_outcomes(shards)} - node_outcomes = [ - outcome_by_node.get(nodeid, {"nodeid": nodeid, "outcome": "missing", "reason": "shard not completed"}) - for nodeid in expected - ] - shard_steps: list[Mapping[str, Any]] = [] - for shard in shards: - raw_step = shard.get("pytest_step") - if isinstance(raw_step, Mapping): - shard_steps.append(raw_step) - if shard_steps: - pytest_step = dict(shard_steps[-1]) - timed_out = next( - ( - step - for step in shard_steps - if str(step.get("diagnosis")) in {"pytest_timeout", "pytest_stall_timeout"} - ), - None, - ) - if timed_out is not None: - pytest_step = dict(timed_out) - selection = { - "selected_count": len(expected), - "selected_nodeids_omitted": 0, - "shard_count": len(shards), - "completed_shard_count": sum(shard.get("status") == SeedShardStatus.COMPLETE.value for shard in shards), - } - else: - omitted = raw_omitted if selection_valid and isinstance(raw_omitted, int) else 1 - database = _testmon_database_state(expected) - node_outcomes = _seed_node_outcomes_from_events( - events_path or Path(".missing-testmon-events"), - expected_nodeids=expected, - database=database, - pytest_step=pytest_step, - use_database_fallback=False, - prior_node_outcomes={ - str(item["nodeid"]): item - for item in prepared.get("prior_node_outcomes", []) - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) - }, - ) - unsuccessful_nodeids = [ - str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in SUCCESSFUL_NODE_OUTCOMES - ] - green_complete = ( - exit_code == 0 - and bool(expected) - and selection_valid - and omitted == 0 - and database["error"] is None - and database["graph_status"] == "complete" - and not database["missing_nodeids"] - and not database["failed_nodeids"] - and database["orphan_execution_edges"] == 0 - and database["orphan_fingerprint_edges"] == 0 - and not unsuccessful_nodeids - ) - identity = prepared.get("identity") - narrow_terminal = isinstance(identity, Mapping) and identity.get("skip_slow") is True - terminal_authorized = _testmon_seed_terminal_authorized(prepared) - release_eligible = green_complete and (not narrow_terminal or terminal_authorized) - terminal_graph = ( - database["error"] is None - and database["graph_status"] == GraphStatus.COMPLETE.value - and not database["missing_nodeids"] - and database["orphan_execution_edges"] == 0 - and database["orphan_fingerprint_edges"] == 0 - and all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) - ) - outcome = _seed_attempt_outcome( - release_eligible=release_eligible, - terminal_graph=terminal_graph, - exit_code=exit_code, - pytest_step=pytest_step, - ) - shard_ledger = ( - shards if sharded else seed_shard_plan(expected, shard_size=max(1, len(expected))) if expected else [] - ) - if not sharded and shard_ledger: - shard_ledger[0].update( - { - "status": ( - SeedShardStatus.COMPLETE.value - if all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) - else SeedShardStatus.INCOMPLETE.value - ), - "node_outcomes": node_outcomes, - } - ) - seed_scope = ( - VerificationScope.NARROW_TERMINAL.value if narrow_terminal else VerificationScope.RELEASE_BASELINE.value - ) - attempt_candidate = { - **dict(prepared), - "status": "complete" if release_eligible else "reusable" if terminal_graph else "incomplete", - "outcome": outcome.value, - "exit_code": exit_code, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, - "selection": { - **selection, - # A resumed run inherits the complete collection ledger from its - # original selection. The current pytest step may select only a - # subset while it repairs missing graph edges. - "selected_count": len(expected) - if prepared.get("resume") and selection_valid - else selection.get("selected_count"), - "selected_nodeids_omitted": 0 if prepared.get("resume") and selection_valid else omitted, - }, - "shards": shard_ledger, - "node_outcomes": node_outcomes, - "identity": prepared.get("identity"), - "run_id": prepared.get("run_id"), - "artifact_dir": prepared.get("artifact_dir"), - "testmon_data": _file_fingerprint(TESTMON_DATA), - "verification_scope": seed_scope, - "terminal_authorization": (TerminalAuthorization.NARROW_TERMINAL.value if terminal_authorized else None), - "release_baseline_allowed": release_eligible, - } - reusable_stamp = stamp_from_attempt( - attempt_candidate, - TESTMON_DATA, - checkout_root=Path.cwd(), - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - ) - reusable = reusable_stamp is not None - release_permission = bool( - reusable - and reusable_stamp is not None - and reusable_stamp.release_baseline_allowed - and (not narrow_terminal or terminal_authorized) - ) - attempt_status = "complete" if green_complete and release_permission else "reusable" if reusable else "incomplete" - payload = { - **dict(prepared), - "status": attempt_status, - "outcome": outcome.value, - "finished_at": datetime.now(timezone.utc).isoformat(), - "exit_code": exit_code, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() if expected else None, - "selection": { - key: ( - len(expected) - if key == "selected_count" and prepared.get("resume") and selection_valid - else 0 - if key == "selected_nodeids_omitted" and prepared.get("resume") and selection_valid - else selection.get(key) - ) - for key in ( - "selected_count", - "deselected_count", - "selected_nodeids_omitted", - "deselected_nodeids_omitted", - "collection_duration_s", - ) - }, - "shards": shard_ledger, - "database": database, - "node_outcomes": node_outcomes, - "node_outcome_counts": dict( - sorted( - { - outcome: sum(1 for item in node_outcomes if item.get("outcome") == outcome) - for outcome in {str(item.get("outcome")) for item in node_outcomes} - }.items() - ) - ), - "unsuccessful_nodeids": unsuccessful_nodeids, - "testmon_data": _file_fingerprint(TESTMON_DATA), - "pytest_step": dict(pytest_step) if pytest_step is not None else None, - "binding": TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict(), - "verification_scope": seed_scope, - "terminal_authorization": (TerminalAuthorization.NARROW_TERMINAL.value if terminal_authorized else None), - } - payload["release_baseline_allowed"] = release_permission - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - if release_permission and reusable_stamp is not None: - _atomic_write_json(TESTMON_SEED_STAMP, reusable_stamp.as_dict()) - else: - TESTMON_SEED_STAMP.unlink(missing_ok=True) - return payload - - -def _refresh_testmon_selection_attempt( - *, - step: Mapping[str, Any], - run: VerifyRun, - exit_code: int, -) -> None: - """Refresh a reusable red graph after every completed affected run.""" - attempt = _read_testmon_seed_attempt() - if attempt is None or attempt.get("release_baseline_allowed") is True: - return - expected = _testmon_seed_expected_nodeids(attempt) - if not expected: - return - database = _testmon_database_state(expected) - artifact_dir = _safe_testmon_artifact_dir(step.get("artifact_dir")) - events_path = artifact_dir / "events.jsonl" if artifact_dir is not None else Path(".missing-testmon-events") - prior = { - str(item["nodeid"]): item - for item in attempt.get("node_outcomes", []) - if isinstance(item, Mapping) and isinstance(item.get("nodeid"), str) - } - node_outcomes = _seed_node_outcomes_from_events( - events_path, - expected_nodeids=expected, - database=database, - pytest_step=step, - use_database_fallback=False, - prior_node_outcomes=prior, - ) - graph_complete = ( - database.get("graph_status") == GraphStatus.COMPLETE.value - and not database.get("missing_nodeids") - and database.get("error") is None - and database.get("orphan_execution_edges") == 0 - and database.get("orphan_fingerprint_edges") == 0 - ) - terminal = all(item.get("outcome") in TERMINAL_NODE_OUTCOMES for item in node_outcomes) - prior_selection = attempt.get("selection") - payload = { - **attempt, - "status": "reusable" if graph_complete and terminal else "incomplete", - "outcome": ( - SeedAttemptOutcome.RED_BASELINE.value - if graph_complete and terminal - else SeedAttemptOutcome.INCOMPLETE.value - ), - "finished_at": datetime.now(timezone.utc).isoformat(), - "exit_code": exit_code, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - "selection": { - **(dict(prior_selection) if isinstance(prior_selection, Mapping) else {}), - "selected_count": len(expected), - "selected_nodeids_omitted": 0, - }, - "database": database, - "node_outcomes": node_outcomes, - "node_outcome_counts": dict( - sorted( - { - outcome: sum(1 for item in node_outcomes if item.get("outcome") == outcome) - for outcome in {str(item.get("outcome")) for item in node_outcomes} - }.items() - ) - ), - "unsuccessful_nodeids": [ - str(item["nodeid"]) for item in node_outcomes if item.get("outcome") not in SUCCESSFUL_NODE_OUTCOMES - ], - "testmon_data": _file_fingerprint(TESTMON_DATA), - "run_id": run.run_id, - "artifact_dir": str(run.relative_run_dir), - "pytest_step": dict(step), - "release_baseline_allowed": False, - "verification_scope": VerificationScope.AFFECTED.value, - } - raw_binding = attempt.get("binding") - if not isinstance(raw_binding, Mapping): - payload["binding"] = TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())).as_dict() - _atomic_write_json(TESTMON_SEED_ATTEMPT, payload) - - -def _discard_testmon_dependency_authority() -> None: - """Remove a dependency graph learned while checkout authority was unstable.""" - for path in ( - TESTMON_SEED_STAMP, - TESTMON_SEED_ATTEMPT, - TESTMON_DATA, - Path(f"{TESTMON_DATA}-wal"), - Path(f"{TESTMON_DATA}-shm"), - Path(f"{TESTMON_DATA}-journal"), - ): - path.unlink(missing_ok=True) - - -# ── main ──────────────────────────────────────────────────────────── - - -_ACTIVE_VERIFY_RUN: tuple[VerifyRun, float, VerificationScope] | None = None - - -def _planned_verification_scope(args: argparse.Namespace, *, full_pytest: bool) -> VerificationScope: - """Return the immutable scope requested before the runner starts.""" - if args.quick or args.commit: - return VerificationScope.NON_TEST - if full_pytest or args.seed_testmon: - return VerificationScope.NARROW_TERMINAL if args.skip_slow else VerificationScope.RELEASE_BASELINE - return VerificationScope.AFFECTED - - -def _changed_paths_from_testmon_authority(base_commit: str | None, head_commit: str | None) -> tuple[str, ...]: - """Require immutable refs before deriving affected executable paths.""" - if base_commit is None or head_commit is None: - raise PytestResourceError("testmon changed-path authority is unavailable") - return _changed_executable_paths(base_commit, head_commit) + ) + return exit_code def _main(argv: list[str] | None = None) -> int: global _ACTIVE_VERIFY_RUN + started_at = time.monotonic() parser = argparse.ArgumentParser(description="Run the local verification baseline.") parser.add_argument("--quick", action="store_true", help="Skip pytest and run only fast local gates.") parser.add_argument( - "--seed-testmon", + "--all", action="store_true", - help="Run full non-integration pytest with --testmon-noselect to seed/update .cache/testmon/testmondata.", - ) - parser.add_argument( - "--all", action="store_true", help="Force the full non-integration pytest diagnostic instead of testmon." - ) - parser.add_argument( - "--full", action="store_true", help="Alias for --all: run full non-integration pytest diagnostic." + help="Run the complete pytest correctness corpus (excluding performance benchmarks).", ) + parser.add_argument("--full", action="store_true", help="Alias for --all.") parser.add_argument("--commit", action="store_true", help="Pre-commit tier: format + lint + mypy only.") - parser.add_argument( - "--skip-slow", action="store_true", help="Exclude @pytest.mark.slow tests from the pytest step." - ) - parser.add_argument( - "--terminal-authorization", - choices=[TerminalAuthorization.NARROW_TERMINAL.value], - help="Typed authorization for a narrow terminal verification that skips slow tests.", - ) parser.add_argument( "--lab", action="store_true", - help=( - "Run the default pytest-testmon baseline plus verification-lab " - "scenario and verify-slos checks; does not imply --all." - ), + help="Run the native pytest-testmon lifecycle plus verification-lab checks.", ) parser.add_argument("--history", action="store_true", help="Print last 10 verify runs and exit.") parser.add_argument("--json", action="store_true", default=None, help="Write structured JSON to stdout.") args = parser.parse_args(sys.argv[1:] if argv is None else argv) _anchor_verification_paths() - bootstrap_message = maybe_bootstrap_testmon_seed( - ROOT, - protocol_version=TESTMON_SEED_PROTOCOL_VERSION, - ) - if bootstrap_message is not None: - sys.stderr.write(bootstrap_message + "\n") - try: - fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools verify") - except CheckoutImportMismatchError as exc: - sys.stderr.write(f"verify: {exc}\n") - return 125 - polylogue_import_path = fingerprint.polylogue_import_path - environment_fingerprint = fingerprint.as_dict() - sys.stderr.write(f"verify: polylogue package → {polylogue_import_path}\n") - if args.history: + try: + assert_polylogue_matches_checkout(ROOT, context="devtools verify") + except CheckoutImportMismatchError as exc: + sys.stderr.write(f"verify: {exc}\n") + return 125 _print_history() return 0 - # Auto-detect JSON when stdout is not a TTY (agent/pipe context). + full_requested = bool(args.all or args.full) use_json = args.json if args.json is not None else not sys.stdout.isatty() - - tier = "full" - if args.commit: - tier = "commit" - elif args.quick: - tier = "quick" - elif args.seed_testmon: - tier = "seed-testmon" - elif args.all or args.full: - tier = "full" - elif args.lab: - tier = "lab" - else: - tier = "testmon" - + tier = ( + "commit" + if args.commit + else "quick" + if args.quick + else "full" + if full_requested + else "lab" + if args.lab + else "testmon" + ) head = _git_head() - full_pytest = bool(args.all or args.full) - affected_testmon = not (args.quick or args.commit or args.seed_testmon or full_pytest) - planned_verification_scope = _planned_verification_scope(args, full_pytest=full_pytest) - testmon_base_commit = _git_commit("origin/master") if affected_testmon else None - testmon_head_commit = head if affected_testmon else None - if affected_testmon and (testmon_base_commit is None or testmon_head_commit is None): - sys.stderr.write("verify: cannot resolve immutable Git refs for affected-test authority.\n") - return 125 - if args.terminal_authorization is not None and not ((full_pytest or args.seed_testmon) and args.skip_slow): - parser.error("--terminal-authorization requires --all, --full, or --seed-testmon with --skip-slow") - preflight_error = _testmon_preflight( - seed_testmon=bool(args.seed_testmon), - full_pytest=full_pytest, - quick=bool(args.quick), - commit=bool(args.commit), + pytest_enabled = not (args.quick or args.commit) + managed_pytest_enabled = pytest_enabled or args.lab + planned_scope = _planned_verification_scope( + args, + testmon_mode="full" if full_requested else None, ) - if preflight_error is not None: - sys.stderr.write(preflight_error) - return 2 - - t0 = time.monotonic() - mutation_monitor = CheckoutMutationMonitor(ROOT) - start_checkout_mutation_monitor(mutation_monitor) - checkout_fingerprint = worktree_fingerprint(ROOT) verify_run = VerifyRun( tier=tier, argv=list(sys.argv[1:] if argv is None else argv), git_head=head, + ) + _ACTIVE_VERIFY_RUN = _ActiveVerifyRun( + run=verify_run, + started_at=started_at, + verification_scope=planned_scope, + head=head, + ) + + optimization_level = _python_optimization_level() + if managed_pytest_enabled and optimization_level > 0: + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="optimized_python_interpreter", + exit_code=125, + message=( + "Python optimization disables verification assertions; " + f"refusing managed pytest at optimization level {optimization_level}" + ), + use_json=bool(use_json), + ) + + try: + fingerprint = assert_polylogue_matches_checkout(ROOT, context="devtools verify") + except CheckoutImportMismatchError as exc: + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="checkout_import_mismatch", + exit_code=125, + message=str(exc), + use_json=bool(use_json), + ) + polylogue_import_path = fingerprint.polylogue_import_path + environment_fingerprint = fingerprint.as_dict() + verify_run.update_checkout_provenance( polylogue_import_path=str(polylogue_import_path), environment_fingerprint=environment_fingerprint, - worktree_fingerprint=checkout_fingerprint, ) - _ACTIVE_VERIFY_RUN = (verify_run, t0, planned_verification_scope) - seed_identity: dict[str, Any] | None = None - resume_testmon_seed = False - prepared_seed_attempt: dict[str, Any] | None = None - if args.seed_testmon: + sys.stderr.write(f"verify: polylogue package → {polylogue_import_path}\n") + + mutation_monitor = CheckoutMutationMonitor(ROOT) + _start_active_checkout_mutation_monitor(mutation_monitor) + checkout_fingerprint = worktree_fingerprint(ROOT) + _finish_active_checkout_mutation_monitor(mutation_monitor) + mutation_monitor = CheckoutMutationMonitor(ROOT) + _start_active_checkout_mutation_monitor(mutation_monitor) + assert _ACTIVE_VERIFY_RUN is not None + _ACTIVE_VERIFY_RUN.initial_worktree_fingerprint = checkout_fingerprint + verify_run.update_checkout_provenance(worktree_fingerprint=checkout_fingerprint) + + base_commit = _git_commit("origin/master") if pytest_enabled else None + if pytest_enabled and (base_commit is None or head is None): + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="native_git_authority_unavailable", + exit_code=125, + message="cannot resolve immutable Git refs for native affected-test authority.", + use_json=bool(use_json), + mutation_monitor=mutation_monitor, + initial_worktree_fingerprint=checkout_fingerprint, + ) + + relevant_paths: tuple[str, ...] = () + required_executable_paths: tuple[str, ...] = () + preparation_required_executable_paths: tuple[str, ...] = () + runtime_data_paths: tuple[str, ...] = () + preparation: NativeTestmonPreparation | None = None + testmon_mode: str | None = None + native_pytest_environment = _native_pytest_environment(force_release_profile=full_requested) + preparation_mutation_observation: CheckoutMutationObservation | None = None + if pytest_enabled: + assert base_commit is not None + assert head is not None try: - seed_identity = _testmon_seed_identity( - git_head=head, - git_tree=_git_committed_tree(), - skip_slow=bool(args.skip_slow), - lab=bool(args.lab), - terminal_authorization=args.terminal_authorization, + relevant_paths = _changed_test_relevant_paths(base_commit, head) + change_impact = classify_native_testmon_changes(ROOT, relevant_paths) + preparation_required_executable_paths = change_impact.executable_paths + required_executable_paths = tuple( + path for path in preparation_required_executable_paths if (ROOT / path).is_file() ) - except RuntimeError as exc: - finish_checkout_mutation_monitor(mutation_monitor) - sys.stderr.write(f"verify: {exc}\n") - early_payload = verify_run.finish( + runtime_data_paths = change_impact.runtime_data_paths + preparation = prepare_native_testmon_environment( + ROOT, + required_executable_paths=preparation_required_executable_paths, + pytest_profile=_pytest_profile(), + pytest_environment=native_pytest_environment, + deadline_monotonic=started_at + VERIFY_INVOCATION_BUDGET_S, + ) + if ( + preparation.selection_mode == "bootstrap" + and native_pytest_environment["HYPOTHESIS_PROFILE"] != "default" + ): + native_pytest_environment = _native_pytest_environment(force_release_profile=True) + preparation = prepare_native_testmon_environment( + ROOT, + required_executable_paths=preparation_required_executable_paths, + pytest_profile=_pytest_profile(), + pytest_environment=native_pytest_environment, + deadline_monotonic=started_at + VERIFY_INVOCATION_BUDGET_S, + ) + assert _ACTIVE_VERIFY_RUN is not None + _ACTIVE_VERIFY_RUN.owned_native_testmon_state = _open_owned_native_testmon_state(ROOT) + except NativeTestmonDeadlineError as exc: + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="verify_invocation_deadline_exceeded", + exit_code=124, + message=str(exc), + use_json=bool(use_json), + mutation_monitor=mutation_monitor, + initial_worktree_fingerprint=checkout_fingerprint, + ) + except (NativeTestmonRepairError, PytestResourceError) as exc: + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, + diagnosis="native_testmon_preparation_failed", exit_code=125, - duration_s=time.monotonic() - t0, - diagnosis="testmon_environment_identity_unavailable", + message=f"native pytest-testmon preparation failed: {exc}", + use_json=bool(use_json), + mutation_monitor=mutation_monitor, + initial_worktree_fingerprint=checkout_fingerprint, ) - _save_history(early_payload) - return 125 - resume_testmon_seed = _testmon_seed_can_resume(seed_identity) - prepared_seed_attempt = _prepare_testmon_seed_attempt( - identity=seed_identity, - run=verify_run, - resume=resume_testmon_seed, - ) - if resume_testmon_seed: - sys.stderr.write("verify: resuming the matching incomplete pytest-testmon seed\n") + testmon_mode = "full" if full_requested or runtime_data_paths else preparation.selection_mode + if preparation.removed_paths: + sys.stderr.write( + "verify: repaired invalid native pytest-testmon state by removing only " + + ", ".join(str(path) for path in preparation.removed_paths) + + "\n" + ) + if preparation.copied_from is not None: + sys.stderr.write(f"verify: copied matching native pytest-testmon DB from {preparation.copied_from}\n") + elif preparation.selection_mode == "bootstrap": + sys.stderr.write("verify: native pytest-testmon environment is empty; plain verify will build it\n") + if runtime_data_paths and not full_requested: + sys.stderr.write( + "verify: changed package runtime data is outside Python tracing; running the complete corpus: " + + ", ".join(runtime_data_paths) + + "\n" + ) + + planned_scope = _planned_verification_scope(args, testmon_mode=testmon_mode) + assert _ACTIVE_VERIFY_RUN is not None + _ACTIVE_VERIFY_RUN.verification_scope = planned_scope if not use_json: sys.stderr.write("verify: running local verification baseline\n") - - # Resource preflight before heavy steps. - if not args.quick and not args.commit: + if pytest_enabled: _warn_low_memory() - exit_code = 0 try: steps = build_verify_steps( quick=bool(args.quick), commit=bool(args.commit), lab=bool(args.lab), - skip_slow=bool(args.skip_slow), - seed_testmon=bool(args.seed_testmon), - resume_testmon_seed=resume_testmon_seed, - full_pytest=full_pytest, - broad_testmon=( - _default_testmon_is_broad_change(testmon_base_commit, testmon_head_commit) - if testmon_base_commit is not None and testmon_head_commit is not None - else False - ), + testmon_mode=testmon_mode or "affected", + testmon_environment=preparation.environment_name if preparation is not None else "", ) - except PytestResourceError as exc: - finish_checkout_mutation_monitor(mutation_monitor) - sys.stderr.write(f"verify: {exc}\n") - early_payload = verify_run.finish( - exit_code=125, - duration_s=time.monotonic() - t0, + except (PytestResourceError, ValueError) as exc: + return _finalize_preflight_failure( + verify_run, + started_at=started_at, + tier=tier, + head=head, + verification_scope=planned_scope, diagnosis="pytest_resource_preflight_failed", + exit_code=125, + message=str(exc), + use_json=bool(use_json), + mutation_monitor=mutation_monitor, + initial_worktree_fingerprint=checkout_fingerprint, ) - _save_history(early_payload) - return 125 + # Git probes and native testmon preparation can refresh the index as part + # of their own read path. Retain that interval's observation: a tracked + # file can be edited and restored while the graph is prepared, which makes + # any resulting selection unsuitable as exact-head authority. + preparation_mutation_observation = _finish_active_checkout_mutation_monitor(mutation_monitor) + mutation_monitor = CheckoutMutationMonitor(ROOT) + _start_active_checkout_mutation_monitor(mutation_monitor) step_results: list[dict[str, Any]] = [] - pending_testmon_stamp: TestmonSeedStamp | None = None - pending_affected_coverage: tuple[tuple[str, ...], int] | None = None - pending_selection_refresh: tuple[dict[str, Any], int] | None = None - testmon_graph_touched = False - changed_path_authority_failed = False + exit_code = 0 + native_graph_touched = False for label, cmd in steps: + remaining = _remaining_invocation_budget(started_at) + if remaining <= 0: + deadline_step = { + "name": label, + "duration_s": 0.0, + "exit": 124, + "diagnosis": "verify_invocation_deadline_exceeded", + "timeout_s": VERIFY_INVOCATION_BUDGET_S, + } + step_results.append(deadline_step) + exit_code = 124 + break if label.startswith("pytest"): - _warn_low_memory() # check again right before the heavy step - rc, elapsed, metadata = _run(label, cmd, run=verify_run) - if label in {"pytest testmon", "pytest testmon (broad)"} or label.startswith("pytest seed-testmon"): - testmon_graph_touched = True - if rc == 0 and label in {"pytest testmon", "pytest testmon (broad)"}: - raw_stamp = _read_json_artifact(TESTMON_SEED_STAMP) - try: - current_stamp = ( - TestmonSeedStamp.from_mapping(raw_stamp, protocol_version=TESTMON_SEED_PROTOCOL_VERSION) - if isinstance(raw_stamp, Mapping) - else None - ) - except ValueError: - current_stamp = None - if current_stamp is not None: - refreshed_stamp = refresh_stamp(current_stamp, TESTMON_DATA) - if refreshed_stamp is not None: - pending_testmon_stamp = refreshed_stamp - try: - executable_paths = _changed_paths_from_testmon_authority(testmon_base_commit, testmon_head_commit) - except PytestResourceError as exc: - changed_path_authority_failed = True - executable_paths = () - rc = 125 - metadata["diagnosis"] = "testmon_changed_path_authority_unavailable" - metadata["error"] = str(exc) - pending_testmon_stamp = None - sys.stderr.write( - "verify: changed-path authority became unavailable after pytest; " - "discarding the affected dependency graph.\n" - ) - selected_count = metadata.get("selected_count") - if selected_count == 0 and executable_paths: - coverage = _matching_testmon_coverage(executable_paths) - if coverage is None: - rc = 5 - metadata["diagnosis"] = "zero_testmon_selection_for_executable_change" - metadata["zero_selection_changed_paths"] = list(executable_paths) - sys.stderr.write( - "verify: pytest-testmon selected zero tests for executable changes and no " - "matching successful coverage receipt exists; refresh the seed or repair " - "dependency capture: " + ", ".join(executable_paths) + "\n" - ) - else: - metadata["zero_selection_coverage"] = coverage - elif isinstance(selected_count, int) and selected_count > 0: - pending_affected_coverage = (tuple(executable_paths), selected_count) + _warn_low_memory() + rc, elapsed, metadata = _run(label, cmd, run=verify_run, timeout_s=remaining) step_result: dict[str, Any] = {"name": label, "duration_s": round(elapsed, 2), "exit": rc} step_result.update(metadata) + if label.startswith("pytest native parallel"): + step_result["semantic_lane"] = "parallel" + native_graph_touched = True + elif label.startswith("pytest native serial"): + step_result["semantic_lane"] = "serial" + native_graph_touched = True step_results.append(step_result) - if args.seed_testmon and label.startswith("pytest seed-testmon collect"): - if rc != 0: - exit_code = rc - break - artifact_dir = _safe_testmon_artifact_dir(metadata.get("artifact_dir")) - selection = _read_json_artifact(artifact_dir / "selection.json") if artifact_dir is not None else None - assert prepared_seed_attempt is not None - prepared_seed_attempt = _prepare_testmon_seed_shards( - prepared_seed_attempt, - selection=selection if isinstance(selection, Mapping) else None, - ) - expected = _testmon_seed_expected_nodeids(prepared_seed_attempt) - shards = validate_seed_shard_ledger(prepared_seed_attempt.get("shards"), expected_nodeids=expected) - if shards is None: - exit_code = 5 - step_result["exit"] = 5 - step_result["diagnosis"] = "testmon_seed_collection_incomplete" - sys.stderr.write("verify: pytest-testmon collection did not produce a complete shard plan.\n") - break - for shard in shards: - if shard.get("status") == SeedShardStatus.COMPLETE.value: - continue - shard_index = int(shard["index"]) - shard_label = f"pytest seed-testmon shard {shard_index}/{len(shards)}" - shard_args_path = verify_run.run_dir / "seed-shards" / f"{shard_index:04d}.args" - try: - shard_cmd = _seed_shard_command(cmd, shard, nodeids_file=shard_args_path) - except (OSError, PytestResourceError) as exc: - resource_failure_result = { - "name": shard_label, - "duration_s": 0.0, - "exit": 125, - "diagnosis": ( - "pytest_resource_refusal" - if isinstance(exc, PytestResourceError) - else "testmon_seed_args_file_write_failed" - ), - "error": str(exc), - "shard_index": shard_index, - "shard_count": len(shards), - "shard_nodeid_count": len(shard["nodeids"]), - } - step_results.append(resource_failure_result) - prepared_seed_attempt = _checkpoint_testmon_seed_shard( - prepared=prepared_seed_attempt, - shard_index=shard_index, - step=resource_failure_result, - ) - exit_code = 125 - break - _warn_low_memory() - shard_rc, shard_elapsed, shard_metadata = _run(shard_label, shard_cmd, run=verify_run) - shard_result: dict[str, Any] = { - "name": shard_label, - "duration_s": round(shard_elapsed, 2), - "exit": shard_rc, - "shard_index": shard_index, - "shard_count": len(shards), - "shard_nodeid_count": len(shard["nodeids"]), - } - shard_result.update(shard_metadata) - step_results.append(shard_result) - prepared_seed_attempt = _checkpoint_testmon_seed_shard( - prepared=prepared_seed_attempt, - shard_index=shard_index, - step=shard_result, - ) - checkpointed_shards = prepared_seed_attempt.get("shards") - if ( - not isinstance(checkpointed_shards, list) - or shard_index > len(checkpointed_shards) - or not isinstance(checkpointed_shards[shard_index - 1], Mapping) - ): - raise RuntimeError("testmon seed shard checkpoint is malformed") - shard_complete = checkpointed_shards[shard_index - 1].get("status") == SeedShardStatus.COMPLETE.value - if shard_rc != 0: - stop_seed = _seed_shard_failure_requires_stop( - shard_result, - shard_complete=shard_complete, - ) - if exit_code == 0 or stop_seed: - # A later infrastructure failure is the terminal - # condition even when an earlier shard recorded - # ordinary red-test evidence. - exit_code = shard_rc - if stop_seed: - break + if rc == 0: continue - if label in {"pytest testmon", "pytest testmon (broad)"} and not args.seed_testmon and not full_pytest: - pending_selection_refresh = (step_result, rc) - if rc != 0: + if exit_code == 0 or rc in {2, 3, 4, 124, 125, 130}: exit_code = rc - if rc == 130 or _stop_after_failed_step(label): - break + if label.startswith("pytest native parallel") and not _native_lane_failure_requires_stop(step_result): + continue + if label.startswith("pytest") or rc == 130 or _stop_after_failed_step(label): + break + assert mutation_monitor is not None final_head = _git_head() final_checkout_fingerprint = worktree_fingerprint(ROOT) - mutation_observation = finish_checkout_mutation_monitor(mutation_monitor) + mutation_observation = _finish_active_checkout_mutation_monitor(mutation_monitor) checkout_stable = True checkout_fingerprint_unavailable = ( head is None @@ -3763,96 +3160,163 @@ def _main(argv: list[str] | None = None) -> int: final_checkout_fingerprint, } ) - if changed_path_authority_failed or checkout_fingerprint_unavailable or mutation_observation.unavailable: + if ( + checkout_fingerprint_unavailable + or mutation_observation.unavailable + or preparation_mutation_observation.unavailable + ): checkout_stable = False diagnosis = ( - "testmon_changed_path_authority_unavailable" - if changed_path_authority_failed - else ( - "checkout_fingerprint_unavailable" - if checkout_fingerprint_unavailable - else "checkout_mutation_monitor_unavailable" - ) + "checkout_fingerprint_unavailable" + if checkout_fingerprint_unavailable + else "checkout_mutation_monitor_unavailable" ) + elif ( + final_head != head + or preparation_mutation_observation.changed + or mutation_observation.changed + or final_checkout_fingerprint != checkout_fingerprint + ): + checkout_stable = False + diagnosis = "checkout_changed_during_verification" + else: + diagnosis = None + if diagnosis is not None: + stability_step = { + "name": "checkout stability", + "duration_s": 0.0, + "exit": 125, + "diagnosis": diagnosis, + "initial_git_head": head, + "final_git_head": final_head, + "initial_worktree_fingerprint": checkout_fingerprint, + "final_worktree_fingerprint": final_checkout_fingerprint, + "transient_checkout_mutation": (preparation_mutation_observation.changed or mutation_observation.changed), + "checkout_mutation_path": ( + preparation_mutation_observation.observed_path or mutation_observation.observed_path + ), + } + step_results.append(stability_step) + exit_code = 125 + sys.stderr.write("verify: checkout contents were not stable for exact-head evidence.\n") + if native_graph_touched: + try: + removed = remove_invalid_native_testmon_state(ROOT) + except NativeTestmonRepairError as exc: + stability_step["testmon_cleanup_error"] = str(exc) + else: + stability_step["testmon_cleanup_paths"] = [str(path) for path in removed] + + native_state = None + if preparation is not None: + native_state = _native_environment_after_run( + preparation, + required_executable_paths=required_executable_paths, + ) + if not native_state.valid: + graph_step = { + "name": "pytest native graph validation", + "duration_s": 0.0, + "exit": 5, + "diagnosis": "native_testmon_graph_invalid", + "reason": native_state.reason, + "missing_executable_paths": list(native_state.missing_executable_paths), + } + step_results.append(graph_step) + if exit_code == 0: + exit_code = 5 + if native_state.missing_executable_paths: + sys.stderr.write( + "verify: changed executable modules have no runtime dependency edge: " + + ", ".join(native_state.missing_executable_paths) + + "\n" + ) + _close_active_native_testmon_state() + + total_duration = round(time.monotonic() - started_at, 2) + deadline_recorded = any(step.get("diagnosis") == "verify_invocation_deadline_exceeded" for step in step_results) + if total_duration > VERIFY_INVOCATION_BUDGET_S and not deadline_recorded: step_results.append( { - "name": "checkout stability", + "name": "verify invocation deadline", "duration_s": 0.0, - "exit": 125, - "diagnosis": diagnosis, - "initial_git_head": head, - "final_git_head": final_head, - "initial_worktree_fingerprint": checkout_fingerprint, - "final_worktree_fingerprint": final_checkout_fingerprint, + "exit": 124, + "diagnosis": "verify_invocation_deadline_exceeded", + "timeout_s": VERIFY_INVOCATION_BUDGET_S, } ) - if exit_code == 0: - exit_code = 125 - sys.stderr.write(f"verify: {diagnosis.replace('_', ' ')}; evidence is not exact-head.\n") - elif final_head != head or mutation_observation.changed or final_checkout_fingerprint != checkout_fingerprint: - checkout_stable = False + exit_code = 124 + deadline_recorded = True + pytest_aggregate: dict[str, Any] | None = None + native_environment = native_state.environment if native_state is not None else None + if preparation is not None: + pytest_aggregate = aggregate_native_testmon_run( + ROOT, + steps=step_results, + environment_name=preparation.environment_name, + corpus_nodeids=native_environment.nodeids if native_environment is not None else (), + environment_status=native_state.status if native_state is not None else "unavailable", + environment_reason=native_state.reason if native_state is not None else "post-run inspection unavailable", + selection_mode=testmon_mode or "affected", + invocation_duration_s=total_duration, + budget_s=VERIFY_INVOCATION_BUDGET_S, + ) + + # Aggregation and final graph inspection are part of the same invocation + # deadline as collection and execution. Recompute once after aggregation + # so a run cannot gain release authority by crossing the budget during + # finalization rather than during a pytest lane. + finalized_duration = round(time.monotonic() - started_at, 2) + if finalized_duration > total_duration: + total_duration = finalized_duration + if total_duration > VERIFY_INVOCATION_BUDGET_S and not deadline_recorded: step_results.append( { - "name": "checkout stability", + "name": "verify invocation deadline", "duration_s": 0.0, - "exit": 125, - "diagnosis": "checkout_changed_during_verification", - "initial_git_head": head, - "final_git_head": final_head, - "initial_worktree_fingerprint": checkout_fingerprint, - "final_worktree_fingerprint": final_checkout_fingerprint, - "transient_checkout_mutation": mutation_observation.changed, - "checkout_mutation_path": mutation_observation.observed_path, + "exit": 124, + "diagnosis": "verify_invocation_deadline_exceeded", + "timeout_s": VERIFY_INVOCATION_BUDGET_S, } ) - if exit_code == 0: - exit_code = 125 - sys.stderr.write("verify: checkout contents changed during verification; evidence is not exact-head.\n") - - seed_receipt: dict[str, Any] | None = None - if checkout_stable: - if pending_testmon_stamp is not None: - _atomic_write_json(TESTMON_SEED_STAMP, pending_testmon_stamp.as_dict()) - if pending_affected_coverage is not None: - executable_paths, selected_count = pending_affected_coverage - _record_testmon_affected_coverage( - executable_paths=executable_paths, - selected_count=selected_count, - run_id=verify_run.run_id, - ) - if pending_selection_refresh is not None: - step_result, selection_exit_code = pending_selection_refresh - _refresh_testmon_selection_attempt( - step=step_result, - run=verify_run, - exit_code=selection_exit_code, - ) - if prepared_seed_attempt is not None: - seed_receipt = _finalize_testmon_seed_attempt( - prepared=prepared_seed_attempt, - step_results=step_results, - exit_code=exit_code, - ) - if exit_code == 0 and seed_receipt["status"] != "complete": - exit_code = 5 - sys.stderr.write( - "verify: pytest passed but the testmon dependency baseline is incomplete; " - f"inspect {TESTMON_SEED_ATTEMPT}.\n" - ) - elif testmon_graph_touched: - _discard_testmon_dependency_authority() - if prepared_seed_attempt is not None: - seed_receipt = { - "status": "discarded", - "outcome": SeedAttemptOutcome.INCOMPLETE.value, - "resume": False, - "expected_count": len(_testmon_seed_expected_nodeids(prepared_seed_attempt)), - "release_baseline_allowed": False, - } + exit_code = 124 + deadline_recorded = True + if pytest_aggregate is not None: + pytest_aggregate["wall_s"] = total_duration + pytest_aggregate["deadline"] = { + "budget_s": VERIFY_INVOCATION_BUDGET_S, + "met": total_duration <= VERIFY_INVOCATION_BUDGET_S, + } - total_duration = round(time.monotonic() - t0, 2) + release_baseline_allowed = _release_baseline_allowed( + selection_mode=testmon_mode, + verification_scope=planned_scope, + exit_code=exit_code, + checkout_stable=checkout_stable, + aggregate=pytest_aggregate, + ) + verification_scope = planned_scope + if testmon_mode == "affected": + release_baseline_allowed = False + + checkout_diagnosis = next( + ( + str(step["diagnosis"]) + for step in reversed(step_results) + if step.get("name") == "checkout stability" and isinstance(step.get("diagnosis"), str) + ), + None, + ) + pytest_diagnosis = next( + ( + str(step["diagnosis"]) + for step in reversed(step_results) + if str(step.get("name", "")).startswith("pytest") and step.get("exit") != 0 + ), + None, + ) + run_diagnosis = checkout_diagnosis or pytest_diagnosis - # Build history entry. history_entry: dict[str, Any] = { "timestamp": datetime.now(timezone.utc).isoformat(), "git_head": head, @@ -3865,109 +3329,58 @@ def _main(argv: list[str] | None = None) -> int: "artifact_dir": str(verify_run.relative_run_dir), "steps": step_results, "total_duration_s": total_duration, + "invocation_budget_s": VERIFY_INVOCATION_BUDGET_S, "exit_code": exit_code, + "verification_scope": verification_scope.value, + "release_baseline_allowed": release_baseline_allowed, } - checkout_stability_diagnosis = next( - ( - str(step["diagnosis"]) - for step in reversed(step_results) - if step.get("name") == "checkout stability" and "diagnosis" in step - ), - None, - ) - fallback_pytest_diagnosis = next( - ( - str(step["diagnosis"]) - for step in reversed(step_results) - if str(step.get("name", "")).startswith("pytest") and "diagnosis" in step - ), - None, - ) - pytest_diagnosis = next( - ( - str(step["diagnosis"]) - for step in reversed(step_results) - if str(step.get("name", "")).startswith("pytest") and step.get("exit") == exit_code and "diagnosis" in step - ), - fallback_pytest_diagnosis, - ) - run_diagnosis = checkout_stability_diagnosis or pytest_diagnosis + if preparation is not None: + history_entry["testmon_environment"] = { + "name": preparation.environment_name, + "selection_mode": testmon_mode, + "copied_from": str(preparation.copied_from) if preparation.copied_from is not None else None, + "required_executable_paths": list(required_executable_paths), + "bootstrap_trigger_paths": list(preparation_required_executable_paths), + "runtime_data_paths": list(runtime_data_paths), + } + if pytest_aggregate is not None: + history_entry["pytest_aggregate"] = pytest_aggregate if run_diagnosis is not None: history_entry["diagnosis"] = run_diagnosis - if seed_receipt is not None: - history_entry["testmon_seed"] = { - "status": seed_receipt["status"], - "outcome": seed_receipt["outcome"], - "resume": seed_receipt["resume"], - "expected_count": seed_receipt["expected_count"], - "attempt_path": str(TESTMON_SEED_ATTEMPT), - "stamp_path": str(TESTMON_SEED_STAMP) if seed_receipt["release_baseline_allowed"] else None, - "release_baseline_allowed": seed_receipt["release_baseline_allowed"], - } - - verification_scope = planned_verification_scope - if args.quick or args.commit: - # Non-test verification is intentionally not release authority, but it - # is still a typed verification receipt. ``None`` made merge-gate - # treat an explicit quick receipt as malformed instead of as a valid - # non-release gate. - release_baseline_allowed: bool | None = False - elif full_pytest or args.seed_testmon: - narrow_terminal = bool(args.skip_slow) - authorized_narrow_terminal = args.terminal_authorization == TerminalAuthorization.NARROW_TERMINAL.value - if full_pytest: - release_baseline_allowed = exit_code == 0 and (not narrow_terminal or authorized_narrow_terminal) - else: - release_baseline_allowed = _testmon_release_baseline_permission() and ( - not narrow_terminal or authorized_narrow_terminal - ) - else: - release_baseline_allowed = _testmon_release_baseline_permission() - history_entry["verification_scope"] = verification_scope.value - history_entry["release_baseline_allowed"] = release_baseline_allowed - history_entry["terminal_authorization"] = args.terminal_authorization - if release_baseline_allowed is False and tier in {"testmon", "lab", "seed-testmon"}: - sys.stderr.write( - "verify: affected-test selection is usable, but the current testmon state does not grant " - "release-baseline permission.\n" - ) - - if use_json: - _print_json(history_entry) - else: - if exit_code == 0: - # Compare against last run, flag regressions. - flags = _compare_against_last(step_results) - sys.stderr.write(f"\nverify: all checks passed ({total_duration:.1f}s total)") - if flags: - sys.stderr.write(" — " + "; ".join(flags) if len(flags) == 1 else "") - sys.stderr.write("\n") - for flag in flags: - sys.stderr.write(flag + "\n") - else: - sys.stderr.write("\n") - else: - sys.stderr.write(f"\nverify: FAILED ({total_duration:.1f}s) — fix before pushing\n") - # Persist history and stamp. - _save_history(history_entry) - verify_run.finish( + finalized_payload = verify_run.finish( exit_code=exit_code, duration_s=total_duration, diagnosis=run_diagnosis, verification_scope=verification_scope.value, release_baseline_allowed=release_baseline_allowed, - terminal_authorization=args.terminal_authorization, + final_git_head=final_head, final_worktree_fingerprint=final_checkout_fingerprint, - checkout_mutation_path=mutation_observation.observed_path, + checkout_mutation_path=(preparation_mutation_observation.observed_path or mutation_observation.observed_path), + checkout_diagnosis=checkout_diagnosis, + pytest_aggregate=pytest_aggregate, + invocation_budget_s=VERIFY_INVOCATION_BUDGET_S, ) + history_entry["pytest_aggregate"] = finalized_payload["pytest_aggregate"] + if use_json: + _print_json(history_entry) + elif exit_code == 0: + flags = _compare_against_last(step_results) + sys.stderr.write(f"\nverify: all checks passed ({total_duration:.1f}s total)") + if flags: + sys.stderr.write("\n") + for flag in flags: + sys.stderr.write(flag + "\n") + else: + sys.stderr.write("\n") + else: + sys.stderr.write(f"\nverify: FAILED ({total_duration:.1f}s); fix before pushing\n") + + _save_history(history_entry) + if exit_code == 0: _stamp_head() - - # Notify only on failure. Passing runs stay silent — the terminal - # already shows the green summary and a desktop popup per run is - # spammy when verify is invoked on every push. - if exit_code != 0: + else: _notify( _format_completion_notification( exit_code=exit_code, @@ -3975,36 +3388,68 @@ def _main(argv: list[str] | None = None) -> int: step_results=step_results, ) ) - return exit_code def _finalize_verify_runner_exception( - run: VerifyRun, - exc: Exception, + active: _ActiveVerifyRun, + exc: BaseException, *, - run_started: float, - verification_scope: VerificationScope, use_json: bool, ) -> int: """Leave typed, durable failed evidence when verification orchestration raises.""" - diagnosis = "verify_runner_exception" - run.finish_interrupted_steps( - exit_code=125, - diagnosis=diagnosis, - termination_reason="runner_exception", - ) + interrupted = isinstance(exc, KeyboardInterrupt) + diagnosis = "verify_interrupted" if interrupted else "verify_runner_exception" + exit_code = 130 if interrupted else 125 + run = active.run + try: + final_head = _git_head() + except Exception: + final_head = None try: final_worktree_fingerprint = worktree_fingerprint(ROOT) except Exception: final_worktree_fingerprint = "unavailable" + mutation_observation = None + if active.mutation_monitor is not None: + try: + mutation_observation = _finish_active_checkout_mutation_monitor(active.mutation_monitor) + except Exception: + mutation_observation = None + _close_active_native_testmon_state() + run.finish_interrupted_steps( + exit_code=exit_code, + diagnosis=diagnosis, + termination_reason="operator_interrupt" if interrupted else "runner_exception", + ) + if ( + active.head is None + or final_head is None + or active.initial_worktree_fingerprint in {None, "unavailable"} + or final_worktree_fingerprint == "unavailable" + or mutation_observation is None + or mutation_observation.unavailable + ): + checkout_diagnosis = "checkout_fingerprint_unavailable" + elif ( + final_head != active.head + or mutation_observation.changed + or final_worktree_fingerprint != active.initial_worktree_fingerprint + ): + checkout_diagnosis = "checkout_changed_during_verification" + else: + checkout_diagnosis = None payload = run.finish( - exit_code=125, - duration_s=time.monotonic() - run_started, + exit_code=exit_code, + duration_s=time.monotonic() - active.started_at, diagnosis=diagnosis, - verification_scope=verification_scope.value, + verification_scope=active.verification_scope.value, release_baseline_allowed=False, + final_git_head=final_head, final_worktree_fingerprint=final_worktree_fingerprint, + checkout_mutation_path=(mutation_observation.observed_path if mutation_observation is not None else None), + checkout_diagnosis=checkout_diagnosis, + invocation_budget_s=VERIFY_INVOCATION_BUDGET_S, ) payload["exception_type"] = type(exc).__name__ payload["error"] = str(exc) @@ -4012,26 +3457,39 @@ def _finalize_verify_runner_exception( if use_json: _print_json(payload) sys.stderr.write(f"verify: unexpected runner exception: {exc}\n") - return 125 + return exit_code @finalize_checkout_mutation_monitors def main(argv: list[str] | None = None) -> int: global _ACTIVE_VERIFY_RUN _ACTIVE_VERIFY_RUN = None + raw_argv = list(sys.argv[1:] if argv is None else argv) + native_pytest_enabled = not any(flag in raw_argv for flag in ("--quick", "--commit", "--history")) + lock = _native_testmon_lifecycle_lock(ROOT) if native_pytest_enabled else contextlib.nullcontext() try: - return _main(argv) - except Exception as exc: - if _ACTIVE_VERIFY_RUN is None: - raise - raw_argv = sys.argv[1:] if argv is None else argv - run, run_started, verification_scope = _ACTIVE_VERIFY_RUN - return _finalize_verify_runner_exception( - run, - exc, - run_started=run_started, - verification_scope=verification_scope, - use_json="--json" in raw_argv, - ) - finally: - _ACTIVE_VERIFY_RUN = None + with lock: + try: + return _main(argv) + except KeyboardInterrupt as exc: + if _ACTIVE_VERIFY_RUN is None: + raise + return _finalize_verify_runner_exception( + _ACTIVE_VERIFY_RUN, + exc, + use_json="--json" in raw_argv, + ) + except Exception as exc: + if _ACTIVE_VERIFY_RUN is None: + raise + return _finalize_verify_runner_exception( + _ACTIVE_VERIFY_RUN, + exc, + use_json="--json" in raw_argv, + ) + finally: + _close_active_native_testmon_state() + _ACTIVE_VERIFY_RUN = None + except PytestResourceError as exc: + sys.stderr.write(f"verify: {exc}\n") + return 125 diff --git a/devtools/verify_runs.py b/devtools/verify_runs.py index ec9d1bee52..adf32bb4b8 100644 --- a/devtools/verify_runs.py +++ b/devtools/verify_runs.py @@ -100,6 +100,157 @@ def _trailing_history_record(descriptor: int, *, end: int) -> tuple[int, bytes]: return 0, b"".join(reversed(suffix)) +def _history_pytest_aggregate(entry: Mapping[str, Any]) -> dict[str, Any]: + existing = entry.get("pytest_aggregate") + if isinstance(existing, Mapping): + return dict(existing) + + def optional_int(value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + def max_optional(current: int | None, value: object) -> int | None: + candidate = optional_int(value) + if candidate is None: + return current + return candidate if current is None else max(current, candidate) + + pytest_steps = [ + step + for step in entry.get("steps", ()) + if isinstance(step, Mapping) and str(step.get("name", "")).startswith("pytest") + ] + outcomes: dict[str, int] = {} + lanes: list[dict[str, Any]] = [] + selected_count = 0 + terminal_count = 0 + collection_wall_s = 0.0 + cleanup_complete = True + containment_complete = True + peak_rss: int | None = None + peak_pss: int | None = None + peak_swap: int | None = None + peak_storage: int | None = None + read_bytes = 0 + write_bytes = 0 + for step in pytest_steps: + statistics = step.get("statistics") + stats = statistics if isinstance(statistics, Mapping) else {} + raw_outcomes = stats.get("outcomes") + if isinstance(raw_outcomes, Mapping): + for outcome, count in raw_outcomes.items(): + if isinstance(count, int): + outcomes[str(outcome)] = outcomes.get(str(outcome), 0) + count + raw_node_count = stats.get("node_count") + if not isinstance(raw_node_count, int): + raw_node_count = step.get("count") + node_count = raw_node_count if isinstance(raw_node_count, int) else 0 + raw_selected = step.get("selected_count") + selected = raw_selected if isinstance(raw_selected, int) else node_count + selected_count += selected + terminal_count += node_count + collection_duration = step.get("collection_duration_s") + if isinstance(collection_duration, int | float): + collection_wall_s += float(collection_duration) + cleanup = stats.get("cleanup") + step_cleanup = cleanup.get("complete") if isinstance(cleanup, Mapping) else None + cleanup_complete = cleanup_complete and step_cleanup is True + containment_complete = containment_complete and isinstance(step.get("containment_mode"), str) + resources = stats.get("resources") + resource_values = resources if isinstance(resources, Mapping) else {} + peak_rss = max_optional(peak_rss, resource_values.get("peak_tree_rss_kb")) + peak_pss = max_optional(peak_pss, resource_values.get("peak_tree_pss_kb")) + peak_swap = max_optional(peak_swap, resource_values.get("peak_tree_swap_pss_kb")) + storage = stats.get("storage") + storage_values = storage if isinstance(storage, Mapping) else {} + peak_storage = max_optional(peak_storage, storage_values.get("basetemp_allocated_bytes_max")) + read_bytes += optional_int(resource_values.get("tree_read_bytes_delta")) or 0 + write_bytes += optional_int(resource_values.get("tree_write_bytes_delta")) or 0 + lanes.append( + { + "lane": "focused" if entry.get("tier") == "focused-test" else "pytest", + "exit_code": step.get("exit"), + "duration_s": step.get("duration_s"), + "selected_count": selected, + "terminal_count": node_count, + "collection_duration_s": collection_duration, + "cleanup_complete": step_cleanup, + "containment_complete": isinstance(step.get("containment_mode"), str), + "containment_mode": step.get("containment_mode"), + "external_addopts_neutralized": False, + "external_plugins_neutralized": False, + "closed_world_collection": False, + } + ) + + no_pytest = not pytest_steps + corpus_digest: str | None = None + exit_code = entry.get("exit_code") + raw_budget = entry.get("invocation_budget_s") + invocation_budget = float(raw_budget) if isinstance(raw_budget, int | float) else None + raw_wall = entry.get("total_duration_s", entry.get("duration_s", 0.0)) + wall_s = float(raw_wall) if isinstance(raw_wall, int | float) else 0.0 + deadline_met = entry.get("diagnosis") != "verify_invocation_deadline_exceeded" + if invocation_budget is not None: + deadline_met = deadline_met and wall_s <= invocation_budget + return { + "schema_version": 1, + "environment": { + "name": None, + "digest": None, + "status": "not-applicable", + "reason": "run did not use the native pytest-testmon lifecycle", + "native_corpus_count": 0, + "native_corpus_digest": corpus_digest, + }, + "corpus": {"count": terminal_count, "digest": corpus_digest}, + "selection_mode": "focused" if entry.get("tier") == "focused-test" else "none", + "external_addopts_neutralized": False, + "external_plugins_neutralized": False, + "closed_world_collection": False, + "lanes": lanes, + "selected_union_count": selected_count, + "terminal_union_count": terminal_count, + "duplicate_outcome_count": 0, + "outcomes": outcomes, + "missing_terminal_count": 0, + "missing_terminal_sample": [], + "non_green_count": sum(count for outcome, count in outcomes.items() if outcome not in _GREEN_TERMINAL_OUTCOMES), + "non_green_sample": [], + "complete_corpus_covered": False, + "terminal_green": bool(pytest_steps) and exit_code == 0, + "collection_wall_s": round(collection_wall_s, 4), + "resources": { + "peak_tree_rss_kb": peak_rss, + "peak_tree_pss_kb": peak_pss, + "peak_tree_swap_pss_kb": peak_swap, + "peak_storage_bytes": peak_storage, + "read_bytes": read_bytes, + "write_bytes": write_bytes, + }, + "cleanup": {"complete": True if no_pytest else cleanup_complete}, + "containment": {"complete": True if no_pytest else containment_complete}, + "deadline": {"budget_s": invocation_budget, "met": deadline_met}, + "wall_s": wall_s, + } + + +def normalize_verify_history_entry(entry: Mapping[str, Any]) -> dict[str, Any]: + """Return the single timestamped history schema shared by every run kind.""" + normalized = dict(entry) + normalized["history_schema_version"] = 1 + normalized["timestamp"] = next( + ( + value + for key in ("timestamp", "finished_at", "started_at") + if isinstance((value := normalized.get(key)), str) and value + ), + datetime.now(UTC).isoformat(), + ) + normalized.setdefault("total_duration_s", normalized.get("duration_s", 0.0)) + normalized["pytest_aggregate"] = _history_pytest_aggregate(normalized) + return normalized + + def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTORY_PATH) -> None: """Append one complete invocation to the cross-worktree run history. @@ -109,7 +260,7 @@ def append_verify_history(entry: Mapping[str, Any], *, path: Path = VERIFY_HISTO durable index used to find and compare them. """ path.parent.mkdir(parents=True, exist_ok=True) - payload = (json.dumps(dict(entry), ensure_ascii=False) + "\n").encode() + payload = (json.dumps(normalize_verify_history_entry(entry), ensure_ascii=False) + "\n").encode() descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_APPEND, 0o600) try: fcntl.flock(descriptor, fcntl.LOCK_EX) @@ -287,6 +438,7 @@ def __init__(self, root: Path) -> None: self._git_current_ref_path: Path | None = None self._git_current_ref_was_loose: bool | None = None self._git_authority_paths: dict[Path, str] = {} + self._git_authority_signatures: dict[Path, str | None] = {} self._directory_topology_fingerprint: frozenset[str] | None = None def start(self) -> None: @@ -425,6 +577,11 @@ def walk_error(_error: OSError) -> None: if self._git_index_path is not None: self._git_authority_paths[self._git_index_path] = ".git/index" self._git_authority_paths.update(self._resolve_git_head_paths()) + for authority_path in self._git_authority_paths: + # The protected startup topology recheck discovers this set again. + # Retain each first baseline so it cannot adopt a write that the + # active watcher observed before processing its coalesced event. + self._git_authority_signatures.setdefault(authority_path, self._authority_signature(authority_path)) for authority_path in self._git_authority_paths: watched_parent = authority_path.parent while not watched_parent.exists() and watched_parent != watched_parent.parent: @@ -434,6 +591,14 @@ def walk_error(_error: OSError) -> None: self._directory_topology_fingerprint = self._directory_topology(directories) return directories + @staticmethod + def _authority_signature(path: Path) -> str | None: + """Identify authority semantics while ignoring byte-identical rewrites.""" + try: + return hashlib.sha256(path.read_bytes()).hexdigest() + except OSError: + return None + def _directory_topology(self, directories: Sequence[Path]) -> frozenset[str]: """Fingerprint source directory membership without trusting pre-watch state.""" return frozenset( @@ -561,6 +726,21 @@ def _is_within_ignored_root(relative: Path, ignored_roots: frozenset[Path]) -> b def _record_change(self, candidate: Path) -> None: if not candidate.is_absolute(): candidate = self.root / candidate + nested_authorities = [ + (authority_path, label) + for authority_path, label in self._git_authority_paths.items() + if candidate != authority_path + and authority_path.is_relative_to(candidate) + and not (label == ".git/packed-refs" and self._git_current_ref_was_loose is True) + ] + if nested_authorities: + for authority_path, label in nested_authorities: + if self._authority_signature(authority_path) != self._git_authority_signatures.get(authority_path): + with self._state_lock: + self._changed = True + self._observed_path = label + return + return for authority_path, label in self._git_authority_paths.items(): if label == ".git/packed-refs" and self._git_current_ref_was_loose is True: # packed-refs is shared by linked worktrees. When this @@ -570,11 +750,6 @@ def _record_change(self, candidate: Path) -> None: # is removed or replaced. Preserve the startup state so a # packed-to-loose transition cannot hide its own first event. continue - if candidate != authority_path and authority_path.is_relative_to(candidate): - with self._state_lock: - self._changed = True - self._observed_path = label - return if candidate.parent != authority_path.parent: continue if candidate.name == f"{authority_path.name}.lock": @@ -583,6 +758,10 @@ def _record_change(self, candidate: Path) -> None: # its authority file. return if candidate.name == authority_path.name: + # The watch event is itself evidence that Git replaced the + # authority file. The bytes may already have been restored by + # the time the coalesced event reaches this thread, so a + # signature comparison here would discard a real mutation. with self._state_lock: self._changed = True self._observed_path = label @@ -779,6 +958,50 @@ def _distribution(values: list[float]) -> dict[str, float | int | None]: } +def _counter_delta(resources: list[dict[str, Any]], key: str) -> int: + """Return the observed delta for one monotonically increasing resource counter.""" + values = [int(row[key]) for row in resources if isinstance(row.get(key), int)] + return max(values) - min(values) if values else 0 + + +def _phase_outcome(phases: Mapping[str, object], name: str) -> str | None: + value = phases.get(name) + if isinstance(value, Mapping): + outcome = value.get("outcome") + return outcome if isinstance(outcome, str) else None + return value if isinstance(value, str) else None + + +def _collapse_terminal_outcome(phases: Mapping[str, object]) -> str: + """Collapse setup, call, and teardown evidence to one terminal outcome.""" + setup = _phase_outcome(phases, "setup") + call = _phase_outcome(phases, "call") + teardown = _phase_outcome(phases, "teardown") + if setup == "failed" or teardown == "failed": + return "error" + if call is not None: + return call + if setup in {"skipped", "xfailed", "xpassed"}: + return setup + if teardown in {"skipped", "xfailed", "xpassed"}: + return teardown + return "interrupted" + + +def _merge_terminal_outcomes( + canonical: Mapping[str, str], + phase_reports: Mapping[str, Mapping[str, object]], + *, + nodeids: Sequence[str] = (), +) -> dict[str, str]: + """Prefer canonical outcomes and fill omitted nodes from phase evidence.""" + all_nodeids = set(nodeids) | set(canonical) | set(phase_reports) + return { + nodeid: canonical[nodeid] if nodeid in canonical else _collapse_terminal_outcome(phase_reports.get(nodeid, {})) + for nodeid in all_nodeids + } + + def aggregate_pytest_statistics( step_dir: Path, *, @@ -870,27 +1093,8 @@ def aggregate_pytest_statistics( bucket = phase_outcomes[when] bucket[outcome] = bucket.get(outcome, 0) + 1 - for nodeid in nodes: - node_reports = reports_by_node.get(nodeid, {}) - setup = node_reports.get("setup", {}).get("outcome") - call = node_reports.get("call", {}).get("outcome") - teardown = node_reports.get("teardown", {}).get("outcome") - canonical_outcome = canonical_outcomes.get(nodeid) - if canonical_outcome is not None: - terminal = canonical_outcome - elif setup == "failed" or teardown == "failed": - terminal = "error" - elif isinstance(call, str): - terminal = call - elif setup in {"skipped", "xfailed", "xpassed"}: - terminal = str(setup) - elif teardown in {"skipped", "xfailed", "xpassed"}: - terminal = str(teardown) - else: - # A test may have emitted its start event just before an interrupt - # or forced containment cleanup. Keep that missing terminal phase - # visible so outcome totals still account for every started node. - terminal = "interrupted" + terminal_outcomes = _merge_terminal_outcomes(canonical_outcomes, reports_by_node, nodeids=tuple(nodes)) + for terminal in terminal_outcomes.values(): outcomes[terminal] = outcomes.get(terminal, 0) + 1 resources: list[dict[str, Any]] = [] @@ -964,6 +1168,12 @@ def aggregate_pytest_statistics( ), default=None, ), + "peak_tree_swap_pss_kb": max( + (int(row["tree_swap_pss_kb"]) for row in resources if isinstance(row.get("tree_swap_pss_kb"), int)), + default=None, + ), + "tree_read_bytes_delta": _counter_delta(resources, "tree_read_bytes"), + "tree_write_bytes_delta": _counter_delta(resources, "tree_write_bytes"), }, "cleanup": { "complete": True @@ -976,9 +1186,252 @@ def aggregate_pytest_statistics( } +_GREEN_TERMINAL_OUTCOMES = frozenset({"passed", "skipped", "xfailed", "xpassed"}) + + +def _safe_step_dir(root: Path, raw: object) -> Path | None: + if not isinstance(raw, str) or not raw: + return None + relative = Path(raw) + if relative.is_absolute() or ".." in relative.parts: + return None + candidate = (root / relative).resolve() + try: + candidate.relative_to(root.resolve()) + except ValueError: + return None + return candidate + + +def _read_json_object(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def _terminal_outcomes_by_node(step_dir: Path) -> dict[str, str]: + canonical = _read_json_object(step_dir / PYTEST_CANONICAL_REPORT_NAME) + raw_tests = canonical.get("tests") if canonical is not None else None + canonical_outcomes: dict[str, str] = {} + if isinstance(raw_tests, list): + canonical_outcomes = { + str(test["nodeid"]): str(test["outcome"]) + for test in raw_tests + if isinstance(test, dict) and isinstance(test.get("nodeid"), str) and isinstance(test.get("outcome"), str) + } + + reports: dict[str, dict[str, object]] = {} + events_path = step_dir / "events.jsonl" + if events_path.exists(): + for line in events_path.read_text(encoding="utf-8", errors="replace").splitlines(): + with contextlib.suppress(json.JSONDecodeError): + row = json.loads(line) + if not isinstance(row, dict) or row.get("event") != "test_report": + continue + nodeid = row.get("nodeid") + when = row.get("when") + outcome = row.get("outcome") + if isinstance(nodeid, str) and when in {"setup", "call", "teardown"} and isinstance(outcome, str): + reports.setdefault(nodeid, {})[str(when)] = outcome + return _merge_terminal_outcomes(canonical_outcomes, reports) + + +def aggregate_native_testmon_run( + root: Path, + *, + steps: Sequence[Mapping[str, Any]], + environment_name: str, + corpus_nodeids: Sequence[str], + environment_status: str = "valid", + environment_reason: str | None = None, + selection_mode: str, + invocation_duration_s: float, + budget_s: float, +) -> dict[str, Any]: + """Build one compact, durable aggregate for the two semantic pytest lanes.""" + lanes: list[dict[str, Any]] = [] + selected_union: set[str] = set() + outcome_by_node: dict[str, str] = {} + duplicate_outcomes: set[str] = set() + outcomes: dict[str, int] = {} + collection_wall_s = 0.0 + peak_rss_kb: int | None = None + peak_pss_kb: int | None = None + peak_swap_pss_kb: int | None = None + peak_storage_bytes: int | None = None + read_bytes = 0 + write_bytes = 0 + cleanup_complete = True + containment_complete = True + selection_complete = True + external_addopts_neutralized = True + external_plugins_neutralized = True + closed_world_collection = True + for step in steps: + lane = step.get("semantic_lane") + if lane not in {"parallel", "serial"}: + continue + step_dir = _safe_step_dir(root, step.get("artifact_dir")) + selection = _read_json_object(step_dir / "selection.json") if step_dir is not None else None + containment_receipt = _read_json_object(step_dir / "containment.json") if step_dir is not None else None + selected = selection.get("selected_nodeids") if selection is not None else None + omitted = selection.get("selected_nodeids_omitted") if selection is not None else None + if not isinstance(selected, list) or not all(isinstance(nodeid, str) for nodeid in selected) or omitted != 0: + selection_complete = False + selected = [] + selected_union.update(selected) + lane_outcomes = _terminal_outcomes_by_node(step_dir) if step_dir is not None else {} + for nodeid, outcome in lane_outcomes.items(): + if nodeid in outcome_by_node: + duplicate_outcomes.add(nodeid) + outcome_by_node[nodeid] = outcome + statistics = step.get("statistics") + resources = statistics.get("resources") if isinstance(statistics, Mapping) else None + storage = statistics.get("storage") if isinstance(statistics, Mapping) else None + cleanup = statistics.get("cleanup") if isinstance(statistics, Mapping) else None + + def _peak(current: int | None, value: object) -> int | None: + return max(current or 0, value) if isinstance(value, int) else current + + if isinstance(resources, Mapping): + peak_rss_kb = _peak(peak_rss_kb, resources.get("peak_tree_rss_kb")) + peak_pss_kb = _peak(peak_pss_kb, resources.get("peak_tree_pss_kb")) + peak_swap_pss_kb = _peak(peak_swap_pss_kb, resources.get("peak_tree_swap_pss_kb")) + lane_read = resources.get("tree_read_bytes_delta") + lane_write = resources.get("tree_write_bytes_delta") + read_bytes += lane_read if isinstance(lane_read, int) else 0 + write_bytes += lane_write if isinstance(lane_write, int) else 0 + if isinstance(storage, Mapping): + peak_storage_bytes = _peak(peak_storage_bytes, storage.get("basetemp_allocated_bytes_max")) + lane_cleanup = cleanup.get("complete") if isinstance(cleanup, Mapping) else None + cleanup_complete = cleanup_complete and lane_cleanup is True + lane_addopts_neutralized = step.get("external_addopts_neutralized") is True + external_addopts_neutralized = external_addopts_neutralized and lane_addopts_neutralized + lane_plugins_neutralized = step.get("external_plugins_neutralized") is True + external_plugins_neutralized = external_plugins_neutralized and lane_plugins_neutralized + lane_closed_world_collection = step.get("closed_world_collection") is True + closed_world_collection = closed_world_collection and lane_closed_world_collection + lane_containment_complete = bool( + containment_receipt is not None + and containment_receipt.get("status") == "finished" + and containment_receipt.get("controller_group_alive") is False + and containment_receipt.get("termination_reason") is None + and containment_receipt.get("escalated_to_sigkill") is False + and not bool(step.get("termination_reason")) + and step.get("containment_escalated_to_sigkill") is not True + ) + containment_complete = containment_complete and lane_containment_complete + collection_duration = step.get("collection_duration_s") + if isinstance(collection_duration, (int, float)): + collection_wall_s += float(collection_duration) + lanes.append( + { + "lane": lane, + "exit_code": step.get("exit"), + "duration_s": step.get("duration_s"), + "collection_duration_s": collection_duration, + "selected_count": step.get("selected_count"), + "terminal_count": len(lane_outcomes), + "cleanup_complete": lane_cleanup, + "containment_mode": step.get("containment_mode"), + "containment_complete": lane_containment_complete, + "external_addopts_neutralized": lane_addopts_neutralized, + "external_plugins_neutralized": lane_plugins_neutralized, + "closed_world_collection": lane_closed_world_collection, + } + ) + + for outcome in outcome_by_node.values(): + outcomes[outcome] = outcomes.get(outcome, 0) + 1 + native_corpus = tuple(sorted(set(corpus_nodeids))) + complete_mode = selection_mode in {"bootstrap", "full"} + corpus = native_corpus + corpus_set = set(corpus) + lane_names = [lane["lane"] for lane in lanes] + complete_corpus_covered = ( + complete_mode + and selection_complete + and bool(corpus) + and external_addopts_neutralized + and external_plugins_neutralized + and closed_world_collection + and selected_union == corpus_set + and set(outcome_by_node) == corpus_set + and not duplicate_outcomes + and len(lane_names) == 2 + and lane_names.count("parallel") == 1 + and lane_names.count("serial") == 1 + ) + missing_terminal = tuple(sorted(corpus_set - set(outcome_by_node))) if complete_mode else () + non_green = tuple( + sorted(nodeid for nodeid, outcome in outcome_by_node.items() if outcome not in _GREEN_TERMINAL_OUTCOMES) + ) + terminal_green = complete_corpus_covered and not missing_terminal and not non_green + cleanup_complete = bool(lanes) and cleanup_complete + containment_complete = bool(lanes) and containment_complete + external_addopts_neutralized = bool(lanes) and external_addopts_neutralized + external_plugins_neutralized = bool(lanes) and external_plugins_neutralized + closed_world_collection = bool(lanes) and closed_world_collection + return { + "schema_version": 1, + "environment": { + "name": environment_name, + "digest": environment_name.removeprefix("polylogue-"), + "status": environment_status, + "reason": environment_reason, + "native_corpus_count": len(native_corpus), + "native_corpus_digest": hashlib.sha256("\n".join(native_corpus).encode()).hexdigest(), + }, + "corpus": { + "count": len(corpus), + "digest": hashlib.sha256("\n".join(corpus).encode()).hexdigest(), + }, + "selection_mode": selection_mode, + "external_addopts_neutralized": external_addopts_neutralized, + "external_plugins_neutralized": external_plugins_neutralized, + "closed_world_collection": closed_world_collection, + "lanes": lanes, + "outcomes": outcomes, + "selected_union_count": len(selected_union), + "terminal_union_count": len(outcome_by_node), + "missing_terminal_count": len(missing_terminal), + "missing_terminal_sample": list(missing_terminal[:20]), + "non_green_count": len(non_green), + "non_green_sample": list(non_green[:20]), + "duplicate_outcome_count": len(duplicate_outcomes), + "complete_corpus_covered": complete_corpus_covered, + "terminal_green": terminal_green, + "wall_s": round(invocation_duration_s, 4), + "collection_wall_s": round(collection_wall_s, 4), + "resources": { + "peak_tree_rss_kb": peak_rss_kb, + "peak_tree_pss_kb": peak_pss_kb, + "peak_tree_swap_pss_kb": peak_swap_pss_kb, + "peak_storage_bytes": peak_storage_bytes, + "read_bytes": read_bytes, + "write_bytes": write_bytes, + }, + "cleanup": {"complete": cleanup_complete}, + "containment": {"complete": containment_complete}, + "deadline": { + "budget_s": budget_s, + "met": invocation_duration_s <= budget_s, + }, + } + + def git_dirty(cwd: Path | None = None) -> bool: try: - result = subprocess.run(["git", "status", "--short"], capture_output=True, text=True, timeout=5, cwd=cwd) + result = subprocess.run( + ["git", "status", "--short"], + capture_output=True, + text=True, + timeout=5, + cwd=cwd, + env=_read_only_git_env(), + ) except (OSError, subprocess.TimeoutExpired): return True return bool(result.stdout.strip()) @@ -998,6 +1451,7 @@ def git_head(cwd: Path | None = None) -> str | None: text=True, timeout=5, cwd=cwd, + env=_read_only_git_env(), ) except (OSError, subprocess.TimeoutExpired): return None @@ -1085,6 +1539,22 @@ def write(self) -> None: if not _current_owner_is_other_live_run(current_path): _write_json(current_path, self._payload) + def update_checkout_provenance( + self, + *, + polylogue_import_path: str | None = None, + environment_fingerprint: Mapping[str, Any] | None = None, + worktree_fingerprint: str | None = None, + ) -> None: + """Persist provenance as each preflight authority becomes available.""" + if polylogue_import_path is not None: + self._payload["polylogue_import_path"] = polylogue_import_path + if environment_fingerprint is not None: + self._payload["environment_fingerprint"] = dict(environment_fingerprint) + if worktree_fingerprint is not None: + self._payload["worktree_fingerprint"] = worktree_fingerprint + self.write() + def start_step(self, *, label: str, cmd: list[str]) -> PytestStepArtifacts: index = len(self._payload["steps"]) + 1 step_id = f"{index:02d}-{_slug(label)}" @@ -1130,7 +1600,7 @@ def finish_step(self, *, step_id: str, result: dict[str, Any]) -> dict[str, Any] if not str(step.get("name", "")).startswith("pytest"): break # An interrupted runner never returns through the normal - # post-subprocess merge. Fold shards here, before every + # post-subprocess merge. Fold worker evidence here, before every # aggregation path, so completed worker evidence survives. with contextlib.suppress(OSError): merge_worker_events(step_dir / "events", step_dir / "events.jsonl") @@ -1181,10 +1651,12 @@ def finish( diagnosis: str | None = None, verification_scope: str | None = None, release_baseline_allowed: bool | None = None, - terminal_authorization: str | None = None, + final_git_head: str | None = None, final_worktree_fingerprint: str | None = None, checkout_mutation_path: str | None = None, checkout_diagnosis: str | None = None, + pytest_aggregate: Mapping[str, Any] | None = None, + invocation_budget_s: float | None = None, ) -> dict[str, Any]: self._payload["finished_at"] = utc_now() self._payload["duration_s"] = round(duration_s, 2) @@ -1192,16 +1664,21 @@ def finish( self._payload["status"] = "success" if exit_code == 0 else "failed" if diagnosis: self._payload["diagnosis"] = diagnosis + self._payload["final_git_head"] = final_git_head if final_worktree_fingerprint is not None: self._payload["final_worktree_fingerprint"] = final_worktree_fingerprint if checkout_mutation_path is not None: self._payload["checkout_mutation_path"] = checkout_mutation_path if checkout_diagnosis is not None: self._payload["checkout_diagnosis"] = checkout_diagnosis + if invocation_budget_s is not None: + self._payload["invocation_budget_s"] = invocation_budget_s + if pytest_aggregate is not None: + self._payload["pytest_aggregate"] = dict(pytest_aggregate) if verification_scope is not None: self._payload["verification_scope"] = verification_scope self._payload["release_baseline_allowed"] = release_baseline_allowed - self._payload["terminal_authorization"] = terminal_authorization + self._payload.setdefault("pytest_aggregate", _history_pytest_aggregate(self._payload)) self.write() return dict(self._payload) diff --git a/docs/devtools.md b/docs/devtools.md index 47847f455a..6602040cab 100644 --- a/docs/devtools.md +++ b/docs/devtools.md @@ -56,7 +56,6 @@ They are not a proof ledger or end-user archive workflow. | Command | Role | | --- | --- | | `devtools lab provider completeness` | Inspect detector, parser, fixture, schema, docs, ImportExplain, and caveat coverage before claiming a provider/importer mode is product-ready. | -| `devtools lab testmon-proof` | Validate the affected-test harness itself: a disposable copy of a real Polylogue module and existing route test is seeded, semantically mutated, edge-severed, restored, and checked for bounded unrelated-change selection. | | `devtools lab snapshot read-surface` | Freeze archive read-surface behavior before archive work, then compare candidate archives against the captured envelope baseline. | | `devtools lab policy schema-versioning` | Enforce the policy boundary documented in docs/internals.md § 'Schema Versioning Model'. Durable tiers use explicit additive migrations with a backup gate; derived tiers are rebuilt or blue-green replaced from source evidence. | | `devtools lab policy bead-graph` | Run before shipping a bead-state delta. With no source option it checks live `bd` state; `--export .beads/issues.jsonl` validates the branch snapshot without importing it into the shared database. The gate reads dependency records only and does not make prose, labels, or campaign-specific edge lists machine authority. | @@ -148,7 +147,6 @@ These are the commands worth remembering during normal repo work: | `devtools lab schema roundtrip` | Verify committed provider schema packages reload and roundtrip cleanly. | | `devtools lab smoke` | Run direct archive and reader smoke sets. | | `devtools lab snapshot read-surface` | Capture and compare archive read-surface snapshots. | -| `devtools lab testmon-proof` | Prove real testmon affected selection against a semantic production mutation. | ### Verification diff --git a/docs/visual-evidence.md b/docs/visual-evidence.md index 56b9bdf69e..638f01cd0d 100644 --- a/docs/visual-evidence.md +++ b/docs/visual-evidence.md @@ -84,7 +84,7 @@ wrapper reads those executed artifacts into its report. The manifests use `schema_version: 1`, `evidence_kind: browserless-dom`, the command, artifact id, fixture id, route, and structural checks asserted by that test. -Both suites are part of the standard non-integration test run. There is no +Both suites are part of the standard correctness corpus. There is no browser binary or Playwright dependency in these fast lanes: they use Python's standard `http.server`, `urllib.request`, and `html.parser` against the real `DaemonAPIHTTPServer`. The `devtools lab smoke` command is the diff --git a/pyproject.toml b/pyproject.toml index 74da3f7584..1f4c970b48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ dev-common = [ "pytest-asyncio>=1.4.0", # Async test support "pytest-xdist>=3.5.0", # Parallel test execution "pytest-randomly>=4.1.0", # Detect order-dependent tests; adds --randomly-seed - "pytest-testmon>=2.1.3", # Per-test affected selection via dependency database + "pytest-testmon>=2.2.0,<2.3.0", # Bootstrap reads the tested DATA_VERSION-14 schema "pytest-json-report>=1.5.0", # Structured JSON report for verify pipeline + dashboard (#1026, #998) "pytest-benchmark>=5.0", # Microbenchmark suite (run with --benchmark-enable -p no:xdist) "hypothesis>=6.165.2", @@ -180,14 +180,10 @@ markers = [ "integration: marks integration tests (pipeline, CLI end-to-end)", "benchmark: marks benchmark tests (run with --benchmark-enable -p no:xdist -p no:randomly)", "contract: marks public contract tests that may emit bounded evidence artifacts", - "scale(level): parametric scale marker (small/medium/large/stretch)", - "scale_small: small-tier scale fixture (~100 convs / ~1k msgs); default verify gate (#1183)", - "scale_medium: medium-tier scale fixture (~1k convs / ~10k msgs); devtools verify --lab gate (#1183)", - "scale_large: large-tier scale fixture (~10k convs / ~100k msgs); nightly CI / explicit campaigns only (#1183)", "machine_contract: marks root CLI JSON success/failure contract tests", "query_routing: marks query-first CLI routing and read-surface tests", "evidence_law: marks generated-surface completeness, anti-dead-code, and evidence-quality laws", - "tui: marks Textual dashboard interaction tests", + "tui: marks Textual dashboard interaction tests (category only; these run in the parallel correctness lane)", "chaos: marks ingestion hostility, interruption, and chronology tests", "live: marks operator-run live archive validation lanes", "xdist_group(group_name): distribute parametrized tests within a file across workers (#1026)", diff --git a/tests/benchmarks/test_daemon_convergence.py b/tests/benchmarks/test_daemon_convergence.py index e821f56520..d9675e6783 100644 --- a/tests/benchmarks/test_daemon_convergence.py +++ b/tests/benchmarks/test_daemon_convergence.py @@ -167,25 +167,8 @@ def __init__(self, archive_root: Path, db_path: Path) -> None: # ── Benchmark tests ───────────────────────────────────────────────── -# Tiers whose per-iteration runtime exceeds the default benchmark budget -# (multiple minutes per repeat in CI) are routed to the ``scale_large`` -# nightly marker. The xxl mega-session tier ingests 100k messages from a -# single file and is the canonical regression probe for #1244 / #845-A. -_NIGHTLY_TIERS = {"xxl-mega-session"} - - -def _tier_params() -> list[Any]: - params: list[Any] = [] - for tier in _SCALE_TIERS: - if tier in _NIGHTLY_TIERS: - params.append(pytest.param(tier, marks=[pytest.mark.scale_large])) - else: - params.append(pytest.param(tier)) - return params - - @pytest.mark.benchmark -@pytest.mark.parametrize("tier", _tier_params()) +@pytest.mark.parametrize("tier", _SCALE_TIERS) def test_convergence_scale_tier(benchmark, tier: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: # type: ignore[no-untyped-def] """Measure convergence throughput at each scale tier.""" corpus_root = _generate_corpus(tmp_path, tier) @@ -339,7 +322,6 @@ def test_convergence_large_session_memory( @pytest.mark.benchmark -@pytest.mark.scale_large def test_convergence_huge_session_memory_bounded( benchmark: BenchmarkFixture, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/benchmarks/test_scale_tiers.py b/tests/benchmarks/test_scale_tiers.py index fdba53bec9..b362014670 100644 --- a/tests/benchmarks/test_scale_tiers.py +++ b/tests/benchmarks/test_scale_tiers.py @@ -1,29 +1,16 @@ -"""Tiered scale tests demonstrating the small/medium/large fixture contract. +"""Query probes over the benchmark runner's shared database sizes. Each test exercises a measured surface (FTS5 search, hybrid RRF search, -list_sessions) against the corresponding scale fixture and asserts -basic shape so a regression in the seeder or query path surfaces even -when ``--benchmark-disable`` is in effect. - -Marker plumbing (issue #1183): - - ``@pytest.mark.scale_small`` — default ``devtools verify`` gate. - ``@pytest.mark.scale_medium`` — ``devtools verify --lab`` only. - ``@pytest.mark.scale_large`` — nightly CI / explicit campaigns only. - -The default ``devtools verify`` pytest step passes ``-m "not scale_medium -and not scale_large"`` so only the small tier runs in the inner loop. -Tests that want measured timings should also carry ``@pytest.mark.benchmark`` -and run under the direct pytest-benchmark nightly workflow. +list_sessions) against the benchmark fixtures and asserts basic shape. The +entire file is an explicit performance-plugin surface; native correctness +verification excludes ``tests/benchmarks`` by path instead of tier markers. """ from __future__ import annotations from pathlib import Path -import pytest - -from tests.benchmarks.helpers import open_bench_store +from tests.benchmarks.helpers import BenchmarkFixture, open_bench_store def _fts_query(db_path: Path, term: str, *, limit: int = 20) -> int: @@ -40,43 +27,39 @@ def _list_query(db_path: Path, *, limit: int = 20) -> int: # --------------------------------------------------------------------------- -# Small tier — runs in default `devtools verify` baseline. +# Small benchmark database. # --------------------------------------------------------------------------- -@pytest.mark.scale_small -def test_scale_small_fts_search_returns_results(tier_small_db: Path) -> None: +def test_bench_1k_fts_search_returns_results(bench_db_1k: Path) -> None: """FTS5 search over the small tier returns at most ``limit`` rows.""" - hits = _fts_query(tier_small_db, "analysis", limit=10) + hits = _fts_query(bench_db_1k, "analysis", limit=10) assert 0 <= hits <= 10 -@pytest.mark.scale_small -def test_scale_small_list_sessions(tier_small_db: Path) -> None: +def test_bench_1k_list_sessions(bench_db_1k: Path) -> None: """``list_summaries`` returns up to ``limit`` entries from the small tier.""" - rows = _list_query(tier_small_db, limit=10) + rows = _list_query(bench_db_1k, limit=10) assert 0 < rows <= 10 # --------------------------------------------------------------------------- -# Medium tier — runs in `devtools verify --lab`. +# Medium benchmark database. # --------------------------------------------------------------------------- -@pytest.mark.scale_medium -def test_scale_medium_fts_search_returns_results(tier_medium_db: Path) -> None: - hits = _fts_query(tier_medium_db, "analysis", limit=20) +def test_bench_10k_fts_search_returns_results(bench_db_10k: Path) -> None: + hits = _fts_query(bench_db_10k, "analysis", limit=20) assert 0 <= hits <= 20 -@pytest.mark.scale_medium -def test_scale_medium_list_sessions(tier_medium_db: Path) -> None: - rows = _list_query(tier_medium_db, limit=20) +def test_bench_10k_list_sessions(bench_db_10k: Path) -> None: + rows = _list_query(bench_db_10k, limit=20) assert 0 < rows <= 20 # --------------------------------------------------------------------------- -# Large tier — runs in nightly CI / explicit campaigns only. +# Large benchmark database — direct file/node campaigns only. # # The fixture creation cost dominates wall-clock time for this tier # (~tens of seconds to minutes depending on host). The tests below are @@ -86,13 +69,11 @@ def test_scale_medium_list_sessions(tier_medium_db: Path) -> None: # --------------------------------------------------------------------------- -@pytest.mark.scale_large -def test_scale_large_fts_search_returns_results(tier_large_db: Path) -> None: - hits = _fts_query(tier_large_db, "analysis", limit=50) +def test_bench_50k_fts_search_returns_results(benchmark: BenchmarkFixture, bench_db_50k: Path) -> None: + hits = benchmark(lambda: _fts_query(bench_db_50k, "analysis", limit=50)) assert 0 <= hits <= 50 -@pytest.mark.scale_large -def test_scale_large_list_sessions(tier_large_db: Path) -> None: - rows = _list_query(tier_large_db, limit=50) +def test_bench_50k_list_sessions(benchmark: BenchmarkFixture, bench_db_50k: Path) -> None: + rows = benchmark(lambda: _list_query(bench_db_50k, limit=50)) assert 0 < rows <= 50 diff --git a/tests/benchmarks/test_schema_linear_scaling.py b/tests/benchmarks/test_schema_linear_scaling.py index 59784ad5f9..656dcf4ce6 100644 --- a/tests/benchmarks/test_schema_linear_scaling.py +++ b/tests/benchmarks/test_schema_linear_scaling.py @@ -12,8 +12,6 @@ from collections.abc import Callable from pathlib import Path -import pytest - from polylogue.schemas.validation.corpus import verify_raw_corpus from polylogue.schemas.validation.requests import SchemaVerificationRequest @@ -26,7 +24,6 @@ def _measure(db_path: Path, record_limit: int | None) -> float: return (time.perf_counter() - start) * 1000 -@pytest.mark.scale_small def test_schema_check_completes_quickly(named_seeded_archive: Callable[[str], Path]) -> None: """Smoke: verify_raw_corpus finishes and returns a valid report.""" db = named_seeded_archive("schema-small") @@ -34,7 +31,6 @@ def test_schema_check_completes_quickly(named_seeded_archive: Callable[[str], Pa assert ms < 30_000, f"10-record corpus took {ms:.0f} ms; expected <30s" -@pytest.mark.scale_medium def test_schema_check_linear_scaling(named_seeded_archive: Callable[[str], Path]) -> None: """Wall time must grow sub-quadratically across record limits. diff --git a/tests/conftest.py b/tests/conftest.py index 211ed0d8f3..7bcc93383d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,7 +42,7 @@ from devtools.verify_runs import pytest_basetemp_claim_path as _basetemp_claim_path # Resolve (but don't yet raise on) the polylogue-vs-checkout mismatch check -# before the first `from polylogue...` import below: a shared/editable venv's +# before test execution can import product modules: a shared/editable venv's # `.pth` entry can point at a different checkout than the one this pytest # process is actually running from (e.g. a linked git worktree reusing the # main checkout's `.venv`), and whichever tree `import polylogue` resolves to @@ -57,15 +57,8 @@ except CheckoutImportMismatchError as _checkout_exc: _CHECKOUT_GUARD_ERROR = _checkout_exc -from polylogue.archive.models import Session -from polylogue.scenarios import CorpusSpec, build_default_corpus_specs -from polylogue.storage.runtime import RawSessionRecord -from tests.infra.builders import make_conv, make_msg -from tests.infra.timeout_policy import timeout_marker_error - pytest_plugins = ( "tests.infra.corpus_fixtures", - "tests.infra.scale_fixtures", "tests.infra.frozen_clock", "tests.infra.clock_guard", ) @@ -84,15 +77,13 @@ if TYPE_CHECKING: from click.testing import CliRunner + from polylogue.archive.models import Session from polylogue.config import Source from polylogue.storage.repository import SessionRepository + from polylogue.storage.runtime import RawSessionRecord from polylogue.storage.sqlite import SQLiteBackend from tests.infra.storage_records import SessionBuilder -# --------------------------------------------------------------------------- -# Scale markers for data-gravity and long-haul validation (Workstream H) -# --------------------------------------------------------------------------- - def _set_managed_pytest_identity(identity: tuple[str, str] | None) -> None: """Expose only the managed identity owned by the active invocation.""" @@ -150,20 +141,6 @@ def pytest_configure(config: pytest.Config) -> None: # devtools/checkout_guard.py for the full hazard writeup). raise pytest.UsageError(f"pytest: {_CHECKOUT_GUARD_ERROR}") from _CHECKOUT_GUARD_ERROR sys.stderr.write(f"pytest: polylogue package → {resolved_polylogue_path()} (checkout: {_TESTS_REPO_ROOT})\n") - config.addinivalue_line("markers", "scale(level): parametric scale marker (small/medium/large/stretch)") - # Tiered scale markers (issue #1183); definitions also live in - # pyproject.toml `markers` so xfail_strict + filterwarnings agree. - config.addinivalue_line( - "markers", "scale_small: small-tier scale fixture (~100 convs / ~1k msgs); default verify gate (#1183)" - ) - config.addinivalue_line( - "markers", "scale_medium: medium-tier scale fixture (~1k convs / ~10k msgs); verify --lab gate (#1183)" - ) - config.addinivalue_line( - "markers", - "scale_large: large-tier scale fixture (~10k convs / ~100k msgs); nightly CI / campaigns only (#1183)", - ) - if config.option.basetemp is not None: configured_basetemp = str(config.option.basetemp) run_id = os.environ.get("POLYLOGUE_PYTEST_RUN_ID") @@ -257,6 +234,8 @@ def pytest_unconfigure(config: pytest.Config) -> None: def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: """Reject unbounded or effectively disabled per-test timeout markers.""" + from tests.infra.timeout_policy import timeout_marker_error + for item in items: marker = item.get_closest_marker("timeout") if marker is None: @@ -584,7 +563,7 @@ def _reclaim_test_tmp_path( # survive the host-configuration scrub below. They are test-run evidence # plumbing, not operator configuration; removing them after collection makes # setup/call reports disappear from the event ledger while teardown still gets -# recorded, which makes interrupted seed shards look falsely successful. +# recorded, which makes interrupted native runs look falsely successful. _MANAGED_VERIFY_ENV = frozenset( { "POLYLOGUE_VERIFY_RUN_ID", @@ -865,6 +844,8 @@ def workspace_env( monkeypatch: pytest.MonkeyPatch, empty_archive_template: Path, ) -> dict[str, Path]: + from tests.infra.archive_templates import clone_archive_template + data_dir = tmp_path / "data" state_dir = tmp_path / "state" archive_root = tmp_path / "archive" @@ -876,7 +857,7 @@ def workspace_env( # contract strictness. Keep validation deterministic and opt-in per test. monkeypatch.setenv("POLYLOGUE_SCHEMA_VALIDATION", "off") - _clone_archive_template(empty_archive_template, archive_root) + clone_archive_template(empty_archive_template, archive_root) return { "archive_root": archive_root, @@ -937,6 +918,8 @@ def cli_workspace( Returns: dict with paths: archive_root, data_root, inbox_dir, db_path """ + from tests.infra.archive_templates import clone_archive_template + # Create directory structure data_dir = tmp_path / "data" state_dir = tmp_path / "state" @@ -958,7 +941,7 @@ def cli_workspace( monkeypatch.setenv("POLYLOGUE_FORCE_PLAIN", "1") # Plain output for tests monkeypatch.setenv("POLYLOGUE_SCHEMA_VALIDATION", "off") - _clone_archive_template(empty_archive_template, archive_root) + clone_archive_template(empty_archive_template, archive_root) return { "archive_root": archive_root, @@ -970,28 +953,6 @@ def cli_workspace( } -def _clone_archive_template(source: Path, destination: Path) -> None: - """Clone one immutable empty archive into a test-private workspace.""" - destination.mkdir(parents=True, exist_ok=True) - try: - subprocess.run( - ["cp", "-a", "--reflink=auto", f"{source}/.", str(destination)], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=10, - ) - except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): - shutil.copytree(source, destination, dirs_exist_ok=True) - - bootstrap_marker = destination / ".maintenance-state" / "durable-change-trains" / ".bootstrap" - if bootstrap_marker.is_file(): - from polylogue.storage.sqlite.durable_change_train import _record_fresh_durable_bootstrap - - bootstrap_marker.unlink() - _record_fresh_durable_bootstrap(destination) - - @pytest.fixture(scope="session") def empty_archive_template( tmp_path_factory: pytest.TempPathFactory, @@ -1283,6 +1244,8 @@ def sample_session() -> Session: Replaces duplicate fixtures in: test_projections.py """ + from tests.infra.builders import make_conv, make_msg + messages = [ make_msg(id="m1", role="user", text="User question", timestamp="2024-01-01T10:00:00"), make_msg(id="m2", role="assistant", text="Assistant response", timestamp="2024-01-01T10:01:00"), @@ -1335,6 +1298,7 @@ def raw_synthetic_samples() -> list[RawSessionRecord]: import hashlib from datetime import datetime, timezone + from polylogue.scenarios import build_default_corpus_specs from polylogue.schemas.synthetic import SyntheticCorpus from polylogue.storage.runtime import RawSessionRecord @@ -1390,6 +1354,7 @@ def test_something(synthetic_source): source = synthetic_source("claude-code", count=3) """ from polylogue.config import Source + from polylogue.scenarios import CorpusSpec from polylogue.schemas.synthetic import SyntheticCorpus from tests.infra.source_builders import SyntheticAntigravityLanguageServerClient diff --git a/tests/infra/scale_fixtures.py b/tests/infra/scale_fixtures.py deleted file mode 100644 index e1d262cbd7..0000000000 --- a/tests/infra/scale_fixtures.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Tiered scale fixtures for performance/regression tests (issue #1183). - -Provides three explicit tiers with documented session/message counts -and pytest markers so each tier runs in the right gate: - - ``tier_small_db`` — ~100 sessions / ~1,000 messages - Marker: ``@pytest.mark.scale_small`` - Default ``devtools verify`` includes these. - ``tier_medium_db`` — ~1,000 sessions / ~10,000 messages - Marker: ``@pytest.mark.scale_medium`` - ``devtools verify --lab`` includes these. - ``tier_large_db`` — ~10,000 sessions / ~100,000 messages - Marker: ``@pytest.mark.scale_large`` - Nightly CI / explicit campaigns only. - -The factories return cached SQLite paths per test session — generating -the large tier is expensive (~minutes), so tests within a session share -one fixture instance. Each tier uses the same realistic distribution -helpers used by ``tests/benchmarks/conftest.py``. - -Growth-shape rule: tests that assert latency across tiers must compare -ratios (large/medium, medium/small) rather than absolute milliseconds. -Absolute budgets bake in host-machine assumptions; ratio budgets stay -portable. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path - -import pytest - - -@dataclass(frozen=True) -class ScaleTier: - """Declarative scale-tier definition. - - Attributes: - name: tier label (``small``/``medium``/``large``) - target_messages: approximate number of messages to seed; the seeder - stops once the running total reaches this floor. - target_sessions: approximate number of sessions expected - for this tier (informational; the seeder is driven by message - count and the realistic-distribution profile). - marker: pytest marker name registered in ``pyproject.toml``. - """ - - name: str - target_messages: int - target_sessions: int - marker: str - - -SCALE_SMALL = ScaleTier( - name="small", - target_messages=1_000, - target_sessions=100, - marker="scale_small", -) - -SCALE_MEDIUM = ScaleTier( - name="medium", - target_messages=10_000, - target_sessions=1_000, - marker="scale_medium", -) - -SCALE_LARGE = ScaleTier( - name="large", - target_messages=100_000, - target_sessions=10_000, - marker="scale_large", -) - -ALL_TIERS: tuple[ScaleTier, ...] = (SCALE_SMALL, SCALE_MEDIUM, SCALE_LARGE) -ALL_MARKERS: tuple[str, ...] = tuple(t.marker for t in ALL_TIERS) - - -def _seed_tier_db(db_path: Path, tier: ScaleTier, *, seed: int = 1183) -> dict[str, int]: - """Seed ``db_path`` with realistic data for the requested tier. - - Defers the heavy import to fixture invocation so unit collection stays - fast — ``tests.benchmarks.conftest`` pulls in storage backends, the - rebuild_index path, and the synthetic corpus generator. - """ - from tests.benchmarks.conftest import _seed_realistic_db # local import: heavy - - return _seed_realistic_db(db_path, target_messages=tier.target_messages, seed=seed) - - -# --------------------------------------------------------------------------- -# Session-scoped tier factories -# -# Each fixture is session-scoped so the (expensive) seeding only happens -# once per test session. The factory returns a ``Path`` to the seeded -# SQLite database; tests open it read-only via the standard -# ``open_bench_store`` / repository helpers. -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope="session") -def tier_small_db(tmp_path_factory: pytest.TempPathFactory) -> Path: - """Small-tier SQLite DB: ~100 convs / ~1k messages. Default verify gate.""" - db_path = tmp_path_factory.mktemp("scale_small") / "small.db" - _seed_tier_db(db_path, SCALE_SMALL) - return db_path - - -@pytest.fixture(scope="session") -def tier_medium_db(tmp_path_factory: pytest.TempPathFactory) -> Path: - """Medium-tier SQLite DB: ~1k convs / ~10k messages. Lab gate.""" - db_path = tmp_path_factory.mktemp("scale_medium") / "medium.db" - _seed_tier_db(db_path, SCALE_MEDIUM) - return db_path - - -@pytest.fixture(scope="session") -def tier_large_db(tmp_path_factory: pytest.TempPathFactory) -> Path: - """Large-tier SQLite DB: ~10k convs / ~100k messages. Nightly gate.""" - db_path = tmp_path_factory.mktemp("scale_large") / "large.db" - _seed_tier_db(db_path, SCALE_LARGE) - return db_path - - -# Public re-exports for documentation/registry consumers. -__all__ = [ - "ALL_MARKERS", - "ALL_TIERS", - "SCALE_LARGE", - "SCALE_MEDIUM", - "SCALE_SMALL", - "ScaleTier", - "tier_large_db", - "tier_medium_db", - "tier_small_db", -] - - -# Factory protocol re-exports for `Callable[..., Path]`-typed call sites. -_FactoryReExports: tuple[Callable[..., Path], ...] = () diff --git a/tests/integration/devtools/test_native_testmon_lifecycle.py b/tests/integration/devtools/test_native_testmon_lifecycle.py new file mode 100644 index 0000000000..6ac254571a --- /dev/null +++ b/tests/integration/devtools/test_native_testmon_lifecycle.py @@ -0,0 +1,1606 @@ +from __future__ import annotations + +import json +import os +import signal +import sqlite3 +import subprocess +import sys +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest + +import devtools.verify as verify +from devtools.testmon_bootstrap import ( + TESTMON_DATA_RELPATH, + NativeTestmonRepairError, + inspect_native_testmon_environment, + prepare_native_testmon_environment, +) +from devtools.testmon_bootstrap import ( + testmon_environment_digest as _testmon_environment_digest, +) +from devtools.verify_runs import PYTEST_CANONICAL_REPORT_NAME + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +pytestmark = [ + pytest.mark.uses_real_clock("coordinates real pytest subprocesses and an interrupt deadline"), + pytest.mark.timeout(300), +] + + +@dataclass(frozen=True) +class LaneResult: + completed: subprocess.CompletedProcess[str] + artifact_dir: Path + selection: dict[str, object] + + +def _git(repo: Path, *args: str) -> str: + completed = subprocess.run( + ["git", *args], + cwd=repo, + capture_output=True, + text=True, + check=True, + ) + return completed.stdout.strip() + + +def _init_repo(root: Path, *, conftest: str = "") -> None: + (root / "tests").mkdir(parents=True) + (root / ".gitignore").write_text( + ".artifacts/\n.benchmarks/\n.cache/\n.coverage*\n.pytest_cache/\n__pycache__/\n", + encoding="utf-8", + ) + (root / "pyproject.toml").write_text( + """ +[tool.pytest.ini_options] +addopts = "-p no:randomly" +cache_dir = ".cache/pytest" +markers = [ + "load_sensitive: serial native-testmon lane", + "tui: Textual interaction category", +] +""".lstrip(), + encoding="utf-8", + ) + (root / "tests" / "conftest.py").write_text(conftest, encoding="utf-8") + _git(root, "init", "-q") + _git(root, "config", "user.email", "tests@example.invalid") + _git(root, "config", "user.name", "Polylogue Tests") + + +def _commit_all(root: Path, message: str) -> str: + _git(root, "add", ".") + _git(root, "commit", "-qm", message) + return _git(root, "rev-parse", "HEAD") + + +def _pytest_environment(repo: Path) -> dict[str, str]: + (repo / TESTMON_DATA_RELPATH).parent.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + for key in tuple(env): + if key.startswith("PYTEST_"): + env.pop(key) + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + env["TESTMON_DATAFILE"] = str(repo / TESTMON_DATA_RELPATH) + env["PYTHONPATH"] = os.pathsep.join((str(repo), str(PROJECT_ROOT), env.get("PYTHONPATH", ""))) + env["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] = "50000" + return env + + +def _run_lane( + repo: Path, + *, + environment_name: str, + mode: str, + lane: str, + workers: int = 0, + timeout: float = 30, + base_marker: str | None = None, +) -> LaneResult: + artifact_dir = repo / ".artifacts" / f"{mode}-{lane}-{uuid.uuid4().hex}" + artifact_dir.mkdir(parents=True) + env = _pytest_environment(repo) + env.update( + { + "POLYLOGUE_PYTEST_EVENTS_DIR": str(artifact_dir / "events"), + "POLYLOGUE_PYTEST_SELECTION_PATH": str(artifact_dir / "selection.json"), + "POLYLOGUE_PYTEST_SUMMARY_PATH": str(artifact_dir / "summary.json"), + } + ) + semantic_marker = "not load_sensitive" if lane == "parallel" else "load_sensitive" + marker = semantic_marker if base_marker is None else f"({base_marker}) and ({semantic_marker})" + selection = "--testmon-forceselect" if mode == "affected" else "--testmon-noselect" + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "--tb=short", + "--override-ini=addopts=", + "--testmon", + f"--testmon-env={environment_name}", + selection, + "-m", + marker, + "-p", + "devtools.pytest_progress_plugin", + "-p", + "pytest-testmon", + "-p", + "pytest_jsonreport", + "-p", + "xdist", + "--json-report", + "--json-report-omit=collectors,log,streams,warnings", + f"--json-report-file={artifact_dir / PYTEST_CANONICAL_REPORT_NAME}", + "-n", + str(workers), + ] + completed = subprocess.run(command, cwd=repo, env=env, capture_output=True, text=True, timeout=timeout) + selection_path = artifact_dir / "selection.json" + if not selection_path.is_file(): + raise AssertionError( + f"native pytest lane produced no selection artifact (returncode={completed.returncode})\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + selection_payload = json.loads(selection_path.read_text(encoding="utf-8")) + return LaneResult(completed, artifact_dir, selection_payload) + + +def _run_plain_verify_corpus( + repo: Path, + *, + mode: str, + environment_name: str, + base_marker: str | None = None, +) -> tuple[LaneResult, LaneResult]: + parallel = _run_lane( + repo, + environment_name=environment_name, + mode=mode, + lane="parallel", + workers=2, + base_marker=base_marker, + ) + serial = _run_lane( + repo, + environment_name=environment_name, + mode=mode, + lane="serial", + base_marker=base_marker, + ) + return parallel, serial + + +def _run_production_verify( + repo: Path, + *args: str, + allow_rejection: bool = False, + environment_overrides: dict[str, str] | None = None, + interpreter_args: tuple[str, ...] = (), +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Run the production verifier orchestration against a tiny fixture corpus. + + The subprocess keeps the real native preparation, two-lane runner, + containment, deadline, aggregate, invocation receipt, and XDG history. + Only unrelated static gates are filtered so this fixture need not copy the + entire Polylogue source tree. + """ + state_root = repo.parent / f"{repo.name}-verify-state" + receipt = state_root / "receipts" / f"{uuid.uuid4().hex}.json" + invocation_id = uuid.uuid4().hex + driver = """ +import os +import shutil +import sys +from pathlib import Path +from types import SimpleNamespace + +import devtools.verify as verify + +root = Path(sys.argv[1]).resolve() +real_build = verify.build_verify_steps +real_env_for_pytest_step = verify.env_for_pytest_step +pytest_index = root / ".git" / "pytest-index" +shutil.copy2(root / ".git" / "index", pytest_index) + +def native_steps_only(**kwargs): + return [step for step in real_build(**kwargs) if step[0].startswith("pytest native")] + +def fixture_env_for_pytest_step(env, **kwargs): + child_env = real_env_for_pytest_step(env, **kwargs) + child_env["GIT_INDEX_FILE"] = str(pytest_index) + return child_env + +verify.ROOT = root +verify.build_verify_steps = native_steps_only +verify.env_for_pytest_step = fixture_env_for_pytest_step +verify.assert_polylogue_matches_checkout = lambda *_args, **_kwargs: SimpleNamespace( + polylogue_import_path=root / "polylogue" / "__init__.py", + as_dict=lambda: {"checkout_root": str(root), "test_fixture": True}, +) +os.chdir(root) +raise SystemExit(verify.main(sys.argv[2:])) +""" + env = os.environ.copy() + env.update( + { + "PYTHONPATH": str(PROJECT_ROOT), + "XDG_STATE_HOME": str(state_root / "xdg-state"), + "POLYLOGUE_PYTEST_WORKERS": "1", + "POLYLOGUE_VERIFICATION_INVOCATION_ID": invocation_id, + "POLYLOGUE_VERIFICATION_RECEIPT_PATH": str(receipt), + "GIT_OPTIONAL_LOCKS": "0", + } + ) + if environment_overrides is not None: + env.update(environment_overrides) + try: + completed = subprocess.run( + [ + sys.executable, + *interpreter_args, + "-c", + driver, + str(repo), + *args, + "--json", + ], + cwd=PROJECT_ROOT, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + except subprocess.TimeoutExpired as exc: + stdout_value: object = exc.stdout + stderr_value: object = exc.stderr + stdout = ( + stdout_value.decode(errors="replace") + if isinstance(stdout_value, bytes) + else stdout_value + if isinstance(stdout_value, str) + else "" + ) + stderr = ( + stderr_value.decode(errors="replace") + if isinstance(stderr_value, bytes) + else stderr_value + if isinstance(stderr_value, str) + else "" + ) + pytest.fail(f"production verify fixture timed out\nstdout:\n{stdout}\nstderr:\n{stderr}") + if not receipt.exists(): + pytest.fail( + f"production verify wrote no invocation receipt\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + pytest.fail( + f"production verify emitted no JSON payload ({exc})\n" + f"returncode={completed.returncode}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + if completed.returncode == 125 and not allow_rejection: + pytest.fail( + f"production verify rejected the fixture checkout\npayload:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + try: + persisted = json.loads(receipt.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + pytest.fail( + f"production verify wrote invalid receipt JSON ({exc})\n" + f"returncode={completed.returncode}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + assert persisted["invocation_id"] == invocation_id + assert persisted["pytest_aggregate"] == payload["pytest_aggregate"] + for authority_field in ("diagnosis", "exit_code", "release_baseline_allowed"): + assert persisted.get(authority_field) == payload.get(authority_field) + return completed, payload + + +def _selected(*results: LaneResult) -> set[str]: + selected: set[str] = set() + for result in results: + raw_nodeids = result.selection.get("selected_nodeids") + assert isinstance(raw_nodeids, list) + assert all(isinstance(nodeid, str) for nodeid in raw_nodeids) + selected.update(raw_nodeids) + return selected + + +def test_empty_plain_verify_bootstraps_then_warm_verify_is_affected(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + """ +import pytest + +def test_parallel_owner(): + from app import answer + assert answer() == 42 + +@pytest.mark.load_sensitive +def test_serial_owner(): + from app import answer + assert answer() == 42 +""".lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + + preparation = prepare_native_testmon_environment(repo) + assert preparation.selection_mode == "bootstrap" + first = _run_plain_verify_corpus( + repo, + mode=preparation.selection_mode, + environment_name=preparation.environment_name, + ) + assert [result.completed.returncode for result in first] == [0, 0] + state = inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=preparation.environment_name, + required_executable_paths=("app.py",), + ) + assert state.valid + assert state.environment is not None + assert state.environment.corpus_count == 2 + + warm = prepare_native_testmon_environment(repo, required_executable_paths=("app.py",)) + assert warm.selection_mode == "affected" + second = _run_plain_verify_corpus(repo, mode="affected", environment_name=warm.environment_name) + assert [result.completed.returncode for result in second] == [0, 0] + assert _selected(*second) == set() + + +def test_production_plain_verify_owns_bootstrap_warm_selection_deadline_and_history(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + integration_dir = repo / "tests" / "integration" + integration_dir.mkdir() + (integration_dir / "test_app.py").write_text( + """ +import pytest + +def test_parallel_owner(): + from polylogue.app import answer + assert answer() == 42 + +@pytest.mark.load_sensitive +def test_serial_owner(): + from polylogue.app import answer + assert answer() == 42 +""".lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + first, bootstrap = _run_production_verify(repo) + + assert first.returncode == 0, f"{first.stderr}\n{json.dumps(bootstrap, indent=2, sort_keys=True)}" + assert bootstrap["testmon_environment"]["selection_mode"] == "bootstrap" + aggregate = bootstrap["pytest_aggregate"] + assert aggregate["complete_corpus_covered"] is True + assert aggregate["terminal_green"] is True + assert aggregate["cleanup"] == {"complete": True} + assert aggregate["containment"] == {"complete": True} + assert aggregate["deadline"] == {"budget_s": 3600.0, "met": True} + lane_steps = [step for step in bootstrap["steps"] if step.get("semantic_lane")] + assert [step["semantic_lane"] for step in lane_steps] == ["parallel", "serial"] + environments = { + arg + for step in lane_steps + for arg in step["statistics"]["command"] + if isinstance(arg, str) and arg.startswith("--testmon-env=") + } + assert environments == {f"--testmon-env={bootstrap['testmon_environment']['name']}"} + lane_timeouts = [step["timeout_s"] for step in lane_steps] + assert 0 < lane_timeouts[1] < lane_timeouts[0] <= 3600 + history_path = repo.parent / "repo-verify-state" / "xdg-state" / "polylogue" / "devtools" / "verify-history.jsonl" + history = [json.loads(line) for line in history_path.read_text(encoding="utf-8").splitlines()] + assert history[-1]["pytest_aggregate"] == aggregate + + second, warm = _run_production_verify(repo) + + assert second.returncode == 0, second.stderr + assert warm["testmon_environment"]["selection_mode"] == "affected" + assert warm["release_baseline_allowed"] is False + assert warm["pytest_aggregate"]["selected_union_count"] == 0 + + (package / "app.py").write_text("def answer() -> int:\n return 0\n", encoding="utf-8") + third, mutated = _run_production_verify(repo) + + assert third.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "affected" + assert mutated["pytest_aggregate"]["terminal_union_count"] == 2 + assert mutated["release_baseline_allowed"] is False + assert "assert 0 == 42" in third.stderr + + +def test_production_verify_all_grants_release_authority_after_complete_two_lane_run(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_release.py").write_text( + "import pytest\n\n" + "def test_parallel_release_owner():\n" + " from polylogue.app import answer\n" + " assert answer() == 42\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_release_owner():\n" + " from polylogue.app import answer\n" + " assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + completed, payload = _run_production_verify(repo, "--all") + + assert completed.returncode == 0, completed.stderr + assert payload["tier"] == "full" + assert payload["testmon_environment"]["selection_mode"] == "full" + assert payload["verification_scope"] == "release-baseline" + assert payload["release_baseline_allowed"] is True + assert payload["worktree_fingerprint"] == payload["final_worktree_fingerprint"] + lanes = [step for step in payload["steps"] if step.get("semantic_lane")] + assert [step["semantic_lane"] for step in lanes] == ["parallel", "serial"] + assert [step["name"] for step in lanes] == ["pytest native parallel (full)", "pytest native serial (full)"] + for step in lanes: + assert "--testmon-noselect" in step["statistics"]["command"] + assert "--testmon-forceselect" not in step["statistics"]["command"] + assert "--override-ini=addopts=" in step["statistics"]["command"] + assert step["external_addopts_neutralized"] is True + assert step["external_plugins_neutralized"] is True + assert step["closed_world_collection"] is True + aggregate = payload["pytest_aggregate"] + assert aggregate["external_addopts_neutralized"] is True + assert aggregate["external_plugins_neutralized"] is True + assert aggregate["closed_world_collection"] is True + assert aggregate["selection_mode"] == "full" + assert aggregate["environment"]["native_corpus_count"] == 2 + assert aggregate["corpus"]["count"] == 2 + assert aggregate["selected_union_count"] == 2 + assert aggregate["terminal_union_count"] == 2 + assert aggregate["missing_terminal_count"] == 0 + assert aggregate["complete_corpus_covered"] is True + assert aggregate["terminal_green"] is True + assert aggregate["cleanup"] == {"complete": True} + assert aggregate["containment"] == {"complete": True} + assert aggregate["deadline"] == {"budget_s": 3600.0, "met": True} + + +@pytest.mark.parametrize(("verify_args", "selection_mode"), [((), "bootstrap"), (("--all",), "full")]) +def test_release_native_runs_override_a_reduced_hypothesis_profile( + tmp_path: Path, + verify_args: tuple[str, ...], + selection_mode: str, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo( + repo, + conftest=( + "import os\n" + "from hypothesis import settings\n\n" + "settings.register_profile('default', max_examples=100)\n" + "settings.register_profile('verify', max_examples=10)\n" + "settings.load_profile(os.environ.get('HYPOTHESIS_PROFILE', 'default'))\n" + ), + ) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (repo / "tests" / "test_hypothesis_profile.py").write_text( + "import pytest\n" + "from hypothesis import settings\n\n" + "def test_parallel_release_profile_is_complete():\n" + " assert settings().max_examples == 100\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_release_profile_is_complete():\n" + " assert settings().max_examples == 100\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + completed, payload = _run_production_verify( + repo, + *verify_args, + environment_overrides={"HYPOTHESIS_PROFILE": "verify"}, + ) + + assert completed.returncode == 0, completed.stderr + assert payload["testmon_environment"]["selection_mode"] == selection_mode + assert payload["release_baseline_allowed"] is True + + +@pytest.mark.parametrize( + ("environment_addopts", "configured_addopts"), + [ + ("--setup-only", None), + ("--collect-only", None), + ("tests/test_release.py::test_parallel_body_must_run", None), + ("--ignore-glob=tests/**", None), + ("--ignore-glob tests/**", None), + (None, "--setup-only --ignore-glob=tests/**"), + ("-ra --strict-markers", None), + ], + ids=( + "setup-only", + "collect-only", + "positional-node", + "ignore-glob-equal", + "ignore-glob-split", + "configured", + "harmless", + ), +) +def test_production_verify_all_neutralizes_external_pytest_addopts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + environment_addopts: str | None, + configured_addopts: str | None, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (repo / "tests" / "test_release.py").write_text( + "import pytest\n\n" + "def test_parallel_body_must_run():\n" + " assert False, 'parallel body executed'\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_body_must_run():\n" + " assert False, 'serial body executed'\n", + encoding="utf-8", + ) + if configured_addopts is not None: + config = repo / "pyproject.toml" + config.write_text( + config.read_text(encoding="utf-8").replace( + 'addopts = "-p no:randomly"', + f'addopts = "{configured_addopts}"', + ), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + if environment_addopts is None: + monkeypatch.delenv("PYTEST_ADDOPTS", raising=False) + else: + monkeypatch.setenv("PYTEST_ADDOPTS", environment_addopts) + + completed, payload = _run_production_verify(repo, "--all") + + assert completed.returncode == 1 + assert payload["release_baseline_allowed"] is False + aggregate = payload["pytest_aggregate"] + assert aggregate["external_addopts_neutralized"] is True + assert aggregate["external_plugins_neutralized"] is True + assert aggregate["selected_union_count"] == 2 + assert aggregate["terminal_union_count"] == 2 + assert aggregate["outcomes"] == {"failed": 2} + lanes = [step for step in payload["steps"] if step.get("semantic_lane")] + assert [step["external_addopts_neutralized"] for step in lanes] == [True, True] + assert [step["external_plugins_neutralized"] for step in lanes] == [True, True] + assert all("--override-ini=addopts=" in step["statistics"]["command"] for step in lanes) + assert "parallel body executed" in completed.stderr + assert "serial body executed" in completed.stderr + + +def test_production_verify_all_drops_pythonpath_startup_injection(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + release_source = ( + "import pytest\n\n" + "def test_parallel_passes():\n" + " assert True\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_passes():\n" + " assert True\n\n" + "def test_omitted_failure():\n" + " assert False, 'PYTHONPATH startup injection did not narrow execution'\n" + ) + (repo / "tests" / "test_release.py").write_text(release_source, encoding="utf-8") + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + (repo / "sitecustomize.py").write_text( + 'import os\nos.environ["PYTEST_ADDOPTS"] = "-k passes"\nos.environ["HYPOTHESIS_PROFILE"] = "narrow"\n', + encoding="utf-8", + ) + + completed, payload = _run_production_verify(repo, "--all") + + assert completed.returncode == 1 + assert payload["release_baseline_allowed"] is False + aggregate = payload["pytest_aggregate"] + assert aggregate["selected_union_count"] == 3 + assert aggregate["terminal_union_count"] == 3 + assert aggregate["outcomes"] == {"failed": 1, "passed": 2} + assert aggregate["terminal_green"] is False + assert "PYTHONPATH startup injection did not narrow execution" in completed.stderr + + +def test_plain_native_lane_environment_removes_ambient_pytest_variables( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("PYTEST_ADDOPTS", "--collect-only") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_plugin") + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "0") + monkeypatch.setenv("PYTEST_CURRENT_TEST", "ambient test identity") + + environment = _pytest_environment(tmp_path) + + assert {key for key in environment if key.startswith("PYTEST_")} == {"PYTEST_DISABLE_PLUGIN_AUTOLOAD"} + assert environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] == "1" + + +@pytest.mark.parametrize( + ("environment_overrides", "interpreter_args", "verify_args"), + [ + ({"PYTHONOPTIMIZE": "1"}, (), ("--all",)), + ({}, ("-O",), ("--all",)), + ({}, ("-OO",), ("--all",)), + ({"PYTHONOPTIMIZE": "1"}, (), ("--quick", "--lab")), + ({"PYTHONOPTIMIZE": "1"}, (), ("--commit", "--lab")), + ], + ids=("pythonoptimize", "dash-o", "dash-oo", "quick-lab", "commit-lab"), +) +def test_production_verify_rejects_optimized_managed_pytest_interpreter( + tmp_path: Path, + environment_overrides: dict[str, str], + interpreter_args: tuple[str, ...], + verify_args: tuple[str, ...], +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "invariant.py").write_text( + "def require_failure():\n assert False, 'product assertion executed'\n", + encoding="utf-8", + ) + (repo / "tests" / "test_release.py").write_text( + "import pytest\n" + "from polylogue.invariant import require_failure\n\n" + "def test_parallel_product_assertion():\n" + " require_failure()\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_lane():\n" + " require_failure()\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + completed, payload = _run_production_verify( + repo, + *verify_args, + allow_rejection=True, + environment_overrides=environment_overrides, + interpreter_args=interpreter_args, + ) + + assert completed.returncode == 125 + assert payload["diagnosis"] == "optimized_python_interpreter" + assert payload["exit_code"] == 125 + assert payload["release_baseline_allowed"] is False + assert payload["pytest_aggregate"]["selection_mode"] == "none" + assert "Python optimization disables verification assertions" in completed.stderr + + +@pytest.mark.parametrize("ambient_addopts", ["--collect-only", "--setup-only"]) +def test_production_affected_verify_neutralizes_execution_suppressing_addopts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ambient_addopts: str, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + tests = repo / "tests" / "test_affected.py" + tests.write_text( + "import pytest\n\n" + "def test_parallel_body():\n" + " assert True\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_body():\n" + " assert True\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + monkeypatch.setenv("PYTEST_ADDOPTS", ambient_addopts) + + seeded, bootstrap = _run_production_verify(repo) + + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["testmon_environment"]["selection_mode"] == "bootstrap" + tests.write_text( + "import pytest\n\n" + "def test_parallel_body():\n" + " assert False, 'affected parallel body executed'\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_body():\n" + " assert False, 'affected serial body executed'\n", + encoding="utf-8", + ) + + completed, payload = _run_production_verify(repo) + + assert completed.returncode == 1 + assert payload["testmon_environment"]["selection_mode"] == "affected" + assert payload["release_baseline_allowed"] is False + aggregate = payload["pytest_aggregate"] + assert aggregate["selected_union_count"] == 2 + assert aggregate["terminal_union_count"] == 2 + assert aggregate["outcomes"] == {"failed": 2} + lanes = [step for step in payload["steps"] if step.get("semantic_lane")] + assert [step["external_addopts_neutralized"] for step in lanes] == [True, True] + assert [step["closed_world_collection"] for step in lanes] == [True, True] + assert "affected parallel body executed" in completed.stderr + assert "affected serial body executed" in completed.stderr + + +def test_production_verify_all_owns_complete_test_root_over_configured_testpaths( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + narrowed = repo / "tests" / "narrowed" + narrowed.mkdir() + (narrowed / "test_owned.py").write_text( + "import pytest\n\n" + "def test_parallel_owned():\n" + " assert True\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_owned():\n" + " assert True\n", + encoding="utf-8", + ) + outside = repo / "tests" / "outside" + outside.mkdir() + (outside / "test_omitted_failure.py").write_text( + "class TestOmitted:\n" + " def test_must_not_be_omitted(self):\n" + " assert False, 'outside configured discovery executed'\n", + encoding="utf-8", + ) + (repo / "ambient_narrow.py").write_text( + "def pytest_ignore_collect(collection_path, config):\n return 'outside' in collection_path.parts\n", + encoding="utf-8", + ) + config = repo / "pyproject.toml" + config.write_text( + config.read_text(encoding="utf-8").replace( + 'cache_dir = ".cache/pytest"', + 'cache_dir = ".cache/pytest"\n' + 'testpaths = ["tests/narrowed"]\n' + 'python_files = ["test_owned.py"]\n' + 'python_classes = ["Owned"]\n' + 'python_functions = ["test_*_owned"]\n' + 'norecursedirs = ["outside"]', + ), + encoding="utf-8", + ) + _commit_all(repo, "narrow discovery fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_narrow") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + completed, payload = _run_production_verify(repo, "--all") + + assert completed.returncode == 1 + assert payload["release_baseline_allowed"] is False + aggregate = payload["pytest_aggregate"] + assert aggregate["external_plugins_neutralized"] is True + assert aggregate["closed_world_collection"] is True + assert aggregate["corpus"]["count"] == 3 + assert aggregate["selected_union_count"] == 3 + assert aggregate["terminal_union_count"] == 3 + assert aggregate["outcomes"] == {"failed": 1, "passed": 2} + assert aggregate["complete_corpus_covered"] is True + assert aggregate["terminal_green"] is False + lanes = [step for step in payload["steps"] if step.get("semantic_lane")] + assert [step["external_plugins_neutralized"] for step in lanes] == [True, True] + assert [step["closed_world_collection"] for step in lanes] == [True, True] + assert all(step["statistics"]["command"].count("tests") == 1 for step in lanes) + assert "outside configured discovery executed" in completed.stderr + + +def test_runtime_json_only_mutation_forces_complete_native_selection(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + semantic = repo / "polylogue" / "archive" / "semantic" + data = semantic / "data" + data.mkdir(parents=True) + for package in (repo / "polylogue", repo / "polylogue" / "archive", semantic): + (package / "__init__.py").write_text("", encoding="utf-8") + pricing_data = data / "litellm_model_prices.json" + pricing_data.write_text('{"test-model": {"input_cost_per_token": 42}}\n', encoding="utf-8") + (semantic / "pricing.py").write_text( + "import json\n" + "from pathlib import Path\n\n" + "def input_price() -> int:\n" + " path = Path(__file__).parent / 'data' / 'litellm_model_prices.json'\n" + " return int(json.loads(path.read_text(encoding='utf-8'))['test-model']['input_cost_per_token'])\n", + encoding="utf-8", + ) + (repo / "tests" / "test_pricing.py").write_text( + "import pytest\n\n" + "def test_parallel_pricing_owner():\n" + " from polylogue.archive.semantic.pricing import input_price\n" + " assert input_price() == 42\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_pricing_owner():\n" + " from polylogue.archive.semantic.pricing import input_price\n" + " assert input_price() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["testmon_environment"]["selection_mode"] == "bootstrap" + + pricing_data.write_text('{"test-model": {"input_cost_per_token": 0}}\n', encoding="utf-8") + completed, mutated = _run_production_verify(repo) + + assert completed.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "full" + assert mutated["testmon_environment"]["runtime_data_paths"] == [ + "polylogue/archive/semantic/data/litellm_model_prices.json" + ] + assert mutated["pytest_aggregate"]["selected_union_count"] == 2 + assert mutated["pytest_aggregate"]["terminal_union_count"] == 2 + assert mutated["release_baseline_allowed"] is False + assert [step["semantic_lane"] for step in mutated["steps"] if step.get("semantic_lane")] == [ + "parallel", + "serial", + ] + assert "assert 0 == 42" in completed.stderr + + +def test_production_verify_test_runtime_data_mutation_executes_and_fails(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + data = repo / "tests" / "data" / "expected.txt" + data.parent.mkdir() + data.write_text("42\n", encoding="utf-8") + (repo / "tests" / "test_data.py").write_text( + "import pytest\n" + "from pathlib import Path\n\n" + "def expected() -> int:\n" + " return int((Path(__file__).parent / 'data' / 'expected.txt').read_text())\n\n" + "def test_parallel_data_owner():\n" + " assert expected() == 42\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_data_owner():\n" + " assert expected() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["testmon_environment"]["selection_mode"] == "bootstrap" + + data.write_text("0\n", encoding="utf-8") + completed, mutated = _run_production_verify(repo) + + assert completed.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "full" + assert mutated["testmon_environment"]["runtime_data_paths"] == ["tests/data/expected.txt"] + assert mutated["pytest_aggregate"]["selected_union_count"] == 2 + assert mutated["pytest_aggregate"]["terminal_union_count"] == 2 + assert "assert 0 == 42" in completed.stderr + + +def test_production_verify_deleted_module_rebuilds_and_fails_dependents(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["pytest_aggregate"]["selected_union_count"] == 1 + + _git(repo, "rm", "polylogue/app.py") + _commit_all(repo, "delete production module") + completed, mutated = _run_production_verify(repo) + + assert completed.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "bootstrap" + assert mutated["testmon_environment"]["required_executable_paths"] == [] + assert mutated["testmon_environment"]["bootstrap_trigger_paths"] == ["polylogue/app.py"] + assert mutated["pytest_aggregate"]["selected_union_count"] == 1 + assert mutated["pytest_aggregate"]["terminal_union_count"] == 1 + assert "ModuleNotFoundError" in completed.stderr + + +def test_production_verify_moved_module_rebuilds_and_fails_dependents(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + assert bootstrap["pytest_aggregate"]["selected_union_count"] == 1 + + _git(repo, "mv", "polylogue/app.py", "polylogue/renamed.py") + _commit_all(repo, "move production module") + completed, mutated = _run_production_verify(repo) + + assert completed.returncode == 1 + assert mutated["testmon_environment"]["selection_mode"] == "bootstrap" + assert mutated["testmon_environment"]["required_executable_paths"] == ["polylogue/renamed.py"] + assert mutated["testmon_environment"]["bootstrap_trigger_paths"] == [ + "polylogue/app.py", + "polylogue/renamed.py", + ] + assert mutated["pytest_aggregate"]["selected_union_count"] == 1 + assert mutated["pytest_aggregate"]["terminal_union_count"] == 1 + assert "ModuleNotFoundError" in completed.stderr + + +def test_production_verify_deleted_module_with_updated_imports_rebuilds_successfully(tmp_path: Path) -> None: + """Keeping the deleted path in post-run requirements makes the updated import fail.""" + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "old.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (package / "app.py").write_text("from polylogue.old import answer\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, _bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + + _git(repo, "rm", "polylogue/old.py") + (package / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + _commit_all(repo, "delete module and update imports") + completed, rebuilt = _run_production_verify(repo) + + assert completed.returncode == 0, completed.stderr + environment = rebuilt["testmon_environment"] + assert environment["selection_mode"] == "bootstrap" + assert environment["required_executable_paths"] == ["polylogue/app.py"] + assert environment["bootstrap_trigger_paths"] == ["polylogue/app.py", "polylogue/old.py"] + assert rebuilt["pytest_aggregate"]["terminal_green"] is True + + +def test_production_verify_moved_module_with_updated_imports_rebuilds_successfully(tmp_path: Path) -> None: + """Keeping the old path as a graph requirement makes the updated move fail.""" + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + package = repo / "polylogue" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "old.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.old import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + seeded, _bootstrap = _run_production_verify(repo) + assert seeded.returncode == 0, seeded.stderr + + _git(repo, "mv", "polylogue/old.py", "polylogue/renamed.py") + (repo / "tests" / "test_app.py").write_text( + "def test_answer():\n from polylogue.renamed import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(repo, "move module and update imports") + completed, rebuilt = _run_production_verify(repo) + + assert completed.returncode == 0, completed.stderr + environment = rebuilt["testmon_environment"] + assert environment["selection_mode"] == "bootstrap" + assert environment["required_executable_paths"] == ["polylogue/renamed.py", "tests/test_app.py"] + assert environment["bootstrap_trigger_paths"] == [ + "polylogue/old.py", + "polylogue/renamed.py", + "tests/test_app.py", + ] + assert rebuilt["pytest_aggregate"]["terminal_green"] is True + + +def test_empty_linked_worktree_with_empty_main_self_bootstraps(tmp_path: Path) -> None: + main = tmp_path / "main" + main.mkdir() + _init_repo(main) + (main / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + (main / "tests" / "test_app.py").write_text("def test_value():\n from app import VALUE\n assert VALUE == 1\n") + _commit_all(main, "fixture") + lane = tmp_path / "lane" + _git(main, "worktree", "add", "-qb", "lane", str(lane)) + + preparation = prepare_native_testmon_environment(lane) + + assert preparation.linked_worktree + assert preparation.main_checkout == main + assert preparation.copied_from is None + assert preparation.selection_mode == "bootstrap" + results = _run_plain_verify_corpus(lane, mode="bootstrap", environment_name=preparation.environment_name) + assert [result.completed.returncode for result in results] == [0, 0] + + +def test_matching_main_copy_then_product_mutation_selects_and_fails_owner(tmp_path: Path) -> None: + main = tmp_path / "main" + main.mkdir() + _init_repo(main) + (main / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (main / "tests" / "test_app.py").write_text( + "def test_answer():\n from app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + _commit_all(main, "fixture") + main_preparation = prepare_native_testmon_environment(main) + _run_plain_verify_corpus(main, mode="bootstrap", environment_name=main_preparation.environment_name) + lane = tmp_path / "lane" + _git(main, "worktree", "add", "-qb", "lane", str(lane)) + (lane / "app.py").write_text("def answer() -> int:\n return 0\n", encoding="utf-8") + + preparation = prepare_native_testmon_environment(lane, required_executable_paths=("app.py",)) + + assert preparation.selection_mode == "affected" + assert preparation.copied_from == main / TESTMON_DATA_RELPATH + result = _run_lane( + lane, + environment_name=preparation.environment_name, + mode="affected", + lane="parallel", + ) + assert result.completed.returncode == 1 + assert result.selection["selected_nodeids"] == ["tests/test_app.py::test_answer"] + assert "assert 0 == 42" in result.completed.stdout + + +def test_interrupted_bootstrap_native_state_resumes_failed_and_unfinished_only(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + started = repo / "started" + (repo / "tests" / "test_a_completed.py").write_text( + "def test_completed():\n assert True\n", + encoding="utf-8", + ) + (repo / "tests" / "test_b_failed.py").write_text( + "def test_failed():\n assert False\n", + encoding="utf-8", + ) + (repo / "tests" / "test_c_unfinished.py").write_text( + f""" +import pathlib +import time + +def test_unfinished(): + marker = pathlib.Path({str(started)!r}) + if marker.exists(): + return + marker.write_text('started') + time.sleep(30) +""".lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + environment_name = _testmon_environment_digest(repo) + env = _pytest_environment(repo) + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:randomly", + "--testmon", + f"--testmon-env={environment_name}", + "--testmon-noselect", + "tests", + ] + process = subprocess.Popen(command, cwd=repo, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + deadline = time.monotonic() + 10 + while not started.exists() and time.monotonic() < deadline: + time.sleep(0.02) + if not started.exists(): + process.send_signal(signal.SIGINT) + stdout, stderr = process.communicate(timeout=10) + raise AssertionError(f"interrupted fixture never started\nstdout:\n{stdout}\nstderr:\n{stderr}") + process.send_signal(signal.SIGINT) + process.communicate(timeout=10) + + state = inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=environment_name, + ) + assert state.valid + resumed = _run_lane(repo, environment_name=environment_name, mode="affected", lane="parallel") + assert resumed.completed.returncode == 1 + assert _selected(resumed) == { + "tests/test_b_failed.py::test_failed", + "tests/test_c_unfinished.py::test_unfinished", + } + assert "tests/test_a_completed.py::test_completed" not in _selected(resumed) + + +def test_node_add_delete_converges_without_a_custom_ledger(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + tests = repo / "tests" / "test_nodes.py" + tests.write_text("def test_existing():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + preparation = prepare_native_testmon_environment(repo) + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=preparation.environment_name) + + tests.write_text("def test_existing():\n assert True\n\ndef test_new():\n assert True\n", encoding="utf-8") + added = _run_plain_verify_corpus(repo, mode="affected", environment_name=preparation.environment_name) + assert "tests/test_nodes.py::test_new" in _selected(*added) + tests.write_text("def test_new():\n assert True\n", encoding="utf-8") + _run_plain_verify_corpus(repo, mode="affected", environment_name=preparation.environment_name) + + state = inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=preparation.environment_name, + ) + assert state.valid + assert state.environment is not None + assert state.environment.nodeids == ("tests/test_nodes.py::test_new",) + + +def test_collection_only_executable_dependency_blocks_until_test_executes_it(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo, conftest="from app import answer\n") + (repo / "app.py").write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + test_file = repo / "tests" / "test_app.py" + test_file.write_text("def test_unrelated():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + preparation = prepare_native_testmon_environment(repo) + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=preparation.environment_name) + + blind = inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=preparation.environment_name, + required_executable_paths=("app.py",), + ) + assert not blind.valid + assert blind.missing_executable_paths == ("app.py",) + + (repo / "tests" / "conftest.py").write_text("", encoding="utf-8") + test_file.write_text( + "def test_owner():\n from app import answer\n assert answer() == 42\n", + encoding="utf-8", + ) + repaired = prepare_native_testmon_environment(repo, required_executable_paths=("app.py",)) + assert repaired.selection_mode == "bootstrap" + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=repaired.environment_name) + assert inspect_native_testmon_environment( + repo / TESTMON_DATA_RELPATH, + environment_name=repaired.environment_name, + required_executable_paths=("app.py",), + ).valid + + +def test_removed_environment_or_dependency_edge_invalidates_native_state(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + (repo / "tests" / "test_app.py").write_text("def test_value():\n from app import VALUE\n assert VALUE == 1\n") + _commit_all(repo, "fixture") + preparation = prepare_native_testmon_environment(repo) + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=preparation.environment_name) + data = repo / TESTMON_DATA_RELPATH + + with sqlite3.connect(data) as connection: + connection.execute("DELETE FROM environment WHERE environment_name = ?", (preparation.environment_name,)) + assert not inspect_native_testmon_environment(data, environment_name=preparation.environment_name).valid + + rebuilt = prepare_native_testmon_environment(repo) + _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=rebuilt.environment_name) + with sqlite3.connect(data) as connection: + connection.execute( + """ + DELETE FROM test_execution_file_fp + WHERE fingerprint_id IN (SELECT id FROM file_fp WHERE filename = 'app.py') + """ + ) + missing = inspect_native_testmon_environment( + data, + environment_name=rebuilt.environment_name, + required_executable_paths=("app.py",), + ) + assert not missing.valid + assert missing.missing_executable_paths == ("app.py",) + + +def test_neutralized_environment_and_declared_plugin_identity_are_owned( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + test_file = repo / "tests" / "test_identity.py" + test_file.write_text("def test_initial():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + + initial = prepare_native_testmon_environment(repo) + assert initial.selection_mode == "bootstrap" + assert [ + result.completed.returncode + for result in _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=initial.environment_name) + ] == [0, 0] + + monkeypatch.setenv("PYTEST_ADDOPTS", "--setup-only") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_plugin") + environment_unchanged = prepare_native_testmon_environment(repo) + assert environment_unchanged.selection_mode == "affected" + assert environment_unchanged.environment_name == initial.environment_name + test_file.write_text( + "import pytest\n\n" + "def test_parallel_body_must_run():\n" + " assert False, 'ambient pytest controls suppressed the parallel body'\n\n" + "@pytest.mark.load_sensitive\n" + "def test_serial_body_must_run():\n" + " assert False, 'ambient pytest controls suppressed the serial body'\n", + encoding="utf-8", + ) + ambient_results = _run_plain_verify_corpus( + repo, + mode="bootstrap", + environment_name=environment_unchanged.environment_name, + ) + assert [result.completed.returncode for result in ambient_results] == [1, 1] + assert _selected(*ambient_results) == { + "tests/test_identity.py::test_parallel_body_must_run", + "tests/test_identity.py::test_serial_body_must_run", + } + monkeypatch.delenv("PYTEST_ADDOPTS") + monkeypatch.delenv("PYTEST_PLUGINS") + + plugin = repo / "local_plugin.py" + plugin.write_text( + "import pytest\n\n@pytest.fixture\ndef native_identity():\n return 'v1'\n", + encoding="utf-8", + ) + (repo / "tests" / "conftest.py").write_text('pytest_plugins = ("local_plugin",)\n', encoding="utf-8") + test_file.write_text( + "def test_initial():\n assert True\n\ndef test_plugin(native_identity):\n assert native_identity == 'v1'\n", + encoding="utf-8", + ) + plugin_changed = prepare_native_testmon_environment(repo) + assert plugin_changed.selection_mode == "bootstrap" + assert plugin_changed.environment_name != environment_unchanged.environment_name + plugin_results = _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=plugin_changed.environment_name) + assert [result.completed.returncode for result in plugin_results] == [0, 0] + assert "tests/test_identity.py::test_plugin" in _selected(*plugin_results) + + plugin.write_text( + "import pytest\n\n@pytest.fixture\ndef native_identity():\n return 'v2'\n", + encoding="utf-8", + ) + test_file.write_text(test_file.read_text(encoding="utf-8").replace("'v1'", "'v2'"), encoding="utf-8") + plugin_mutated = prepare_native_testmon_environment(repo) + assert plugin_mutated.selection_mode == "bootstrap" + assert plugin_mutated.environment_name != plugin_changed.environment_name + assert [ + result.completed.returncode + for result in _run_plain_verify_corpus(repo, mode="bootstrap", environment_name=plugin_mutated.environment_name) + ] == [0, 0] + + +def test_production_verify_reports_stdout_when_json_payload_is_invalid( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def invalid_json_result(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + environment = kwargs["env"] + assert isinstance(environment, dict) + receipt = Path(environment["POLYLOGUE_VERIFICATION_RECEIPT_PATH"]) + receipt.parent.mkdir(parents=True, exist_ok=True) + receipt.write_text("{}\n", encoding="utf-8") + return subprocess.CompletedProcess(command, 1, "not JSON", "verifier diagnostics") + + monkeypatch.setattr(subprocess, "run", invalid_json_result) + + with pytest.raises(pytest.fail.Exception) as failure: + _run_production_verify(tmp_path) + + assert "production verify emitted no JSON payload" in str(failure.value) + assert "stdout:\nnot JSON" in str(failure.value) + assert "stderr:\nverifier diagnostics" in str(failure.value) + + +def test_production_verify_reports_stdout_when_receipt_json_is_invalid( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def invalid_receipt_result(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + environment = kwargs["env"] + assert isinstance(environment, dict) + receipt = Path(environment["POLYLOGUE_VERIFICATION_RECEIPT_PATH"]) + receipt.parent.mkdir(parents=True, exist_ok=True) + receipt.write_text("not JSON", encoding="utf-8") + return subprocess.CompletedProcess(command, 1, "{}", "verifier diagnostics") + + monkeypatch.setattr(subprocess, "run", invalid_receipt_result) + + with pytest.raises(pytest.fail.Exception) as failure: + _run_production_verify(tmp_path) + + assert "production verify wrote invalid receipt JSON" in str(failure.value) + assert "stdout:\n{}" in str(failure.value) + assert "stderr:\nverifier diagnostics" in str(failure.value) + + +@pytest.mark.parametrize( + "declaration", + [ + "from plugin_config import plugin_names\n\npytest_plugins = plugin_names\n", + 'from plugin_config import plugin_names\n\nglobals()["pytest_plugins"] = plugin_names\n', + ], +) +def test_production_verify_fails_closed_on_dynamic_pytest_plugins(tmp_path: Path, declaration: str) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "plugin_config.py").write_text('plugin_names = ("local_plugin",)\n', encoding="utf-8") + (repo / "local_plugin.py").write_text("VALUE = 1\n", encoding="utf-8") + (repo / "tests" / "conftest.py").write_text(declaration, encoding="utf-8") + (repo / "tests" / "test_body.py").write_text("def test_body():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + origin = tmp_path / "origin.git" + _git(origin.parent, "init", "--bare", "-q", str(origin)) + _git(repo, "remote", "add", "origin", str(origin)) + _git(repo, "branch", "-M", "master") + _git(repo, "push", "-qu", "origin", "master") + + completed, payload = _run_production_verify(repo, allow_rejection=True) + + assert completed.returncode == 125 + assert payload["diagnosis"] == "native_testmon_preparation_failed" + assert payload["release_baseline_allowed"] is False + assert "pytest_plugins declaration must" in completed.stderr + + +def test_managed_native_launch_keeps_state_inode_bound_during_parent_replacement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "tests" / "test_body.py").write_text("def test_body():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + prepare_native_testmon_environment(repo) + external_cache = tmp_path / "external-cache" + external_cache.mkdir() + monkeypatch.setattr(verify, "ROOT", repo) + monkeypatch.setattr(verify, "TESTMON_DATA", repo / TESTMON_DATA_RELPATH) + + def replace_parent( + _label: str, + _cmd: list[str], + **kwargs: object, + ) -> tuple[int, float, dict[str, object]]: + bound_data = kwargs["native_testmon_data"] + assert isinstance(bound_data, Path) + (repo / ".cache").rename(repo / ".cache-owned") + (repo / ".cache").symlink_to(external_cache, target_is_directory=True) + bound_data.write_text("bound database", encoding="utf-8") + return 0, 0.01, {} + + with pytest.MonkeyPatch.context() as patcher: + patcher.setattr(verify, "_run_step", replace_parent) + assert verify._run("pytest native parallel (affected)", ["pytest"])[0] == 0 + + assert (repo / ".cache-owned" / "testmon" / "testmondata").read_text(encoding="utf-8") == "bound database" + assert list(external_cache.iterdir()) == [] + + +def test_managed_native_routes_reject_replaced_cache_parent( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + (repo / "tests" / "test_body.py").write_text("def test_body():\n assert True\n", encoding="utf-8") + _commit_all(repo, "fixture") + preparation = prepare_native_testmon_environment(repo) + owned_cache = repo / ".cache" + owned_cache.rename(repo / ".cache-owned") + external_cache = tmp_path / "external-cache" + external_cache.mkdir() + sentinel = external_cache / "sentinel" + sentinel.write_text("external", encoding="utf-8") + owned_cache.symlink_to(external_cache, target_is_directory=True) + monkeypatch.setattr(verify, "ROOT", repo) + monkeypatch.setattr(verify, "TESTMON_DATA", repo / TESTMON_DATA_RELPATH) + + with pytest.raises(NativeTestmonRepairError, match="refusing symlinked owned testmon parent"): + verify._run("pytest native parallel (affected)", ["pytest"]) + with pytest.raises(NativeTestmonRepairError, match="refusing symlinked owned testmon parent"): + verify._native_environment_after_run(preparation, required_executable_paths=()) + + assert sentinel.read_text(encoding="utf-8") == "external" + assert list(external_cache.iterdir()) == [sentinel] + + +def test_runtime_helper_mutation_stays_incremental_and_selects_owner(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _init_repo(repo) + infra = repo / "tests" / "infra" + infra.mkdir() + (repo / "tests" / "__init__.py").write_text("", encoding="utf-8") + (infra / "__init__.py").write_text("", encoding="utf-8") + helper = infra / "runtime_helper.py" + helper.write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + (repo / "tests" / "test_runtime_helper.py").write_text( + """ +import pytest + +def test_runtime_helper_owner(): + from tests.infra.runtime_helper import answer + assert answer() == 42 + +@pytest.mark.load_sensitive +def test_runtime_helper_serial_owner(): + from tests.infra.runtime_helper import answer + assert answer() == 42 +""".lstrip(), + encoding="utf-8", + ) + _commit_all(repo, "fixture") + + preparation = prepare_native_testmon_environment(repo) + bootstrap = _run_plain_verify_corpus( + repo, + mode="bootstrap", + environment_name=preparation.environment_name, + ) + assert [result.completed.returncode for result in bootstrap] == [0, 0] + + helper.write_text("def answer() -> int:\n return 0\n", encoding="utf-8") + affected = prepare_native_testmon_environment(repo) + assert affected.selection_mode == "affected" + results = _run_plain_verify_corpus(repo, mode="affected", environment_name=affected.environment_name) + + assert [result.completed.returncode for result in results] == [1, 1] + assert _selected(*results) == { + "tests/test_runtime_helper.py::test_runtime_helper_owner", + "tests/test_runtime_helper.py::test_runtime_helper_serial_owner", + } diff --git a/tests/integration/devtools/test_testmon_seed_recovery.py b/tests/integration/devtools/test_testmon_seed_recovery.py deleted file mode 100644 index 5968d77279..0000000000 --- a/tests/integration/devtools/test_testmon_seed_recovery.py +++ /dev/null @@ -1,185 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import os -import sqlite3 -import subprocess -import sys -from pathlib import Path - -import pytest - -from devtools import testmon_bootstrap, testmon_state, verify -from devtools.testmon_state import file_fingerprint, inspect_testmon_database, seed_shard_plan -from devtools.verify_runs import CheckoutMutationObservation - - -def test_real_testmon_graph_copies_and_rebinds_in_a_temporary_lane( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - class _StableMutationMonitor: - def __init__(self, _root: Path) -> None: - pass - - def start(self) -> None: - pass - - def finish(self) -> CheckoutMutationObservation: - return CheckoutMutationObservation(changed=False, unavailable=False) - - source = tmp_path / "source" - source.mkdir() - (source / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") - (source / "test_sample.py").write_text( - "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 2\n", - encoding="utf-8", - ) - data = source / ".cache" / "testmon" / "testmondata" - data.parent.mkdir(parents=True) - env = os.environ.copy() - env["TESTMON_DATAFILE"] = str(data) - run = subprocess.run( - [sys.executable, "-m", "pytest", "-q", "--testmon", "--testmon-noselect"], - cwd=source, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode != 0 - expected = ("test_sample.py::test_passed", "test_sample.py::test_failed") - assert inspect_testmon_database(data, expected).usable_for_selection - runtime_identity = testmon_state.testmon_runtime_identity(source) - assert runtime_identity is not None - dependency_environment, pytest_harness = runtime_identity - shards = seed_shard_plan(expected, shard_size=len(expected)) - shards[0]["status"] = "complete" - shards[0]["node_outcomes"] = [ - {"nodeid": expected[0], "outcome": "passed"}, - {"nodeid": expected[1], "outcome": "failed"}, - ] - attempt = { - "protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, - "status": "reusable", - "outcome": "red-baseline", - "identity": { - "git_head": "head", - "worktree_fingerprint": "source-tree", - "python": sys.version, - "skip_slow": False, - "lab": False, - "dependency_environment": dependency_environment, - "pytest_harness": pytest_harness, - }, - "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, - "expected_nodeids": list(expected), - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - "node_outcomes": [ - {"nodeid": expected[0], "outcome": "passed"}, - {"nodeid": expected[1], "outcome": "failed"}, - ], - "shards": shards, - "exit_code": 1, - "run_id": "real-testmon", - "artifact_dir": ".cache/verify/runs/real-testmon", - "testmon_data": file_fingerprint(data), - } - artifact_dir = source / ".cache" / "verify" / "runs" / "real-testmon" - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": "real-testmon", - "checkout_root": str(source.resolve()), - "artifact_dir": ".cache/verify/runs/real-testmon", - } - ), - encoding="utf-8", - ) - source_attempt = source / ".cache" / "testmon" / "seed-attempt.json" - source_attempt.parent.mkdir(parents=True, exist_ok=True) - source_attempt.write_text(json.dumps(attempt), encoding="utf-8") - - lane = tmp_path / "lane" - lane.mkdir() - (lane / "test_sample.py").write_text( - "def test_passed():\n assert 1 == 1\n\ndef test_failed():\n assert 1 == 1\n", - encoding="utf-8", - ) - monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, source)) - copy_calls: list[tuple[Path, Path]] = [] - original_copy = testmon_bootstrap._atomic_copy_sqlite_db - - def counted_copy(src: Path, dst: Path) -> None: - copy_calls.append((src, dst)) - original_copy(src, dst) - - monkeypatch.setattr(testmon_bootstrap, "_atomic_copy_sqlite_db", counted_copy) - (lane / "pyproject.toml").write_text('[project]\nname = "polylogue"\n', encoding="utf-8") - (lane / "polylogue" / "cli").mkdir(parents=True) - (lane / "polylogue" / "__init__.py").write_text("", encoding="utf-8") - (lane / "polylogue" / "cli" / "click_app.py").write_text("", encoding="utf-8") - - local_data = lane / ".cache" / "testmon" / "testmondata" - local_stamp = lane / ".cache" / "testmon" / "seed.json" - local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" - - monkeypatch.chdir(lane) - monkeypatch.setattr(verify, "ROOT", lane) - monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: lane / "polylogue" / "__init__.py") - monkeypatch.setattr("devtools.checkout_guard._is_linked_worktree", lambda _root: True) - monkeypatch.setattr("devtools.checkout_guard._python_environment_root", lambda _executable: lane) - monkeypatch.setattr(verify, "build_verify_steps", lambda **_kwargs: [("pytest testmon", ["pytest"])]) - run_count = 0 - - def fake_run(*_args: object, **_kwargs: object) -> tuple[int, float, dict[str, object]]: - nonlocal run_count - run_count += 1 - if run_count == 1: - with sqlite3.connect(local_data) as connection: - connection.execute("update test_execution set failed = 0 where test_name = ?", (expected[1],)) - return 1, 0.01, {"selected_count": 1} - return 0, 0.01, {"selected_count": 1} - - monkeypatch.setattr(verify, "_run", fake_run) - monkeypatch.setattr(verify, "_git_head", lambda: "head") - monkeypatch.setattr(verify, "_git_commit", lambda _ref: "base") - monkeypatch.setattr(verify, "_default_testmon_is_broad_change", lambda _base_commit, _head_commit: False) - monkeypatch.setattr(verify, "_changed_executable_paths", lambda _base_commit, _head_commit: ()) - monkeypatch.setattr(verify, "_stamp_head", lambda: None) - monkeypatch.setattr(verify, "worktree_fingerprint", lambda *_args: "stable") - monkeypatch.setattr(verify, "CheckoutMutationMonitor", _StableMutationMonitor) - - assert verify.main([]) == 1 - result = json.loads(capsys.readouterr().out) - assert local_data.is_file() - assert local_attempt.is_file() - assert not local_stamp.exists() - assert result["steps"][0]["selected_count"] == 1 - assert result["release_baseline_allowed"] is False - assert verify._testmon_release_baseline_permission() is False - assert not local_stamp.exists() - assert len(copy_calls) == 1 - assert (lane / ".cache" / "verify" / "current-run.json").is_file() - refreshed_attempt = json.loads(local_attempt.read_text()) - assert refreshed_attempt["testmon_data"] == file_fingerprint(local_data) - current_run = json.loads((lane / ".cache" / "verify" / "current-run.json").read_text()) - assert refreshed_attempt["run_id"] == current_run["run_id"] - - assert verify.main([]) == 0 - second = json.loads(capsys.readouterr().out) - assert second["steps"][0]["selected_count"] == 1 - assert len(copy_calls) == 1 - - assert verify.main([]) == 0 - third = json.loads(capsys.readouterr().out) - assert third["steps"][0]["selected_count"] == 1 - assert len(copy_calls) == 1 - - with sqlite3.connect(local_data) as connection: - connection.execute("delete from test_execution_file_fp") - assert verify.main([]) == 2 diff --git a/tests/unit/devtools/test_checkout_guard.py b/tests/unit/devtools/test_checkout_guard.py index a5ed1b0916..18f15ae9ee 100644 --- a/tests/unit/devtools/test_checkout_guard.py +++ b/tests/unit/devtools/test_checkout_guard.py @@ -9,7 +9,6 @@ from __future__ import annotations -import hashlib import json from pathlib import Path @@ -17,7 +16,6 @@ import devtools.click_dispatch as click_dispatch import devtools.run_tests as run_tests -import devtools.testmon_state as testmon_state import devtools.verify as verify import polylogue from devtools.checkout_guard import ( @@ -166,85 +164,6 @@ def _fake_linked_checkout(tmp_path: Path) -> Path: return root -def _write_in_progress_seed_attempt(root: Path, *, status: str = "running", **overrides: object) -> Path: - payload: dict[str, object] = { - "protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, - "status": status, - "identity": { - "git_head": "head", - "worktree_fingerprint": "fingerprint", - "python": "3.14", - "skip_slow": True, - "lab": False, - }, - "resume": False, - "expected_nodeids": [], - "expected_count": 0, - "started_at": "2026-08-05T12:00:00+00:00", - "run_id": "seed-testmon-20260805T120000Z", - "artifact_dir": ".cache/verify/runs/seed-testmon-20260805T120000Z", - "testmon_data_before": "missing", - } - if status == "reusable": - nodeid = "tests/test.py::test_one" - runtime_identity = testmon_state.testmon_runtime_identity(root) - assert runtime_identity is not None - dependency_environment, pytest_harness = runtime_identity - payload.update( - { - "identity": { - "git_head": "head", - "worktree_fingerprint": "fingerprint", - "python": "3.14", - "skip_slow": True, - "lab": False, - "dependency_environment": dependency_environment, - "pytest_harness": pytest_harness, - }, - "expected_nodeids": [nodeid], - "expected_count": 1, - "expected_digest": hashlib.sha256(nodeid.encode()).hexdigest(), - "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, - "node_outcomes": [{"nodeid": nodeid, "outcome": "failed"}], - "exit_code": 1, - "testmon_data": "fingerprint", - "release_baseline_allowed": False, - "verification_scope": "affected", - "binding": { - "mode": "exact", - "checkout_root": str(root.resolve()), - "source_checkout_root": None, - }, - } - ) - payload.update(overrides) - attempt = root / ".cache" / "testmon" / "seed-attempt.json" - attempt.parent.mkdir(parents=True, exist_ok=True) - attempt.write_text(json.dumps(payload)) - if status == "reusable": - run_dir = root / ".cache" / "verify" / "runs" / str(payload["run_id"]) - run_dir.mkdir(parents=True, exist_ok=True) - (run_dir / "run.json").write_text( - json.dumps( - { - "run_id": payload["run_id"], - "checkout_root": str(root.resolve()), - "artifact_dir": f".cache/verify/runs/{payload['run_id']}", - } - ) - ) - (root / ".cache" / "verify" / "current-run.json").write_text( - json.dumps( - { - "run_id": payload["run_id"], - "checkout_root": str(root.resolve()), - "artifact_dir": f".cache/verify/runs/{payload['run_id']}", - } - ) - ) - return attempt - - def test_checkout_environment_fingerprint_accepts_clean_linked_worktree(tmp_path: Path) -> None: root = _fake_linked_checkout(tmp_path) fingerprint = checkout_environment_fingerprint( @@ -258,114 +177,40 @@ def test_checkout_environment_fingerprint_accepts_clean_linked_worktree(tmp_path assert fingerprint.python_environment_root == root -def test_checkout_environment_fingerprint_accepts_current_in_progress_seed_attempt( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_checkout_guard_leaves_derived_native_testmon_state_for_verify_to_repair( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: root = _fake_linked_checkout(tmp_path) - attempt = _write_in_progress_seed_attempt(root) - (root / ".cache" / "testmon" / "testmondata").write_text("partial") + data = root / ".cache" / "testmon" / "testmondata" + data.parent.mkdir(parents=True) + data.write_bytes(b"interrupted or invalid derived SQLite state") package_path = root / "polylogue" / "__init__.py" monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) fingerprint = assert_polylogue_matches_checkout( root, - context="seed-testmon bootstrap", + context="plain verify repair", python_executable=root / ".venv" / "bin" / "python", ) assert fingerprint.clean - assert fingerprint.testmon_state_origin is None - assert attempt.is_file() + assert data.read_bytes().startswith(b"interrupted") -def test_checkout_environment_fingerprint_accepts_finalized_selection_attempt( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - root = _fake_linked_checkout(tmp_path) - attempt = _write_in_progress_seed_attempt(root, status="reusable") - (root / ".cache" / "testmon" / "testmondata").write_text("complete graph") - package_path = root / "polylogue" / "__init__.py" - monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) - - fingerprint = assert_polylogue_matches_checkout( - root, - context="affected selection", - python_executable=root / ".venv" / "bin" / "python", - ) - - assert fingerprint.clean - assert fingerprint.testmon_state_origin is None - assert attempt.is_file() - - -@pytest.mark.parametrize( - ("status", "overrides"), - [ - ("running", {"identity": {}}), - ("running", {"expected_count": 1}), - ("running", {"artifact_dir": "/foreign/.cache/verify/runs/seed"}), - ("complete", {}), - ], -) -def test_checkout_environment_fingerprint_rejects_invalid_or_completed_seed_attempt( +def test_checkout_preflight_reports_runtime_provenance_but_not_testmon_state( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - status: str, - overrides: dict[str, object], -) -> None: - root = _fake_linked_checkout(tmp_path) - attempt = _write_in_progress_seed_attempt(root, status=status, **overrides) - (root / ".cache" / "testmon" / "testmondata").write_text("foreign or incomplete") - package_path = root / "polylogue" / "__init__.py" - monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) - - with pytest.raises(CheckoutEnvironmentMismatchError) as excinfo: - assert_polylogue_matches_checkout( - root, - context="invalid testmon state", - python_executable=root / ".venv" / "bin" / "python", - ) - - message = str(excinfo.value) - assert str(attempt.parent) in message - assert "no verifiable checkout-root marker" in message - - -def test_checkout_environment_fingerprint_requires_provenance_for_completed_seed( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - root = _fake_linked_checkout(tmp_path) - seed_dir = root / ".cache" / "testmon" - seed_dir.mkdir(parents=True) - (seed_dir / "testmondata").write_text("complete") - (seed_dir / "seed.json").write_text( - json.dumps({"protocol_version": verify.TESTMON_SEED_PROTOCOL_VERSION, "status": "complete"}) - ) - package_path = root / "polylogue" / "__init__.py" - monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) - - with pytest.raises(CheckoutEnvironmentMismatchError) as excinfo: - assert_polylogue_matches_checkout( - root, - context="completed testmon state", - python_executable=root / ".venv" / "bin" / "python", - ) - - assert str(seed_dir / "seed.json") in str(excinfo.value) - - -def test_checkout_preflight_reports_seeded_artifacts_and_main_interpreter( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: root = _fake_linked_checkout(tmp_path) main = tmp_path / "main-checkout" - main_venv = main / ".venv" / "bin" - main_venv.mkdir(parents=True) - main_python = main_venv / "python" + main_python = main / ".venv" / "bin" / "python" + main_python.parent.mkdir(parents=True) (root / ".venv").mkdir() (root / "node_modules").mkdir() - (root / ".cache" / "testmon").mkdir(parents=True) - (root / ".cache" / "testmon" / "seed.json").write_text(json.dumps({"status": "complete"})) + data = root / ".cache" / "testmon" / "testmondata" + data.parent.mkdir(parents=True) + data.write_bytes(b"repairable") (root / ".cache" / "verify").mkdir(parents=True) verify_marker = root / ".cache" / "verify" / "current-run.json" verify_marker.write_text(json.dumps({"checkout_root": str(main)})) @@ -373,16 +218,14 @@ def test_checkout_preflight_reports_seeded_artifacts_and_main_interpreter( monkeypatch.setattr("devtools.checkout_guard.resolved_polylogue_path", lambda: package_path) with pytest.raises(CheckoutEnvironmentMismatchError) as excinfo: - assert_polylogue_matches_checkout(root, context="seeded lane", python_executable=main_python) + assert_polylogue_matches_checkout(root, context="lane", python_executable=main_python) message = str(excinfo.value) assert str(main_python) in message assert str(root / ".venv") in message assert str(root / "node_modules") in message - assert str(root / ".cache" / "testmon" / "seed.json") in message assert str(verify_marker) in message - assert "direnv allow" in message - assert "remediation" in message + assert str(data) not in message def test_verify_run_persists_environment_fingerprint(tmp_path: Path) -> None: diff --git a/tests/unit/devtools/test_durable_schema_policy_gate.py b/tests/unit/devtools/test_durable_schema_policy_gate.py index 8e1388cbca..1415ceb58b 100644 --- a/tests/unit/devtools/test_durable_schema_policy_gate.py +++ b/tests/unit/devtools/test_durable_schema_policy_gate.py @@ -94,7 +94,7 @@ def test_schema_versioning_policy_runs_exactly_once_in_every_noncommit_fast_gate for label, _command in verify.build_verify_steps( quick=quick, lab=lab, - skip_slow=True, + testmon_environment="env-digest" if not quick else "", ) ] assert labels.count("lab policy schema-versioning") == 1 @@ -104,7 +104,6 @@ def test_schema_versioning_policy_runs_exactly_once_in_every_noncommit_fast_gate for label, _command in verify.build_verify_steps( quick=True, lab=False, - skip_slow=True, commit=True, ) ] diff --git a/tests/unit/devtools/test_merge_boundary.py b/tests/unit/devtools/test_merge_boundary.py index a906cc16db..a5cc84bf0a 100644 --- a/tests/unit/devtools/test_merge_boundary.py +++ b/tests/unit/devtools/test_merge_boundary.py @@ -11,7 +11,7 @@ import pytest -from devtools import merge_boundary, merge_gate, pr_scope +from devtools import click_dispatch, merge_boundary, merge_gate, pr_scope from devtools.checkout_guard import checkout_environment_fingerprint from tests.infra.frozen_clock import FrozenClock @@ -26,10 +26,19 @@ @pytest.fixture(autouse=True) -def _scope_bead_record(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - beads_dir = tmp_path / ".beads" - beads_dir.mkdir() - (beads_dir / "issues.jsonl").write_text(json.dumps(_SCOPE_BEAD) + "\n") +def _scope_bead_record(monkeypatch: pytest.MonkeyPatch) -> None: + """Give merge-flow tests a canonical Bead snapshot without faking validation. + + Production scope validation reads the candidate and prospective records from + committed Git revisions. These unit tests deliberately use placeholder PR + SHAs, so inject that storage boundary and retain the real carrier, + disposition, digest, and merge-gate validation paths. + """ + + def canonical_records(_revision: str) -> dict[str, dict[str, object]]: + return {_SCOPE_BEAD["id"]: dict(_SCOPE_BEAD)} + + monkeypatch.setattr(pr_scope, "_bead_records_at", canonical_records) monkeypatch.setattr(pr_scope, "changed_bead_ids", lambda **_kwargs: []) @@ -137,7 +146,6 @@ def _run(cmd: list[str], **kwargs: Any) -> MagicMock: "exit_code": local_exit, "verification_scope": "affected", "release_baseline_allowed": False, - "terminal_authorization": None, } ) ) @@ -156,7 +164,6 @@ def _write_terminal_receipt( head: str = "merged-master", scope: str = "release-baseline", release_allowed: bool = True, - terminal_authorization: str | None = None, ) -> None: env = kwargs["env"] assert isinstance(env, dict) @@ -170,7 +177,6 @@ def _write_terminal_receipt( "exit_code": 0, "verification_scope": scope, "release_baseline_allowed": release_allowed, - "terminal_authorization": terminal_authorization, } ) ) @@ -206,6 +212,50 @@ def merge(pr: int, **_kwargs: object) -> int: assert captured == [3948] +def test_workspace_merge_dispatches_documented_direct_pr_form(monkeypatch: pytest.MonkeyPatch) -> None: + captured: list[tuple[int, bool]] = [] + + def merge(pr: int, **kwargs: object) -> int: + captured.append((pr, bool(kwargs["dry_run"]))) + return 0 + + monkeypatch.setattr(merge_boundary, "cmd_merge", merge) + + assert click_dispatch._dispatch(["workspace", "merge", "3952", "--dry-run"]) == 0 + assert captured == [(3952, True)] + + +def test_workspace_merge_preserves_train_status_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[bool] = [] + + def train_status(as_json: bool) -> int: + calls.append(as_json) + return 0 + + monkeypatch.setattr(merge_boundary, "cmd_train_status", train_status) + + assert click_dispatch._dispatch(["workspace", "merge", "train-status", "--json"]) == 0 + assert calls == [True] + + +def test_workspace_merge_preserves_record_full_verify_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: + captured: list[tuple[str, str]] = [] + + def record(command: str, target_sha: str, **_kwargs: object) -> int: + captured.append((command, target_sha)) + return 0 + + monkeypatch.setattr(merge_boundary, "_reconciled_terminal_verify_snapshot", lambda: {}) + monkeypatch.setattr(merge_boundary, "_fetched_current_default_branch_sha", lambda: "master-sha") + monkeypatch.setattr(merge_boundary, "_run_post_merge_terminal_verify", record) + + assert ( + click_dispatch._dispatch(["workspace", "merge", "record-full-verify", "--command", "devtools verify --all"]) + == 0 + ) + assert captured == [("devtools verify --all", "master-sha")] + + # --------------------------------------------------------------------------- # cmd_merge # --------------------------------------------------------------------------- @@ -903,6 +953,48 @@ def test_train_status_blocks_when_pr_merged_after_last_full_verify( assert merge_boundary.cmd_train_status(as_json=False) == 1 +def test_train_status_requires_release_baseline_guidance( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.chdir(tmp_path) + merge_boundary._append_merge_entry(1, "sha1", "some title") + + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + output = capsys.readouterr().out + assert "devtools verify --all" in output + assert "narrower agreed selection" not in output + assert "does not grant the release-baseline authority" in output + + +def test_train_status_reads_historical_scope_without_granting_release_authority( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Rejecting unknown historical scopes would make old ledgers unreadable.""" + monkeypatch.chdir(tmp_path) + merge_boundary._write_ledger( + { + "merges": [], + "last_full_verify": { + "at": 1000.0, + "verification_started_at": 1000.0, + "duration_s": 1.0, + "command": "devtools verify --affected", + "exit_code": 0, + "verification_scope": "narrow-terminal", + "release_baseline_allowed": True, + "merge_sequence": 0, + "accepted": True, + }, + } + ) + merge_boundary._append_merge_entry(1, "sha1", "some title") + + assert merge_boundary._read_ledger()["last_full_verify"]["verification_scope"] == "narrow-terminal" + assert merge_boundary._pending_prs_since_last_full_verify(merge_boundary._read_ledger()) + assert merge_boundary.cmd_train_status(as_json=False) == 1 + + def test_train_status_rejects_untyped_accepted_terminal_ledger(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.chdir(tmp_path) merge_boundary._write_ledger( @@ -1011,47 +1103,6 @@ def test_record_full_verify_rejects_success_without_structured_release_permissio assert merge_boundary.cmd_train_status(as_json=False) == 1 -def test_record_full_verify_rejects_skip_slow_without_typed_authorization( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.chdir(tmp_path) - merge_boundary._append_merge_entry(1, "sha1", "some title") - monkeypatch.setattr( - subprocess, - "run", - lambda _cmd, **_kwargs: MagicMock( - returncode=0, - stdout=json.dumps({"verification_scope": "narrow-terminal", "release_baseline_allowed": False}), - stderr="", - ), - ) - - assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 1 - assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is False - assert merge_boundary.cmd_train_status(as_json=False) == 1 - - -def test_record_full_verify_accepts_explicit_typed_narrow_terminal_authorization( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.chdir(tmp_path) - merge_boundary._append_merge_entry(1, "sha1", "some title") - - def run(_cmd: list[str], **kwargs: Any) -> MagicMock: - _write_terminal_receipt( - kwargs, - scope="narrow-terminal", - terminal_authorization="narrow-terminal", - ) - return MagicMock(returncode=0, stdout="", stderr="") - - monkeypatch.setattr(subprocess, "run", run) - - assert merge_boundary.cmd_record_full_verify("devtools verify --all --skip-slow", target_sha="merged-master") == 0 - assert merge_boundary._read_ledger()["last_full_verify"]["accepted"] is True - assert merge_boundary.cmd_train_status(as_json=False) == 0 - - def test_record_full_verify_rejects_untyped_scope_even_when_permission_is_true( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/unit/devtools/test_merge_gate.py b/tests/unit/devtools/test_merge_gate.py index 3d955674e9..86c29526a1 100644 --- a/tests/unit/devtools/test_merge_gate.py +++ b/tests/unit/devtools/test_merge_gate.py @@ -126,7 +126,6 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: "exit_code": local_exit, "verification_scope": "affected", "release_baseline_allowed": False, - "terminal_authorization": None, } ) ) @@ -227,7 +226,7 @@ def test_record_persists_receipt_keyed_to_current_head_sha(monkeypatch: pytest.M assert receipt["exit_code"] == 0 -@pytest.mark.parametrize("command", ["devtools verify", "devtools verify --lab", "devtools verify --json --skip-slow"]) +@pytest.mark.parametrize("command", ["devtools verify", "devtools verify --lab", "devtools verify --json"]) def test_check_accepts_affected_receipt_without_release_baseline_permission( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, command: str ) -> None: @@ -258,9 +257,7 @@ def test_check_rejects_successful_non_test_receipt(monkeypatch: pytest.MonkeyPat assert merge_gate.cmd_check(42, max_age_s=3600, poll_rounds=1, poll_interval_s=0, as_json=False) == 1 -@pytest.mark.parametrize( - "command", ["devtools verify --all", "devtools verify --full", "devtools verify --seed-testmon"] -) +@pytest.mark.parametrize("command", ["devtools verify --all", "devtools verify --full"]) def test_check_blocks_full_receipt_without_release_baseline_permission( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, command: str ) -> None: @@ -299,7 +296,6 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: "exit_code": 0, "verification_scope": "release-baseline", "release_baseline_allowed": True, - "terminal_authorization": None, } ), stderr="", @@ -319,9 +315,8 @@ def test_record_consumes_receipt_after_streamed_verifier_progress( pr_view: dict[str, object] = {"headRefOid": "abc123", "headRefName": "feature/x"} base = cast(Callable[..., MagicMock], _fake_run(pr_view, [], local_head_sha="abc123")) payload = { - "verification_scope": "narrow-terminal", + "verification_scope": "affected", "release_baseline_allowed": False, - "terminal_authorization": "narrow-terminal", } def _run(cmd: list[str], **kwargs: object) -> MagicMock: @@ -346,9 +341,8 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: monkeypatch.setattr(subprocess, "run", _run) assert merge_gate.cmd_record(42, "devtools test tests/unit/foo.py") == 0 receipt = json.loads(merge_gate._receipt_path(42).read_text()) - assert receipt["verification_scope"] == "narrow-terminal" + assert receipt["verification_scope"] == "affected" assert receipt["release_baseline_allowed"] is False - assert receipt["terminal_authorization"] == "narrow-terminal" def test_record_consumes_exact_invocation_receipt_when_verifier_writes_only_stderr( @@ -375,7 +369,6 @@ def _run(cmd: list[str], **kwargs: object) -> MagicMock: "exit_code": 0, "verification_scope": "affected", "release_baseline_allowed": False, - "terminal_authorization": None, } ) ) diff --git a/tests/unit/devtools/test_project_motd.py b/tests/unit/devtools/test_project_motd.py index 44b821b051..d602c88843 100644 --- a/tests/unit/devtools/test_project_motd.py +++ b/tests/unit/devtools/test_project_motd.py @@ -37,7 +37,7 @@ def test_render_motd_contains_expected_sections( assert ( "ready devtools render all --check · devtools verify --quick · devtools release build-package" in rendered ) - assert "test pytest -q --ignore=tests/integration" in rendered + assert "test devtools verify" in rendered assert "roots keep .venv/ .direnv/ · cache .cache/ · outputs .local/ · build .local/result" in rendered assert "dirty · 1 staged · 2 modified · 3 untracked" in rendered diff --git a/tests/unit/devtools/test_run_tests.py b/tests/unit/devtools/test_run_tests.py index c57c78538a..ee4eccdbc8 100644 --- a/tests/unit/devtools/test_run_tests.py +++ b/tests/unit/devtools/test_run_tests.py @@ -267,6 +267,8 @@ def interrupt(*_args: Any, **_kwargs: Any) -> tuple[int, float, dict[str, Any]]: for payload in (history, run_payload, current_payload, receipt_payload): assert payload["diagnosis"] == "pytest_interrupted" assert payload["checkout_diagnosis"] == "checkout_changed_during_focused_test" + assert payload["pytest_aggregate"]["selection_mode"] == "focused" + assert history["pytest_aggregate"] == receipt_payload["pytest_aggregate"] def test_normalize_selection_paths_preserves_pytest_path_option_semantics( diff --git a/tests/unit/devtools/test_testmon_bootstrap.py b/tests/unit/devtools/test_testmon_bootstrap.py index 2fde1706a0..bee477e725 100644 --- a/tests/unit/devtools/test_testmon_bootstrap.py +++ b/tests/unit/devtools/test_testmon_bootstrap.py @@ -1,830 +1,477 @@ -"""Tests for the worktree testmon-seed bootstrap (devtools/testmon_bootstrap.py). - -Covers polylogue-mq4vx: a fresh agent worktree lane starts with no local -`.cache/testmon/testmondata`, either paying the full `--seed-testmon` cost -again or hitting the unseeded-refusal preflight in `devtools/verify.py`. The -main checkout's testmondata is copyable (file_fp entries are relative paths -with per-file checksums, so a stale copy self-invalidates changed files), so -`maybe_bootstrap_testmon_seed` copies it in before that preflight runs. - -These tests target `decide_testmon_bootstrap` (the pure decision) and -`bootstrap_testmon_seed_files` (the copy action) directly with tmp dirs -- -not the full `devtools verify` pipeline, per the bootstrap's own module -docstring contract. -""" - from __future__ import annotations -import hashlib -import json -import sqlite3 -from collections.abc import Callable +import os from pathlib import Path -from typing import cast import pytest -import devtools.checkout_guard as checkout_guard -import devtools.testmon_bootstrap as testmon_bootstrap -import devtools.verify as verify from devtools.testmon_bootstrap import ( - BootstrapDecision, - bootstrap_testmon_seed_files, - decide_testmon_bootstrap, -) -from devtools.testmon_state import ( - BaselineStatus, - BindingMode, - CollectionStatus, - GraphInspection, - GraphStatus, - file_fingerprint, -) -from devtools.testmon_state import ( - TestmonBinding as _TestmonBinding, + NativeTestmonDeadlineError, + NativeTestmonRepairError, + _testmon_schema_version, + classify_native_testmon_changes, + classify_source_ast, + executable_python_paths, + inspect_native_testmon_environment, + prepare_native_testmon_environment, + remove_invalid_native_testmon_state, + validate_native_testmon_state_ownership, ) -from devtools.testmon_state import ( - TestmonIdentity as _TestmonIdentity, -) -from devtools.testmon_state import ( - TestmonSeedStamp as _TestmonSeedStamp, +from devtools.testmon_bootstrap import ( + testmon_environment_digest as _testmon_environment_digest, ) -PROTOCOL_VERSION = 4 - - -def _write_valid_seed_stamp(path: Path, *, protocol_version: int = PROTOCOL_VERSION) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - data = path.parent / "testmondata" - if not data.exists(): - _write_sqlite_db(data) - with sqlite3.connect(data) as conn: - nodeids = tuple(row[0] for row in conn.execute("select test_name from test_execution")) - graph = GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()) - stamp = _TestmonSeedStamp( - protocol_version, - CollectionStatus.COMPLETE, - nodeids, - 0, - BaselineStatus.GREEN, - True, - 0, - graph, - _TestmonIdentity("head", "tree", "python", True, False, None, "narrow-terminal"), - _TestmonBinding(BindingMode.EXACT, str(path.parent.resolve())), - file_fingerprint(data), - "seed", - ".cache/verify/runs/seed", + +def test_ast_classification_distinguishes_declarations_from_execution(tmp_path: Path) -> None: + declarations = tmp_path / "types.py" + declarations.write_text( + '"""Types only."""\nname: str\n\nclass Record:\n identifier: int\n\n def label(self) -> str: ...\n', + encoding="utf-8", ) - artifact_dir = path.parent / ".cache" / "verify" / "runs" / "seed" - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": "seed", - "checkout_root": str(path.parent.resolve()), - "artifact_dir": ".cache/verify/runs/seed", - } - ) + executable = tmp_path / "runtime.py" + executable.write_text("VALUE: int = 3\n", encoding="utf-8") + + assert classify_source_ast(declarations) == "declaration-only" + assert classify_source_ast(executable) == "executable" + + +def test_ast_classification_treats_type_checking_guards_as_declarations(tmp_path: Path) -> None: + declarations = tmp_path / "protocols.py" + declarations.write_text( + "from typing import TYPE_CHECKING\n\n" + "if TYPE_CHECKING:\n" + " from polylogue.archive.models import Session\n\n" + "class SessionReader:\n" + " session: 'Session'\n" + " def read(self) -> 'Session': ...\n", + encoding="utf-8", ) - path.write_text(json.dumps(stamp.as_dict())) + assert classify_source_ast(declarations) == "declaration-only" -def _write_sqlite_db(path: Path, *, rows: tuple[str, ...] = ("a", "b")) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - conn = sqlite3.connect(path) - try: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute( - "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, environment_id INTEGER, test_name TEXT, failed INTEGER)" - ) - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - conn.executemany("INSERT INTO file_fp(filename, fsha) VALUES (?, ?)", [(row, f"sha-{row}") for row in rows]) - conn.executemany("INSERT INTO test_execution(test_name, failed) VALUES (?, 0)", [(row,) for row in rows]) - conn.executemany( - "INSERT INTO test_execution_file_fp VALUES (?, ?)", - [(index, index) for index, _row in enumerate(rows, start=1)], - ) - conn.commit() - finally: - conn.close() + declarations.write_text(declarations.read_text(encoding="utf-8") + "\nVALUE = build_runtime_value()\n") + assert classify_source_ast(declarations) == "executable" -def _red_attempt_decision(tmp_path: Path) -> tuple[BootstrapDecision, Path, Path, Path, Path]: - main_root = tmp_path / "main" - main_data = main_root / "testmondata" - _write_sqlite_db(main_data, rows=("tests/test.py::test_passed", "tests/test.py::test_failed")) - attempt = main_root / "seed-attempt.json" - attempt.write_text( - json.dumps( - { - "protocol_version": PROTOCOL_VERSION, - "status": "reusable", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - }, - "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, - "expected_nodeids": ["tests/test.py::test_passed", "tests/test.py::test_failed"], - "expected_count": 2, - "expected_digest": hashlib.sha256( - "\n".join(sorted(["tests/test.py::test_passed", "tests/test.py::test_failed"])).encode() - ).hexdigest(), - "testmon_data": file_fingerprint(main_data), - "node_outcomes": [ - {"nodeid": "tests/test.py::test_passed", "outcome": "passed"}, - {"nodeid": "tests/test.py::test_failed", "outcome": "failed"}, - ], - "exit_code": 1, - "run_id": "red-run", - "artifact_dir": ".cache/verify/runs/red-run", - } - ) - ) - artifact = main_root / ".cache" / "verify" / "runs" / "red-run" - artifact.mkdir(parents=True, exist_ok=True) - (artifact / "run.json").write_text( - json.dumps( - { - "run_id": "red-run", - "checkout_root": str(main_root.resolve()), - "artifact_dir": ".cache/verify/runs/red-run", - } - ) - ) - lane = tmp_path / "lane" - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=lane / "testmondata", - local_seed_stamp=lane / "seed.json", - local_seed_attempt=lane / "seed-attempt.json", - main_testmon_data=main_data, - main_seed_stamp=main_root / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, - main_checkout_root=main_root, - local_checkout_root=lane, - ) - return decision, lane / "testmondata", lane / "seed.json", lane / "seed-attempt.json", lane - - -def test_not_a_linked_worktree_never_bootstraps(tmp_path: Path) -> None: - """The main checkout itself must never "bootstrap from itself".""" - decision = decide_testmon_bootstrap( - is_linked_worktree=False, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=tmp_path / "main" / "seed.json", - protocol_version=PROTOCOL_VERSION, - ) - assert decision == BootstrapDecision(False, decision.reason) - assert not decision.should_bootstrap - - -def test_local_seed_already_present_skips_bootstrap(tmp_path: Path) -> None: - """A worktree that already seeded itself must not be clobbered by main's copy.""" - local_data = tmp_path / "local" / "testmondata" - local_stamp = tmp_path / "local" / "seed.json" - _write_sqlite_db(local_data) - _write_valid_seed_stamp(local_stamp) - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - assert "already has" in decision.reason - - -def test_invalid_local_seed_does_not_block_valid_main_bootstrap(tmp_path: Path) -> None: - local_data = tmp_path / "local" / "testmondata" - local_stamp = tmp_path / "local" / "seed.json" - _write_sqlite_db(local_data) - _write_valid_seed_stamp(local_stamp) - local_data.write_bytes(local_data.read_bytes() + b"stale") - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert decision.should_bootstrap +def test_ast_classification_treats_ordinary_imports_as_executable(tmp_path: Path) -> None: + """Removing ordinary-import execution from the classifier makes this fail.""" + module = tmp_path / "runtime.py" + module.write_text("from package.runtime import value\n", encoding="utf-8") + assert classify_source_ast(module) == "executable" -def test_main_seed_absent_skips_bootstrap(tmp_path: Path) -> None: - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=tmp_path / "main" / "seed.json", - protocol_version=PROTOCOL_VERSION, + +def test_pure_enum_contracts_are_non_traceable_runtime_inputs(tmp_path: Path) -> None: + module = tmp_path / "devtools" / "verification_contracts.py" + module.parent.mkdir() + module.write_text( + "from enum import StrEnum\n\n" + "class VerificationScope(StrEnum):\n" + " AFFECTED = 'affected'\n" + " RELEASE_BASELINE = 'release-baseline'\n", + encoding="utf-8", ) - assert not decision.should_bootstrap - assert "testmondata file is missing" in decision.reason + assert classify_source_ast(module) == "declaration-only" + impact = classify_native_testmon_changes(tmp_path, ("devtools/verification_contracts.py",)) -def test_main_seed_stamp_wrong_protocol_version_skips_bootstrap(tmp_path: Path) -> None: - main_stamp = tmp_path / "main" / "seed.json" - _write_valid_seed_stamp(main_stamp, protocol_version=PROTOCOL_VERSION + 1) + assert impact.executable_paths == () + assert impact.runtime_data_paths == ("devtools/verification_contracts.py",) - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - assert "stale" in decision.reason or "no validated" in decision.reason - - -def test_main_seed_stamp_incomplete_status_skips_bootstrap(tmp_path: Path) -> None: - main_stamp = tmp_path / "main" / "seed.json" - main_stamp.parent.mkdir(parents=True, exist_ok=True) - main_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "incomplete"})) - _write_sqlite_db(tmp_path / "main" / "testmondata") - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - - -def test_main_seed_stamp_unreadable_json_skips_bootstrap(tmp_path: Path) -> None: - main_stamp = tmp_path / "main" / "seed.json" - main_stamp.parent.mkdir(parents=True, exist_ok=True) - main_stamp.write_text("{not valid json") - _write_sqlite_db(tmp_path / "main" / "testmondata") - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - - -def test_valid_seed_stamp_but_missing_testmondata_skips_bootstrap(tmp_path: Path) -> None: - """A seed stamp claims completeness but the db file itself vanished -- don't copy nothing.""" - main_stamp = tmp_path / "main" / "seed.json" - main_stamp.parent.mkdir(parents=True, exist_ok=True) - main_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=tmp_path / "main" / "testmondata", - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert not decision.should_bootstrap - assert "testmondata file is missing" in decision.reason - - -def test_valid_main_seed_and_empty_local_bootstraps(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=tmp_path / "local" / "seed.json", - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert decision.should_bootstrap - assert decision.main_testmon_data == main_data - assert decision.main_seed_stamp == main_stamp +def test_executable_paths_require_current_runtime_modules_and_deleted_modules(tmp_path: Path) -> None: + module = tmp_path / "polylogue" / "runtime.py" + module.parent.mkdir() + module.write_text("VALUE = factory()\n", encoding="utf-8") + malformed = module.with_name("malformed.py") + malformed.write_text("def broken(:\n", encoding="utf-8") -def test_complete_red_attempt_bootstraps_as_selection_only_state(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - _write_sqlite_db(main_data, rows=("tests/test.py::test_passed", "tests/test.py::test_failed")) - attempt = tmp_path / "main" / "seed-attempt.json" - attempt.write_text( - json.dumps( - { - "protocol_version": PROTOCOL_VERSION, - "status": "reusable", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - }, - "selection": {"selected_count": 2, "selected_nodeids_omitted": 0}, - "expected_nodeids": ["tests/test.py::test_passed", "tests/test.py::test_failed"], - "expected_count": 2, - "expected_digest": hashlib.sha256( - "\n".join(sorted(["tests/test.py::test_passed", "tests/test.py::test_failed"])).encode() - ).hexdigest(), - "testmon_data": file_fingerprint(main_data), - "node_outcomes": [ - {"nodeid": "tests/test.py::test_passed", "outcome": "passed"}, - {"nodeid": "tests/test.py::test_failed", "outcome": "failed"}, - ], - "exit_code": 1, - "run_id": "red-run", - "artifact_dir": ".cache/verify/runs/red-run", - } - ) - ) - red_artifact = tmp_path / "main" / ".cache" / "verify" / "runs" / "red-run" - red_artifact.mkdir(parents=True, exist_ok=True) - (red_artifact / "run.json").write_text( - json.dumps( - { - "run_id": "red-run", - "checkout_root": str((tmp_path / "main").resolve()), - "artifact_dir": ".cache/verify/runs/red-run", - } - ) - ) - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "lane" / "testmondata", - local_seed_stamp=tmp_path / "lane" / "seed.json", - main_testmon_data=main_data, - main_seed_stamp=tmp_path / "main" / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, + assert executable_python_paths( + tmp_path, + ("polylogue/runtime.py", "polylogue/malformed.py", "polylogue/deleted.py"), + ) == ( + "polylogue/deleted.py", + "polylogue/malformed.py", + "polylogue/runtime.py", ) - assert decision.should_bootstrap - assert decision.main_seed_attempt == attempt - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - local_attempt = tmp_path / "lane" / "seed-attempt.json" - assert bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - local_seed_attempt=local_attempt, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - assert not local_stamp.exists() - rebound_attempt = json.loads(local_attempt.read_text()) - assert rebound_attempt["artifact_dir"] == ".cache/verify/runs/red-run" - assert rebound_attempt["testmon_data"] == file_fingerprint(local_data) - rebound_receipt = json.loads( - (tmp_path / "lane" / ".cache" / "verify" / "runs" / "red-run" / "run.json").read_text() + +def test_package_runtime_data_changes_force_full_native_selection(tmp_path: Path) -> None: + runtime = tmp_path / "polylogue" / "archive" / "semantic" / "data" / "prices.json" + runtime.parent.mkdir(parents=True) + runtime.write_text("{}\n", encoding="utf-8") + declaration = runtime.with_name("types.pyi") + declaration.write_text("VALUE: int\n", encoding="utf-8") + + impact = classify_native_testmon_changes( + tmp_path, + ( + "polylogue/archive/semantic/data/prices.json", + "polylogue/archive/semantic/data/deleted.json", + "polylogue/archive/semantic/data/types.pyi", + "docs/prices.json", + ), ) - assert rebound_receipt["run_id"] == "red-run" - assert rebound_receipt["checkout_root"] == str((tmp_path / "lane").resolve()) - current_run = json.loads((tmp_path / "lane" / ".cache" / "verify" / "current-run.json").read_text()) - assert current_run["run_id"] == "red-run" - assert current_run["checkout_root"] == str((tmp_path / "lane").resolve()) - - rebound_decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - local_seed_attempt=local_attempt, - main_testmon_data=main_data, - main_seed_stamp=tmp_path / "main" / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, - main_checkout_root=tmp_path / "main", - local_checkout_root=tmp_path / "lane", + + assert impact.executable_paths == () + assert impact.runtime_data_paths == ( + "polylogue/archive/semantic/data/deleted.json", + "polylogue/archive/semantic/data/prices.json", + "polylogue/archive/semantic/data/types.pyi", ) - assert not rebound_decision.should_bootstrap - assert "checkout-bound selection attempt" in rebound_decision.reason -def test_complete_typed_markerless_green_attempt_bootstraps_only_as_selection_state(tmp_path: Path) -> None: - main_root = tmp_path / "main" - main_data = main_root / "testmondata" - _write_sqlite_db(main_data, rows=("tests/test.py::test_passed",)) - attempt = main_root / "seed-attempt.json" - attempt.write_text( - json.dumps( - { - "protocol_version": PROTOCOL_VERSION, - "status": "complete", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - "terminal_authorization": None, - }, - "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, - "expected_nodeids": ["tests/test.py::test_passed"], - "expected_count": 1, - "expected_digest": hashlib.sha256(b"tests/test.py::test_passed").hexdigest(), - "node_outcomes": [{"nodeid": "tests/test.py::test_passed", "outcome": "passed"}], - "exit_code": 0, - "verification_scope": "release-baseline", - "release_baseline_allowed": True, - "run_id": "green-run", - "artifact_dir": ".cache/verify/runs/green-run", - "testmon_data": file_fingerprint(main_data), - } - ) - ) - artifact = main_root / ".cache" / "verify" / "runs" / "green-run" - artifact.mkdir(parents=True) - (artifact / "run.json").write_text( - json.dumps( - { - "run_id": "green-run", - "checkout_root": str(main_root.resolve()), - "artifact_dir": ".cache/verify/runs/green-run", - } - ) +def test_test_runtime_data_changes_force_full_native_selection(tmp_path: Path) -> None: + runtime = tmp_path / "tests" / "data" / "payload.json" + runtime.parent.mkdir(parents=True) + runtime.write_text("{}\n", encoding="utf-8") + + impact = classify_native_testmon_changes( + tmp_path, + ( + "tests/data/payload.json", + "tests/data/deleted.json", + "docs/payload.json", + ), ) - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "lane" / "testmondata", - local_seed_stamp=tmp_path / "lane" / "seed.json", - main_testmon_data=main_data, - main_seed_stamp=main_root / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, + assert impact.executable_paths == () + assert impact.runtime_data_paths == ( + "tests/data/deleted.json", + "tests/data/payload.json", ) - assert decision.should_bootstrap - assert decision.selection_only - assert bootstrap_testmon_seed_files( - decision, - local_testmon_data=tmp_path / "lane" / "testmondata", - local_seed_stamp=tmp_path / "lane" / "seed.json", - local_seed_attempt=tmp_path / "lane" / "seed-attempt.json", - checkout_root=tmp_path / "lane", - inherited_from=main_root, + +def test_benchmarks_are_not_required_graph_paths_and_packaging_is_untraceable(tmp_path: Path) -> None: + """Restoring benchmark graph edges or dropping packaging inputs makes this fail.""" + benchmark = tmp_path / "tests" / "benchmarks" / "test_scale.py" + benchmark.parent.mkdir(parents=True) + benchmark.write_text("def test_scale(): pass\n", encoding="utf-8") + packaging_python = tmp_path / "packaging" / "hatch_build.py" + packaging_python.parent.mkdir() + packaging_python.write_text("def build_hook(): pass\n", encoding="utf-8") + + impact = classify_native_testmon_changes( + tmp_path, + ( + "tests/benchmarks/test_scale.py", + "tests/benchmarks/deleted.py", + "packaging/polylogue.nix", + "packaging/hatch_build.py", + "docs/release.md", + ), ) - assert not (tmp_path / "lane" / "seed.json").exists() + + assert impact.executable_paths == () + assert impact.runtime_data_paths == ("packaging/hatch_build.py", "packaging/polylogue.nix") -def test_markerless_complete_bootstrap_passes_guard_and_verify_preflight( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_testmon_schema_matches_the_tested_dependency_contract(tmp_path: Path) -> None: + """Changing the pinned testmon schema or required columns makes this fail.""" + import testmon.db + + database = tmp_path / "testmondata" + db = testmon.db.DB(str(database)) + try: + assert _testmon_schema_version() == 14 + assert tuple(db.con.execute("PRAGMA user_version").fetchone()) == (14,) + for table, expected in { + "environment": {"id", "environment_name", "system_packages", "python_version"}, + "file_fp": {"id", "filename", "method_checksums", "mtime", "fsha"}, + "test_execution": {"id", "environment_id", "test_name", "duration", "failed", "forced"}, + "test_execution_file_fp": {"test_execution_id", "fingerprint_id"}, + }.items(): + columns = {row[1] for row in db.con.execute(f"PRAGMA table_info({table})")} + assert expected <= columns + finally: + db.con.close() + + +def test_environment_digest_changes_with_collection_semantics( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - main_root = tmp_path / "main" - main_data = main_root / "testmondata" - nodeid = "tests/test.py::test_passed" - _write_sqlite_db(main_data, rows=(nodeid,)) - attempt = main_root / "seed-attempt.json" - attempt.write_text( - json.dumps( - { - "protocol_version": PROTOCOL_VERSION, - "status": "complete", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - "terminal_authorization": None, - }, - "selection": {"selected_count": 1, "selected_nodeids_omitted": 0}, - "expected_nodeids": [nodeid], - "expected_count": 1, - "expected_digest": hashlib.sha256(nodeid.encode()).hexdigest(), - "node_outcomes": [{"nodeid": nodeid, "outcome": "passed"}], - "exit_code": 0, - "verification_scope": "release-baseline", - "release_baseline_allowed": True, - "run_id": "green-run", - "artifact_dir": ".cache/verify/runs/green-run", - "testmon_data": file_fingerprint(main_data), - } - ) - ) - artifact = main_root / ".cache" / "verify" / "runs" / "green-run" - artifact.mkdir(parents=True) - (artifact / "run.json").write_text( - json.dumps( + (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-q'\n", encoding="utf-8") + initial = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + + (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-ra'\n", encoding="utf-8") + config_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + monkeypatch.setattr("devtools.testmon_bootstrap._installed_distributions", lambda: (("pytest", "changed"),)) + distributions_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + monkeypatch.setenv("POLYLOGUE_CI", "testmon-digest-contract") + managed_environment_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=include") + profile_changed = _testmon_environment_digest(tmp_path, pytest_profile="slow=exclude") + + assert ( + len( { - "run_id": "green-run", - "checkout_root": str(main_root.resolve()), - "artifact_dir": ".cache/verify/runs/green-run", + initial, + config_changed, + distributions_changed, + managed_environment_changed, + profile_changed, } ) + == 5 ) - lane = tmp_path / "lane" - lane.mkdir() - (lane / ".git").write_text("gitdir: /main/.git/worktrees/lane\n") - (lane / ".venv" / "bin").mkdir(parents=True) - package = lane / "polylogue" - package.mkdir() - (package / "__init__.py").write_text("") - local_data = lane / ".cache" / "testmon" / "testmondata" - local_stamp = lane / ".cache" / "testmon" / "seed.json" - local_attempt = lane / ".cache" / "testmon" / "seed-attempt.json" - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - main_testmon_data=main_data, - main_seed_stamp=main_root / "seed.json", - main_seed_attempt=attempt, - protocol_version=PROTOCOL_VERSION, - ) - assert decision.selection_only - assert bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - local_seed_attempt=local_attempt, - checkout_root=lane, - inherited_from=main_root, - ) - monkeypatch.setattr(checkout_guard, "_is_linked_worktree", lambda _root: True) - fingerprint = checkout_guard.checkout_environment_fingerprint( - lane, - polylogue_import_path=package / "__init__.py", - python_executable=lane / ".venv" / "bin" / "python", - ) - assert not fingerprint.clean - monkeypatch.setattr(verify, "ROOT", lane) - monkeypatch.setattr(verify, "TESTMON_DATA", local_data) - monkeypatch.setattr(verify, "TESTMON_SEED_STAMP", local_stamp) - monkeypatch.setattr(verify, "TESTMON_SEED_ATTEMPT", local_attempt) - assert verify._testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is not None - assert json.loads(local_attempt.read_text())["release_baseline_allowed"] is False - - -def test_local_seed_missing_only_stamp_still_bootstraps(tmp_path: Path) -> None: - """Partial local state (e.g. a stale stamp with no db, or vice versa) still needs a fresh copy.""" - local_stamp = tmp_path / "local" / "seed.json" - local_stamp.parent.mkdir(parents=True, exist_ok=True) - local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "usable"})) - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - - decision = decide_testmon_bootstrap( - is_linked_worktree=True, - local_testmon_data=tmp_path / "local" / "testmondata", - local_seed_stamp=local_stamp, - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - protocol_version=PROTOCOL_VERSION, - ) - assert decision.should_bootstrap - - -def test_bootstrap_seed_files_copies_db_and_stamp(tmp_path: Path) -> None: - main_data = tmp_path / "main?fragment#1" / "testmondata" - main_stamp = tmp_path / "main?fragment#1" / "seed.json" - _write_sqlite_db(main_data, rows=("x", "y", "z")) - _write_valid_seed_stamp(main_stamp) - local_data = tmp_path / "local" / "testmondata" - local_stamp = tmp_path / "local" / "seed.json" - - decision = BootstrapDecision( - True, - "test", - main_testmon_data=main_data, - main_seed_stamp=main_stamp, - ) - assert bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "local", - inherited_from=tmp_path / "main?fragment#1", - ) +def test_environment_digest_changes_when_root_conftest_is_added(tmp_path: Path) -> None: + initial = _testmon_environment_digest(tmp_path) - local_payload = json.loads(local_stamp.read_text()) - source_payload = json.loads(main_stamp.read_text()) - comparable_keys = set(source_payload) - {"binding", "testmon_data"} - assert {key: local_payload[key] for key in comparable_keys} == {key: source_payload[key] for key in comparable_keys} - assert local_payload["binding"]["checkout_root"] == str(tmp_path / "local") - assert local_payload["binding"]["source_checkout_root"] == str(tmp_path / "main?fragment#1") - conn = sqlite3.connect(local_data) - try: - rows = conn.execute("SELECT filename, fsha FROM file_fp ORDER BY filename").fetchall() - finally: - conn.close() - assert rows == [("x", "sha-x"), ("y", "sha-y"), ("z", "sha-z")] - # No temp files left behind. - assert sorted(p.name for p in local_data.parent.iterdir()) == [".cache", "seed.json", "testmondata"] - - -def test_bootstrap_seed_files_marks_destination_and_source_checkout(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - - bootstrap_testmon_seed_files( - BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", + (tmp_path / "conftest.py").write_text( + "def pytest_collection_modifyitems(items):\n items.reverse()\n", + encoding="utf-8", ) - payload = json.loads(local_stamp.read_text()) - assert payload["binding"]["checkout_root"] == str((tmp_path / "lane").resolve()) - assert payload["binding"]["source_checkout_root"] == str((tmp_path / "main").resolve()) - source = json.loads(main_stamp.read_text()) - assert {key: payload[key] for key in source if key not in {"binding", "testmon_data"}} == { - key: source[key] for key in source if key not in {"binding", "testmon_data"} - } - - -def test_bootstrap_seed_files_rejects_paths_outside_or_colliding_with_destination(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - decision = BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp) - local_data = tmp_path / "lane" / "testmondata" - - assert not bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=tmp_path / "outside" / "seed.json", - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", + assert _testmon_environment_digest(tmp_path) != initial + + +def test_environment_digest_ignores_benchmark_conftest_and_its_plugin_declaration(tmp_path: Path) -> None: + benchmark_conftest = tmp_path / "tests" / "benchmarks" / "conftest.py" + benchmark_conftest.parent.mkdir(parents=True) + benchmark_conftest.write_text("pytest_plugins = dynamic_plugin_names\n", encoding="utf-8") + + initial = _testmon_environment_digest(tmp_path) + benchmark_conftest.write_text("pytest_plugins = other_dynamic_plugin_names\n", encoding="utf-8") + + assert _testmon_environment_digest(tmp_path) == initial + + +def test_inactive_runtime_helper_does_not_force_fresh_environment(tmp_path: Path) -> None: + helper = tmp_path / "tests" / "infra" / "runtime_helper.py" + helper.parent.mkdir(parents=True) + helper.write_text("def answer() -> int:\n return 41\n", encoding="utf-8") + initial = _testmon_environment_digest(tmp_path) + + helper.write_text("def answer() -> int:\n return 42\n", encoding="utf-8") + + assert _testmon_environment_digest(tmp_path) == initial + + +def test_declared_local_fixture_plugin_changes_environment(tmp_path: Path) -> None: + conftest = tmp_path / "tests" / "conftest.py" + plugin = tmp_path / "tests" / "infra" / "fixture_plugin.py" + plugin.parent.mkdir(parents=True) + conftest.write_text('pytest_plugins = ("tests.infra.fixture_plugin",)\n', encoding="utf-8") + plugin.write_text("import pytest\n\n@pytest.fixture\ndef value():\n return 1\n", encoding="utf-8") + initial = _testmon_environment_digest(tmp_path) + + plugin.write_text("import pytest\n\n@pytest.fixture\ndef value():\n return 2\n", encoding="utf-8") + + assert _testmon_environment_digest(tmp_path) != initial + + +def test_dynamic_pytest_plugin_declaration_fails_closed(tmp_path: Path) -> None: + conftest = tmp_path / "tests" / "conftest.py" + conftest.parent.mkdir(parents=True) + conftest.write_text( + "from plugin_config import plugin_names\n\npytest_plugins = plugin_names\n", + encoding="utf-8", ) - assert not (tmp_path / "outside" / "seed.json").exists() - assert not local_data.exists() - - assert not bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_data, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", + + with pytest.raises(NativeTestmonRepairError, match="must be a literal string/list/tuple"): + _testmon_environment_digest(tmp_path) + + conftest.write_text( + 'pytest_plugins = []\npytest_plugins.append("tests.infra.fixture_plugin")\n', + encoding="utf-8", ) - assert not local_data.exists() - - -def test_bootstrap_seed_files_keeps_copied_state_when_stamp_turns_invalid(tmp_path: Path) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - main_stamp.parent.mkdir(parents=True, exist_ok=True) - main_stamp.write_text("{concurrent rewrite") - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - - stamped = bootstrap_testmon_seed_files( - BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", + with pytest.raises(NativeTestmonRepairError, match="must be one literal assignment"): + _testmon_environment_digest(tmp_path) + + conftest.write_text( + 'globals()["pytest_plugins"] = ("tests.infra.fixture_plugin",)\n', + encoding="utf-8", ) + with pytest.raises(NativeTestmonRepairError, match="must be one literal assignment"): + _testmon_environment_digest(tmp_path) - assert stamped is False - assert not local_data.exists() - assert not local_stamp.exists() + +def test_environment_digest_ignores_neutralized_pytest_plugins( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plugin = tmp_path / "local_plugin.py" + plugin.write_text("VALUE = 'v1'\n", encoding="utf-8") + monkeypatch.setenv("PYTEST_PLUGINS", "local_plugin") + + initial = _testmon_environment_digest(tmp_path) + plugin.write_text("VALUE = 'v2'\n", encoding="utf-8") + + assert _testmon_environment_digest(tmp_path) == initial -def test_bootstrap_graph_mismatch_publishes_no_destination_state( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("addopts", ["-p local_plugin", "-p=local_plugin"]) +def test_environment_digest_ignores_plugins_from_neutralized_pytest_addopts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + addopts: str, ) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - monkeypatch.setattr(testmon_bootstrap, "refresh_stamp", lambda *_args, **_kwargs: None) - - assert not bootstrap_testmon_seed_files( - BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - assert not local_data.exists() - assert not local_stamp.exists() - assert not (tmp_path / "lane" / ".cache" / "verify" / "current-run.json").exists() + plugin = tmp_path / "local_plugin.py" + plugin.write_text("VALUE = 'v1'\n", encoding="utf-8") + monkeypatch.setenv("PYTEST_ADDOPTS", addopts) + + initial = _testmon_environment_digest(tmp_path) + plugin.write_text("VALUE = 'v2'\n", encoding="utf-8") + assert _testmon_environment_digest(tmp_path) == initial -def test_bootstrap_receipt_rebind_failure_publishes_no_destination_state( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + +def test_environment_digest_ignores_neutralized_pytest_addopts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - main_data = tmp_path / "main" / "testmondata" - main_stamp = tmp_path / "main" / "seed.json" - _write_sqlite_db(main_data) - _write_valid_seed_stamp(main_stamp) - local_data = tmp_path / "lane" / "testmondata" - local_stamp = tmp_path / "lane" / "seed.json" - monkeypatch.setattr(testmon_bootstrap, "_rebind_run_receipt", lambda **_kwargs: False) - - assert not bootstrap_testmon_seed_files( - BootstrapDecision(True, "test", main_testmon_data=main_data, main_seed_stamp=main_stamp), - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - checkout_root=tmp_path / "lane", - inherited_from=tmp_path / "main", - ) - assert not local_data.exists() - assert not local_stamp.exists() - assert not (tmp_path / "lane" / ".cache" / "verify" / "current-run.json").exists() + initial = _testmon_environment_digest(tmp_path) + monkeypatch.setenv("PYTEST_ADDOPTS", "--setup-only --ignore-glob=tests/**") -def test_bootstrap_rebound_attempt_failure_publishes_no_destination_state( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + assert _testmon_environment_digest(tmp_path) == initial + + +def test_environment_digest_stops_at_invocation_deadline(tmp_path: Path) -> None: + with pytest.raises(NativeTestmonDeadlineError, match="invocation deadline"): + _testmon_environment_digest(tmp_path, deadline_monotonic=0.0) + + +def test_plugin_declaration_discovery_stops_at_invocation_deadline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - decision, local_data, local_stamp, local_attempt, lane = _red_attempt_decision(tmp_path) - original_stamp_from_attempt = cast(Callable[..., object], testmon_bootstrap.__dict__["stamp_from_attempt"]) - calls = 0 - - def fail_rebound_attempt(*args: object, **kwargs: object) -> object: - nonlocal calls - calls += 1 - if calls == 2: - return None - return original_stamp_from_attempt(*args, **kwargs) - - monkeypatch.setattr(testmon_bootstrap, "stamp_from_attempt", fail_rebound_attempt) - - assert not bootstrap_testmon_seed_files( - decision, - local_testmon_data=local_data, - local_seed_stamp=local_stamp, - local_seed_attempt=local_attempt, - checkout_root=lane, - inherited_from=tmp_path / "main", - ) - assert not local_data.exists() - assert not local_stamp.exists() - assert not local_attempt.exists() - assert not (lane / ".cache" / "verify" / "current-run.json").exists() + plugin_declaration = tmp_path / "tests" / "test_plugins.py" + plugin_declaration.parent.mkdir() + plugin_declaration.write_text('pytest_plugins = ("fixture_plugin",)\n', encoding="utf-8") + clock = {"value": 0.0} + original_read_text = Path.read_text + + def expire_after_plugin_discovery( + path: Path, + encoding: str | None = None, + errors: str | None = None, + ) -> str: + source = original_read_text(path, encoding=encoding, errors=errors) + if path == plugin_declaration: + clock["value"] = 1.0 + return source + + monkeypatch.setattr("devtools.testmon_bootstrap.time.monotonic", lambda: clock["value"]) + monkeypatch.setattr(Path, "read_text", expire_after_plugin_discovery) + + with pytest.raises(NativeTestmonDeadlineError, match="invocation deadline"): + _testmon_environment_digest(tmp_path, deadline_monotonic=0.5) + +def test_invalid_cleanup_removes_only_owned_sqlite_and_sidecars(tmp_path: Path) -> None: + state_dir = tmp_path / ".cache" / "testmon" + state_dir.mkdir(parents=True) + owned = [state_dir / "testmondata", state_dir / "testmondata-wal", state_dir / "testmondata-shm"] + unrelated = state_dir / "keep.txt" + for path in (*owned, unrelated): + path.write_text(path.name, encoding="utf-8") -def test_maybe_bootstrap_does_not_migrate_an_untyped_legacy_local_stamp( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + removed = remove_invalid_native_testmon_state(tmp_path) + + assert set(removed) == set(owned) + assert unrelated.read_text(encoding="utf-8") == "keep.txt" + + +def test_invalid_cleanup_refuses_directory_at_database_path(tmp_path: Path) -> None: + (tmp_path / ".cache" / "testmon" / "testmondata").mkdir(parents=True) + + with pytest.raises(NativeTestmonRepairError, match="refusing to remove directory"): + remove_invalid_native_testmon_state(tmp_path) + + +@pytest.mark.parametrize("symlinked_parent", [".cache", ".cache/testmon"]) +def test_invalid_cleanup_refuses_symlinked_state_parents( + tmp_path: Path, + symlinked_parent: str, ) -> None: - lane = tmp_path / "lane" - main = tmp_path / "main" - local_data = lane / "cache" / "testmondata" - local_stamp = lane / "cache" / "seed.json" - _write_sqlite_db(local_data) - local_stamp.parent.mkdir(parents=True, exist_ok=True) - local_stamp.write_text(json.dumps({"protocol_version": PROTOCOL_VERSION, "status": "complete"})) - monkeypatch.setattr(testmon_bootstrap, "_git_worktree_info", lambda _root: (True, main)) - - message = testmon_bootstrap.maybe_bootstrap_testmon_seed( - lane, - testmon_data_relpath="cache/testmondata", - seed_stamp_relpath="cache/seed.json", - protocol_version=PROTOCOL_VERSION, - ) + outside = tmp_path / "outside" + outside.mkdir() + sentinel = outside / "testmon" / "testmondata" if symlinked_parent == ".cache" else outside / "testmondata" + sentinel.parent.mkdir(parents=True, exist_ok=True) + sentinel.write_text("external state", encoding="utf-8") + parent = tmp_path / symlinked_parent + parent.parent.mkdir(parents=True, exist_ok=True) + parent.symlink_to(outside, target_is_directory=True) + + with pytest.raises(NativeTestmonRepairError, match="symlinked owned testmon parent"): + remove_invalid_native_testmon_state(tmp_path) - assert message is None - assert json.loads(local_stamp.read_text())["status"] == "complete" + assert sentinel.read_text(encoding="utf-8") == "external state" -def test_bootstrap_seed_files_noop_when_decision_says_no(tmp_path: Path) -> None: - local_data = tmp_path / "local" / "testmondata" - local_stamp = tmp_path / "local" / "seed.json" - decision = BootstrapDecision(False, "not needed") +def test_native_preparation_rejects_symlinked_state_parent_before_inspection(tmp_path: Path) -> None: + outside = tmp_path / "outside" + outside.mkdir() + cache = tmp_path / ".cache" + cache.mkdir() + (cache / "testmon").symlink_to(outside, target_is_directory=True) - bootstrap_testmon_seed_files(decision, local_testmon_data=local_data, local_seed_stamp=local_stamp) + with pytest.raises(NativeTestmonRepairError, match="symlinked owned testmon parent"): + prepare_native_testmon_environment(tmp_path) - assert not local_data.exists() - assert not local_stamp.exists() + +def test_native_inspection_rejects_symlinked_database_before_sqlite_open(tmp_path: Path) -> None: + outside = tmp_path / "outside.db" + outside.write_text("external state", encoding="utf-8") + data = tmp_path / "testmondata" + data.symlink_to(outside) + + state = inspect_native_testmon_environment(data, environment_name="owned-environment") + + assert state.status == "invalid" + assert state.reason == "native testmon database is not a single-link regular file" + assert outside.read_text(encoding="utf-8") == "external state" + + +def test_native_inspection_rejects_symlinked_sidecar_before_sqlite_open(tmp_path: Path) -> None: + data = tmp_path / "testmondata" + data.write_text("not opened", encoding="utf-8") + outside = tmp_path / "outside-wal" + outside.write_text("external sidecar", encoding="utf-8") + sidecar = Path(f"{data}-wal") + sidecar.symlink_to(outside) + + state = inspect_native_testmon_environment(data, environment_name="owned-environment") + + assert state.status == "invalid" + assert state.reason == f"native testmon sidecar is not a single-link regular file: {sidecar}" + assert outside.read_text(encoding="utf-8") == "external sidecar" + + +@pytest.mark.parametrize("suffix", ("", "-wal")) +def test_native_testmon_ownership_rejects_hardlinked_database_and_sidecars(tmp_path: Path, suffix: str) -> None: + state_dir = tmp_path / ".cache" / "testmon" + state_dir.mkdir(parents=True) + outside = tmp_path / f"outside{suffix}" + outside.write_text("external state", encoding="utf-8") + owned = state_dir / f"testmondata{suffix}" + os.link(outside, owned) + + with pytest.raises(NativeTestmonRepairError, match="single-link regular file"): + validate_native_testmon_state_ownership(tmp_path) + + assert outside.read_text(encoding="utf-8") == "external state" + + +@pytest.mark.parametrize("suffix", ("", "-wal")) +def test_native_inspection_rejects_hardlinked_database_and_sidecars(tmp_path: Path, suffix: str) -> None: + data = tmp_path / "testmondata" + if suffix: + data.write_text("not opened", encoding="utf-8") + outside = tmp_path / f"outside{suffix}" + outside.write_text("external state", encoding="utf-8") + owned = Path(f"{data}{suffix}") + os.link(outside, owned) + + state = inspect_native_testmon_environment(data, environment_name="owned-environment") + + assert state.status == "invalid" + subject = "database" if not suffix else "sidecar" + assert state.reason == f"native testmon {subject} is not a single-link regular file" + ( + "" if not suffix else f": {owned}" + ) + assert outside.read_text(encoding="utf-8") == "external state" diff --git a/tests/unit/devtools/test_testmon_mutation_proof.py b/tests/unit/devtools/test_testmon_mutation_proof.py deleted file mode 100644 index 3d1217701b..0000000000 --- a/tests/unit/devtools/test_testmon_mutation_proof.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Real-route tests for the bounded testmon anti-vacuity proof.""" - -from __future__ import annotations - -from devtools.testmon_mutation_proof import _TARGET_NODEID, run_proof - - -def test_real_testmon_mutation_proof() -> None: - result = run_proof() - - assert result.ok, result.failure - assert result.target_nodeid == _TARGET_NODEID - assert _TARGET_NODEID in result.selected_nodeids - assert result.mutation_exit_code != 0 - assert result.restored_exit_code == 0 - assert result.severed_edge_rejected - assert result.unrelated_selected_count < result.total_seeded_nodes - assert result.cleanup_complete diff --git a/tests/unit/devtools/test_testmon_state.py b/tests/unit/devtools/test_testmon_state.py deleted file mode 100644 index 3df6ce4440..0000000000 --- a/tests/unit/devtools/test_testmon_state.py +++ /dev/null @@ -1,455 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import sqlite3 -from pathlib import Path -from unittest.mock import patch - -import pytest - -import devtools.testmon_state as testmon_state -from devtools.testmon_state import ( - BaselineStatus, - GraphStatus, - file_fingerprint, - inspect_testmon_database, - stamp_from_attempt, - validate_stamp, -) -from devtools.testmon_state import ( - TestmonSeedStamp as _TestmonSeedStamp, -) - -PROTOCOL = 4 -NODEIDS = ("tests/test_seed.py::test_passed", "tests/test_seed.py::test_failed") - - -def _write_graph(path: Path, *, failed: bool = False, with_edges: bool = True) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(path) as connection: - connection.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - connection.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - connection.execute("CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT, failed INTEGER)") - connection.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - for index, nodeid in enumerate(NODEIDS, start=1): - connection.execute("INSERT INTO file_fp VALUES (?, ?, ?)", (index, f"file-{index}.py", f"sha-{index}")) - connection.execute( - "INSERT INTO test_execution VALUES (?, ?, ?)", - (index, nodeid, int(failed and index == 2)), - ) - if with_edges: - connection.execute("INSERT INTO test_execution_file_fp VALUES (?, ?)", (index, index)) - - -def _attempt(data: Path, *, outcomes: tuple[str, str] = ("passed", "failed")) -> dict[str, object]: - artifact_dir = data.parent / ".cache" / "verify" / "runs" / "run-red" - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": "run-red", - "checkout_root": str(data.parent.resolve()), - "artifact_dir": ".cache/verify/runs/run-red", - } - ) - ) - return { - "protocol_version": PROTOCOL, - "status": "reusable", - "identity": { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - "terminal_authorization": "narrow-terminal", - }, - "selection": { - "selected_count": len(NODEIDS), - "selected_nodeids_omitted": 0, - }, - "expected_nodeids": list(NODEIDS), - "expected_count": len(NODEIDS), - "expected_digest": hashlib.sha256("\n".join(sorted(NODEIDS)).encode()).hexdigest(), - "verification_scope": "narrow-terminal", - "release_baseline_allowed": False, - "node_outcomes": [ - {"nodeid": nodeid, "outcome": outcome} for nodeid, outcome in zip(NODEIDS, outcomes, strict=True) - ], - "exit_code": 1, - "run_id": "run-red", - "artifact_dir": ".cache/verify/runs/run-red", - "testmon_data": file_fingerprint(data), - } - - -def test_failed_complete_graph_is_selection_only_and_rebindable(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data, failed=True) - - stamp = stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert stamp is not None - assert stamp.baseline_status is BaselineStatus.RED - assert stamp.affected_selection_allowed - assert not stamp.release_baseline_allowed - - passed_outcomes = stamp_from_attempt( - _attempt(data, outcomes=("passed", "passed")), data, checkout_root=tmp_path, protocol_version=PROTOCOL - ) - assert passed_outcomes is not None - assert passed_outcomes.baseline_status is BaselineStatus.RED - assert not passed_outcomes.release_baseline_allowed - - stamp_path = tmp_path / "seed.json" - stamp_path.write_text(json.dumps(stamp.as_dict())) - assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_seed_shard_ledger_rejects_duplicate_nodes_across_shards() -> None: - shard = { - "index": 1, - "nodeids": [NODEIDS[0]], - "nodeid_count": 1, - "nodeid_digest": hashlib.sha256(NODEIDS[0].encode()).hexdigest(), - "status": "complete", - "node_outcomes": [{"nodeid": NODEIDS[0], "outcome": "passed"}], - } - duplicate = {**shard, "index": 2} - - assert testmon_state.validate_seed_shard_ledger([shard, duplicate], expected_nodeids=[NODEIDS[0]]) is None - - -def test_seed_shard_ledger_accepts_expected_and_unexpected_xfail_outcomes() -> None: - nodes = sorted(NODEIDS) - shard = { - "index": 1, - "nodeids": nodes, - "nodeid_count": len(nodes), - "nodeid_digest": hashlib.sha256("\n".join(nodes).encode()).hexdigest(), - "status": "complete", - "node_outcomes": [ - {"nodeid": nodes[0], "outcome": "xfailed"}, - {"nodeid": nodes[1], "outcome": "xpassed"}, - ], - } - - assert testmon_state.validate_seed_shard_ledger([shard], expected_nodeids=nodes) == [shard] - - -def test_expected_failure_and_non_strict_xpass_preserve_green_baseline(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data, outcomes=("xfailed", "xpassed")) - attempt.update(status="complete", exit_code=0, release_baseline_allowed=True, verification_scope="release-baseline") - identity = attempt["identity"] - assert isinstance(identity, dict) - identity.update(skip_slow=False, terminal_authorization=None) - - green = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert green is not None - assert green.baseline_status is BaselineStatus.GREEN - serialized = green.as_dict() - assert _TestmonSeedStamp.from_mapping(serialized, protocol_version=PROTOCOL).baseline_status is BaselineStatus.GREEN - - # pytest controls strict-xpass behavior through its process exit code. A - # strict xpass therefore remains reusable graph evidence but is red. - attempt.update(status="reusable", exit_code=1, release_baseline_allowed=False) - strict = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert strict is not None - assert strict.baseline_status is BaselineStatus.RED - assert strict.affected_selection_allowed - - -def test_testmon_database_canonicalizes_xdist_group_names(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - with sqlite3.connect(data) as connection: - connection.execute( - "UPDATE test_execution SET test_name = ? WHERE test_name = ?", - (f"{NODEIDS[0]}@web-reader", NODEIDS[0]), - ) - - graph = inspect_testmon_database(data, NODEIDS) - - assert graph.missing_nodeids == () - assert graph.recorded_count == len(NODEIDS) - - -def test_omitted_interrupted_and_uncovered_nodes_fail_closed(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - omitted = _attempt(data) - omitted["selection"] = {"selected_count": 1, "selected_nodeids_omitted": 1} - assert stamp_from_attempt(omitted, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - interrupted = _attempt(data, outcomes=("passed", "interrupted")) - assert stamp_from_attempt(interrupted, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - data.unlink() - _write_graph(data, with_edges=False) - assert stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_current_protocol_rejects_a_reusable_attempt_with_a_nonreusable_outcome( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - data = tmp_path / "testmondata" - _write_graph(data, failed=True) - attempt = _attempt(data) - attempt["protocol_version"] = 6 - identity = attempt["identity"] - assert isinstance(identity, dict) - identity["dependency_environment"] = "dependencies" - identity["pytest_harness"] = "harness" - monkeypatch.setattr(testmon_state, "testmon_runtime_identity", lambda _root: ("dependencies", "harness")) - - attempt["outcome"] = "resource-timeout" - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=6) is None - - attempt["outcome"] = "red-baseline" - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=6) is not None - - -def test_stamp_from_attempt_does_not_reopen_the_validated_database(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data) - - with patch("devtools.testmon_state.file_fingerprint", return_value=attempt["testmon_data"]) as fingerprint: - stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert stamp is not None - fingerprint.assert_called_once_with(data) - - -def test_incomplete_attempt_fails_closed(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data, outcomes=("passed", "passed")) - attempt["exit_code"] = 0 - attempt["release_baseline_allowed"] = True - attempt["status"] = "incomplete" - - stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert stamp is None - - attempt["status"] = "complete" - completed = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert completed is not None - assert completed.baseline_status is BaselineStatus.GREEN - assert completed.release_baseline_allowed - - -def test_reusable_attempt_rejects_a_changed_dependency_or_pytest_harness( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """Reusable graphs belong to the environment that captured them.""" - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data) - identity = attempt["identity"] - assert isinstance(identity, dict) - identity["dependency_environment"] = "dependency-environment" - identity["pytest_harness"] = "pytest-harness" - attempt["protocol_version"] = 5 - monkeypatch.setattr( - testmon_state, - "testmon_runtime_identity", - lambda _root: ("dependency-environment", "pytest-harness"), - raising=False, - ) - - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=5) is not None - - identity["dependency_environment"] = "different-environment" - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=5) is None - - -def test_runtime_identity_includes_test_behavior_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setattr(testmon_state, "_installed_distributions", lambda: (("pytest", "9"),)) - monkeypatch.setenv("HYPOTHESIS_PROFILE", "ci") - monkeypatch.setenv("POLYLOGUE_CI", "1") - first = testmon_state.testmon_runtime_identity(tmp_path) - - monkeypatch.setenv("HYPOTHESIS_PROFILE", "default") - second = testmon_state.testmon_runtime_identity(tmp_path) - - assert first is not None - assert second is not None - assert first[0] == second[0] - assert first[1] != second[1] - - -def test_green_skipped_slow_attempt_without_typed_terminal_authority_is_selection_only(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data, outcomes=("passed", "passed")) - attempt["status"] = "complete" - attempt["exit_code"] = 0 - raw_identity = attempt["identity"] - assert isinstance(raw_identity, dict) - identity = dict(raw_identity) - identity["terminal_authorization"] = None - attempt["identity"] = identity - attempt["verification_scope"] = "narrow-terminal" - attempt["release_baseline_allowed"] = False - - stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - - assert stamp is not None - assert stamp.baseline_status is BaselineStatus.RED - assert stamp.affected_selection_allowed - assert not stamp.release_baseline_allowed - - -def test_typed_complete_markerless_attempt_is_selection_only(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data, failed=False) - attempt = _attempt(data, outcomes=("passed", "passed")) - attempt["status"] = "complete" - attempt["exit_code"] = 0 - raw_identity = attempt["identity"] - assert isinstance(raw_identity, dict) - identity = dict(raw_identity) - identity["skip_slow"] = False - identity["terminal_authorization"] = None - attempt["identity"] = identity - attempt["verification_scope"] = "release-baseline" - attempt["release_baseline_allowed"] = True - - published = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - markerless = stamp_from_attempt( - attempt, - data, - checkout_root=tmp_path, - protocol_version=PROTOCOL, - published_marker=False, - ) - - assert published is not None - assert published.release_baseline_allowed - assert markerless is not None - assert markerless.baseline_status is BaselineStatus.RED - assert markerless.affected_selection_allowed - assert not markerless.release_baseline_allowed - - -def test_malformed_sqlite_and_stale_stamp_fail_closed(tmp_path: Path) -> None: - malformed = tmp_path / "malformed" - malformed.write_bytes(b"not sqlite") - inspection = inspect_testmon_database(malformed, NODEIDS) - assert inspection.status is GraphStatus.INVALID - - data = tmp_path / "testmondata" - _write_graph(data) - stamp = stamp_from_attempt(_attempt(data), data, checkout_root=tmp_path, protocol_version=PROTOCOL) - assert stamp is not None - stamp_path = tmp_path / "seed.json" - stamp_path.write_text(json.dumps(stamp.as_dict())) - data.write_bytes(data.read_bytes() + b"stale") - assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_attempt_and_green_stamp_artifacts_fail_closed_when_malformed(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data, outcomes=("passed", "passed")) - attempt["exit_code"] = 0 - attempt["release_baseline_allowed"] = True - attempt["artifact_dir"] = "/tmp/outside-testmon-run" - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - attempt["artifact_dir"] = ".cache/verify/runs/run-red" - attempt["status"] = "complete" - stamp = stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) - assert stamp is not None - receipt = tmp_path / ".cache" / "verify" / "runs" / "run-red" / "run.json" - receipt.unlink() - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - receipt.write_text( - json.dumps( - { - "run_id": "wrong-run", - "checkout_root": str(tmp_path.resolve()), - "artifact_dir": ".cache/verify/runs/run-red", - } - ) - ) - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - receipt.write_text( - json.dumps( - { - "run_id": "run-red", - "checkout_root": str(tmp_path.resolve()), - "artifact_dir": ".cache/verify/runs/run-red", - } - ) - ) - stamp_path = tmp_path / ".cache" / "testmon" / "seed.json" - stamp_path.parent.mkdir(parents=True) - payload = stamp.as_dict() - payload["baseline"]["exit_code"] = 1 - stamp_path.write_text(json.dumps(payload)) - assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - payload = stamp.as_dict() - payload["graph"]["failed_nodeids"] = [NODEIDS[0]] - stamp_path.write_text(json.dumps(payload)) - assert validate_stamp(stamp_path, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_malformed_sqlite_values_fail_closed(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - with sqlite3.connect(data) as connection: - connection.execute("update test_execution set failed = 'bad' where id = 1") - - inspection = inspect_testmon_database(data, NODEIDS) - - assert inspection.status is GraphStatus.INVALID - - -def test_sqlite_paths_with_uri_characters_are_inspected_safely(tmp_path: Path) -> None: - data = tmp_path / "checkout?fragment#1" / "testmondata" - _write_graph(data) - - inspection = inspect_testmon_database(data, NODEIDS) - - assert inspection.status is GraphStatus.COMPLETE - - -@pytest.mark.parametrize("filename", ["../outside.py", "/tmp/outside.py"]) -def test_unsafe_testmon_fingerprint_paths_fail_closed(tmp_path: Path, filename: str) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - with sqlite3.connect(data) as connection: - connection.execute("update file_fp set filename = ? where id = 1", (filename,)) - - assert inspect_testmon_database(data, NODEIDS).status is GraphStatus.INVALID - - -def test_attempt_status_must_be_promotable(tmp_path: Path) -> None: - data = tmp_path / "testmondata" - _write_graph(data) - attempt = _attempt(data) - attempt["status"] = "running" - - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - attempt = _attempt(data) - attempt["run_id"] = None - assert stamp_from_attempt(attempt, data, checkout_root=tmp_path, protocol_version=PROTOCOL) is None - - -def test_stamp_parser_rejects_untyped_or_non_graph_state() -> None: - try: - _TestmonSeedStamp.from_mapping({"protocol_version": PROTOCOL, "status": "complete"}, protocol_version=PROTOCOL) - except ValueError: - pass - else: - raise AssertionError("legacy green-looking stamp must not be accepted") diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 9e3ab287df..147af7967e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -1,12 +1,10 @@ from __future__ import annotations import fcntl -import hashlib import json import os import platform import shutil -import sqlite3 import subprocess import sys import threading @@ -21,26 +19,9 @@ import watchfiles from devtools import run_tests, verify, verify_runs -from devtools.testmon_state import ( - BaselineStatus, - BindingMode, - CollectionStatus, - GraphInspection, - GraphStatus, - file_fingerprint, -) -from devtools.testmon_state import ( - TestmonBinding as _TestmonBinding, -) -from devtools.testmon_state import ( - TestmonIdentity as _TestmonIdentity, -) -from devtools.testmon_state import ( - TestmonSeedStamp as _TestmonSeedStamp, -) -from devtools.testmon_state import ( - testmon_runtime_identity as _testmon_runtime_identity, -) +from devtools.checkout_guard import CheckoutImportMismatchError +from devtools.testmon_bootstrap import NativeTestmonDeadlineError, NativeTestmonRepairError, executable_python_paths +from devtools.verification_contracts import VerificationScope from devtools.verify import ( PYTEST_CONTAINMENT_PATH, PYTEST_EVENTS_PATH, @@ -49,34 +30,18 @@ PYTEST_PROGRESS_PATH, PYTEST_REPORT_PATH, ROOT, - TESTMON_AFFECTED_STAMP, - TESTMON_DATA, - TESTMON_SEED_ATTEMPT, - TESTMON_SEED_PROTOCOL_VERSION, - TESTMON_SEED_SHARD_SIZE, - TESTMON_SEED_STAMP, _anchor_verification_paths, - _checkpoint_testmon_seed_shard, - _finalize_testmon_seed_attempt, - _flatten_seed_outcomes, _format_completion_notification, - _matching_testmon_coverage, + _native_lane_failure_requires_stop, _parse_pytest_test_count, - _prepare_testmon_seed_attempt, - _prepare_testmon_seed_shards, _pytest_command_metadata, _pytest_metadata_from_report, _pytest_stall_timeout_s, _pytest_timeout_s, _read_pytest_report, - _record_testmon_affected_coverage, + _release_baseline_allowed, _run, - _seed_node_outcomes_from_events, - _seed_shard_command, _stop_after_failed_step, - _testmon_database_state, - _testmon_preflight, - _testmon_seed_can_resume, build_verify_steps, main, ) @@ -89,6 +54,7 @@ VerifyRun, adaptive_pytest_runtime_policy, adaptive_pytest_worker_count, + aggregate_native_testmon_run, aggregate_pytest_statistics, append_verify_history, apply_managed_pytest_runtime_policy, @@ -108,17 +74,10 @@ @pytest.fixture(autouse=True) def _isolate_verify_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Keep supervisor and testmon receipts private to each test.""" + """Keep supervisor and native testmon state private to each test.""" monkeypatch.chdir(tmp_path) - for name in ( - "TESTMON_DATA", - "TESTMON_SEED_STAMP", - "TESTMON_SEED_ATTEMPT", - "TESTMON_AFFECTED_STAMP", - ): - isolated = tmp_path / ".cache" / "testmon" / getattr(verify, name).name - monkeypatch.setattr(verify, name, isolated) - monkeypatch.setattr(sys.modules[__name__], name, isolated) + isolated = tmp_path / ".cache" / "testmon" / "testmondata" + monkeypatch.setattr(verify, "TESTMON_DATA", isolated) def _pytest_marker_expr(command: list[str]) -> str: @@ -128,80 +87,8 @@ def _pytest_marker_expr(command: list[str]) -> str: return command[marker_indexes[-1] + 1] -def _testmon_runtime_identity_fields(checkout_root: Path = ROOT) -> dict[str, str]: - runtime_identity = _testmon_runtime_identity(checkout_root) - assert runtime_identity is not None - dependency_environment, pytest_harness = runtime_identity - return {"dependency_environment": dependency_environment, "pytest_harness": pytest_harness} - - -def _write_real_testmon_state(nodeids: tuple[str, ...] = ("tests/test_a.py::test_one",)) -> Path: - TESTMON_DATA.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(TESTMON_DATA) as conn: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute("CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT, failed INTEGER)") - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - for index, nodeid in enumerate(nodeids, start=1): - conn.execute("INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", (index, nodeid, f"sha-{index}")) - conn.execute("INSERT INTO test_execution(id, test_name, failed) VALUES (?, ?, 0)", (index, nodeid)) - conn.execute("INSERT INTO test_execution_file_fp VALUES (?, ?)", (index, index)) - stamp = _TestmonSeedStamp( - TESTMON_SEED_PROTOCOL_VERSION, - CollectionStatus.COMPLETE, - nodeids, - 0, - BaselineStatus.GREEN, - True, - 0, - GraphInspection(GraphStatus.COMPLETE, len(nodeids), len(nodeids), (), 0, 0, None, ()), - _TestmonIdentity( - "current-head", - "covered", - "python", - True, - False, - None, - "narrow-terminal", - **_testmon_runtime_identity_fields(), - ), - _TestmonBinding(BindingMode.EXACT, str(ROOT.resolve())), - file_fingerprint(TESTMON_DATA), - "seed", - ".cache/verify/runs/seed", - ) - TESTMON_SEED_STAMP.parent.mkdir(parents=True, exist_ok=True) - artifact_dir = ROOT / ".cache" / "verify" / "runs" / "seed" - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": "seed", - "checkout_root": str(ROOT.resolve()), - "artifact_dir": ".cache/verify/runs/seed", - } - ) - ) - TESTMON_SEED_STAMP.write_text(json.dumps(stamp.as_dict())) - return TESTMON_DATA - - -def _write_run_receipt(root: Path, run_id: str) -> None: - artifact_dir = root / ".cache" / "verify" / "runs" / run_id - artifact_dir.mkdir(parents=True, exist_ok=True) - (artifact_dir / "run.json").write_text( - json.dumps( - { - "run_id": run_id, - "checkout_root": str(root.resolve()), - "artifact_dir": f".cache/verify/runs/{run_id}", - } - ) - ) - - def test_quick_verify_omits_pytest() -> None: - steps = build_verify_steps(quick=True, lab=False, skip_slow=False) + steps = build_verify_steps(quick=True, lab=False) labels = [label for label, _command in steps] assert labels == [ @@ -218,494 +105,91 @@ def test_quick_verify_omits_pytest() -> None: ] -def test_default_verify_uses_adaptive_pytest_testmon(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 8) - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - - label, command = steps[-1] - assert label == "pytest testmon" - assert "--testmon" in command - assert "--testmon-noselect" not in command - assert "--testmon-forceselect" in command - assert "-n" in command - assert command[command.index("-n") + 1] == "8" - assert "--dist=loadgroup" in command - - -def test_broad_default_verify_uses_parallel_testmon(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 8) - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, broad_testmon=True) - - label, command = steps[-1] - assert label == "pytest testmon (broad)" - assert "--testmon" in command - assert "--testmon-forceselect" in command - assert command[command.index("-n") + 1] == "8" - - -def test_pytest_step_requests_structured_json_report() -> None: - """Every pytest invocation must emit the report consumed by verify and dashboards (#1026).""" - for kwargs in ( - {"seed_testmon": True}, - {"full_pytest": True}, - {}, # default testmon - ): - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, **kwargs) - pytest_steps = [(label, command) for label, command in steps if label.startswith("pytest")] - assert pytest_steps, kwargs - # Every pytest lane emits a structured JSON report. - for label, command in pytest_steps: - assert "--json-report" in command, f"{label}: {command}" - assert any(arg.startswith("--json-report-file=") for arg in command), label - assert command[command.index("-p") + 1] == "devtools.pytest_progress_plugin" - # The canonical report path consumed by verify/dashboards is emitted by - # the primary lane; the #1775 isolated lane writes its own file. - expected_target = f"--json-report-file={PYTEST_REPORT_PATH}" - assert any(expected_target in command for _label, command in pytest_steps), kwargs - - -def test_seed_testmon_runs_full_collection_without_selection(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("POLYLOGUE_PYTEST_WORKERS", raising=False) +@pytest.mark.parametrize( + ("mode", "selection_flag"), + [("affected", "--testmon-forceselect"), ("bootstrap", "--testmon-noselect"), ("full", "--testmon-noselect")], +) +def test_native_testmon_uses_exactly_two_semantic_lanes( + monkeypatch: pytest.MonkeyPatch, + mode: str, + selection_flag: str, +) -> None: monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 8) - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, seed_testmon=True) - - label, command = steps[-1] - assert label == "pytest seed-testmon collect" - assert "--collect-only" in command - assert command[command.index("--ignore=tests/benchmarks")] == "--ignore=tests/benchmarks" - assert "--testmon" not in command - assert "-n" in command - assert command[command.index("-n") + 1] == "0" - - -def test_seed_testmon_caps_adaptive_workers(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("POLYLOGUE_PYTEST_WORKERS", raising=False) - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 12) - - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, seed_testmon=True) - - label, command = steps[-1] - assert label == "pytest seed-testmon collect" - assert command[command.index("-n") + 1] == "0" - - -def test_seed_shards_are_deterministic_and_use_managed_xdist(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _environment: 64) - expected = sorted(f"tests/test_seed.py::test_{index:03d}" for index in range(TESTMON_SEED_SHARD_SIZE + 2)) - prepared = _prepare_testmon_seed_shards( - {"resume": False, "expected_nodeids": []}, - selection={ - "selected_count": len(expected), - "selected_nodeids": list(reversed(expected)), - "selected_nodeids_omitted": 0, - }, - ) - - shards = prepared["shards"] - assert [shard["nodeid_count"] for shard in shards] == [TESTMON_SEED_SHARD_SIZE, 2] - assert shards[0]["nodeids"] == expected[:TESTMON_SEED_SHARD_SIZE] - assert shards[1]["nodeids"] == expected[TESTMON_SEED_SHARD_SIZE:] - nodeids_file = tmp_path / "seed-shard.args" - command = _seed_shard_command(["pytest", "--collect-only", "-n", "0"], shards[0], nodeids_file=nodeids_file) - assert "--collect-only" not in command - assert command[command.index("-n") + 1] == "10" - assert "--testmon" in command - assert "--testmon-noselect" in command - assert "--dist=loadgroup" in command - assert command.count("-n") == 1 - assert command[command.index("-n") + 1] == "10" - assert command[-1] == f"@{nodeids_file}" - assert nodeids_file.read_text().splitlines() == expected[:TESTMON_SEED_SHARD_SIZE] - - -def test_seed_outcomes_normalize_xdist_group_suffix(tmp_path: Path) -> None: - expected = ["tests/test_seed.py::test_grouped"] - events = tmp_path / "events.jsonl" - events.write_text( - json.dumps( - { - "event": "test_report", - "nodeid": f"{expected[0]}@web-reader", - "when": "call", - "outcome": "passed", - } - ) - + "\n" - ) - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=expected, - database={"node_outcomes": {}}, - pytest_step=None, + steps = build_verify_steps( + quick=False, + lab=False, + testmon_mode=mode, + testmon_environment="env-digest", ) - assert outcomes == [ - { - "nodeid": expected[0], - "outcome": "passed", - "reason": "test call passed", - "started": False, - "finished": False, - "phases": [{"when": "call", "outcome": "passed", "duration_s": None}], - } - ] - - -def test_seed_shard_checkpoint_preserves_completed_shards_for_resume( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - expected = ["tests/test_seed.py::test_b", "tests/test_seed.py::test_a"] - ordered = sorted(expected) - prepared = _prepare_testmon_seed_shards( - {"resume": False, "expected_nodeids": []}, - selection={"selected_count": 2, "selected_nodeids": expected, "selected_nodeids_omitted": 0}, - ) - prepared["shards"] = [ - { - **prepared["shards"][0], - "nodeids": [ordered[0]], - "nodeid_count": 1, - "nodeid_digest": hashlib.sha256(ordered[0].encode()).hexdigest(), - }, - { - **prepared["shards"][0], - "index": 2, - "nodeids": [ordered[1]], - "nodeid_count": 1, - "nodeid_digest": hashlib.sha256(ordered[1].encode()).hexdigest(), - "status": "pending", - "node_outcomes": [], - }, + pytest_steps = [(label, command) for label, command in steps if label.startswith("pytest")] + assert [label for label, _command in pytest_steps] == [ + f"pytest native parallel ({mode})", + f"pytest native serial ({mode})", ] - _atomic_payload = { - **prepared, - "expected_nodeids": ordered, - "expected_count": 2, - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - } - artifact_dir = tmp_path / "shard-1" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": 1, - "selected_nodeids": [f"{ordered[0]}@web-reader"], - "selected_nodeids_omitted": 0, - } - ) - ) - (artifact_dir / "events.jsonl").write_text( - json.dumps( - { - "event": "test_report", - "nodeid": f"{ordered[0]}@web-reader", - "when": "call", - "outcome": "xfailed", - } - ) - + "\n" - ) - - checkpointed = _checkpoint_testmon_seed_shard( - prepared=_atomic_payload, - shard_index=1, - step={"name": "pytest seed-testmon shard 1/2", "exit": 0, "artifact_dir": str(artifact_dir)}, - ) - - assert checkpointed["shards"][0]["status"] == "complete" - assert checkpointed["shards"][1]["status"] == "pending" - assert json.loads(TESTMON_SEED_ATTEMPT.read_text())["shards"][0]["node_outcomes"][0]["outcome"] == "xfailed" - resumed = _prepare_testmon_seed_attempt( - identity={ - "git_head": "head", - "git_tree": "tree", - "worktree_fingerprint": "fingerprint", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - run=VerifyRun(tier="seed-testmon", argv=[], git_head="head", polylogue_import_path="polylogue"), - resume=True, - ) - assert resumed["shards"][0]["status"] == "complete" - assert resumed["shards"][1]["status"] == "pending" - - -def test_seed_shard_checkpoint_does_not_trust_preexisting_testmon_rows( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - expected = ["tests/test_seed.py::test_only_database_row"] - prepared = _prepare_testmon_seed_shards( - {"resume": False, "expected_nodeids": []}, - selection={"selected_count": 1, "selected_nodeids": expected, "selected_nodeids_omitted": 0}, - ) - artifact_dir = tmp_path / "shard-1" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps({"selected_count": 1, "selected_nodeids": expected, "selected_nodeids_omitted": 0}) - ) - (artifact_dir / "events.jsonl").write_text("") - monkeypatch.setattr( - "devtools.verify._testmon_database_state", - lambda _nodeids: { - "recorded_count": 1, - "failed_count": 0, - "dependency_edge_count": 0, - "missing_nodeids": [], - "failed_nodeids": [], - "node_outcomes": {expected[0]: "passed"}, - "error": None, - "graph_status": "complete", - "orphan_execution_edges": 0, - "orphan_fingerprint_edges": 0, - }, - ) - - checkpointed = _checkpoint_testmon_seed_shard( - prepared=prepared, - shard_index=1, - step={"name": "pytest seed-testmon shard 1/1", "exit": 0, "artifact_dir": str(artifact_dir)}, - ) - - shard = checkpointed["shards"][0] - assert shard["status"] == "incomplete" - assert shard["node_outcomes"][0]["outcome"] == "missing" - - -def test_seed_outcome_does_not_infer_call_success_from_teardown( - tmp_path: Path, + parallel = pytest_steps[0][1] + serial = pytest_steps[1][1] + assert _pytest_marker_expr(parallel) == "not load_sensitive" + assert _pytest_marker_expr(serial) == "load_sensitive" + assert parallel[parallel.index("-n") + 1] == "8" + assert serial[serial.index("-n") + 1] == "0" + for _label, command in pytest_steps: + assert "--testmon" in command + assert "--testmon-env=env-digest" in command + assert selection_flag in command + assert "--json-report" in command + assert command[command.index("-p") + 1] == "devtools.pytest_progress_plugin" + assert verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS in command + assert all(arg in command for arg in verify._PYTEST_CLOSED_WORLD_COLLECTION_ARGS) + assert all(arg in command for arg in verify._PYTEST_MANAGED_PLUGIN_ARGS) + assert command.count("tests") == 1 + assert "--override-ini=python_files=test_*.py *_test.py fuzz_*.py" in command + assert "--override-ini=python_classes=Test" in command + assert "--override-ini=python_functions=test" in command + assert "--override-ini=norecursedirs=" in command + + +@pytest.mark.parametrize("mode", ["affected", "bootstrap", "full"]) +def test_native_lane_command_contract_rejects_unowned_selectors( + monkeypatch: pytest.MonkeyPatch, + mode: str, ) -> None: - nodeid = "tests/test_seed.py::test_call_missing" - events = tmp_path / "events.jsonl" - events.write_text( - json.dumps({"event": "test_started", "nodeid": nodeid}) - + "\n" - + json.dumps({"event": "test_report", "nodeid": nodeid, "when": "teardown", "outcome": "passed"}) - + "\n" - + json.dumps({"event": "test_finished", "nodeid": nodeid}) - + "\n" - ) - - without_database = _seed_node_outcomes_from_events( - events, - expected_nodeids=[nodeid], - database={"node_outcomes": {}}, - pytest_step={"exit": 0}, - ) - assert without_database[0]["outcome"] == "missing" - - with_failed_database = _seed_node_outcomes_from_events( - events, - expected_nodeids=[nodeid], - database={"node_outcomes": {nodeid: "failed"}}, - pytest_step={"exit": 1}, - ) - assert with_failed_database[0]["outcome"] == "failed" - - -def test_seed_shard_failure_remains_visible_and_blocks_release(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - expected = ["tests/test_seed.py::test_failed"] - prepared = _prepare_testmon_seed_shards( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "run_id": "sharded-failure", - "artifact_dir": ".cache/verify/runs/sharded-failure", - }, - selection={"selected_count": 1, "selected_nodeids": expected, "selected_nodeids_omitted": 0}, - ) - artifact_dir = tmp_path / "shard-failure" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps({"selected_count": 1, "selected_nodeids": expected, "selected_nodeids_omitted": 0}) - ) - (artifact_dir / "events.jsonl").write_text( - json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "failed"}) + "\n" - ) - TESTMON_DATA.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("create table environment (id integer primary key, environment_name text)") - connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") - connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") - connection.execute("create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)") - connection.execute("insert into test_execution values (1, ?, 1)", (expected[0],)) - connection.execute("insert into file_fp values (1, 'test_seed.py', 'sha')") - connection.execute("insert into test_execution_file_fp values (1, 1)") - _write_run_receipt(tmp_path, "sharded-failure") - - checkpointed = _checkpoint_testmon_seed_shard( - prepared=prepared, - shard_index=1, - step={"name": "pytest seed-testmon shard 1/1", "exit": 1, "artifact_dir": str(artifact_dir)}, - ) - receipt = _finalize_testmon_seed_attempt( - prepared=checkpointed, - step_results=[{"name": "pytest seed-testmon shard 1/1", "exit": 1, "artifact_dir": str(artifact_dir)}], - exit_code=1, - ) - - assert receipt["shards"][0]["status"] == "complete" - assert receipt["unsuccessful_nodeids"] == expected - assert receipt["release_baseline_allowed"] is False - - -def test_resumed_seed_uses_affected_selection_for_remaining_tests() -> None: + monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 2) steps = build_verify_steps( quick=False, lab=False, - skip_slow=False, - seed_testmon=True, - resume_testmon_seed=True, + testmon_mode=mode, + testmon_environment="env-digest", ) + label, command = next(step for step in steps if "parallel" in step[0]) - label, command = steps[-1] - assert label == "pytest seed-testmon collect (resume)" - assert "--collect-only" in command - assert "--testmon" not in command - - -def test_full_verify_includes_full_pytest_without_testmon(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("POLYLOGUE_PYTEST_WORKERS", raising=False) - monkeypatch.setattr("devtools.verify.adaptive_pytest_worker_count", lambda _env: 8) - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, full_pytest=True) - - # #1775: the full diagnostic runs as two lanes — a parallel bulk lane plus a - # single-process isolated lane for load-sensitive/tui tests. Neither uses - # testmon; the bulk lane keeps xdist parallelism, the isolated lane forces -n 0. - labels = [label for label, _command in steps] - assert labels[-2:] == ["pytest full (parallel)", "pytest load-sensitive (isolated)"] - - bulk_label, bulk_command = steps[-2] - assert bulk_label == "pytest full (parallel)" - assert "--testmon" not in bulk_command - assert "-n" in bulk_command - assert bulk_command[bulk_command.index("-n") + 1] == "8" - assert "--dist=loadgroup" in bulk_command - - isolated_label, isolated_command = steps[-1] - assert isolated_label == "pytest load-sensitive (isolated)" - assert "--testmon" not in isolated_command - assert isolated_command[isolated_command.index("-n") + 1] == "0" - - -def test_seed_collection_refuses_parallel_worker_overrides(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("POLYLOGUE_PYTEST_WORKERS", "4") - - steps = build_verify_steps(quick=False, lab=False, skip_slow=False, seed_testmon=True) - - label, command = steps[-1] - assert label == "pytest seed-testmon collect" - assert command[command.index("-n") + 1] == "0" - - -def test_seed_defaults_to_managed_scratch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - shm, scratch = _patch_basetemp_roots(monkeypatch, tmp_path, realm_mounted=True) - _patch_resource_capacity(monkeypatch, shm=shm, scratch=scratch, available_mb=8192) - for name in ( - "POLYLOGUE_PYTEST_BASETEMP_ROOT", - "POLYLOGUE_PYTEST_TMPFS", - "POLYLOGUE_PYTEST_TMPFS_MAX_MB", - "POLYLOGUE_PYTEST_BASETEMP_REQUIRED_MB", - "POLYLOGUE_PYTEST_BASETEMP_MIN_FREE_MB", - ): - monkeypatch.delenv(name, raising=False) - completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") - - with ( - patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed) as run, - patch("devtools.verify._read_pytest_report", return_value=None), + assert verify._native_pytest_command_is_closed_world(label, command) + for narrowed in ( + [*command, "--ignore=tests/unit"], + [*command, "tests/unit"], + [*command, "--deselect=tests/test_failure.py::test_failure"], + [*command, "--override-ini=python_files=test_owned.py"], ): - rc, _elapsed, metadata = _run("pytest seed-testmon", ["pytest", "--testmon", "--testmon-noselect"]) - - assert rc == 0 - assert metadata["pytest_tmpfs"] is False - assert run.call_args.kwargs["env"]["POLYLOGUE_PYTEST_TMPFS"] == "0" - assert run.call_args.kwargs["env"]["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(scratch) - assert run.call_args.kwargs["env"]["POLYLOGUE_PYTEST_SELECTION_NODEID_LIMIT"] == "50000" - - -def test_default_testmon_worker_count_can_be_overridden(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("POLYLOGUE_PYTEST_WORKERS", "3") - - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - - label, command = steps[-1] - assert label == "pytest testmon" - assert command[command.index("-n") + 1] == "3" - - -def test_marker_filters_keep_testmon_selection_forced() -> None: - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - - label, command = steps[-1] - assert label == "pytest testmon" - marker_expr = _pytest_marker_expr(command) - assert "not benchmark" in marker_expr - assert "not scale_medium" in marker_expr - assert "not scale_large" in marker_expr - assert "--testmon-forceselect" in command - - -def test_skip_slow_composes_with_forced_testmon_selection() -> None: - steps = build_verify_steps(quick=False, lab=False, skip_slow=True) + assert not verify._native_pytest_command_is_closed_world(label, narrowed) - label, command = steps[-1] - assert label == "pytest testmon" - # Scale-tier policy (#1183): the default verify gate filters out - # ``scale_medium``/``scale_large``; ``--skip-slow`` composes with that - # filter via ``and`` rather than replacing it. - marker_expr = _pytest_marker_expr(command) - assert "not benchmark" in marker_expr - assert "not slow" in marker_expr - assert "not scale_medium" in marker_expr - assert "not scale_large" in marker_expr - assert "--testmon-forceselect" in command +def test_native_corpus_excludes_only_benchmark_directory() -> None: + complete_steps = build_verify_steps( + quick=False, + lab=True, + testmon_environment="env-digest", + ) -def test_default_verify_excludes_medium_and_large_scale_markers() -> None: - """Default verify pytest step deselects the medium/large scale tiers (#1183).""" - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - - label, command = steps[-1] - assert label == "pytest testmon" - marker_expr = _pytest_marker_expr(command) - assert "not benchmark" in marker_expr - assert "not scale_medium" in marker_expr - assert "not scale_large" in marker_expr - # ``scale_small`` is *not* excluded — it runs in the default gate. - assert "scale_small" not in marker_expr - - -def test_lab_verify_includes_medium_scale_marker() -> None: - """``--lab`` lets ``scale_medium`` into the pytest step but still gates ``scale_large`` (#1183).""" - steps = build_verify_steps(quick=False, lab=True, skip_slow=False) - - pytest_step = next((label, command) for label, command in steps if label.startswith("pytest")) - label, command = pytest_step - marker_expr = _pytest_marker_expr(command) - assert "not benchmark" in marker_expr - assert "not scale_large" in marker_expr - assert "not scale_medium" not in marker_expr - assert "scale_small" not in marker_expr + complete_command = next(command for label, command in complete_steps if "parallel" in label) + complete_expr = _pytest_marker_expr(complete_command) + assert complete_expr == "not load_sensitive" + assert "--ignore=tests/benchmarks" in complete_command + assert "--ignore=tests/integration" not in complete_command def test_lab_verify_delegates_to_lab_smoke() -> None: - steps = build_verify_steps(quick=True, lab=True, skip_slow=False) + steps = build_verify_steps(quick=True, lab=True) labels = [label for label, _command in steps] assert "lab smoke" in labels @@ -717,175 +201,6 @@ def test_lab_verify_delegates_to_lab_smoke() -> None: ) -def test_testmon_preflight_requires_seed_when_database_missing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert "devtools verify --seed-testmon" in message - - -def test_testmon_preflight_requires_seed_stamp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert ".cache/testmon/seed.json" in message - - -def test_testmon_preflight_accepts_seeded_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - _write_real_testmon_state() - - assert _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) is None - - -def test_testmon_preflight_rejects_stale_database_fingerprint( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - monkeypatch.chdir(tmp_path) - _write_real_testmon_state() - TESTMON_DATA.write_bytes(TESTMON_DATA.read_bytes() + b"stale") - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert "stale" in message - assert capsys.readouterr().err == "" - - -def test_testmon_preflight_rejects_malformed_sqlite_state( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("mutated") - seed_stamp = tmp_path / ".cache" / "testmon" / "seed.json" - seed_stamp.parent.mkdir(parents=True, exist_ok=True) - seed_stamp.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "usable", - } - ) - ) - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert "stale" in message or "malformed" in message - assert capsys.readouterr().err == "" - - -def test_testmon_preflight_rejects_incomplete_seed_receipt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - seed_stamp = tmp_path / ".cache" / "testmon" / "seed.json" - seed_stamp.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "incomplete", - "git_head": "current-head", - "testmon_data": hashlib.sha256(b"partial").hexdigest(), - } - ) - ) - - message = _testmon_preflight(seed_testmon=False, full_pytest=False, quick=False, commit=False) - - assert message is not None - assert "stale" in message or "malformed" in message - - -def test_matching_incomplete_seed_is_resumable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - identity = { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": True, - "lab": False, - "terminal_authorization": None, - } - TESTMON_SEED_ATTEMPT.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "incomplete", - "identity": identity, - "expected_nodeids": ["tests/unit/test_example.py::test_one"], - "expected_count": 1, - "expected_digest": hashlib.sha256(b"tests/unit/test_example.py::test_one").hexdigest(), - "run_id": "interrupted", - "started_at": "2026-08-05T12:00:00+00:00", - "testmon_data_before": "partial", - } - ) - ) - - assert _testmon_seed_can_resume(identity) is True - assert _testmon_seed_can_resume({**identity, "git_head": "other", "git_tree": "tree-hash"}) is True - assert _testmon_seed_can_resume({**identity, "git_tree": "different-tree"}) is False - assert _testmon_seed_can_resume({**identity, "worktree_fingerprint": "changed"}) is False - assert _testmon_seed_can_resume({**identity, "skip_slow": False}) is False - - -def test_two_interrupted_resumes_flatten_all_carried_outcomes(tmp_path: Path) -> None: - monkeypatch = pytest.MonkeyPatch() - monkeypatch.chdir(tmp_path) - try: - expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - identity = { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": False, - "lab": False, - "terminal_authorization": None, - } - TESTMON_SEED_ATTEMPT.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "incomplete", - "identity": identity, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - "node_outcomes": [{"nodeid": expected[0], "outcome": "passed"}], - } - ) - ) - first = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="head", root=tmp_path) - _prepare_testmon_seed_attempt(identity=identity, run=first, resume=True) - first_payload = json.loads(TESTMON_SEED_ATTEMPT.read_text()) - first_payload["status"] = "incomplete" - first_payload["node_outcomes"] = [{"nodeid": expected[1], "outcome": "passed"}] - TESTMON_SEED_ATTEMPT.write_text(json.dumps(first_payload)) - - second = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="head", root=tmp_path) - prepared = _prepare_testmon_seed_attempt(identity=identity, run=second, resume=True) - - assert {item["nodeid"] for item in prepared["prior_node_outcomes"]} == set(expected) - assert {item["outcome"] for item in prepared["prior_node_outcomes"]} == {"passed"} - assert _flatten_seed_outcomes(prepared) == prepared["prior_node_outcomes"] - finally: - monkeypatch.undo() - - def test_focused_run_can_record_typed_affected_scope(tmp_path: Path) -> None: run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) @@ -1120,39 +435,137 @@ def test_aggregate_pytest_statistics_recognizes_completed_empty_report(tmp_path: assert result["outcomes"] == {} -def test_verify_run_statistics_only_cover_pytest_steps(tmp_path: Path) -> None: - run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) - artifacts = run.start_step(label="ruff check", cmd=["ruff", "check"]) - - run.finish_step(step_id=artifacts.step_id, result={"exit": 0, "duration_s": 0.1}) - - step = run._payload["steps"][0] - assert "statistics" not in step - assert not artifacts.statistics_path.exists() - - -def test_verify_run_embeds_compact_statistics_before_worktree_cleanup(tmp_path: Path) -> None: - run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) - artifacts = run.start_step(label="pytest focused", cmd=["pytest", "tests/unit/example.py"]) - artifacts.events_merged_path.write_text( +def test_aggregate_pytest_statistics_merges_partial_canonical_and_event_outcomes(tmp_path: Path) -> None: + """Returning early on a partial canonical report drops the event-only node.""" + step = tmp_path / "step" + step.mkdir() + (step / "events.jsonl").write_text( json.dumps( { "event": "test_report", - "nodeid": "tests/unit/example.py::test_one", + "nodeid": "event-only", "when": "call", - "duration_s": 0.25, "outcome": "passed", - "worker_id": "controller", + "duration_s": 0.1, } ) + "\n" ) - - run.finish_step(step_id=artifacts.step_id, result={"exit": 0, "duration_s": 0.25}) - payload = run.finish(exit_code=0, duration_s=0.25) - shutil.rmtree(run.run_dir) - - statistics = payload["steps"][0]["statistics"] + (step / "pytest-report.json").write_text( + json.dumps( + { + "tests": [ + { + "nodeid": "canonical-only", + "outcome": "xfailed", + "call": {"outcome": "skipped", "duration": 0.1}, + } + ] + } + ) + ) + + result = aggregate_pytest_statistics(step) + + assert result["node_count"] == 2 + assert result["outcomes"] == {"passed": 1, "xfailed": 1} + + +def test_native_aggregate_requires_both_lanes_to_neutralize_external_addopts(tmp_path: Path) -> None: + """A missing lane invariant cannot become release authority through green outcomes.""" + for lane, nodeid, outcome in (("parallel", "a", "passed"), ("serial", "b", "skipped")): + step = tmp_path / lane + step.mkdir() + (step / "selection.json").write_text(json.dumps({"selected_nodeids": [nodeid], "selected_nodeids_omitted": 0})) + (step / "events.jsonl").write_text( + json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": outcome}) + "\n" + ) + (step / "containment.json").write_text( + json.dumps( + { + "status": "finished", + "controller_group_alive": False, + "termination_reason": None, + "escalated_to_sigkill": False, + } + ) + ) + + result = aggregate_native_testmon_run( + tmp_path, + steps=[ + { + "semantic_lane": "parallel", + "artifact_dir": "parallel", + "exit": 0, + "external_addopts_neutralized": True, + "external_plugins_neutralized": True, + "closed_world_collection": True, + "statistics": {"cleanup": {"complete": True}}, + }, + { + "semantic_lane": "serial", + "artifact_dir": "serial", + "exit": 0, + "statistics": {"cleanup": {"complete": True}}, + }, + ], + environment_name="polylogue-test", + corpus_nodeids=("a", "b"), + selection_mode="bootstrap", + invocation_duration_s=0.1, + budget_s=10.0, + ) + + assert result["external_addopts_neutralized"] is False + assert result["lanes"][0]["external_addopts_neutralized"] is True + assert result["lanes"][1]["external_addopts_neutralized"] is False + assert result["external_plugins_neutralized"] is False + assert result["lanes"][0]["external_plugins_neutralized"] is True + assert result["lanes"][1]["external_plugins_neutralized"] is False + assert result["closed_world_collection"] is False + assert result["lanes"][0]["closed_world_collection"] is True + assert result["lanes"][1]["closed_world_collection"] is False + assert result["selected_union_count"] == 2 + assert result["terminal_union_count"] == 2 + assert result["non_green_count"] == 0 + assert result["complete_corpus_covered"] is False + assert result["terminal_green"] is False + + +def test_verify_run_statistics_only_cover_pytest_steps(tmp_path: Path) -> None: + run = VerifyRun(tier="quick", argv=["--quick"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="ruff check", cmd=["ruff", "check"]) + + run.finish_step(step_id=artifacts.step_id, result={"exit": 0, "duration_s": 0.1}) + + step = run._payload["steps"][0] + assert "statistics" not in step + assert not artifacts.statistics_path.exists() + + +def test_verify_run_embeds_compact_statistics_before_worktree_cleanup(tmp_path: Path) -> None: + run = VerifyRun(tier="focused-test", argv=["tests/unit/example.py"], git_head="head", root=tmp_path) + artifacts = run.start_step(label="pytest focused", cmd=["pytest", "tests/unit/example.py"]) + artifacts.events_merged_path.write_text( + json.dumps( + { + "event": "test_report", + "nodeid": "tests/unit/example.py::test_one", + "when": "call", + "duration_s": 0.25, + "outcome": "passed", + "worker_id": "controller", + } + ) + + "\n" + ) + + run.finish_step(step_id=artifacts.step_id, result={"exit": 0, "duration_s": 0.25}) + payload = run.finish(exit_code=0, duration_s=0.25) + shutil.rmtree(run.run_dir) + + statistics = payload["steps"][0]["statistics"] assert statistics["node_count"] == 1 assert statistics["phases"]["call"]["p50_s"] == 0.25 @@ -1384,45 +797,6 @@ def _write_worker_facts(*_args: object, artifacts: PytestStepArtifacts, **_kwarg assert selection["worker_id"] == "runner" -def test_verify_main_records_containment_failure_as_terminal_history( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - history_path = tmp_path / "verify-history.jsonl" - monkeypatch.setattr(verify, "HISTORY_PATH", history_path) - - with ( - patch("devtools.verify._anchor_verification_paths"), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify.build_verify_steps", return_value=[("pytest containment", ["pytest", "-n", "0"])]), - patch("devtools.verify.apply_managed_pytest_runtime_policy", return_value=({}, None)), - patch( - "devtools.verify._run_pytest_with_heartbeat", - side_effect=verify.PytestContainmentError("owned child still running"), - ), - patch("devtools.verify.cleanup_managed_pytest_basetemp") as cleanup, - patch("devtools.verify._notify"), - ): - rc = main(["--json"]) - - history = json.loads(history_path.read_text(encoding="utf-8")) - run_json = next((tmp_path / ".cache" / "verify" / "runs").glob("*/run.json")) - run_payload = json.loads(run_json.read_text(encoding="utf-8")) - payload = json.loads(capsys.readouterr().out) - - assert rc == 125 - cleanup.assert_not_called() - assert payload["diagnosis"] == "pytest_containment_unproven" - assert history["exit_code"] == 125 - assert history["diagnosis"] == "pytest_containment_unproven" - assert run_payload["status"] == "failed" - assert run_payload["steps"][0]["status"] == "failed" - - def test_print_history_accepts_verify_and_focused_run_records( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], @@ -1475,6 +849,49 @@ def test_verify_history_appends_concurrent_records_without_interleaving(tmp_path rows = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] assert sorted(row["sequence"] for row in rows) == list(range(64)) + assert all(row["history_schema_version"] == 1 for row in rows) + assert all(isinstance(row["timestamp"], str) for row in rows) + assert all(row["pytest_aggregate"]["selection_mode"] == "none" for row in rows) + + +def test_verify_history_normalizes_focused_and_quick_runs_to_one_aggregate_schema(tmp_path: Path) -> None: + history = tmp_path / "state" / "verify-history.jsonl" + finished_at = "2026-08-13T17:30:00+00:00" + focused = { + "tier": "focused-test", + "finished_at": finished_at, + "duration_s": 1.25, + "exit_code": 0, + "steps": [ + { + "name": "pytest focused", + "exit": 0, + "duration_s": 1.0, + "selected_count": 2, + "collection_duration_s": 0.1, + "containment_mode": "systemd-scope", + "statistics": { + "node_count": 2, + "outcomes": {"passed": 2}, + "cleanup": {"complete": True}, + "resources": {"peak_tree_rss_kb": 512, "tree_read_bytes_delta": 64}, + }, + } + ], + } + quick = {"tier": "quick", "timestamp": "2026-08-13T17:31:00+00:00", "exit_code": 0, "steps": []} + + append_verify_history(focused, path=history) + append_verify_history(quick, path=history) + + focused_row, quick_row = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] + assert focused_row["timestamp"] == finished_at + assert focused_row["total_duration_s"] == 1.25 + assert focused_row["pytest_aggregate"]["selection_mode"] == "focused" + assert focused_row["pytest_aggregate"]["outcomes"] == {"passed": 2} + assert focused_row["pytest_aggregate"]["resources"]["peak_tree_rss_kb"] == 512 + assert quick_row["pytest_aggregate"]["selection_mode"] == "none" + assert quick_row["pytest_aggregate"]["lanes"] == [] def test_verify_history_repairs_or_frames_an_incomplete_trailing_record(tmp_path: Path) -> None: @@ -1488,7 +905,9 @@ def test_verify_history_repairs_or_frames_an_incomplete_trailing_record(tmp_path append_verify_history({"sequence": 2}, path=history) rows = [json.loads(line) for line in history.read_text(encoding="utf-8").splitlines()] - assert rows == [{"sequence": 0}, {"sequence": 1}, {"sequence": 2}] + assert rows[0] == {"sequence": 0} + assert [row["sequence"] for row in rows[1:]] == [1, 2] + assert all(row["history_schema_version"] == 1 for row in rows[1:]) def test_verify_history_append_reads_only_the_trailing_record( @@ -1512,7 +931,9 @@ def measured_read(descriptor: int, count: int) -> bytes: append_verify_history({"sequence": 1}, path=history) assert bytes_read < 128 * 1024 - assert json.loads(history.read_text(encoding="utf-8").splitlines()[-1]) == {"sequence": 1} + row = json.loads(history.read_text(encoding="utf-8").splitlines()[-1]) + assert row["sequence"] == 1 + assert row["history_schema_version"] == 1 def test_compare_against_last_skips_intervening_focused_history(monkeypatch: pytest.MonkeyPatch) -> None: @@ -1557,200 +978,6 @@ def test_compare_against_last_selects_prior_run_independently_per_step(monkeypat assert "pytest testmon" in flags[0] -def test_running_seed_recovers_ledger_from_selection_artifact(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - artifact_dir = tmp_path / ".cache" / "verify" / "runs" / "interrupted" - step_dir = artifact_dir / "steps" / "17-pytest-seed-testmon" - step_dir.mkdir(parents=True) - expected = ["tests/unit/test_example.py::test_one"] - (step_dir / "selection.json").write_text( - json.dumps({"selected_nodeids": expected, "selected_nodeids_omitted": 0, "selected_count": 1}) - ) - identity = { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": True, - "lab": False, - "terminal_authorization": None, - } - TESTMON_SEED_ATTEMPT.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": identity, - "expected_nodeids": [], - "artifact_dir": str(artifact_dir.relative_to(tmp_path)), - } - ) - ) - - assert _testmon_seed_can_resume({**identity, "git_head": "fixed", "git_tree": "tree-hash"}) is True - - run = VerifyRun(tier="seed-testmon", argv=["--seed-testmon"], git_head="fixed") - prepared = _prepare_testmon_seed_attempt( - identity={**identity, "git_head": "fixed", "git_tree": "tree-hash"}, run=run, resume=True - ) - - assert prepared["expected_nodeids"] == expected - assert prepared["expected_count"] == 1 - assert prepared["expected_digest"] == hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest() - persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) - assert persisted["expected_digest"] == prepared["expected_digest"] - - -def test_seed_resume_rejects_selection_artifact_outside_checkout( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_text("partial") - outside = tmp_path.parent / "outside-testmon-artifacts" - step_dir = outside / "steps" / "17-pytest-seed-testmon" - step_dir.mkdir(parents=True) - (step_dir / "selection.json").write_text( - json.dumps( - { - "selected_nodeids": ["tests/unit/test_example.py::test_one"], - "selected_nodeids_omitted": 0, - "selected_count": 1, - } - ) - ) - identity = { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "3.13", - "skip_slow": True, - "lab": False, - } - TESTMON_SEED_ATTEMPT.write_text( - json.dumps( - { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": identity, - "expected_nodeids": [], - "artifact_dir": str(outside), - } - ) - ) - - assert _testmon_seed_can_resume(identity) is False - - -def test_resumed_seed_does_not_reuse_an_unexecuted_database_row(tmp_path: Path) -> None: - monkeypatch = pytest.MonkeyPatch() - monkeypatch.chdir(tmp_path) - try: - expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) - ) - (artifact_dir / "events.jsonl").write_text( - json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" - ) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("create table environment (id integer primary key, environment_name text)") - connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") - connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") - connection.execute( - "create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)" - ) - connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) - connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) - connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) - prepared = { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": True, - "expected_nodeids": expected, - "run_id": "resume", - "artifact_dir": ".cache/verify/runs/resume", - } - _write_run_receipt(tmp_path, "resume") - - receipt = _finalize_testmon_seed_attempt( - prepared=prepared, - step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) - - assert receipt["status"] == "incomplete" - assert {item["nodeid"]: item["outcome"] for item in receipt["node_outcomes"]} == { - expected[0]: "passed", - expected[1]: "missing", - } - - (artifact_dir / "selection.json").write_text(json.dumps({})) - (artifact_dir / "events.jsonl").write_text( - "\n".join( - json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) - for nodeid in expected - ) - + "\n" - ) - missing_selection = _finalize_testmon_seed_attempt( - prepared=prepared, - step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) - assert missing_selection["status"] == "incomplete" - finally: - monkeypatch.undo() - - -def test_testmon_database_state_reports_missing_and_failed_nodes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as conn: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute( - "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" - ) - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - conn.executemany( - "INSERT INTO test_execution(test_name, failed) VALUES (?, ?)", - [("tests/test_a.py::test_ok", 0), ("tests/test_b.py::test_failed", 1)], - ) - conn.executemany( - "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", - [(1, "a.py", "a"), (2, "b.py", "b")], - ) - conn.executemany("INSERT INTO test_execution_file_fp VALUES (?, ?)", [(1, 1), (2, 2)]) - - state = _testmon_database_state( - ["tests/test_a.py::test_ok", "tests/test_b.py::test_failed", "tests/test_c.py::test_missing"] - ) - - assert state["recorded_count"] == 2 - assert state["failed_nodeids"] == ["tests/test_b.py::test_failed"] - assert state["missing_nodeids"] == ["tests/test_c.py::test_missing"] - assert state["node_outcomes"] == { - "tests/test_a.py::test_ok": "passed", - "tests/test_b.py::test_failed": "failed", - "tests/test_c.py::test_missing": "missing", - } - - def test_worktree_fingerprint_hashes_untracked_file_contents(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], check=True) subprocess.run(["git", "config", "user.email", "tests@example.invalid"], check=True) @@ -1816,7 +1043,8 @@ def test_changed_paths_keep_start_time_base_when_remote_ref_advances( assert pinned_base == base subprocess.run(["git", "update-ref", "refs/remotes/origin/master", "HEAD"], cwd=tmp_path, check=True) - assert verify._changed_executable_paths(pinned_base, feature_head) == ("polylogue/example.py",) + changed = verify._changed_test_relevant_paths(pinned_base, feature_head) + assert executable_python_paths(tmp_path, changed) == ("polylogue/example.py",) def test_changed_paths_include_untracked_executable_files( @@ -1838,7 +1066,8 @@ def test_changed_paths_include_untracked_executable_files( untracked.write_text("value = 1\n", encoding="utf-8") monkeypatch.setattr(verify, "ROOT", tmp_path) - assert verify._changed_executable_paths(head, head) == ("devtools/new_command.py",) + changed = verify._changed_test_relevant_paths(head, head) + assert executable_python_paths(tmp_path, changed) == ("devtools/new_command.py",) def test_changed_paths_include_executable_rename_sources( @@ -1865,7 +1094,8 @@ def test_changed_paths_include_executable_rename_sources( ).stdout.strip() monkeypatch.setattr(verify, "ROOT", tmp_path) - assert verify._changed_executable_paths(base, head) == ("polylogue/example.py",) + changed = verify._changed_test_relevant_paths(base, head) + assert "polylogue/example.py" in changed def test_changed_paths_parse_non_ascii_names_without_git_quoting( @@ -1886,7 +1116,8 @@ def test_changed_paths_parse_non_ascii_names_without_git_quoting( source.write_text("value = 2\n", encoding="utf-8") monkeypatch.setattr(verify, "ROOT", tmp_path) - assert verify._changed_executable_paths(base, base) == ("polylogue/café.py",) + changed = verify._changed_test_relevant_paths(base, base) + assert executable_python_paths(tmp_path, changed) == ("polylogue/café.py",) def test_git_head_uses_bounded_authoritative_probe() -> None: @@ -2214,12 +1445,66 @@ def test_checkout_mutation_monitor_observes_transient_index_authority_change(tmp hidden.parent.mkdir() hidden.write_text("secret authority\n", encoding="utf-8") subprocess.run(["git", "add", "-f", "ignored/hidden.py"], cwd=tmp_path, check=True) + deadline = time.monotonic() + 1 + while not monitor._changed and time.monotonic() < deadline: + time.sleep(0.001) + assert monitor._changed, "monitor did not witness the changed index authority before it was restored" subprocess.run(["git", "reset", "-q", "--", "ignored/hidden.py"], cwd=tmp_path, check=True) observation = monitor.finish() assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") +@pytest.mark.uses_real_clock("coordinates a deliberately coalesced Git index watcher event") +def test_checkout_mutation_monitor_keeps_coalesced_index_authority_event( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Restoring signature comparison would miss the delayed authority event.""" + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + index = tmp_path / ".git" / "index" + baseline = index.read_bytes() + release_event = threading.Event() + + def delayed_watch(*_paths: Path, **kwargs: object) -> object: + yield set() + assert release_event.wait(timeout=1) + yield {(watchfiles.Change.modified, str(index))} + stop_event = kwargs["stop_event"] + assert isinstance(stop_event, threading.Event) + stop_event.wait(timeout=1) + + monkeypatch.setattr(watchfiles, "watch", delayed_watch) + monitor = CheckoutMutationMonitor(tmp_path) + monitor.start() + tracked.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + index.write_bytes(baseline) + release_event.set() + observation = monitor.finish() + + assert observation == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") + + +def test_checkout_mutation_monitor_keeps_initial_authority_signature_across_topology_recheck(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + tracked = tmp_path / "tracked.py" + tracked.write_text("value = 1\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + + monitor = CheckoutMutationMonitor(tmp_path) + monitor._watched_directories() + tracked.write_text("value = 2\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.py"], cwd=tmp_path, check=True) + monitor._watched_directories() + monitor._record_change(tmp_path / ".git") + + assert monitor.finish() == CheckoutMutationObservation(changed=True, unavailable=False, observed_path=".git/index") + + @pytest.mark.uses_real_clock("waits for the filesystem watcher to witness a branch-ref replacement") def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) @@ -2244,6 +1529,10 @@ def test_checkout_mutation_monitor_observes_transient_head_ref_change(tmp_path: monitor = CheckoutMutationMonitor(tmp_path) monitor.start() subprocess.run(["git", "update-ref", branch, first], cwd=tmp_path, check=True) + deadline = time.monotonic() + 1 + while not monitor._changed and time.monotonic() < deadline: + time.sleep(0.001) + assert monitor._changed, "monitor did not witness the changed ref authority before it was restored" subprocess.run(["git", "update-ref", branch, second], cwd=tmp_path, check=True) observation = monitor.finish() @@ -2395,7 +1684,7 @@ def test_checkout_mutation_monitor_ignores_uncommitted_git_index_lock(tmp_path: assert observation == CheckoutMutationObservation(changed=False, unavailable=False) -def test_checkout_mutation_monitor_treats_every_ready_index_event_as_authority_change( +def test_checkout_mutation_monitor_records_semantically_unchanged_index_replacement( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -2406,6 +1695,9 @@ def test_checkout_mutation_monitor_treats_every_ready_index_event_as_authority_c def portable_watch(*_paths: Path, **_kwargs: object) -> object: yield set() + replacement = index.with_suffix(".replacement") + replacement.write_bytes(index.read_bytes()) + replacement.replace(index) yield {(watchfiles.Change.modified, str(index))} stop_event = _kwargs["stop_event"] assert isinstance(stop_event, threading.Event) @@ -2477,597 +1769,30 @@ def broken_watch(*_paths: Path, **_kwargs: object) -> object: assert observation == CheckoutMutationObservation(changed=False, unavailable=True) -def test_seed_receipt_classifies_every_node_terminal_outcome( - tmp_path: Path, -) -> None: - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - expected = [ - "tests/test_seed.py::test_passed", - "tests/test_seed.py::test_failed", - "tests/test_seed.py::test_error", - "tests/test_seed.py::test_timeout", - "tests/test_seed.py::test_worker_crash", - "tests/test_seed.py::test_missing", - ] - (artifact_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": len(expected), - "deselected_count": 0, - "selected_nodeids": expected, - "selected_nodeids_omitted": 0, - } - ) - ) - events = [ - {"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}, - { - "event": "test_report", - "nodeid": expected[1], - "when": "call", - "outcome": "failed", - "longrepr": "assert false", - }, - { - "event": "test_report", - "nodeid": expected[2], - "when": "setup", - "outcome": "failed", - "longrepr": "fixture exploded", - }, - { - "event": "test_report", - "nodeid": expected[3], - "when": "call", - "outcome": "failed", - "longrepr": "Failed: Timeout > 10s", - }, - {"event": "test_started", "nodeid": expected[4]}, - ] - (artifact_dir / "events.jsonl").write_text("".join(json.dumps(event) + "\n" for event in events)) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as conn: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute( - "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" - ) - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - conn.executemany( - "INSERT INTO test_execution(test_name, failed) VALUES (?, ?)", - [(nodeid, int(nodeid != expected[0])) for nodeid in expected[:-1]], - ) - conn.executemany( - "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", - [(index, f"file-{index}.py", f"sha-{index}") for index, _nodeid in enumerate(expected[:-1], start=1)], - ) - conn.executemany( - "INSERT INTO test_execution_file_fp VALUES (?, ?)", - [(index, index) for index, _nodeid in enumerate(expected[:-1], start=1)], - ) - - _write_run_receipt(tmp_path, "run-mixed") - receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-mixed", - "artifact_dir": ".cache/verify/runs/run-mixed", - }, - step_results=[ - { - "name": "pytest seed-testmon", - "artifact_dir": str(artifact_dir), - "exit": 1, - "diagnosis": "xdist_worker_crash", - } - ], - exit_code=1, +def test_classify_late_sigterm_after_pytest_success_summary() -> None: + diagnosis = classify_pytest_result( + returncode=-15, + termination_reason=None, + report_present=False, + summary={"exitstatus": 0}, + progress_event="finished", ) - assert receipt["status"] == "incomplete" - assert {item["nodeid"]: item["outcome"] for item in receipt["node_outcomes"]} == { - expected[0]: "passed", - expected[1]: "failed", - expected[2]: "error", - expected[3]: "timeout", - expected[4]: "worker_crash", - expected[5]: "missing", - } - assert receipt["node_outcome_counts"] == { - "error": 1, - "failed": 1, - "missing": 1, - "passed": 1, - "timeout": 1, - "worker_crash": 1, - } - + assert diagnosis == "report_missing_after_sessionfinish_success" -def test_seed_node_outcomes_preserve_interrupted_active_node(tmp_path: Path) -> None: - events = tmp_path / "events.jsonl" - events.write_text(json.dumps({"event": "test_started", "nodeid": "tests/test_a.py::test_active"}) + "\n") - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=["tests/test_a.py::test_active"], - database={"node_outcomes": {"tests/test_a.py::test_active": "missing"}}, - pytest_step={"diagnosis": "terminated by signal"}, +def test_resource_sampler_records_process_tree_sample(tmp_path: Path) -> None: + path = tmp_path / "resources.jsonl" + sampler = ResourceSampler( + root_pid=os.getpid(), + run_id="test-run", + root=tmp_path, + env={"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)}, + output_path=path, ) - assert outcomes[0]["outcome"] == "interrupted" - - -def test_seed_node_outcomes_keep_unconfirmed_teardown_incomplete(tmp_path: Path) -> None: - """A terminal teardown does not prove that the missing call phase passed.""" - events = tmp_path / "events.jsonl" - events.write_text( - "\n".join( - [ - json.dumps({"event": "test_started", "nodeid": "tests/test_a.py::test_finished"}), - json.dumps( - { - "event": "test_report", - "nodeid": "tests/test_a.py::test_finished", - "when": "teardown", - "outcome": "passed", - } - ), - json.dumps({"event": "test_finished", "nodeid": "tests/test_a.py::test_finished"}), - ] - ) - + "\n" - ) - - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=["tests/test_a.py::test_finished"], - database={"node_outcomes": {"tests/test_a.py::test_finished": "missing"}}, - pytest_step={"diagnosis": "pytest_failed"}, - ) - - assert outcomes[0]["outcome"] == "missing" - assert outcomes[0]["reason"] == "passing teardown without call report or testmon result" - - -def test_seed_resource_timeout_has_a_distinct_typed_terminal_outcome( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A supervisor resource stop must not collapse into generic incompleteness.""" - monkeypatch.chdir(tmp_path) - expected = ["tests/test_seed.py::test_active"] - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": len(expected), - "deselected_count": 0, - "selected_nodeids": expected, - "selected_nodeids_omitted": 0, - } - ) - ) - (artifact_dir / "events.jsonl").write_text(json.dumps({"event": "test_started", "nodeid": expected[0]}) + "\n") - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("create table environment (id integer primary key, environment_name text)") - connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") - connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") - connection.execute("create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)") - _write_run_receipt(tmp_path, "run-resource-timeout") - - receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-resource-timeout", - "artifact_dir": ".cache/verify/runs/run-resource-timeout", - }, - step_results=[ - { - "name": "pytest seed-testmon", - "artifact_dir": str(artifact_dir), - "exit": 124, - "diagnosis": "pytest_terminated", - "termination_reason": "pytest tmpfs budget exceeded: 512.0 MiB > 500 MiB", - } - ], - exit_code=124, - ) - - assert receipt["status"] == "incomplete" - assert receipt["outcome"] == "resource-timeout" - assert receipt["release_baseline_allowed"] is False - - -def test_seed_node_outcomes_accept_setup_skip_as_terminal_skip(tmp_path: Path) -> None: - events = tmp_path / "events.jsonl" - events.write_text( - json.dumps( - { - "event": "test_report", - "nodeid": "tests/test_a.py::test_setup_skip", - "when": "setup", - "outcome": "skipped", - } - ) - + "\n" - ) - - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=["tests/test_a.py::test_setup_skip"], - database={"node_outcomes": {"tests/test_a.py::test_setup_skip": "missing"}}, - pytest_step={}, - use_database_fallback=False, - ) - - assert outcomes == [ - { - "nodeid": "tests/test_a.py::test_setup_skip", - "outcome": "skipped", - "reason": "test setup or teardown skipped", - "started": False, - "finished": False, - "phases": [{"when": "setup", "outcome": "skipped", "duration_s": None}], - } - ] - - -def test_seed_node_outcomes_preserve_call_and_fixture_xfail_xpass(tmp_path: Path) -> None: - """Durable pytest reports, including fixture ``pytest.xfail()``, finish seed nodes.""" - events = tmp_path / "events.jsonl" - nodes = [ - "tests/test_a.py::test_call_xfailed", - "tests/test_a.py::test_call_xpassed", - "tests/test_a.py::test_setup_xfailed", - ] - events.write_text( - "\n".join( - json.dumps(event) - for event in ( - {"event": "test_report", "nodeid": nodes[0], "when": "call", "outcome": "xfailed"}, - {"event": "test_report", "nodeid": nodes[1], "when": "call", "outcome": "xpassed"}, - {"event": "test_report", "nodeid": nodes[2], "when": "setup", "outcome": "xfailed"}, - ) - ) - + "\n" - ) - - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=nodes, - database={"node_outcomes": {}}, - pytest_step={}, - use_database_fallback=False, - ) - - assert {item["nodeid"]: item["outcome"] for item in outcomes} == dict( - zip(nodes, ("xfailed", "xpassed", "xfailed"), strict=True) - ) - - -def test_resumed_seed_carries_forward_prior_terminal_outcome(tmp_path: Path) -> None: - events = tmp_path / "events.jsonl" - events.write_text( - json.dumps( - {"event": "test_report", "nodeid": "tests/test_a.py::test_repaired", "when": "call", "outcome": "passed"} - ) - + "\n" - ) - outcomes = _seed_node_outcomes_from_events( - events, - expected_nodeids=[ - "tests/test_a.py::test_repaired", - "tests/test_b.py::test_prior", - "tests/test_c.py::test_expected_failure", - ], - database={"node_outcomes": {"tests/test_b.py::test_prior": "passed"}}, - pytest_step={}, - use_database_fallback=False, - prior_node_outcomes={ - "tests/test_b.py::test_prior": {"nodeid": "tests/test_b.py::test_prior", "outcome": "passed"}, - "tests/test_c.py::test_expected_failure": { - "nodeid": "tests/test_c.py::test_expected_failure", - "outcome": "xfailed", - }, - }, - ) - - assert {item["nodeid"]: item["outcome"] for item in outcomes} == { - "tests/test_a.py::test_repaired": "passed", - "tests/test_b.py::test_prior": "passed", - "tests/test_c.py::test_expected_failure": "xfailed", - } - - -def test_seed_completion_requires_full_failure_free_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] - (artifact_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": 2, - "deselected_count": 0, - "selected_nodeids": expected, - "selected_nodeids_omitted": 0, - } - ) - ) - (artifact_dir / "events.jsonl").write_text( - "".join( - json.dumps({"event": "test_report", "nodeid": nodeid, "when": "call", "outcome": "passed"}) + "\n" - for nodeid in expected - ) - ) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as conn: - conn.execute("CREATE TABLE environment (id INTEGER PRIMARY KEY, environment_name TEXT)") - conn.execute("CREATE TABLE file_fp (id INTEGER PRIMARY KEY, filename TEXT, fsha TEXT)") - conn.execute( - "CREATE TABLE test_execution (id INTEGER PRIMARY KEY, test_name TEXT NOT NULL, failed INTEGER NOT NULL)" - ) - conn.execute("CREATE TABLE test_execution_file_fp (test_execution_id INTEGER, fingerprint_id INTEGER)") - conn.executemany( - "INSERT INTO test_execution(test_name, failed) VALUES (?, 0)", - [(nodeid,) for nodeid in expected], - ) - conn.executemany( - "INSERT INTO file_fp(id, filename, fsha) VALUES (?, ?, ?)", - [(index, f"file-{index}.py", f"sha-{index}") for index, _nodeid in enumerate(expected, start=1)], - ) - conn.executemany( - "INSERT INTO test_execution_file_fp VALUES (?, ?)", - [(index, index) for index, _nodeid in enumerate(expected, start=1)], - ) - - _write_run_receipt(tmp_path, "run-1") - _write_run_receipt(tmp_path, "run-stale-db") - _write_run_receipt(tmp_path, "run-orphaned") - receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-1", - "artifact_dir": ".cache/verify/runs/run-1", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 0}], - exit_code=0, - ) - - assert receipt["status"] == "complete" - assert receipt["expected_count"] == 2 - stamp = json.loads((tmp_path / ".cache" / "testmon" / "seed.json").read_text()) - assert stamp["status"] == "usable" - assert stamp["collection"]["expected_count"] == 2 - - _write_run_receipt(tmp_path, "run-authorized") - authorized_receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - "terminal_authorization": "narrow-terminal", - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-authorized", - "artifact_dir": ".cache/verify/runs/run-authorized", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 0}], - exit_code=0, - ) - assert authorized_receipt["status"] == "complete" - assert authorized_receipt["release_baseline_allowed"] is True - - _write_run_receipt(tmp_path, "run-red") - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("update test_execution set failed = 1 where test_name = ?", (expected[0],)) - red_receipt = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-red", - "artifact_dir": ".cache/verify/runs/run-red", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir), "exit": 1}], - exit_code=1, - ) - assert red_receipt["status"] == "reusable" - assert red_receipt["release_baseline_allowed"] is False - persisted_attempt = json.loads((tmp_path / ".cache" / "testmon" / "seed-attempt.json").read_text()) - assert persisted_attempt["release_baseline_allowed"] is False - assert not (tmp_path / ".cache" / "testmon" / "seed.json").exists() - - (artifact_dir / "events.jsonl").write_text("") - stale_database = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-stale-db", - "artifact_dir": ".cache/verify/runs/run-stale-db", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) - assert stale_database["status"] == "incomplete" - - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("insert into test_execution_file_fp values (999, 1)") - orphaned = _finalize_testmon_seed_attempt( - prepared={ - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": True, - "lab": False, - }, - "resume": False, - "expected_nodeids": [], - "run_id": "run-orphaned", - "artifact_dir": ".cache/verify/runs/run-orphaned", - }, - step_results=[{"name": "pytest seed-testmon", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) - assert orphaned["status"] == "incomplete" - - -def test_resumed_seed_persists_full_selection_before_stamp_publication( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - expected = ["tests/test_a.py::test_one", "tests/test_b.py::test_two"] - artifact_dir = tmp_path / "artifacts" - artifact_dir.mkdir() - (artifact_dir / "selection.json").write_text( - json.dumps({"selected_count": 1, "selected_nodeids": [expected[0]], "selected_nodeids_omitted": 0}) - ) - (artifact_dir / "events.jsonl").write_text( - json.dumps({"event": "test_report", "nodeid": expected[0], "when": "call", "outcome": "passed"}) + "\n" - ) - TESTMON_DATA.parent.mkdir(parents=True) - with sqlite3.connect(TESTMON_DATA) as connection: - connection.execute("create table environment (id integer primary key, environment_name text)") - connection.execute("create table file_fp (id integer primary key, filename text, fsha text)") - connection.execute("create table test_execution (id integer primary key, test_name text, failed integer)") - connection.execute("create table test_execution_file_fp (test_execution_id integer, fingerprint_id integer)") - connection.executemany("insert into test_execution values (?, ?, 0)", [(1, expected[0]), (2, expected[1])]) - connection.executemany("insert into file_fp values (?, ?, ?)", [(1, "a.py", "a"), (2, "b.py", "b")]) - connection.executemany("insert into test_execution_file_fp values (?, ?)", [(1, 1), (2, 2)]) - _write_run_receipt(tmp_path, "resumed") - prepared = { - "protocol_version": TESTMON_SEED_PROTOCOL_VERSION, - "status": "running", - "identity": { - "git_head": "head", - "git_tree": "tree-hash", - "worktree_fingerprint": "tree", - "python": "python", - "skip_slow": False, - "lab": False, - "terminal_authorization": None, - **_testmon_runtime_identity_fields(Path.cwd()), - }, - "resume": True, - "expected_nodeids": expected, - "expected_count": len(expected), - "expected_digest": hashlib.sha256("\n".join(sorted(expected)).encode()).hexdigest(), - "prior_node_outcomes": [{"nodeid": expected[1], "outcome": "passed"}], - "run_id": "resumed", - "artifact_dir": ".cache/verify/runs/resumed", - } - original_write = verify._atomic_write_json - - def crash_before_stamp(path: Path, payload: object) -> None: - if path == TESTMON_SEED_STAMP: - raise RuntimeError("simulated crash before seed publication") - assert isinstance(payload, dict) - original_write(path, payload) - - with patch("devtools.verify._atomic_write_json", side_effect=crash_before_stamp): - with pytest.raises(RuntimeError, match="before seed publication"): - _finalize_testmon_seed_attempt( - prepared=prepared, - step_results=[{"name": "pytest seed-testmon (resume)", "artifact_dir": str(artifact_dir)}], - exit_code=0, - ) - - persisted = json.loads(TESTMON_SEED_ATTEMPT.read_text()) - assert persisted["status"] == "complete" - assert persisted["expected_count"] == len(expected) - assert persisted["selection"]["selected_count"] == len(expected) - assert persisted["selection"]["selected_nodeids_omitted"] == 0 - assert not TESTMON_SEED_STAMP.exists() - - -def test_classify_late_sigterm_after_pytest_success_summary() -> None: - diagnosis = classify_pytest_result( - returncode=-15, - termination_reason=None, - report_present=False, - summary={"exitstatus": 0}, - progress_event="finished", - ) - - assert diagnosis == "report_missing_after_sessionfinish_success" - - -def test_resource_sampler_records_process_tree_sample(tmp_path: Path) -> None: - path = tmp_path / "resources.jsonl" - sampler = ResourceSampler( - root_pid=os.getpid(), - run_id="test-run", - root=tmp_path, - env={"POLYLOGUE_PYTEST_BASETEMP_ROOT": str(tmp_path)}, - output_path=path, - ) - - sample = sampler.sample(event="sample") - summary = sampler.summary() + sample = sampler.sample(event="sample") + summary = sampler.summary() assert sample["process_count"] >= 1 assert sample["tree_rss_kb"] > 0 @@ -4094,6 +2819,7 @@ def test_pytest_workload_receipt_uses_allocated_basetemp_peak() -> None: tmpfs_budget_mb=1, basetemp_cleanup=None, concurrency=1, + timeout_s=3600, ) execute = next(phase for phase in receipt["phases"] if phase["name"] == "execute") @@ -4137,15 +2863,6 @@ def test_cleanup_managed_pytest_basetemp_keeps_seed_cache(tmp_path: Path) -> Non assert seeded.exists() -def test_testmon_preflight_allows_seed_and_full_without_database( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - - assert _testmon_preflight(seed_testmon=True, full_pytest=False, quick=False, commit=False) is None - assert _testmon_preflight(seed_testmon=False, full_pytest=True, quick=False, commit=False) is None - - def test_parse_pytest_test_count_from_summary() -> None: output = "bringing up nodes...\n\n6 passed, 2 skipped, 1 xfailed in 8.49s\n" @@ -4348,6 +3065,129 @@ def apply_policy(env: dict[str, str], **_kwargs: object) -> tuple[dict[str, str] assert captured["POLYLOGUE_PYTEST_EXPLICIT_BASETEMP"] == str(explicit) +@pytest.mark.parametrize("mode", ["affected", "bootstrap", "full"]) +def test_managed_native_lane_removes_environment_addopts_before_pytest( + monkeypatch: pytest.MonkeyPatch, + mode: str, +) -> None: + captured_command: list[str] = [] + captured_env: dict[str, str] = {} + completed = subprocess.CompletedProcess(args=["pytest"], returncode=0, stdout="1 passed in 0.1s\n", stderr="") + monkeypatch.setenv("PYTEST_ADDOPTS", "--setup-only") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_narrow") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + command = next( + command + for label, command in build_verify_steps( + quick=False, + lab=False, + testmon_mode=mode, + testmon_environment="env-digest", + ) + if "parallel" in label + ) + + def run_pytest(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + captured_command.extend(command) + captured_env.update(cast("dict[str, str]", kwargs["env"])) + return completed + + with ( + patch("devtools.verify.apply_managed_pytest_runtime_policy", side_effect=lambda env, **_kwargs: (env, None)), + patch("devtools.verify._run_pytest_with_heartbeat", side_effect=run_pytest), + patch("devtools.verify.cleanup_managed_pytest_basetemp", return_value=None), + patch("devtools.verify._read_pytest_report", return_value=None), + ): + rc, _elapsed, metadata = _run( + f"pytest native parallel ({mode})", + command, + ) + + assert rc == 0 + assert verify._PYTEST_CLEAR_CONFIGURED_ADDOPTS in captured_command + assert "PYTEST_ADDOPTS" not in captured_env + assert "PYTEST_PLUGINS" not in captured_env + assert captured_env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] == "1" + assert metadata["external_addopts_neutralized"] is True + assert metadata["external_plugins_neutralized"] is True + assert metadata["closed_world_collection"] is True + + +@pytest.mark.uses_real_clock("coordinates two contenders for the checkout lifecycle lock") +def test_native_testmon_lifecycle_lock_serializes_checkout_state(tmp_path: Path) -> None: + holder_entered = threading.Event() + release_holder = threading.Event() + contender_entered = threading.Event() + + def hold_lock() -> None: + with verify._native_testmon_lifecycle_lock(tmp_path): + holder_entered.set() + assert release_holder.wait(timeout=2) + + def contend_for_lock() -> None: + with verify._native_testmon_lifecycle_lock(tmp_path): + contender_entered.set() + + with ThreadPoolExecutor(max_workers=2) as pool: + holder = pool.submit(hold_lock) + assert holder_entered.wait(timeout=2) + contender = pool.submit(contend_for_lock) + assert not contender_entered.wait(timeout=0.05) + release_holder.set() + holder.result(timeout=2) + contender.result(timeout=2) + + assert contender_entered.is_set() + + +def test_native_testmon_lifecycle_lock_refuses_symlink_without_touching_target(tmp_path: Path) -> None: + cache = tmp_path / ".cache" + cache.mkdir() + target = tmp_path / "outside-checkout-target" + target.write_text("preserve this file\n", encoding="utf-8") + (cache / "native-testmon-lifecycle.lock").symlink_to(target) + + with pytest.raises(NativeTestmonRepairError, match="native testmon lifecycle lock"): + with verify._native_testmon_lifecycle_lock(tmp_path): + pytest.fail("symlinked lifecycle lock must not be acquired") + + assert target.read_text(encoding="utf-8") == "preserve this file\n" + + +def test_native_testmon_lifecycle_lock_refuses_hardlink_without_touching_target(tmp_path: Path) -> None: + cache = tmp_path / ".cache" + cache.mkdir() + target = tmp_path / "outside-checkout-target" + target.write_text("preserve this file\n", encoding="utf-8") + os.link(target, cache / "native-testmon-lifecycle.lock") + + with pytest.raises(NativeTestmonRepairError, match="single-link regular file"): + with verify._native_testmon_lifecycle_lock(tmp_path): + pytest.fail("hard-linked lifecycle lock must not be acquired") + + assert target.read_text(encoding="utf-8") == "preserve this file\n" + + +@pytest.mark.uses_real_clock("proves lifecycle-lock waiters receive a bounded resource refusal") +def test_native_testmon_lifecycle_lock_times_out_while_holder_remains_active(tmp_path: Path) -> None: + holder_entered = threading.Event() + release_holder = threading.Event() + + def hold_lock() -> None: + with verify._native_testmon_lifecycle_lock(tmp_path): + holder_entered.set() + assert release_holder.wait(timeout=2) + + with ThreadPoolExecutor(max_workers=1) as pool: + holder = pool.submit(hold_lock) + assert holder_entered.wait(timeout=2) + with pytest.raises(PytestResourceError, match="timed out waiting for native testmon lifecycle lock"): + with verify._native_testmon_lifecycle_lock(tmp_path, timeout_s=0.01): + pytest.fail("timed-out contender must not acquire the lock") + release_holder.set() + holder.result(timeout=2) + + def test_run_clears_stale_current_statistics_before_an_interrupted_pytest_step(tmp_path: Path) -> None: stale_statistics = tmp_path / verify_runs.CURRENT_STATISTICS_PATH stale_statistics.parent.mkdir(parents=True) @@ -4459,7 +3299,7 @@ def to_dict(self) -> dict[str, int]: patch("devtools.verify._run_pytest_with_heartbeat", return_value=completed), patch("devtools.verify._read_pytest_report", return_value=None), ): - rc, _elapsed, metadata = _run("pytest seed-testmon", ["pytest", "--testmon", "-n", "4"]) + rc, _elapsed, metadata = _run("pytest native parallel (bootstrap)", ["pytest", "--testmon", "-n", "4"]) assert rc == 0 assert apply_policy.call_args.kwargs["worker_count"] == 4 @@ -4472,12 +3312,10 @@ def to_dict(self) -> dict[str, int]: ("label", "full_suite"), [ ("pytest focused", False), - ("pytest testmon", False), - ("pytest testmon (broad)", True), - ("pytest seed-testmon", True), - ("pytest seed-testmon shard 1/4", True), - ("pytest full (parallel)", True), - ("pytest load-sensitive (isolated)", True), + ("pytest native parallel (affected)", False), + ("pytest native serial (affected)", False), + ("pytest native parallel (bootstrap)", True), + ("pytest native serial (full)", True), ], ) def test_run_scopes_measured_full_suite_basetemp_demand(tmp_path: Path, label: str, full_suite: bool) -> None: @@ -4514,6 +3352,9 @@ def test_bench_slo_forces_nested_pytest_to_managed_scratch( scratch.mkdir() run = VerifyRun(tier="lab", argv=[], git_head=None, root=tmp_path) monkeypatch.setenv("POLYLOGUE_PYTEST_BASETEMP_ROOT", str(inherited_tmpfs_root)) + monkeypatch.setenv("PYTEST_ADDOPTS", "--collect-only") + monkeypatch.setenv("PYTEST_PLUGINS", "ambient_plugin") + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") managed_env = { "POLYLOGUE_PYTEST_TMPFS": "0", "POLYLOGUE_PYTEST_BASETEMP_ROOT": str(scratch), @@ -4536,6 +3377,9 @@ def test_bench_slo_forces_nested_pytest_to_managed_scratch( assert env["POLYLOGUE_PYTEST_RUN_ID"] == run.run_id assert env["POLYLOGUE_PYTEST_TMPFS"] == "0" assert env["POLYLOGUE_PYTEST_BASETEMP_ROOT"] == str(scratch) + assert "PYTEST_ADDOPTS" not in env + assert "PYTEST_PLUGINS" not in env + assert "PYTEST_DISABLE_PLUGIN_AUTOLOAD" not in env def test_run_forces_subprocesses_to_current_checkout(monkeypatch: pytest.MonkeyPatch) -> None: @@ -4543,6 +3387,10 @@ def test_run_forces_subprocesses_to_current_checkout(monkeypatch: pytest.MonkeyP monkeypatch.setenv("POLYLOGUE_REPO_ROOT", "/stale/main") monkeypatch.setenv("PYTHONPYCACHEPREFIX", "/stale/main/.cache/pycache") monkeypatch.setenv("PYTHONPATH", "/stale/main") + monkeypatch.setenv("PYTHONOPTIMIZE", "1") + monkeypatch.setenv("PYTHONHOME", "/stale/python") + monkeypatch.setenv("PYTHONUSERBASE", "/stale/user-site") + monkeypatch.setenv("PYTHONNOUSERSITE", "0") completed = subprocess.CompletedProcess(args=["devtools"], returncode=0, stdout="", stderr="") with patch("devtools.verify.subprocess.run", return_value=completed) as run: @@ -4553,10 +3401,28 @@ def test_run_forces_subprocesses_to_current_checkout(monkeypatch: pytest.MonkeyP assert env["POLYLOGUE_ROOT"] == str(ROOT) assert env["POLYLOGUE_REPO_ROOT"] == str(ROOT) assert env["PYTHONPYCACHEPREFIX"] == str(ROOT / ".cache" / "pycache") - assert env["PYTHONPATH"].split(os.pathsep)[0] == str(ROOT) + assert "PYTHONPATH" not in env + assert "PYTHONOPTIMIZE" not in env + assert "PYTHONHOME" not in env + assert "PYTHONUSERBASE" not in env + assert env["PYTHONNOUSERSITE"] == "1" assert env["POLYLOGUE_PYTEST_EVENTS_PATH"] == str(ROOT / PYTEST_EVENTS_PATH) +def test_mypy_probe_uses_managed_python_startup_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PYTHONPATH", "/caller/sitecustomize") + monkeypatch.setenv("PYTHONOPTIMIZE", "2") + completed = subprocess.CompletedProcess(args=["dmypy", "status"], returncode=1, stdout="", stderr="") + + with patch("devtools.verify.subprocess.run", return_value=completed) as run: + assert verify._mypy_cmd() == ["mypy"] + + assert run.call_args.kwargs["cwd"] == ROOT + env = run.call_args.kwargs["env"] + assert "PYTHONPATH" not in env + assert "PYTHONOPTIMIZE" not in env + + def test_verify_subprocess_env_removes_cloud_basetemp_in_local_worktree(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("POLYLOGUE_PYTEST_BASETEMP_ROOT", "/tmp/polylogue-pytest") completed = subprocess.CompletedProcess(args=["devtools"], returncode=0, stdout="", stderr="") @@ -4969,8 +3835,15 @@ def test_read_pytest_report_parses_valid_payload(tmp_path: Path) -> None: assert parsed == {"summary": {"passed": 3}, "duration": 1.0} -def test_verify_continues_after_failed_cheap_step(capsys: pytest.CaptureFixture[str]) -> None: +def test_verify_continues_after_failed_cheap_step( + capsys: pytest.CaptureFixture[str], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: calls: list[str] = [] + receipt = tmp_path / "quick-receipt.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "quick-failure") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: calls.append(label) @@ -4986,11 +3859,14 @@ def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, flo rc = main(["--quick", "--json"]) assert rc == 1 - assert calls == [label for label, _command in build_verify_steps(quick=True, lab=False, skip_slow=False)] + assert calls == [label for label, _command in build_verify_steps(quick=True, lab=False)] payload = json.loads(capsys.readouterr().out) assert payload["exit_code"] == 1 assert payload["verification_scope"] == "non-test" assert payload["release_baseline_allowed"] is False + assert payload["pytest_aggregate"]["selection_mode"] == "none" + assert payload["pytest_aggregate"]["deadline"] == {"budget_s": 3600.0, "met": True} + assert json.loads(receipt.read_text())["pytest_aggregate"] == payload["pytest_aggregate"] @pytest.mark.parametrize("fingerprints", [("unavailable", "stable"), ("stable", "unavailable")]) @@ -5078,6 +3954,45 @@ def finish(self) -> CheckoutMutationObservation: assert checkout_step["final_git_head"] == "different-head" +def test_verify_keeps_checkout_monitor_active_through_final_authority_samples( + capsys: pytest.CaptureFixture[str], +) -> None: + events: list[str] = [] + + class _OrderedMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + events.append("start") + + def finish(self) -> CheckoutMutationObservation: + events.append("finish") + return CheckoutMutationObservation(changed=False, unavailable=False) + + def git_head() -> str: + events.append("head") + return "stable-head" + + def fingerprint(_root: Path) -> str: + events.append("fingerprint") + return "stable-fingerprint" + + with ( + patch("devtools.verify._run", return_value=(0, 0.01, {})), + patch("devtools.verify._git_head", side_effect=git_head), + patch("devtools.verify.CheckoutMutationMonitor", _OrderedMonitor), + patch("devtools.verify.worktree_fingerprint", side_effect=fingerprint), + patch("devtools.verify._save_history"), + patch("devtools.verify._stamp_head"), + patch("devtools.verify._notify"), + ): + assert main(["--quick", "--json"]) == 0 + + assert events[-3:] == ["head", "fingerprint", "finish"] + assert json.loads(capsys.readouterr().out)["exit_code"] == 0 + + @pytest.mark.parametrize( ("fingerprints", "expected_diagnosis"), [ @@ -5180,513 +4095,513 @@ def finish(self) -> CheckoutMutationObservation: assert durable_payload["final_worktree_fingerprint"] == "stable" -def test_transient_checkout_mutation_discards_testmon_graph_before_publication( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize( + ("raised", "expected_exit", "expected_diagnosis"), + [ + (RuntimeError("fingerprint failed"), 125, "verify_runner_exception"), + (KeyboardInterrupt(), 130, "verify_interrupted"), + ], +) +def test_verify_finalizes_checkout_monitor_when_startup_fingerprint_raises( capsys: pytest.CaptureFixture[str], + raised: BaseException, + expected_exit: int, + expected_diagnosis: str, ) -> None: - class _ChangedMonitor: + events: list[str] = [] + history: dict[str, Any] = {} + head_calls = iter(("initial-head", "final-head")) + fingerprint_calls = 0 + + def _git_head() -> str: + head = next(head_calls) + events.append(head) + return head + + def _worktree_fingerprint(_root: Path) -> str: + nonlocal fingerprint_calls + fingerprint_calls += 1 + if fingerprint_calls == 1: + events.append("startup-fingerprint") + raise raised + events.append("final-fingerprint") + return "final-fingerprint" + + class _ExceptionalExitMonitor: def __init__(self, _root: Path) -> None: pass def start(self) -> None: - pass + events.append("monitor-started") def finish(self) -> CheckoutMutationObservation: - return CheckoutMutationObservation(changed=True, unavailable=False, observed_path="polylogue/example.py") - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(verify, "ROOT", tmp_path) - monkeypatch.setattr(verify, "CheckoutMutationMonitor", _ChangedMonitor) - monkeypatch.setattr( - verify, - "assert_polylogue_matches_checkout", - lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), - ) - TESTMON_DATA.parent.mkdir(parents=True) - TESTMON_DATA.write_bytes(b"transient dependency graph") - TESTMON_SEED_STAMP.write_text("{}", encoding="utf-8") - affected_publish = MagicMock() - selection_publish = MagicMock() - - with ( - patch("devtools.verify._anchor_verification_paths"), - patch("devtools.verify.maybe_bootstrap_testmon_seed", return_value=None), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest", "--testmon"])]), - patch("devtools.verify._run", return_value=(0, 0.01, {"selected_count": 1})), - patch("devtools.verify._changed_executable_paths", return_value=("polylogue/example.py",)), - patch("devtools.verify._record_testmon_affected_coverage", affected_publish), - patch("devtools.verify._refresh_testmon_selection_attempt", selection_publish), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._testmon_release_baseline_permission", return_value=False), - patch("devtools.verify._warn_low_memory"), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - patch("devtools.verify.worktree_fingerprint", return_value="stable"), - ): - assert main(["--json"]) == 125 - - assert not TESTMON_DATA.exists() - assert not TESTMON_SEED_STAMP.exists() - affected_publish.assert_not_called() - selection_publish.assert_not_called() - assert json.loads(capsys.readouterr().out)["diagnosis"] == "checkout_changed_during_verification" - - -def test_verify_stops_after_failed_heavy_step(capsys: pytest.CaptureFixture[str]) -> None: - calls: list[str] = [] - - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - calls.append(label) - return (1 if label.startswith("pytest") else 0), 0.01, {} + events.append("monitor-finished") + return CheckoutMutationObservation(changed=False, unavailable=False) with ( - patch("devtools.verify._run", side_effect=fake_run), - patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), + patch("devtools.verify.CheckoutMutationMonitor", _ExceptionalExitMonitor), + patch("devtools.verify._git_head", side_effect=_git_head), + patch("devtools.verify.worktree_fingerprint", side_effect=_worktree_fingerprint), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), patch("devtools.verify._notify"), - patch("devtools.verify._testmon_preflight", return_value=None), ): - rc = main(["--json"]) - - assert rc == 1 - assert calls[-1].startswith("pytest") - payload = capsys.readouterr().out - assert '"exit_code": 1' in payload + assert main(["--quick", "--json"]) == expected_exit + + assert events == [ + "initial-head", + "monitor-started", + "startup-fingerprint", + "final-head", + "final-fingerprint", + "monitor-finished", + ] + assert history["diagnosis"] == expected_diagnosis + assert history["final_git_head"] == "final-head" + assert history["final_worktree_fingerprint"] == "final-fingerprint" + payload = json.loads(capsys.readouterr().out) + assert payload["diagnosis"] == expected_diagnosis + assert payload["final_git_head"] == "final-head" + assert payload["final_worktree_fingerprint"] == "final-fingerprint" -@pytest.mark.parametrize( - ("shard_results", "expected_exit", "expected_diagnosis", "expected_statuses"), - [ - ( - [(124, "pytest_timeout"), (0, "pytest_passed")], - 124, - "pytest_timeout", - ["incomplete", "pending"], - ), - ( - [(1, "pytest_failed"), (0, "pytest_passed")], - 1, - "pytest_failed", - ["complete", "complete"], - ), - ( - [(1, "pytest_failed"), (124, "pytest_timeout")], - 124, - "pytest_timeout", - ["complete", "incomplete"], - ), - ( - [(1, "pytest_failed"), (0, "pytest_passed")], - 1, - "pytest_failed", - ["incomplete", "pending"], - ), - ], -) -def test_seed_testmon_stops_only_after_infrastructure_failed_shard( +def test_import_guard_failure_writes_normalized_history_and_invocation_receipt( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], - shard_results: list[tuple[int, str]], - expected_exit: int, - expected_diagnosis: str, - expected_statuses: list[str], ) -> None: - nodeids = ["tests/test_seed.py::test_one", "tests/test_seed.py::test_two"] - collection_dir = tmp_path / "collection" - collection_dir.mkdir() - (collection_dir / "selection.json").write_text( - json.dumps( - { - "selected_count": len(nodeids), - "selected_nodeids": nodeids, - "selected_nodeids_omitted": 0, - } - ) - ) - calls: list[str] = [] - checkpointed: list[int] = [] - finalized_shard_statuses: list[str] = [] - - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - del command, kwargs - calls.append(label) - if label == "pytest seed-testmon collect": - return 0, 0.01, {"artifact_dir": str(collection_dir)} - if label.startswith("pytest seed-testmon shard "): - shard_index = int(label.rsplit(" ", 1)[1].split("/", 1)[0]) - shard_exit, diagnosis = shard_results[shard_index - 1] - return shard_exit, 0.01, {"diagnosis": diagnosis} - pytest.fail(f"unexpected seed step: {label}") - - def fake_checkpoint(*, prepared: dict[str, object], shard_index: int, step: dict[str, object]) -> dict[str, object]: - del step - checkpointed.append(shard_index) - raw_shards = prepared["shards"] - assert isinstance(raw_shards, list) - assert all(isinstance(shard, dict) for shard in raw_shards) - shards = [dict(shard) for shard in raw_shards] - shards[shard_index - 1]["status"] = expected_statuses[shard_index - 1] - return {**prepared, "shards": shards} - - def fake_finalize( - *, prepared: dict[str, object], step_results: list[dict[str, object]], exit_code: int - ) -> dict[str, object]: - del step_results - assert exit_code == expected_exit - raw_shards = prepared["shards"] - assert isinstance(raw_shards, list) - assert all(isinstance(shard, dict) for shard in raw_shards) - finalized_shard_statuses.extend(str(shard["status"]) for shard in raw_shards) - return { - "status": "incomplete" if "incomplete" in expected_statuses else "complete", - "outcome": "resource_timeout" if expected_exit == 124 else "red-baseline", - "resume": False, - "expected_count": len(nodeids), - "release_baseline_allowed": False, - } + history: dict[str, Any] = {} + receipt = tmp_path / "invocation-receipt.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "import-mismatch") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) - monkeypatch.setattr(verify, "TESTMON_SEED_SHARD_SIZE", 1) with ( - patch("devtools.verify._anchor_verification_paths"), - patch("devtools.verify.maybe_bootstrap_testmon_seed", return_value=None), - patch("devtools.verify._run", side_effect=fake_run), - patch( - "devtools.verify.build_verify_steps", - return_value=[("pytest seed-testmon collect", ["pytest", "--collect-only"])], - ), patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_committed_tree", return_value="tree"), patch( - "devtools.verify._testmon_seed_identity", - return_value={"git_head": "head", "git_tree": "tree", "skip_slow": False, "lab": False}, + "devtools.verify.assert_polylogue_matches_checkout", + side_effect=CheckoutImportMismatchError("wrong checkout import"), ), - patch("devtools.verify._testmon_seed_can_resume", return_value=False), - patch("devtools.verify._checkpoint_testmon_seed_shard", side_effect=fake_checkpoint), - patch("devtools.verify._finalize_testmon_seed_attempt", side_effect=fake_finalize), - patch("devtools.verify._testmon_release_baseline_permission", return_value=False), - patch("devtools.verify._warn_low_memory"), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), patch("devtools.verify._notify"), ): - rc = main(["--seed-testmon", "--json"]) + assert main(["--quick", "--json"]) == 125 - assert rc == expected_exit - executed_shards = sum(status != "pending" for status in expected_statuses) - assert calls == [ - "pytest seed-testmon collect", - *(f"pytest seed-testmon shard {index}/2" for index in range(1, executed_shards + 1)), - ] - assert checkpointed == list(range(1, executed_shards + 1)) - assert finalized_shard_statuses == expected_statuses - output = json.loads(capsys.readouterr().out) - assert output["exit_code"] == expected_exit - assert output["diagnosis"] == expected_diagnosis + normalized = verify_runs.normalize_verify_history_entry(history) + assert normalized["timestamp"] + assert normalized["pytest_aggregate"]["selection_mode"] == "none" + assert normalized["diagnosis"] == "checkout_import_mismatch" + receipt_payload = json.loads(receipt.read_text()) + assert receipt_payload["diagnosis"] == "checkout_import_mismatch" + assert receipt_payload["pytest_aggregate"] == history["pytest_aggregate"] + assert receipt_payload["pytest_aggregate"]["deadline"] == {"budget_s": 3600.0, "met": True} + assert json.loads(capsys.readouterr().out)["diagnosis"] == "checkout_import_mismatch" -@pytest.mark.parametrize( - ("argv", "expected_scope", "expected_permission"), - [ - (["--all", "--skip-slow"], "narrow-terminal", False), - (["--all", "--skip-slow", "--terminal-authorization", "narrow-terminal"], "narrow-terminal", True), - ], -) -def test_verify_main_types_skip_slow_terminal_authority( - capsys: pytest.CaptureFixture[str], argv: list[str], expected_scope: str, expected_permission: bool +def test_git_authority_failure_writes_history_and_invocation_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - del label, command, kwargs - return 0, 0.01, {} + events: list[str] = [] - with ( - patch("devtools.verify._run", side_effect=fake_run), - patch("devtools.verify.build_verify_steps", return_value=[("pytest full", ["pytest"])]), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._save_history") as save_history, - patch("devtools.verify._stamp_head"), - patch("devtools.verify._notify"), - ): - assert main([*argv, "--json"]) == 0 + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass - payload = json.loads(capsys.readouterr().out) - assert payload["verification_scope"] == expected_scope - assert payload["release_baseline_allowed"] is expected_permission - assert payload["terminal_authorization"] == ("narrow-terminal" if expected_permission else None) - assert save_history.call_args.args[0]["checkout_root"] == str(ROOT.resolve()) + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + events.append("finish") + return CheckoutMutationObservation(changed=False, unavailable=False) + def git_head() -> str: + events.append("head") + return "head" + + def fingerprint(_root: Path) -> str: + events.append("fingerprint") + return "stable" + + history: dict[str, Any] = {} + receipt = tmp_path / "invocation-receipt.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "git-authority") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) -def test_verify_refuses_unbudgeted_pytest_before_running_steps(capsys: pytest.CaptureFixture[str]) -> None: with ( - patch("devtools.verify.build_verify_steps", side_effect=PytestResourceError("only 0.50 GiB available")), - patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._run") as run, - patch("devtools.verify._save_history") as save_history, + patch("devtools.verify._git_head", side_effect=git_head), + patch("devtools.verify._git_commit", return_value=None), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", side_effect=fingerprint), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), + patch("devtools.verify._notify"), ): - rc = main(["--json"]) + assert main(["--json"]) == 125 - assert rc == 125 - run.assert_not_called() - assert save_history.call_args.args[0]["diagnosis"] == "pytest_resource_preflight_failed" - assert "only 0.50 GiB available" in capsys.readouterr().err + assert history["diagnosis"] == "native_git_authority_unavailable" + assert history["final_worktree_fingerprint"] == "stable" + assert events[-3:] == ["head", "fingerprint", "finish"] + assert json.loads(receipt.read_text())["diagnosis"] == "native_git_authority_unavailable" + assert json.loads(capsys.readouterr().out)["diagnosis"] == "native_git_authority_unavailable" -def test_verify_starts_checkout_monitor_before_broad_change_classification( +def test_native_preparation_uses_invocation_deadline_and_records_expiry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - events: list[str] = [] - - class _OrderingMonitor: + class _StableMonitor: def __init__(self, _root: Path) -> None: pass def start(self) -> None: - events.append("monitor-started") + pass def finish(self) -> CheckoutMutationObservation: - events.append("monitor-finished") return CheckoutMutationObservation(changed=False, unavailable=False) - def classify(_base: str, _head: str) -> bool: - assert events == ["monitor-started"] - events.append("classified") - return False + history: dict[str, Any] = {} + receipt = tmp_path / "invocation-receipt.json" + monkeypatch.setenv(verify_runs.VERIFICATION_INVOCATION_ID_ENV, "preparation-deadline") + monkeypatch.setenv(verify_runs.VERIFICATION_RECEIPT_PATH_ENV, str(receipt)) + monkeypatch.setattr(verify, "VERIFY_INVOCATION_BUDGET_S", 42.0) + + def expire_preparation(*_args: object, **kwargs: object) -> object: + assert kwargs["deadline_monotonic"] == 142.0 + raise NativeTestmonDeadlineError("verify invocation deadline expired during native testmon preparation") with ( - patch("devtools.verify.CheckoutMutationMonitor", _OrderingMonitor), + patch("devtools.verify.time.monotonic", return_value=100.0), patch("devtools.verify._git_head", return_value="head"), patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", side_effect=expire_preparation), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), patch("devtools.verify.worktree_fingerprint", return_value="stable"), - patch("devtools.verify._default_testmon_is_broad_change", side_effect=classify), - patch("devtools.verify.build_verify_steps", return_value=[]), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._testmon_release_baseline_permission", return_value=False), - patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), + patch("devtools.verify._save_history", side_effect=lambda entry: history.update(entry)), patch("devtools.verify._notify"), ): - assert main(["--json"]) == 0 + assert main(["--json"]) == 124 - assert events == ["monitor-started", "classified", "monitor-finished"] - assert json.loads(capsys.readouterr().out)["exit_code"] == 0 + normalized = verify_runs.normalize_verify_history_entry(history) + assert normalized["diagnosis"] == "verify_invocation_deadline_exceeded" + assert normalized["pytest_aggregate"]["deadline"] == {"budget_s": 42.0, "met": False} + assert json.loads(receipt.read_text())["exit_code"] == 124 + assert json.loads(capsys.readouterr().out)["exit_code"] == 124 -def test_verify_finalizes_checkout_monitor_when_startup_fingerprint_raises() -> None: - events: list[str] = [] +def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(ROOT / "devtools") - class _ExceptionalExitMonitor: + _anchor_verification_paths() + + assert Path.cwd() == ROOT.resolve() + + +def test_failed_step_stop_policy_distinguishes_cheap_and_heavy_steps() -> None: + assert _stop_after_failed_step("ruff check") is False + assert _stop_after_failed_step("verify layering") is False + assert _stop_after_failed_step("pytest native serial (affected)") is False + assert _stop_after_failed_step("lab smoke") is True + assert _stop_after_failed_step("bench slo") is True + assert _native_lane_failure_requires_stop({"exit": 1, "diagnosis": "pytest_failed"}) is False + assert _native_lane_failure_requires_stop({"exit": 2, "diagnosis": "pytest_collection_failed"}) is True + + +def test_verify_continues_serial_lane_after_parallel_test_failure( + capsys: pytest.CaptureFixture[str], +) -> None: + calls: list[str] = [] + lane_timeouts: list[float] = [] + + class _StableMonitor: def __init__(self, _root: Path) -> None: pass def start(self) -> None: - events.append("monitor-started") + pass def finish(self) -> CheckoutMutationObservation: - events.append("monitor-finished") return CheckoutMutationObservation(changed=False, unavailable=False) + preparation = SimpleNamespace( + environment_name="env", + selection_mode="affected", + removed_paths=(), + copied_from=None, + ) + native_state = SimpleNamespace( + valid=True, + status="valid", + environment=SimpleNamespace(nodeids=("tests/test_parallel.py::test_owner",)), + missing_executable_paths=(), + reason="current", + ) + + def fake_run(label: str, _command: list[str], **_kwargs: object) -> tuple[int, float, dict[str, object]]: + calls.append(label) + lane_timeouts.append(cast(float, _kwargs["timeout_s"])) + if "parallel" in label: + return 1, 0.01, {"diagnosis": "pytest_failed"} + return 0, 0.01, {"diagnosis": "pytest_passed"} + with ( - patch("devtools.verify.CheckoutMutationMonitor", _ExceptionalExitMonitor), patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify.worktree_fingerprint", side_effect=RuntimeError("fingerprint failed")), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", return_value=preparation), + patch("devtools.verify._native_environment_after_run", return_value=native_state), + patch( + "devtools.verify.build_verify_steps", + return_value=[ + ("pytest native parallel (affected)", ["pytest"]), + ("pytest native serial (affected)", ["pytest"]), + ], + ), + patch("devtools.verify._run", side_effect=fake_run), + patch("devtools.verify._remaining_invocation_budget", side_effect=(3500.0, 3200.0)), + patch("devtools.verify.aggregate_native_testmon_run", return_value={"terminal_green": False}), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), + patch("devtools.verify._save_history"), + patch("devtools.verify._notify"), ): - with pytest.raises(RuntimeError, match="fingerprint failed"): - main(["--quick", "--json"]) + assert main(["--json"]) == 1 + + assert calls == ["pytest native parallel (affected)", "pytest native serial (affected)"] + assert lane_timeouts == [3500.0, 3200.0] + assert json.loads(capsys.readouterr().out)["release_baseline_allowed"] is False + + +def test_release_authority_requires_current_complete_green_invocation() -> None: + aggregate = { + "complete_corpus_covered": True, + "terminal_green": True, + "external_addopts_neutralized": True, + "external_plugins_neutralized": True, + "closed_world_collection": True, + "cleanup": {"complete": True}, + "containment": {"complete": True}, + "deadline": {"met": True}, + } - assert events == ["monitor-started", "monitor-finished"] + assert _release_baseline_allowed( + selection_mode="bootstrap", + verification_scope=VerificationScope.RELEASE_BASELINE, + exit_code=0, + checkout_stable=True, + aggregate=aggregate, + ) + assert not _release_baseline_allowed( + selection_mode="affected", + verification_scope=VerificationScope.AFFECTED, + exit_code=0, + checkout_stable=True, + aggregate=aggregate, + ) + for broken in ( + {**aggregate, "complete_corpus_covered": False}, + {**aggregate, "terminal_green": False}, + {**aggregate, "external_addopts_neutralized": False}, + {**aggregate, "external_plugins_neutralized": False}, + {**aggregate, "closed_world_collection": False}, + {**aggregate, "cleanup": {"complete": False}}, + {**aggregate, "containment": {"complete": False}}, + {**aggregate, "deadline": {"met": False}}, + ): + assert not _release_baseline_allowed( + selection_mode="full", + verification_scope=VerificationScope.RELEASE_BASELINE, + exit_code=0, + checkout_stable=True, + aggregate=broken, + ) -def test_verify_finalizes_runner_exception_after_open_step( +def test_preparation_mutation_withholds_release_authority_after_restoration( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: - history: dict[str, Any] = {} + observations = iter( + ( + CheckoutMutationObservation(changed=False, unavailable=False), + CheckoutMutationObservation(changed=True, unavailable=False, observed_path="polylogue/module.py"), + CheckoutMutationObservation(changed=False, unavailable=False), + ) + ) + + class _SequencedMonitor: + def __init__(self, _root: Path) -> None: + pass + + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return next(observations) + + preparation = SimpleNamespace( + environment_name="environment", + selection_mode="affected", + removed_paths=(), + copied_from=None, + ) + native_state = SimpleNamespace( + valid=True, + status="valid", + reason="current", + environment=SimpleNamespace(nodeids=("tests/test_owner.py::test_owner",)), + missing_executable_paths=(), + ) + release_aggregate = { + "complete_corpus_covered": True, + "terminal_green": True, + "external_addopts_neutralized": True, + "external_plugins_neutralized": True, + "closed_world_collection": True, + "cleanup": {"complete": True}, + "containment": {"complete": True}, + "deadline": {"met": True}, + } monkeypatch.setattr(verify, "ROOT", tmp_path) - monkeypatch.chdir(tmp_path) monkeypatch.setattr( verify, "assert_polylogue_matches_checkout", lambda *_args, **_kwargs: SimpleNamespace(polylogue_import_path=tmp_path / "polylogue", as_dict=lambda: {}), ) - monkeypatch.setattr(verify, "maybe_bootstrap_testmon_seed", lambda *_args, **_kwargs: None) - monkeypatch.setattr(verify, "_git_head", lambda: "head") - monkeypatch.setattr(verify, "worktree_fingerprint", lambda _root: "stable") - monitor = MagicMock() - monitor.finish.return_value = CheckoutMutationObservation(changed=False, unavailable=False) - monkeypatch.setattr(verify, "CheckoutMutationMonitor", lambda _root: monitor) - monkeypatch.setattr(verify, "_save_history", lambda payload: history.update(payload)) - monkeypatch.setattr(verify, "build_verify_steps", lambda **_kwargs: [("ruff check", ["ruff", "check"])]) - - def explode(_label: str, command: list[str], **kwargs: Any) -> tuple[int, float, dict[str, Any]]: - run = kwargs["run"] - run.start_step(label="ruff check", cmd=command) - raise RuntimeError("verification runner exploded") - - monkeypatch.setattr(verify, "_run", explode) - monotonic_values = iter((100.0, 107.5)) - monkeypatch.setattr("devtools.verify.time.monotonic", lambda: next(monotonic_values)) - - assert verify.main(["--quick", "--json"]) == 125 - assert history["exit_code"] == 125 - assert history["diagnosis"] == "verify_runner_exception" - assert history["duration_s"] == 7.5 - assert history["verification_scope"] == "non-test" - assert history["steps"][0]["status"] == "failed" - assert history["steps"][0]["exit"] == 125 - - -def test_verify_anchors_relative_state_to_checkout_when_invoked_from_subdirectory( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.chdir(ROOT / "devtools") - - _anchor_verification_paths() - - assert Path.cwd() == ROOT.resolve() - - -def test_verify_rejects_zero_testmon_selection_for_executable_change( - capsys: pytest.CaptureFixture[str], -) -> None: - changed_executable_paths = MagicMock(return_value=("polylogue/example.py",)) - - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - del command, kwargs - return 0, 0.01, ({"selected_count": 0} if label.startswith("pytest") else {}) with ( - patch("devtools.verify._run", side_effect=fake_run), patch("devtools.verify._git_head", return_value="head"), - patch("devtools.verify._git_commit", return_value="pinned-base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), + patch("devtools.verify._git_commit", return_value="base"), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", return_value=preparation), + patch("devtools.verify._open_owned_native_testmon_state", return_value=SimpleNamespace(close=lambda: None)), + patch("devtools.verify._native_environment_after_run", return_value=native_state), + patch("devtools.verify.build_verify_steps", return_value=[]), + patch("devtools.verify.aggregate_native_testmon_run", return_value=release_aggregate), + patch("devtools.verify.CheckoutMutationMonitor", _SequencedMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._changed_executable_paths", changed_executable_paths), - patch("devtools.verify._matching_testmon_coverage", return_value=None), ): - rc = main(["--json"]) + assert main(["--all", "--json"]) == 125 - assert rc == 5 payload = json.loads(capsys.readouterr().out) - pytest_step = next(step for step in payload["steps"] if step["name"].startswith("pytest")) - assert pytest_step["diagnosis"] == "zero_testmon_selection_for_executable_change" - assert pytest_step["zero_selection_changed_paths"] == ["polylogue/example.py"] - changed_executable_paths.assert_called_once_with("pinned-base", "head") + assert payload["release_baseline_allowed"] is False + stability = next(step for step in payload["steps"] if step["name"] == "checkout stability") + assert stability["diagnosis"] == "checkout_changed_during_verification" + assert stability["checkout_mutation_path"] == "polylogue/module.py" -def test_verify_finalizes_and_discards_graph_when_post_pytest_path_authority_fails( +def test_collection_failure_still_persists_native_run_aggregate( capsys: pytest.CaptureFixture[str], ) -> None: - monitor = MagicMock() - monitor.finish.return_value = CheckoutMutationObservation(changed=False, unavailable=False) - discard = MagicMock() + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass + def start(self) -> None: + pass + + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + + preparation = SimpleNamespace( + environment_name="env", + selection_mode="bootstrap", + removed_paths=(), + copied_from=None, + ) + invalid_state = SimpleNamespace( + valid=False, + status="invalid", + reason="native environment has no unique collected corpus", + environment=None, + missing_executable_paths=(), + ) with ( - patch("devtools.verify._run", return_value=(0, 0.01, {"selected_count": 1})), - patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), patch("devtools.verify._git_head", return_value="head"), patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), - patch("devtools.verify._changed_executable_paths", side_effect=PytestResourceError("git unavailable")), - patch("devtools.verify._discard_testmon_dependency_authority", discard), - patch("devtools.verify.CheckoutMutationMonitor", return_value=monitor), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", return_value=preparation), + patch("devtools.verify._native_environment_after_run", return_value=invalid_state), + patch( + "devtools.verify.build_verify_steps", + return_value=[("pytest native parallel (bootstrap)", ["pytest"])], + ), + patch( + "devtools.verify._run", + return_value=(2, 0.01, {"diagnosis": "pytest_collection_failed"}), + ), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), patch("devtools.verify.worktree_fingerprint", return_value="stable"), patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), - patch("devtools.verify._testmon_preflight", return_value=None), ): - rc = main(["--json"]) + assert main(["--json"]) == 2 - assert rc == 125 - monitor.finish.assert_called_once_with() - discard.assert_called_once_with() payload = json.loads(capsys.readouterr().out) - assert payload["diagnosis"] == "testmon_changed_path_authority_unavailable" + aggregate = payload["pytest_aggregate"] + assert aggregate["environment"]["status"] == "invalid" + assert aggregate["environment"]["native_corpus_count"] == 0 + assert aggregate["lanes"][0]["lane"] == "parallel" + assert aggregate["cleanup"]["complete"] is False + assert aggregate["containment"]["complete"] is False -def test_testmon_changed_path_authority_refuses_missing_commit_binding() -> None: - changed_paths = MagicMock() +def test_deadline_starts_before_native_preparation_and_fails_closed_after_steps( + capsys: pytest.CaptureFixture[str], +) -> None: + class _StableMonitor: + def __init__(self, _root: Path) -> None: + pass - with patch("devtools.verify._changed_executable_paths", changed_paths): - with pytest.raises(PytestResourceError, match="changed-path authority is unavailable"): - verify._changed_paths_from_testmon_authority(None, "head") + def start(self) -> None: + pass - changed_paths.assert_not_called() + def finish(self) -> CheckoutMutationObservation: + return CheckoutMutationObservation(changed=False, unavailable=False) + preparation = SimpleNamespace( + environment_name="env", + selection_mode="bootstrap", + removed_paths=(), + copied_from=None, + ) + native_state = SimpleNamespace( + valid=True, + status="valid", + reason="current", + environment=SimpleNamespace(nodeids=("tests/test_owner.py::test_owner",)), + missing_executable_paths=(), + ) + clock = iter((100.0, 3701.0, 3701.0, 3701.0)) -def test_verify_accepts_zero_testmon_selection_after_matching_coverage( - capsys: pytest.CaptureFixture[str], -) -> None: - def fake_run(label: str, command: list[str], **kwargs: object) -> tuple[int, float, dict[str, object]]: - del command, kwargs - return 0, 0.01, ({"selected_count": 0} if label.startswith("pytest") else {}) + def prepare(*_args: object, **_kwargs: object) -> object: + assert time.monotonic() == 3701.0 + return preparation with ( - patch("devtools.verify._run", side_effect=fake_run), - patch("devtools.verify.build_verify_steps", return_value=[("pytest testmon", ["pytest"])]), + patch("devtools.verify.time.monotonic", side_effect=lambda: next(clock)), patch("devtools.verify._git_head", return_value="head"), patch("devtools.verify._git_commit", return_value="base"), - patch("devtools.verify._default_testmon_is_broad_change", return_value=False), + patch("devtools.verify._changed_test_relevant_paths", return_value=()), + patch("devtools.verify.prepare_native_testmon_environment", side_effect=prepare), + patch("devtools.verify._native_environment_after_run", return_value=native_state), + patch("devtools.verify.build_verify_steps", return_value=[]), + patch("devtools.verify.CheckoutMutationMonitor", _StableMonitor), + patch("devtools.verify.worktree_fingerprint", return_value="stable"), patch("devtools.verify._save_history"), - patch("devtools.verify._stamp_head"), patch("devtools.verify._notify"), - patch("devtools.verify._testmon_preflight", return_value=None), - patch("devtools.verify._changed_executable_paths", return_value=("polylogue/example.py",)), - patch("devtools.verify._matching_testmon_coverage", return_value="successful_affected_run"), ): - rc = main(["--json"]) + assert main(["--json"]) == 124 - assert rc == 0 payload = json.loads(capsys.readouterr().out) - pytest_step = next(step for step in payload["steps"] if step["name"].startswith("pytest")) - assert pytest_step["zero_selection_coverage"] == "successful_affected_run" - - -def test_testmon_coverage_receipts_are_content_exact() -> None: - paths = ("polylogue/example.py",) - _write_real_testmon_state() - assert _matching_testmon_coverage(paths) is None - - TESTMON_SEED_STAMP.unlink() - with patch("devtools.verify.worktree_fingerprint", return_value="affected"): - _record_testmon_affected_coverage( - executable_paths=paths, - selected_count=3, - run_id="run-1", - ) - assert TESTMON_AFFECTED_STAMP.exists() - assert _matching_testmon_coverage(paths) == "successful_affected_run" - assert _matching_testmon_coverage(("polylogue/other.py",)) is None - - with patch("devtools.verify.worktree_fingerprint", return_value="changed"): - assert _matching_testmon_coverage(paths) is None - - TESTMON_AFFECTED_STAMP.write_text(json.dumps({"identity": {"worktree_fingerprint": "affected"}})) - with patch("devtools.verify.worktree_fingerprint", return_value="affected"): - assert _matching_testmon_coverage(paths) is None - - -def test_failed_step_stop_policy_distinguishes_cheap_and_heavy_steps() -> None: - assert _stop_after_failed_step("ruff check") is False - assert _stop_after_failed_step("verify layering") is False - assert _stop_after_failed_step("pytest testmon") is True - assert _stop_after_failed_step("lab smoke") is True - assert _stop_after_failed_step("bench slo") is True + assert payload["total_duration_s"] == 3601.0 + assert payload["pytest_aggregate"]["deadline"] == {"budget_s": 3600.0, "met": False} + assert any(step["diagnosis"] == "verify_invocation_deadline_exceeded" for step in payload["steps"]) def test_completion_notification_uses_pytest_count() -> None: @@ -5712,34 +4627,6 @@ def test_completion_notification_omits_unknown_pytest_count() -> None: assert summary == "PASS (118s)" -def test_default_testmon_step_pairs_marker_filter_with_forceselect() -> None: - """#1632: any pytest -m marker filter in the default lane MUST be paired with --testmon-forceselect. - - Without ``--testmon-forceselect``, a marker selector deactivates - pytest-testmon's affected-test selection and the run silently - expands to the whole suite — PR #1550 fixed exactly this regression - after a full week of every default verify running 9.5K tests - instead of the affected subset. This invariant is the regression - guard so the footgun cannot re-land silently again. - """ - steps = build_verify_steps(quick=False, lab=False, skip_slow=False) - label, command = steps[-1] - assert label == "pytest testmon" - if "-m" in command: - assert "--testmon-forceselect" in command, ( - f"marker filter without --testmon-forceselect re-introduces the #1550 silent-deselection footgun: {command}" - ) - - -def test_skip_slow_testmon_step_keeps_forceselect_with_compound_marker() -> None: - """``--skip-slow`` composes the marker; the pairing invariant must still hold.""" - steps = build_verify_steps(quick=False, lab=False, skip_slow=True) - label, command = steps[-1] - assert label == "pytest testmon" - assert "-m" in command - assert "--testmon-forceselect" in command - - def test_verify_does_not_notify_on_pass() -> None: """Passing verify runs stay silent — only failures send a desktop popup.""" diff --git a/tests/unit/infra/test_archive_templates.py b/tests/unit/infra/test_archive_templates.py index 60091c87ad..19ab0c1145 100644 --- a/tests/unit/infra/test_archive_templates.py +++ b/tests/unit/infra/test_archive_templates.py @@ -9,10 +9,57 @@ import subprocess import sys from pathlib import Path +from unittest.mock import patch + +import pytest from tests.infra.archive_templates import clone_archive_template, finalize_archive_template +def test_clone_fallback_is_private_writable_and_symlink_safe(tmp_path: Path) -> None: + """The portable fallback preserves links and thaws only the private clone.""" + template = tmp_path / "template" + template.mkdir() + source_file = template / "source.db" + with contextlib.closing(sqlite3.connect(source_file)) as conn, conn: + conn.execute("CREATE TABLE entries (value TEXT)") + conn.execute("INSERT INTO entries VALUES ('immutable-template')") + (template / "source-link.db").symlink_to(source_file.name) + finalize_archive_template(template) + source_bytes = source_file.read_bytes() + + clone = tmp_path / "clone" + with patch("tests.infra.archive_templates.subprocess.run", side_effect=OSError("cp unavailable")): + clone_archive_template(template, clone) + + assert clone.joinpath("source-link.db").is_symlink() + assert clone.joinpath("source.db").stat().st_mode & stat.S_IWUSR + clone.joinpath("source.db").write_bytes(b"private-mutation") + assert source_file.read_bytes() == source_bytes + + +def test_clone_rebinds_durable_bootstrap_identity(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Reusing the source store or global identity would make this reopen unsafe.""" + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + + template = tmp_path / "template" + clone = tmp_path / "clone" + marker = Path(".maintenance-state/durable-change-trains/.bootstrap") + monkeypatch.setattr("polylogue.paths.archive_root", lambda: tmp_path / "configured") + with ArchiveStore(template): + pass + source_identity = template.joinpath(marker).read_bytes() + + clone_archive_template(template, clone) + + assert clone.joinpath(marker).read_bytes() != source_identity + with ArchiveStore(template): + pass + with ArchiveStore(clone): + pass + assert template.joinpath(marker).read_bytes() == source_identity + + def _leave_crash_recovered_wal(database: Path) -> None: writer = subprocess.Popen( [ diff --git a/tests/unit/storage/test_perf_rescue_1314.py b/tests/unit/storage/test_perf_rescue_1314.py index efef2e6900..a85ac58e75 100644 --- a/tests/unit/storage/test_perf_rescue_1314.py +++ b/tests/unit/storage/test_perf_rescue_1314.py @@ -19,6 +19,7 @@ from __future__ import annotations +import shutil from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path @@ -32,6 +33,16 @@ from polylogue.storage.sqlite.queries.stats import get_origin_metrics_rows from tests.benchmarks.helpers import open_bench_store +pytest_plugins = ("tests.benchmarks.conftest",) + + +@pytest.fixture +def isolated_bench_db_1k(tmp_path: Path, bench_db_1k: Path) -> Path: + """Give each test a private copy of the session-scoped benchmark input.""" + index_db = tmp_path / "index.db" + shutil.copy2(bench_db_1k.parent / "index.db", index_db) + return index_db + @contextmanager def _capture_aiosqlite_sql() -> Iterator[list[str]]: @@ -73,12 +84,11 @@ def test_session_insight_rebuild_page_size_is_at_least_50() -> None: # --------------------------------------------------------------------------- -@pytest.mark.scale_small -def test_search_session_hits_uses_freshness_ledger_before_match(tier_small_db: Path) -> None: +def test_search_session_hits_uses_freshness_ledger_before_match(isolated_bench_db_1k: Path) -> None: """Search should not pay archive-scale COUNT(*) probes after daemon readiness.""" from polylogue.storage.fts.freshness import READY, record_fts_surface_state_async - with open_bench_store(tier_small_db) as store: + with open_bench_store(isolated_bench_db_1k) as store: backend = store.backend async def _run(statements: list[str]) -> None: @@ -102,10 +112,9 @@ async def _run(statements: list[str]) -> None: assert all("count(*) from messages where text is not null" not in sql for sql in lowered[:match_index]) -@pytest.mark.scale_small -def test_search_session_hits_falls_back_to_exact_freshness(tier_small_db: Path) -> None: +def test_search_session_hits_falls_back_to_exact_freshness(isolated_bench_db_1k: Path) -> None: """Absent ledger rows fall back to exact FTS verification before MATCH.""" - with open_bench_store(tier_small_db) as store: + with open_bench_store(isolated_bench_db_1k) as store: backend = store.backend async def _run(statements: list[str]) -> None: @@ -132,12 +141,11 @@ async def _run(statements: list[str]) -> None: # --------------------------------------------------------------------------- -@pytest.mark.scale_small -def test_origin_metrics_reads_sessions_aggregates(tier_small_db: Path) -> None: +def test_origin_metrics_reads_sessions_aggregates(isolated_bench_db_1k: Path) -> None: """Origin metrics must source the per-session pre-aggregates from ``sessions`` rather than scanning ``messages``. """ - with open_bench_store(tier_small_db) as store: + with open_bench_store(isolated_bench_db_1k) as store: backend = store.backend async def _run() -> list[dict[str, object]]: @@ -148,7 +156,7 @@ async def _run() -> list[dict[str, object]]: with _capture_aiosqlite_sql() as statements: rows = store.run(_run()) - assert rows, "scale_small fixture should produce at least one origin row" + assert rows, "1k-message benchmark fixture should produce at least one origin row" joined = "\n".join(statements).lower() assert "from sessions" in joined assert "session_stats" not in joined @@ -177,8 +185,7 @@ async def _run() -> list[dict[str, object]]: # --------------------------------------------------------------------------- -@pytest.mark.scale_small -def test_get_messages_hydration_does_not_call_model_copy(tier_small_db: Path) -> None: +def test_get_messages_hydration_does_not_call_model_copy(isolated_bench_db_1k: Path) -> None: """``get_messages`` must mutate the freshly-constructed MessageRecord instances in place rather than calling pydantic's ``model_copy``. @@ -197,7 +204,7 @@ def _spy(self: Any, *args: Any, **kwargs: Any) -> Any: MessageRecord.model_copy = _spy # type: ignore[method-assign] try: - with open_bench_store(tier_small_db) as store: + with open_bench_store(isolated_bench_db_1k) as store: async def _run() -> int: summaries = await store.repository.list_summaries(limit=5) diff --git a/tests/unit/storage/test_store_ops.py b/tests/unit/storage/test_store_ops.py index 6d905b2875..2715f69f34 100644 --- a/tests/unit/storage/test_store_ops.py +++ b/tests/unit/storage/test_store_ops.py @@ -1790,12 +1790,12 @@ async def test_tag_assignment_roundtrip_and_counts( empty_archive_template: Path, ) -> None: """Strategy-generated tags are retrievable and counted consistently.""" - from tests.conftest import _clone_archive_template from tests.infra.archive_scenarios import archive_for_scenario_db, native_session_id_for + from tests.infra.archive_templates import clone_archive_template with tempfile.TemporaryDirectory() as tmp_dir: archive_root = Path(tmp_dir) / "archive" - _clone_archive_template(empty_archive_template, archive_root) + clone_archive_template(empty_archive_template, archive_root) db_path = archive_root / "index.db" seed_session_graph(db_path, spec.sessions) diff --git a/tests/unit/test_pytest_temp_policy.py b/tests/unit/test_pytest_temp_policy.py index 9b4242851e..261b1679e0 100644 --- a/tests/unit/test_pytest_temp_policy.py +++ b/tests/unit/test_pytest_temp_policy.py @@ -556,7 +556,7 @@ def test_explicit_basetemp_claim_survives_real_pytest_basetemp_replacement(tmp_p "pytest", "--basetemp", str(explicit), - "tests/unit/test_pytest_temp_policy.py::test_archive_template_clone_is_private", + "tests/unit/infra/test_archive_templates.py::test_clone_fallback_is_private_writable_and_symlink_safe", ], cwd=repo_root, env=env, @@ -921,33 +921,3 @@ def test_sessionfinish_retains_explicit_diagnostic_basetemp( conftest.pytest_sessionfinish(cast("pytest.Session", session), 1) assert explicit.exists() - - -def test_archive_template_clone_is_private(tmp_path: Path) -> None: - source = tmp_path / "source" - destination = tmp_path / "destination" - source.mkdir() - (source / "index.db").write_bytes(b"immutable-template") - - conftest._clone_archive_template(source, destination) - (destination / "index.db").write_bytes(b"private-mutation") - - assert (source / "index.db").read_bytes() == b"immutable-template" - - -def test_archive_template_clone_rebinds_durable_identity(tmp_path: Path) -> None: - from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore - - source = tmp_path / "source" - destination = tmp_path / "destination" - marker_relative = Path(".maintenance-state/durable-change-trains/.bootstrap") - with ArchiveStore(source): - pass - source_marker = (source / marker_relative).read_bytes() - - conftest._clone_archive_template(source, destination) - - assert (destination / marker_relative).read_bytes() != source_marker - with ArchiveStore(destination): - pass - assert (source / marker_relative).read_bytes() == source_marker diff --git a/uv.lock b/uv.lock index b7077ff4ae..4f0eb1f92e 100644 --- a/uv.lock +++ b/uv.lock @@ -1374,7 +1374,7 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'dev-common'", specifier = ">=4" }, { name = "pytest-json-report", marker = "extra == 'dev-common'", specifier = ">=1.5.0" }, { name = "pytest-randomly", marker = "extra == 'dev-common'", specifier = ">=4.1.0" }, - { name = "pytest-testmon", marker = "extra == 'dev-common'", specifier = ">=2.1.3" }, + { name = "pytest-testmon", marker = "extra == 'dev-common'", specifier = ">=2.2.0,<2.3.0" }, { name = "pytest-timeout", marker = "extra == 'dev-common'", specifier = ">=2.4.0" }, { name = "pytest-xdist", marker = "extra == 'dev-common'", specifier = ">=3.5.0" }, { name = "python-multipart", specifier = ">=0.0.32" },